diff --git a/docs/getting-started/setup.md b/docs/getting-started/setup.md index cd0bdacb..136385e1 100644 --- a/docs/getting-started/setup.md +++ b/docs/getting-started/setup.md @@ -75,7 +75,7 @@ authenticate until some role exists — this token is what breaks that cycle, and it's the only step that needs the root token. ```bash -# Exactly the four grants the bootstrap unit needs, and nothing else. Each was +# Exactly the six grants the bootstrap unit needs, and nothing else. Each was # derived with `bao -output-policy`, which prints what a command requires # without running it. bao policy write swarm-bootstrap - <<'EOF' @@ -96,6 +96,17 @@ path "sys/auth/cert" { path "auth/cert/certs/swarm-controller" { capabilities = ["create", "update"] } + +# The KV engine the controller writes agent credentials through — also absent +# on a fresh store, and checked the same way. No `sudo` here, unlike the auth +# mount above: enabling a secrets engine does not ask for it. +path "sys/mounts" { + capabilities = ["read"] +} + +path "sys/mounts/secret" { + capabilities = ["create", "update"] +} EOF # A token holding it. `-orphan` so it outlives the session that made it. @@ -105,23 +116,47 @@ bao token create -policy=swarm-bootstrap -ttl=24h -orphan -display-name=swarm-bo Put the token's value at `services.hyperhive.deploy.bao.bootstrapTokenFile` (all-local names that path for you), then rebuild. A one-shot unit **on the host** reads it, writes the `swarm-controller` policy, enables the cert auth -method, and creates the `swarm-controller` role that attaches the two. It runs -there because every API listener demands a client certificate, and the host is -the side that has one. +method, mounts the KV engine the controller stores credentials in, and creates +the `swarm-controller` role that attaches policy to certificate. It runs there +because every API listener demands a client certificate, and the host is the +side that has one. -⚠️ **The grants above are derived, not proven.** They come from -`-output-policy`, not from a swarm that came up on them. The unit has run twice -against a live store and failed both times for reasons of its own — it could -not reach the store from where it then ran — so neither run exercised these -grants at all. +⏱️ **Expect the first attempt to fail if you rebuilt into this.** A rebuild +restarts the store, and the unit races it — the store answers `local node not +active` until it finishes coming up. It retries every 30s and the second +attempt is the one that usually lands. Nothing to do. -**Delete the file only once `bao read auth/cert/certs/swarm-controller` returns -the role.** The unit skips when the token is absent, so a host that has finished -bootstrapping stops carrying the credential — but deleting it before the role +**Confirm with `systemctl status swarm-bao-controller-policy`**, which wants no +token — a successful run logs `Uploaded policy`, `Enabled cert auth method` and +`Data written to: auth/cert/certs/swarm-controller`. ⚠️ Do _not_ reach for `bao +read auth/cert/…` to check: the host's `bao` wrapper carries an address, a CA +and a client certificate but deliberately **no token**, so that read answers +`403` whether or not the role exists. + +**Delete the token file only once that unit has succeeded.** It skips when the +token is absent, so a host that has finished bootstrapping stops carrying the +credential — but deleting it before the role exists leaves the unit skipping forever with nothing to show for it, and looks exactly like a store that was never bootstrapped. The TTL above means a forgotten one expires rather than lingering. +
Already bootstrapped before the KV mount existed? + +A store bootstrapped by an earlier version has the policy, the auth method and +the role, but no `secret/` engine — the controller's first credential write +answers `no handler for route "secret/data/…"`. The bootstrap token cannot fix +it either: the policy it was minted from names nothing under `sys/mounts`. Mount +it once with the root token from `init`: + +```bash +sudo bash -c 'BAO_TOKEN="" bao secrets enable -path=secret kv-v2' +``` + +No rebuild needed — the unit's own check finds the mount on its next run and +leaves it alone. + +
+ Whether anything more is needed depends on `services.hyperhive.deploy.bao.seal`: diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 9d538a65..d5d45470 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -198,11 +198,18 @@ let capabilities = ["list"] } - path "secret/data/swarm/agents/*" { + path "${credentialMountPath}/data/swarm/agents/*" { capabilities = ["create", "update"] } ''; + # The KV v2 engine the controller writes agent credentials through. Named + # once because the grant above and the `secrets enable` in the bootstrap unit + # have to agree: a policy pointing at a mount nobody created is precisely the + # failure this pair exists to avoid. `swarm-secret-client` pins the same + # value on the reading side. + credentialMountPath = "secret"; + # Every listener serves the same identity: they differ in which address # they answer on, not in who they are. Client verification is separate and # optional — a store with no `clientCaFile` still serves TLS, it just does @@ -904,6 +911,23 @@ in # than failing on one that already exists. printf '%s' ${lib.escapeShellArg controllerPolicyText} | bao policy write ${lib.escapeShellArg controllerPolicyName} - + + # The policy grants paths under `${credentialMountPath}/`, and a + # fresh store has no such engine — only a dev-mode one does. Writing + # to an unmounted path answers "route entry not found", so the + # controller's first credential write fails against a grant that + # reads as correct. + # + # Outside the client-CA block below on purpose: this mount is what + # the controller writes *through*, independent of who may log in. + # + # Asked rather than attempted, same as the auth mount: `secrets + # enable` errors on a path already in use. + mounts="$(bao secrets list -format=json)" + case "$mounts" in + *'"${credentialMountPath}/"'*) ;; + *) bao secrets enable -path=${credentialMountPath} kv-v2 ;; + esac '' + lib.optionalString (baoDeploy.clientCaFile != null) '' diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 7a48f032..17e2e210 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -613,6 +613,22 @@ let && lib.hasInfix "auth/cert/certs/swarm-controller" s && lib.hasInfix "/var/lib/swarm-bao-tls/client-ca.pem" s; } + { + # Same shape as the cert mount above, for the engine the controller + # writes credentials through: a fresh store has no `secret/`, so the + # grant would name a mount nobody created and the first write would 404. + # + # ⚠️ Matched on the COMMAND, for the reason the no-client-CA case below + # spells out: the policy text is embedded in this same script and grants + # `secret/data/...`, so any arm keyed on the *path* is satisfied either + # way and could never fail. + name = "the granting unit creates the KV mount the controller writes through"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "bao secrets enable -path=secret kv-v2" s; + } { # Nothing asserted the PKI script before this, so a third leaf could be # added to it and every case still passed — measured, not assumed: the @@ -780,7 +796,12 @@ let in lib.hasInfix "bao policy write" s && !(lib.hasInfix "bao auth enable cert" s) - && !(lib.hasInfix "client-ca.pem" s); + && !(lib.hasInfix "client-ca.pem" s) + # The KV mount is NOT part of what a missing client CA switches off: + # the controller writes through it whether or not anything can log in + # by certificate. Asserted here rather than trusted, because both + # steps live in the same script and one indentation level decides it. + && lib.hasInfix "bao secrets enable -path=secret kv-v2" s; } { # Reads the vhost's rendered `root`, not the option: the UI is served