diff --git a/docs/getting-started/setup.md b/docs/getting-started/setup.md index 7443e2ee..cd0bdacb 100644 --- a/docs/getting-started/setup.md +++ b/docs/getting-started/setup.md @@ -75,7 +75,7 @@ authenticate until some role exists — this token is what breaks that cycle, and it's the only step that needs the root token. ```bash -# Exactly the six grants the bootstrap unit needs, and nothing else. Each was +# Exactly the four grants the bootstrap unit needs, and nothing else. Each was # derived with `bao -output-policy`, which prints what a command requires # without running it. bao policy write swarm-bootstrap - <<'EOF' @@ -96,17 +96,6 @@ path "sys/auth/cert" { path "auth/cert/certs/swarm-controller" { capabilities = ["create", "update"] } - -# The KV engine the controller writes agent credentials through — also absent -# on a fresh store, and checked the same way. No `sudo` here, unlike the auth -# mount above: enabling a secrets engine does not ask for it. -path "sys/mounts" { - capabilities = ["read"] -} - -path "sys/mounts/secret" { - capabilities = ["create", "update"] -} EOF # A token holding it. `-orphan` so it outlives the session that made it. @@ -116,47 +105,23 @@ bao token create -policy=swarm-bootstrap -ttl=24h -orphan -display-name=swarm-bo Put the token's value at `services.hyperhive.deploy.bao.bootstrapTokenFile` (all-local names that path for you), then rebuild. A one-shot unit **on the host** reads it, writes the `swarm-controller` policy, enables the cert auth -method, mounts the KV engine the controller stores credentials in, and creates -the `swarm-controller` role that attaches policy to certificate. It runs there -because every API listener demands a client certificate, and the host is the -side that has one. +method, and creates the `swarm-controller` role that attaches the two. It runs +there because every API listener demands a client certificate, and the host is +the side that has one. -⏱️ **Expect the first attempt to fail if you rebuilt into this.** A rebuild -restarts the store, and the unit races it — the store answers `local node not -active` until it finishes coming up. It retries every 30s and the second -attempt is the one that usually lands. Nothing to do. +⚠️ **The grants above are derived, not proven.** They come from +`-output-policy`, not from a swarm that came up on them. The unit has run twice +against a live store and failed both times for reasons of its own — it could +not reach the store from where it then ran — so neither run exercised these +grants at all. -**Confirm with `systemctl status swarm-bao-controller-policy`**, which wants no -token — a successful run logs `Uploaded policy`, `Enabled cert auth method` and -`Data written to: auth/cert/certs/swarm-controller`. ⚠️ Do _not_ reach for `bao -read auth/cert/…` to check: the host's `bao` wrapper carries an address, a CA -and a client certificate but deliberately **no token**, so that read answers -`403` whether or not the role exists. - -**Delete the token file only once that unit has succeeded.** It skips when the -token is absent, so a host that has finished bootstrapping stops carrying the -credential — but deleting it before the role +**Delete the file only once `bao read auth/cert/certs/swarm-controller` returns +the role.** The unit skips when the token is absent, so a host that has finished +bootstrapping stops carrying the credential — but deleting it before the role exists leaves the unit skipping forever with nothing to show for it, and looks exactly like a store that was never bootstrapped. The TTL above means a forgotten one expires rather than lingering. -
Already bootstrapped before the KV mount existed? - -A store bootstrapped by an earlier version has the policy, the auth method and -the role, but no `secret/` engine — the controller's first credential write -answers `no handler for route "secret/data/…"`. The bootstrap token can't fix it -either: the policy it was minted from names nothing under `sys/mounts`. Mount it -once with the root token from `init`: - -```bash -sudo bash -c 'BAO_TOKEN="" bao secrets enable -path=secret kv-v2' -``` - -No rebuild needed — the unit's own check finds the mount on its next run and -leaves it alone. - -
- Whether anything more is needed depends on `services.hyperhive.deploy.bao.seal`: diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index d5d45470..9d538a65 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -198,18 +198,11 @@ let capabilities = ["list"] } - path "${credentialMountPath}/data/swarm/agents/*" { + path "secret/data/swarm/agents/*" { capabilities = ["create", "update"] } ''; - # The KV v2 engine the controller writes agent credentials through. Named - # once because the grant above and the `secrets enable` in the bootstrap unit - # have to agree: a policy pointing at a mount nobody created is precisely the - # failure this pair exists to avoid. `swarm-secret-client` pins the same - # value on the reading side. - credentialMountPath = "secret"; - # Every listener serves the same identity: they differ in which address # they answer on, not in who they are. Client verification is separate and # optional — a store with no `clientCaFile` still serves TLS, it just does @@ -911,23 +904,6 @@ in # than failing on one that already exists. printf '%s' ${lib.escapeShellArg controllerPolicyText} | bao policy write ${lib.escapeShellArg controllerPolicyName} - - - # The policy grants paths under `${credentialMountPath}/`, and a - # fresh store has no such engine — only a dev-mode one does. Writing - # to an unmounted path answers "route entry not found", so the - # controller's first credential write fails against a grant that - # reads as correct. - # - # Outside the client-CA block below on purpose: this mount is what - # the controller writes *through*, independent of who may log in. - # - # Asked rather than attempted, same as the auth mount: `secrets - # enable` errors on a path already in use. - mounts="$(bao secrets list -format=json)" - case "$mounts" in - *'"${credentialMountPath}/"'*) ;; - *) bao secrets enable -path=${credentialMountPath} kv-v2 ;; - esac '' + lib.optionalString (baoDeploy.clientCaFile != null) '' diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 17e2e210..7a48f032 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -613,22 +613,6 @@ let && lib.hasInfix "auth/cert/certs/swarm-controller" s && lib.hasInfix "/var/lib/swarm-bao-tls/client-ca.pem" s; } - { - # Same shape as the cert mount above, for the engine the controller - # writes credentials through: a fresh store has no `secret/`, so the - # grant would name a mount nobody created and the first write would 404. - # - # ⚠️ Matched on the COMMAND, for the reason the no-client-CA case below - # spells out: the policy text is embedded in this same script and grants - # `secret/data/...`, so any arm keyed on the *path* is satisfied either - # way and could never fail. - name = "the granting unit creates the KV mount the controller writes through"; - ok = - let - s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; - in - lib.hasInfix "bao secrets enable -path=secret kv-v2" s; - } { # Nothing asserted the PKI script before this, so a third leaf could be # added to it and every case still passed — measured, not assumed: the @@ -796,12 +780,7 @@ let in lib.hasInfix "bao policy write" s && !(lib.hasInfix "bao auth enable cert" s) - && !(lib.hasInfix "client-ca.pem" s) - # The KV mount is NOT part of what a missing client CA switches off: - # the controller writes through it whether or not anything can log in - # by certificate. Asserted here rather than trusted, because both - # steps live in the same script and one indentation level decides it. - && lib.hasInfix "bao secrets enable -path=secret kv-v2" s; + && !(lib.hasInfix "client-ca.pem" s); } { # Reads the vhost's rendered `root`, not the option: the UI is served