From 48e6a0b88f7a64cec5d8e6a6448ff03606d7d281 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 11 Sep 2026 00:16:46 +0200 Subject: [PATCH] swarm-bao: create the KV mount the controller writes credentials through MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bootstrap unit writes a policy granting `secret/data/swarm/agents/*` and nothing creates that engine. A fresh OpenBao has no `secret/` — only a dev-mode one does — so `swarm-controller`'s first credential write answers `no handler for route "secret/data/swarm/agents//matrix/". route entry not found.` Measured on the live host at 21:27:27Z; #4171. `git grep` for `secrets enable`, `kv-v2`, `kv_v2` and `sys/mounts` returned zero across the whole tree. Control, so the zero means something: `auth enable` in this same file returns 2 — the same defect was already found and fixed once, for the cert auth mount, with a comment that states the principle. This is the other half of it. The mount name is now bound once and interpolated into both the policy text and the new step, because a grant and a mount that disagree is exactly the failure being fixed. Placed outside the client-CA block: the controller writes *through* this mount regardless of whether anything can log in by certificate. `module-eval` asserts that, since one indentation level decides it. Grants, measured against a real openbao 2.6.2 rather than derived: `-output-policy` asks for `sys/mounts/secret` create+update, and a token holding exactly `sys/mounts` read + `sys/mounts/` create/update enabled the engine — **no `sudo`**, unlike `sys/auth/cert`. Negative control: the same token on an ungranted path got 403, so the grant is what made it work. `setup.md`'s documented policy gains those two. Also from that session, each deciding how this is written: re-enabling an existing path errors (exit 2), so this asks first like the auth mount does; `secrets list -format=json` keys look like `"secret/"`, so the `case` idiom ports over; and `kv put -mount=

` reports `

/data/...`, confirming v2 — the prefix the policy grants and the client writes. setup.md also drops a check that cannot work: it told the operator to confirm with `bao read auth/cert/…`, which 403s because the host wrapper carries no token. `systemctl status swarm-bao-controller-policy` needs no credential and names the three success lines. The first-attempt-after-rebuild race is now written down too — the store is still coming up, and the 30s retry is what lands. Refs #4171. --- docs/getting-started/setup.md | 59 +++++++++++++++++++++++++++------- nix/host-modules/swarm-bao.nix | 26 ++++++++++++++- nix/module-eval.nix | 23 ++++++++++++- 3 files changed, 94 insertions(+), 14 deletions(-) diff --git a/docs/getting-started/setup.md b/docs/getting-started/setup.md index cd0bdacb..136385e1 100644 --- a/docs/getting-started/setup.md +++ b/docs/getting-started/setup.md @@ -75,7 +75,7 @@ authenticate until some role exists — this token is what breaks that cycle, and it's the only step that needs the root token. ```bash -# Exactly the four grants the bootstrap unit needs, and nothing else. Each was +# Exactly the six grants the bootstrap unit needs, and nothing else. Each was # derived with `bao -output-policy`, which prints what a command requires # without running it. bao policy write swarm-bootstrap - <<'EOF' @@ -96,6 +96,17 @@ path "sys/auth/cert" { path "auth/cert/certs/swarm-controller" { capabilities = ["create", "update"] } + +# The KV engine the controller writes agent credentials through — also absent +# on a fresh store, and checked the same way. No `sudo` here, unlike the auth +# mount above: enabling a secrets engine does not ask for it. +path "sys/mounts" { + capabilities = ["read"] +} + +path "sys/mounts/secret" { + capabilities = ["create", "update"] +} EOF # A token holding it. `-orphan` so it outlives the session that made it. @@ -105,23 +116,47 @@ bao token create -policy=swarm-bootstrap -ttl=24h -orphan -display-name=swarm-bo Put the token's value at `services.hyperhive.deploy.bao.bootstrapTokenFile` (all-local names that path for you), then rebuild. A one-shot unit **on the host** reads it, writes the `swarm-controller` policy, enables the cert auth -method, and creates the `swarm-controller` role that attaches the two. It runs -there because every API listener demands a client certificate, and the host is -the side that has one. +method, mounts the KV engine the controller stores credentials in, and creates +the `swarm-controller` role that attaches policy to certificate. It runs there +because every API listener demands a client certificate, and the host is the +side that has one. -⚠️ **The grants above are derived, not proven.** They come from -`-output-policy`, not from a swarm that came up on them. The unit has run twice -against a live store and failed both times for reasons of its own — it could -not reach the store from where it then ran — so neither run exercised these -grants at all. +⏱️ **Expect the first attempt to fail if you rebuilt into this.** A rebuild +restarts the store, and the unit races it — the store answers `local node not +active` until it finishes coming up. It retries every 30s and the second +attempt is the one that usually lands. Nothing to do. -**Delete the file only once `bao read auth/cert/certs/swarm-controller` returns -the role.** The unit skips when the token is absent, so a host that has finished -bootstrapping stops carrying the credential — but deleting it before the role +**Confirm with `systemctl status swarm-bao-controller-policy`**, which wants no +token — a successful run logs `Uploaded policy`, `Enabled cert auth method` and +`Data written to: auth/cert/certs/swarm-controller`. ⚠️ Do _not_ reach for `bao +read auth/cert/…` to check: the host's `bao` wrapper carries an address, a CA +and a client certificate but deliberately **no token**, so that read answers +`403` whether or not the role exists. + +**Delete the token file only once that unit has succeeded.** It skips when the +token is absent, so a host that has finished bootstrapping stops carrying the +credential — but deleting it before the role exists leaves the unit skipping forever with nothing to show for it, and looks exactly like a store that was never bootstrapped. The TTL above means a forgotten one expires rather than lingering. +

Already bootstrapped before the KV mount existed? + +A store bootstrapped by an earlier version has the policy, the auth method and +the role, but no `secret/` engine — the controller's first credential write +answers `no handler for route "secret/data/…"`. The bootstrap token cannot fix +it either: the policy it was minted from names nothing under `sys/mounts`. Mount +it once with the root token from `init`: + +```bash +sudo bash -c 'BAO_TOKEN="" bao secrets enable -path=secret kv-v2' +``` + +No rebuild needed — the unit's own check finds the mount on its next run and +leaves it alone. + +
+ Whether anything more is needed depends on `services.hyperhive.deploy.bao.seal`: diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 9d538a65..d5d45470 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -198,11 +198,18 @@ let capabilities = ["list"] } - path "secret/data/swarm/agents/*" { + path "${credentialMountPath}/data/swarm/agents/*" { capabilities = ["create", "update"] } ''; + # The KV v2 engine the controller writes agent credentials through. Named + # once because the grant above and the `secrets enable` in the bootstrap unit + # have to agree: a policy pointing at a mount nobody created is precisely the + # failure this pair exists to avoid. `swarm-secret-client` pins the same + # value on the reading side. + credentialMountPath = "secret"; + # Every listener serves the same identity: they differ in which address # they answer on, not in who they are. Client verification is separate and # optional — a store with no `clientCaFile` still serves TLS, it just does @@ -904,6 +911,23 @@ in # than failing on one that already exists. printf '%s' ${lib.escapeShellArg controllerPolicyText} | bao policy write ${lib.escapeShellArg controllerPolicyName} - + + # The policy grants paths under `${credentialMountPath}/`, and a + # fresh store has no such engine — only a dev-mode one does. Writing + # to an unmounted path answers "route entry not found", so the + # controller's first credential write fails against a grant that + # reads as correct. + # + # Outside the client-CA block below on purpose: this mount is what + # the controller writes *through*, independent of who may log in. + # + # Asked rather than attempted, same as the auth mount: `secrets + # enable` errors on a path already in use. + mounts="$(bao secrets list -format=json)" + case "$mounts" in + *'"${credentialMountPath}/"'*) ;; + *) bao secrets enable -path=${credentialMountPath} kv-v2 ;; + esac '' + lib.optionalString (baoDeploy.clientCaFile != null) '' diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 7a48f032..17e2e210 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -613,6 +613,22 @@ let && lib.hasInfix "auth/cert/certs/swarm-controller" s && lib.hasInfix "/var/lib/swarm-bao-tls/client-ca.pem" s; } + { + # Same shape as the cert mount above, for the engine the controller + # writes credentials through: a fresh store has no `secret/`, so the + # grant would name a mount nobody created and the first write would 404. + # + # ⚠️ Matched on the COMMAND, for the reason the no-client-CA case below + # spells out: the policy text is embedded in this same script and grants + # `secret/data/...`, so any arm keyed on the *path* is satisfied either + # way and could never fail. + name = "the granting unit creates the KV mount the controller writes through"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "bao secrets enable -path=secret kv-v2" s; + } { # Nothing asserted the PKI script before this, so a third leaf could be # added to it and every case still passed — measured, not assumed: the @@ -780,7 +796,12 @@ let in lib.hasInfix "bao policy write" s && !(lib.hasInfix "bao auth enable cert" s) - && !(lib.hasInfix "client-ca.pem" s); + && !(lib.hasInfix "client-ca.pem" s) + # The KV mount is NOT part of what a missing client CA switches off: + # the controller writes through it whether or not anything can log in + # by certificate. Asserted here rather than trusted, because both + # steps live in the same script and one indentation level decides it. + && lib.hasInfix "bao secrets enable -path=secret kv-v2" s; } { # Reads the vhost's rendered `root`, not the option: the UI is served