From e9cec0da2123c62ee7b96408d3b688e9881198aa Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 27 Sep 2026 03:33:38 +0200 Subject: [PATCH] swarm-bao: write every swarm-* grant as a bao granter, not with a 24h token Every unit that writes a bao policy or cert-auth role ran only while the operator-placed bootstrap token existed, and skipped silently otherwise. The token lives 24h, so on any real swarm a PR adding or changing a grant deployed with its unit skipped, and each one needed a manual token refresh (plus a root `bao policy write` when it added a path). A `bao-granter` principal now writes them. Its leaf is minted by swarm-bao-pki on the store host (0600 root, never copied off it), and its policy covers `swarm-*` policies, `swarm-*` cert-auth roles and `pki/roles/swarm-*` by glob, plus the mount and services-root paths the controller's unit already used. All ten granting units (controller, secret-publisher, matrix-ctl, matrix-token, queue-agent, grafana-oidc, otel-oidc, forwarder-oidc, services-issuer, nats-tls) log in with it instead of reading the token. They keep the 2880 x 30s retry, now require swarm-bao-pki, and when the store refuses the granter they fail and print the one-time step instead of skipping. swarm-bao-granter-role is the one unit left on the token. It enables the auth mounts (moved out of the controller's unit) and writes the granter's own policy and role. The bootstrap policy is renamed `bao-bootstrap` and shrinks to those five stanzas; it is shipped at /etc/hyperhive/bao-bootstrap-policy.hcl. The old name `swarm-bootstrap` matched the granter's own `swarm-*` glob. The granter's CN joins certAuthCns, so no hive can be named into its role. An assertion keeps both pki role names under `swarm-`. With no client CA the granting units no longer render, and a warning says so. module-eval pins the granter's policy stanza by stanza, what it cannot reach, that every call a granting unit makes is granted, and that only swarm-bao-granter-role reads the token. Refs #4704 --- docs/getting-started/setup.md | 128 +++-- nix/host-modules/bao-bootstrap-policy.hcl | 37 ++ .../glue-bao-readers-policy-order.nix | 7 +- nix/host-modules/glue-bao-tls.nix | 12 + .../swarm-bao-bootstrap-policy.hcl | 159 ------ nix/host-modules/swarm-bao.nix | 426 ++++++++++---- nix/host-modules/swarm.nix | 1 + nix/module-eval/bao-grants.nix | 535 +++++++++++++----- nix/module-eval/hive-tls.nix | 6 +- nix/module-eval/name-guards.nix | 17 + nix/module-eval/nats-tls.nix | 8 +- nix/module-eval/swarm-otel-core.nix | 1 + 12 files changed, 896 insertions(+), 441 deletions(-) create mode 100644 nix/host-modules/bao-bootstrap-policy.hcl delete mode 100644 nix/host-modules/swarm-bao-bootstrap-policy.hcl diff --git a/docs/getting-started/setup.md b/docs/getting-started/setup.md index 28457c02..b00365c6 100644 --- a/docs/getting-started/setup.md +++ b/docs/getting-started/setup.md @@ -88,66 +88,108 @@ its DNS name resolves to the bridge from in there: export domain>` to verify the name while connecting on loopback. The host is the shorter path. -While you still hold that root token, mint the one credential the swarm needs -to grant itself anything. Cert auth answers a _role_, so nothing can -authenticate until some role exists — this token is what breaks that cycle, -and it's the only step that needs the root token. +While you still hold that root token, set up the **granter**: the one principal +that writes every `swarm-*` policy and role from then on. Cert auth answers a +_role_, so nothing can authenticate until some role exists. A short-lived +bootstrap token breaks that cycle once, and it's the only step that needs the +root token. -The policy is `nix/host-modules/swarm-bao-bootstrap-policy.hcl` in this -repository, and CI fails when a unit using the token needs a path it lacks. +The policy it carries is `nix/host-modules/bao-bootstrap-policy.hcl`, shipped +on the store's host at `/etc/hyperhive/bao-bootstrap-policy.hcl`. It covers +the auth mounts and the granter's own policy and role, and nothing else. CI +fails when the unit using the token needs a path it lacks. ```bash -# The policy file, copied to wherever you run `bao`. -bao policy write swarm-bootstrap swarm-bao-bootstrap-policy.hcl - -# A token holding it. `-orphan` so it outlives the session that made it. -bao token create -policy=swarm-bootstrap -ttl=24h -orphan -display-name=swarm-bootstrap +sudo -i +read -rs BAO_TOKEN && export BAO_TOKEN # paste the root token from `bao operator init` +bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl +bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token \ + | install -D -m 0600 /dev/stdin /var/lib/swarm-bao-bootstrap/grant.token +unset BAO_TOKEN +systemctl restart swarm-bao-granter-role ``` -Put the token's value at `services.hyperhive.deploy.bao.bootstrapTokenFile` -(all-local names that path for you), then rebuild. A one-shot unit **on the -host** reads it, writes the `swarm-controller` policy, enables the cert auth -method, mounts the KV engine the controller stores credentials in, and creates -the `swarm-controller` role that attaches policy to certificate. It runs there -because every API listener demands a client certificate, and the host is the -side that has one. +The token file is `services.hyperhive.deploy.bao.bootstrapTokenFile`, which +all-local names for you. On a store host that isn't all-local, set it and +rebuild first. + +`swarm-bao-granter-role` runs **on the host**. It enables the cert auth +method, writes the `bao-granter` policy, and creates the `bao-granter` role, +which accepts the leaf `/var/lib/swarm-bao-pki/granter.pem`. Every +`swarm-bao-*-policy` unit then logs in with that leaf. The controller's unit +mounts the KV and pki engines and writes the `swarm-controller` role, and each +sibling unit writes its own principal's policy and role. Every one runs on the +host, because every API listener demands a client certificate and the host is +the side that has one. + +**Confirm with `systemctl status swarm-bao-granter-role`**, which should log +`Uploaded policy: bao-granter` and `Data written to: auth/cert/certs/bao-granter`. +Then restart the granting units that failed while they waited: + +```bash +systemctl reset-failed 'swarm-bao-*-policy.service' +systemctl restart 'swarm-bao-*-policy.service' +systemctl status swarm-bao-controller-policy # Uploaded policy, Data written to: auth/cert/certs/swarm-controller +rm /var/lib/swarm-bao-bootstrap/grant.token +``` + +⚠️ Don't reach for `bao read auth/cert/…` to check. The host's `bao` +wrapper carries an address, a CA and a client certificate but deliberately +**no token**, so that read answers `403` whether or not the role exists. ⏱️ **Expect the first attempt to fail if you rebuilt into this.** A rebuild -restarts the store, and the unit races it — the store answers `local node not -active` until it finishes coming up. It retries every 30s and the second -attempt is the one that usually lands. Nothing to do. +restarts the store, and the units race it: the store answers `local node not +active` until it finishes coming up. They retry every 30s for a day, so a +sealed or late store heals itself. -**Confirm with `systemctl status swarm-bao-controller-policy`**, which wants no -token — a successful run logs `Uploaded policy`, `Enabled cert auth method` and -`Data written to: auth/cert/certs/swarm-controller`. ⚠️ Do _not_ reach for `bao -read auth/cert/…` to check: the host's `bao` wrapper carries an address, a CA -and a client certificate but deliberately **no token**, so that read answers -`403` whether or not the role exists. +Until you set up the granter, each `swarm-bao-*-policy` unit **fails** and logs +the commands above. It never skips. Delete the token file only once +`swarm-bao-granter-role` has succeeded. That unit skips while the file is +absent, which is the steady state afterwards. The TTL above means a forgotten +token expires rather than lingering. -**Delete the token file only once that unit has succeeded.** It skips when the -token is absent, so a host that has finished bootstrapping stops carrying the -credential — but deleting it before the role -exists leaves the unit skipping forever with nothing to show for it, and looks -exactly like a store that was never bootstrapped. The TTL above means a -forgotten one expires rather than lingering. +After that, a new or changed `swarm-*` grant needs no operator step: the unit +that writes it changes, and the deploy restarts it. A root step comes back only +when the granter itself needs a path it lacks, such as a new mount. -
Already bootstrapped before the KV mount existed? +⚠️ The granting units re-run on **boot** and whenever a deploy **changes** +them, not on every deploy. When a grant drifts in the store and its unit stays the +same, the next boot re-asserts it, not the next switch. -A store bootstrapped by an earlier version has the policy, the auth method and -the role, but no `secret/` engine — the controller's first credential write -answers `no handler for route "secret/data/…"`. The bootstrap token can't fix it -either: the policy that minted it names nothing under `sys/mounts`. Mount it -once with the root token from `init`: +
Upgrading a swarm set up with the older swarm-bootstrap policy + +A store set up before the granter existed has every grant, but no `bao-granter` +policy or role. After the deploy that introduces it, each `swarm-bao-*-policy` +unit fails and logs the one-time step. Run the two blocks above as they stand. +The old policy can go, with the root token again: ```bash -sudo bash -c 'BAO_TOKEN="" bao secrets enable -path=secret kv-v2' +bao policy delete swarm-bootstrap ``` -No rebuild needed — the unit's own check finds the mount on its next run and -leaves it alone. -
+**Residual risk, stated plainly.** The granter is root-equivalent. It may write +any `swarm-*` policy with any content, and attach it to a role that accepts any +certificate; no bao ACL can constrain what a policy says. What bounds it: + +- `nix/host-modules/swarm-bao.nix` renders every policy it writes, and + module-eval pins each principal's grants. **Merging a change to that policy + text is granting it**: it takes effect on the next deploy with no bao step, + so code review is the only gate. +- Its key sits permanently at `/var/lib/swarm-bao-pki/granter-key.pem`, `0600` + root in a `0700` directory, readable only by root units on the store's host. + That host already holds `ca-key.pem`, which can mint a leaf with any subject, + and `controller-key.pem`, whose policy is already root-equivalent. Root on + that host gains nothing new. +- **Never copy `granter-key.pem` off the host** the way operators copy the + other leaves in that directory. That hands out root-equivalence. +- Nothing revokes a stolen leaf on its own: the role trusts the CA plus the + subject. Rotate the store's CA, or have root point the `bao-granter` role at + a new `deploy.bao.granterCommonName`. Deleting `granter{,-key}.pem` and + restarting `swarm-bao-pki` mints a new leaf, but doesn't invalidate the old + one. + What else you need depends on `services.hyperhive.deploy.bao.seal`: diff --git a/nix/host-modules/bao-bootstrap-policy.hcl b/nix/host-modules/bao-bootstrap-policy.hcl new file mode 100644 index 00000000..bb2d0abf --- /dev/null +++ b/nix/host-modules/bao-bootstrap-policy.hcl @@ -0,0 +1,37 @@ +# The `bao-bootstrap` policy: what the 24h bootstrap token may do, and nothing +# else. ../../docs/getting-started/setup.md has the operator write it with the +# root token, from the copy ./swarm-bao.nix ships at +# /etc/hyperhive/bao-bootstrap-policy.hcl; `swarm-bao-granter-role` then acts +# with it. Every other grant is written by the `bao-granter` principal this +# creates. +# +# Named outside `swarm-*`, so the granter cannot rewrite the policy the next +# bootstrap token carries. +# +# Each stanza was derived with `bao -output-policy`, which prints what a +# command requires without sending it. ../module-eval/bao-grants.nix reads +# this file and fails when the unit that uses the token calls a path it does +# not grant. + +# The auth mounts. Reading `sys/auth` is how the unit checks, and `sudo` is +# what enabling one costs. +path "sys/auth" { + capabilities = ["read"] +} + +path "sys/auth/cert" { + capabilities = ["create", "update", "sudo"] +} + +path "sys/auth/approle" { + capabilities = ["create", "update", "sudo"] +} + +# The granter's own policy and role, and nothing it may write. +path "sys/policies/acl/bao-granter" { + capabilities = ["create", "update"] +} + +path "auth/cert/certs/bao-granter" { + capabilities = ["create", "update"] +} diff --git a/nix/host-modules/glue-bao-readers-policy-order.nix b/nix/host-modules/glue-bao-readers-policy-order.nix index 018eae19..a87cf48d 100644 --- a/nix/host-modules/glue-bao-readers-policy-order.nix +++ b/nix/host-modules/glue-bao-readers-policy-order.nix @@ -12,9 +12,10 @@ # with no ExecStart, so each gate below restates the one the reader's own # module puts on it. A reader whose gate changes must change here too. # -# Ordering, never a requirement: a policy unit skips once the bootstrap token -# is gone, and a skipped unit counts as done. `wants` as well as `after`, so a -# reader started on its own pulls its policy unit into the same transaction. +# Ordering, never a requirement: a policy unit that failed still counts as +# done, and the reader's own retries carry it past that. `wants` as well as +# `after`, so a reader started on its own pulls its policy unit into the same +# transaction. { lib, config, diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index 4201d3b1..d410d9f9 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -108,6 +108,12 @@ in # on `client.pem`. forwarderOidcClientCertFile = lib.mkDefault "${pkiDir}/forwarder-oidc.pem"; forwarderOidcClientKeyFile = lib.mkDefault "${pkiDir}/forwarder-oidc-key.pem"; + + # The granter: every `swarm-bao-*-policy` unit on this host logs in with + # it. ⚠️ Unlike every other leaf here, never the file an operator copies: + # its policy is root-equivalent and its only reader is this host. + granterClientCertFile = lib.mkDefault "${pkiDir}/granter.pem"; + granterClientKeyFile = lib.mkDefault "${pkiDir}/granter-key.pem"; }; # Idempotent on ABSENCE, never on content. Re-issuing the CA invalidates @@ -248,6 +254,12 @@ in # the file an operator copies. [ -s ${pkiDir}/nats.pem ] || ${signLeaf} ${pkiDir} nats \ ${lib.escapeShellArg deployCfg.bao.natsCommonName} "" clientAuth + + # The granter's, which writes every `swarm-*` grant. Minted here because + # it opens the store for the units that create the roles every other + # leaf logs in with. Stays on this host; see its default above. + [ -s ${pkiDir}/granter.pem ] || ${signLeaf} ${pkiDir} granter \ + ${lib.escapeShellArg deployCfg.bao.granterCommonName} "" clientAuth ''; }; }; diff --git a/nix/host-modules/swarm-bao-bootstrap-policy.hcl b/nix/host-modules/swarm-bao-bootstrap-policy.hcl deleted file mode 100644 index 6d572848..00000000 --- a/nix/host-modules/swarm-bao-bootstrap-policy.hcl +++ /dev/null @@ -1,159 +0,0 @@ -# The `swarm-bootstrap` policy: what the 24h bootstrap token may do, and -# nothing else. ../../docs/getting-started/setup.md has the operator write it -# with the root token; ./swarm-bao.nix's granting units then act with it. -# -# Each stanza was derived with `bao -output-policy`, which prints what a -# command requires without sending it. ../module-eval/bao-grants.nix reads -# this file and fails when a unit that uses the token calls a path it does not -# grant. The pki paths assume the default `servicesPkiMountPath` (`pki`), -# `servicesPkiRoleName` (`swarm-services`) and `natsPkiRoleName` (`swarm-nats`). - -# swarm-bao-controller-policy: the controller's own policy and role. -path "sys/policies/acl/swarm-controller" { - capabilities = ["create", "update"] -} - -path "auth/cert/certs/swarm-controller" { - capabilities = ["create", "update"] -} - -# The auth mounts it creates. Reading `sys/auth` is how the unit checks, and -# `sudo` is what enabling one costs. -path "sys/auth" { - capabilities = ["read"] -} - -path "sys/auth/cert" { - capabilities = ["create", "update", "sudo"] -} - -path "sys/auth/approle" { - capabilities = ["create", "update", "sudo"] -} - -# The KV and PKI engines, checked the same way. Enabling a secrets engine does -# not ask for `sudo`. -path "sys/mounts" { - capabilities = ["read"] -} - -path "sys/mounts/secret" { - capabilities = ["create", "update"] -} - -path "sys/mounts/pki" { - capabilities = ["create", "update"] -} - -path "sys/mounts/pki/tune" { - capabilities = ["create", "update"] -} - -# The services root: generated once, read back on every run, and replaced -# only when it can no longer outlive a leaf. -path "pki/issuers" { - capabilities = ["list"] -} - -path "pki/cert/ca" { - capabilities = ["read"] -} - -path "pki/root" { - capabilities = ["delete", "sudo"] -} - -path "pki/root/generate/internal" { - capabilities = ["create", "update"] -} - -path "pki/roles/swarm-services" { - capabilities = ["create", "update"] -} - -# swarm-bao-secret-publisher-policy -path "sys/policies/acl/swarm-secret-publisher" { - capabilities = ["create", "update"] -} - -path "auth/cert/certs/swarm-secret-publisher" { - capabilities = ["create", "update"] -} - -# swarm-bao-matrix-ctl-policy -path "sys/policies/acl/swarm-matrix-ctl" { - capabilities = ["create", "update"] -} - -path "auth/cert/certs/swarm-matrix-ctl" { - capabilities = ["create", "update"] -} - -# swarm-bao-services-issuer-policy -path "sys/policies/acl/swarm-services-issuer" { - capabilities = ["create", "update"] -} - -path "auth/cert/certs/swarm-services-issuer" { - capabilities = ["create", "update"] -} - -# swarm-bao-grafana-oidc-policy -path "sys/policies/acl/swarm-grafana-oidc" { - capabilities = ["create", "update"] -} - -path "auth/cert/certs/swarm-grafana-oidc" { - capabilities = ["create", "update"] -} - -# swarm-bao-otel-oidc-policy -path "sys/policies/acl/swarm-otel-oidc" { - capabilities = ["create", "update"] -} - -path "auth/cert/certs/swarm-otel-oidc" { - capabilities = ["create", "update"] -} - -# swarm-bao-forwarder-oidc-policy -path "sys/policies/acl/swarm-forwarder-oidc" { - capabilities = ["create", "update"] -} - -path "auth/cert/certs/swarm-forwarder-oidc" { - capabilities = ["create", "update"] -} - -# swarm-bao-nats-tls-policy: the queue's own pki role, beside -# `swarm-services` above, and its policy and login role. -path "pki/roles/swarm-nats" { - capabilities = ["create", "update"] -} - -path "sys/policies/acl/swarm-nats" { - capabilities = ["create", "update"] -} - -path "auth/cert/certs/swarm-nats" { - capabilities = ["create", "update"] -} - -# swarm-bao-matrix-token-policy and swarm-bao-queue-agent-policy write one -# policy and role per hive, `-`, so these two are globs. Each -# stops at its own prefix. -path "sys/policies/acl/swarm-matrix-token-*" { - capabilities = ["create", "update"] -} - -path "auth/cert/certs/swarm-matrix-token-*" { - capabilities = ["create", "update"] -} - -path "sys/policies/acl/swarm-queue-agent-*" { - capabilities = ["create", "update"] -} - -path "auth/cert/certs/swarm-queue-agent-*" { - capabilities = ["create", "update"] -} diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index d44ddce2..ca623c93 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -151,7 +151,7 @@ let # rather than as the missing setting it is. haveServerTls = baoDeploy.serverCertFile != null && baoDeploy.serverKeyFile != null; - # The credential that writes the swarm's first grant. A token and not a + # The credential that writes the granter's role below. A token and not a # certificate: cert auth answers a *role*, so nothing can authenticate here # until some role exists, and whatever creates the first one cannot itself # use one. An operator places it — ../../docs/getting-started/setup.md. @@ -163,6 +163,137 @@ let bootstrapTokenDir = if haveBootstrapToken then builtins.dirOf baoDeploy.bootstrapTokenFile else null; + # The principal every `swarm-bao-*-policy` unit logs in as, so a new or + # changed `swarm-*` grant applies on deploy with no operator step. Policy and + # role share one name, outside both `swarm-*` and `hive-*`: neither the + # granter's globs nor the controller's reach the objects that constrain it. + granterPolicyName = "bao-granter"; + granterCn = baoDeploy.granterCommonName; + + # No CA means no login role can be written, so nothing could log in as the + # granter; the units that need it do not render. + haveGranter = + baoDeploy.granterClientCertFile != null + && baoDeploy.granterClientKeyFile != null + && baoDeploy.clientCaFile != null; + + # ⚠️ ROOT-EQUIVALENT BY CONSTRUCTION. No ACL constrains the body of a policy, + # so a principal that may write `swarm-*` policies and the roles attaching + # them may grant itself anything. What bounds it is that every policy it + # writes is rendered from this file, and that its key never leaves this host. + # + # A trailing `*` in a bao ACL path is a pure string-prefix match, and an + # exact path wins over any prefix. + # + # The first three are the per-principal grants. The rest are what + # `swarm-bao-controller-policy` does besides grants: the KV and pki mounts and + # the services root. No `sys/auth`: the auth mounts are created with the + # bootstrap token by `swarm-bao-granter-role`. + # + # Piped as a shell-quoted argument like `controllerPolicyText`, so + # ../module-eval/bao-grants.nix can read it out of the unit script. + granterPolicyText = '' + path "sys/policies/acl/swarm-*" { + capabilities = ["create", "update"] + } + + path "auth/cert/certs/swarm-*" { + capabilities = ["create", "update"] + } + + path "${servicesPkiMountPath}/roles/swarm-*" { + capabilities = ["create", "update"] + } + + path "sys/mounts" { + capabilities = ["read"] + } + + path "sys/mounts/${credentialMountPath}" { + capabilities = ["create", "update"] + } + + path "sys/mounts/${servicesPkiMountPath}" { + capabilities = ["create", "update"] + } + + path "sys/mounts/${servicesPkiMountPath}/tune" { + capabilities = ["create", "update"] + } + + path "${servicesPkiMountPath}/issuers" { + capabilities = ["list"] + } + + path "${servicesPkiMountPath}/cert/ca" { + capabilities = ["read"] + } + + path "${servicesPkiMountPath}/root" { + capabilities = ["delete", "sudo"] + } + + path "${servicesPkiMountPath}/root/generate/internal" { + capabilities = ["create", "update"] + } + ''; + + # What a granting unit prints when the store refuses the granter: the + # one-time step, runnable as root on this host. + granterSetupSteps = [ + "read -rs BAO_TOKEN && export BAO_TOKEN # the root token from 'bao operator init'" + "bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl" + ] + ++ ( + if haveBootstrapToken then + [ + "bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token | install -D -m 0600 /dev/stdin ${baoDeploy.bootstrapTokenFile}" + "unset BAO_TOKEN" + "systemctl restart swarm-bao-granter-role" + ] + else + [ + "# then set services.hyperhive.deploy.bao.bootstrapTokenFile on this host, deploy, and place a token there:" + "bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token" + ] + ); + + # The login every granting unit starts with. It FAILS rather than skips: a + # grant that was not written is otherwise invisible until whatever needs it + # fails somewhere else. `bao status` exits 0 only on a reachable, unsealed + # store, which separates "the granter is not set up" from "retry later"; + # either way bao's own message follows. + granterLogin = '' + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + if bao status >/dev/null 2>&1; then + echo ${lib.escapeShellArg "the store is unsealed but refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2 + echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2 + ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") granterSetupSteps} + else + echo "the store is sealed or unreachable; retrying." >&2 + fi + cat "$err" >&2 + exit 1 + fi + export BAO_TOKEN + ''; + + # The granter's certificate for the granting units. The `baoCli` wrapper + # only defaults these, so the unit's environment wins. + granterEnv = { + BAO_CLIENT_CERT = baoDeploy.granterClientCertFile; + BAO_CLIENT_KEY = baoDeploy.granterClientKeyFile; + }; + + # `swarm-bao-pki` mints the granter's leaf; the granter's role is written by + # `swarm-bao-granter-role`, which normally skips, hence ordering only there. + granterAfter = [ + "swarm-bao-pki.service" + "swarm-bao-granter-role.service" + ]; + # The name both ends must agree on: the cert-auth role below attaches this # policy by spelling it the same way, and is itself named after it. controllerPolicyName = "swarm-controller"; @@ -546,27 +677,25 @@ let # after it. # # `after` and not `requires`, for the reason the publisher's unit states: the - # controller's unit creates the KV and cert-auth mounts this one writes into, - # but a failed oneshot still counts as finished, so ordering plus this unit's - # own retry is what converges. - # - # The role write is inside the client-CA branch and the policy write is not, - # exactly as the three above: with no CA there is no trust anchor for a login - # role, but the policy it would attach is still worth asserting. + # controller's and the granter's units create the mounts this one writes + # into, but a failed oneshot still counts as finished, so ordering plus this + # unit's own retry is what converges. readerPolicyUnit = description: objects: - lib.mkIf haveBootstrapToken { + lib.mkIf haveGranter { inherit description; after = [ "container@${cfg.machine}.service" "swarm-bao-controller-policy.service" - ]; + ] + ++ granterAfter; + requires = [ "swarm-bao-pki.service" ]; wantedBy = [ "multi-user.target" ]; path = [ baoCli pkgs.coreutils ]; - unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; + environment = granterEnv; # Same unseal wait as its siblings above, for the reason stated there: # under `seal = "shamir"` a human unseals by hand. startLimitBurst = 2880; @@ -580,14 +709,11 @@ let script = '' set -euo pipefail - BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" - export BAO_TOKEN - + ${granterLogin} '' + lib.concatMapStrings readerPolicyWrite objects - + lib.optionalString (baoDeploy.clientCaFile != null) ( - "\n" + lib.concatMapStrings readerRoleWrite objects - ); + + "\n" + + lib.concatMapStrings readerRoleWrite objects; }; # Every listener serves the same identity: they differ in which address @@ -979,19 +1105,22 @@ in default = null; example = "/var/lib/swarm-bao-bootstrap/grant.token"; description = '' - Token used **once per swarm** to write the first authorisation grants, - after which every client authenticates with a certificate instead. + Token used **once per swarm** to create the store's cert-auth mount and + the `bao-granter` policy and role, after which every granting unit + logs in as the granter with a certificate instead. Cert auth answers a *role*, so no client can authenticate until some - role exists — and creating that first one is what this token is for. + role exists — and creating the granter's is what this token is for. It has to come from outside that cycle, which is why an operator places it rather than the deployment minting it. - Produce it from the root token `bao operator init` printed, scoped to - that one policy write and nothing else, then delete it once the swarm - has come up — {file}`docs/getting-started/setup.md` has the commands. - Setting this is what enables the granting unit; leaving it null means - the deployment writes those grants some other way. + Produce it from the root token `bao operator init` printed, under the + `bao-bootstrap` policy shipped at + {file}`/etc/hyperhive/bao-bootstrap-policy.hcl`, then delete it once + `swarm-bao-granter-role` has run — + {file}`docs/getting-started/setup.md` has the commands. Setting this is + what renders `swarm-bao-granter-role`; while it is null, a store whose + granter is not set up has no way to set it up. A path, never a value. ''; @@ -1420,6 +1549,48 @@ in ''; }; + granterCommonName = lib.mkOption { + type = lib.types.str; + default = "bao-granter"; + description = '' + Subject the store's `bao-granter` cert-auth role accepts: the identity + every `swarm-bao-*-policy` unit on the store's host logs in as to write + the `swarm-*` policies, cert-auth roles and pki roles. + + ⚠️ Root-equivalent: it may write a `swarm-*` policy with any content. + Reserved as a hive name by ./swarm.nix, like its siblings. + ''; + }; + + granterClientCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/granter.pem"; + description = '' + Certificate the store's granting units present to the store. Its + subject must be + {option}`services.hyperhive.deploy.bao.granterCommonName`. + + Null, or a null + {option}`services.hyperhive.deploy.bao.clientCaFile`, means this + deployment writes those grants some other way: no granting unit + renders. + + ⚠️ Unlike every other leaf the store's host mints, this one is never + copied to another host; its only reader is that host. + ''; + }; + + granterClientKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/granter-key.pem"; + description = '' + Private key for + {option}`services.hyperhive.deploy.bao.granterClientCertFile`. + ''; + }; + serverCaFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -1635,8 +1806,29 @@ in grant reads every secret in the store; this role reads one path. ''; } + { + # The granter writes pki roles through `roles/swarm-*` and nothing + # else, so a role named otherwise is a 403 at deploy time. + assertion = + !haveGranter + || (lib.hasPrefix "swarm-" servicesPkiRoleName && lib.hasPrefix "swarm-" natsPkiRoleName); + message = '' + services.hyperhive.deploy.bao.servicesPkiRoleName + (${servicesPkiRoleName}) and + services.hyperhive.deploy.bao.natsPkiRoleName (${natsPkiRoleName}) + must both start with `swarm-`: the bao granter that writes them may + write pki roles under that prefix only. + ''; + } ]; + warnings = lib.optional (haveServerTls && baoDeploy.clientCaFile == null) '' + services.hyperhive.deploy.bao.clientCaFile is null, so no cert-auth + role can be written and no client can log in to the swarm secret store. + None of the swarm-bao-*-policy units render: this deployment writes no + bao policy or role. + ''; + # The name every reader dials, made resolvable where the store runs. # Cross-hive traffic always goes via the domain; only what it resolves # to varies, and a multi-host swarm is the operator's upstream DNS. This @@ -1664,6 +1856,10 @@ in # addresses on every command. environment.systemPackages = [ baoCli ]; + # The policy the operator writes with the root token for the one-time + # granter step, on the host where that step runs. + environment.etc."hyperhive/bao-bootstrap-policy.hcl".source = ./bao-bootstrap-policy.hcl; + # The in-container unit plus the host-side ones this module defines. # `swarm-bao-pki` and `swarm-bao-matrix-token` are declared by the glue # modules that create them, per the option's own rule — and a name @@ -1674,6 +1870,7 @@ in "swarm-bao-certs" "swarm-bao-token" "swarm-bao-forwarder-oidc" + "swarm-bao-granter-role" "swarm-bao-controller-policy" "swarm-bao-secret-publisher-policy" "swarm-bao-matrix-ctl-policy" @@ -1965,17 +2162,85 @@ in ''; }; - # The swarm's first grant, written from the HOST. Every API listener sets - # `tls_require_and_verify_client_cert`, so a client needs an identity - # wherever it runs — and only the host has one. The bootstrap token is a - # host path too; the container saw it through a bind mount. - systemd.services.swarm-bao-controller-policy = lib.mkIf haveBootstrapToken { - description = "write the swarm controller's bao policy and cert-auth role"; + # The one unit that still acts with the bootstrap token: it creates the + # auth mounts and the granter's own policy and role, which nothing the + # granter holds may write. Skipped while the token is absent, which is + # the steady state once it has run; the granting units below are the ones + # that fail loudly when it has never run. + # + # Ordering only toward them, never a requirement, for that same reason. + systemd.services.swarm-bao-granter-role = lib.mkIf (haveBootstrapToken && haveGranter) { + description = "write the bao granter's policy and cert-auth role with the bootstrap token"; after = [ "container@${cfg.machine}.service" ]; wantedBy = [ "multi-user.target" ]; - # The wrapper rather than the package: it carries the address, the CA - # and this host's certificate, which is what makes running here cheaper - # than shipping an identity the other way. + path = [ + baoCli + pkgs.coreutils + ]; + # Named but not placed is a legitimate state: all-local supplies the + # path as a default and the operator drops the file there after + # `bao operator init`. Skipping rather than failing is also what makes + # deleting the token at the end of that procedure safe. + unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; + # Same unseal wait as the controller's unit below, for the reason + # stated there. + startLimitBurst = 2880; + startLimitIntervalSec = 90000; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + Restart = "on-failure"; + RestartSec = 30; + }; + script = '' + set -euo pipefail + + BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" + export BAO_TOKEN + + # Every cert-auth role in this file lives under `auth/cert/`, and + # nothing else creates that mount. + # + # Asked rather than attempted: `auth enable` errors on a mount + # that already exists, and recognising that would tie a rebuild + # to an error string we have never seen this store emit. + mounted="$(bao auth list -format=json)" + case "$mounted" in + *'"cert/"'*) ;; + *) bao auth enable cert ;; + esac + + case "$mounted" in + *'"approle/"'*) ;; + *) bao auth enable approle ;; + esac + + printf '%s' ${lib.escapeShellArg granterPolicyText} | + bao policy write ${lib.escapeShellArg granterPolicyName} - + + # The TTL bounds a leaked login token to minutes; the leaf is what + # lives long. + bao write auth/cert/certs/${lib.escapeShellArg granterPolicyName} \ + certificate=@${tlsDir}/client-ca.pem \ + allowed_common_names=${lib.escapeShellArg granterCn} \ + token_policies=${lib.escapeShellArg granterPolicyName} \ + display_name=${lib.escapeShellArg granterCn} \ + token_ttl=15m \ + token_max_ttl=15m + ''; + }; + + # The swarm's first grant, written from the HOST. Every API listener sets + # `tls_require_and_verify_client_cert`, so a client needs an identity + # wherever it runs — and only the host has one: the granter's leaf. + systemd.services.swarm-bao-controller-policy = lib.mkIf haveGranter { + description = "write the swarm controller's bao policy and cert-auth role"; + after = [ "container@${cfg.machine}.service" ] ++ granterAfter; + requires = [ "swarm-bao-pki.service" ]; + wantedBy = [ "multi-user.target" ]; + # The wrapper rather than the package: it carries the address and the + # CA, which is what makes running here cheaper than shipping an + # identity the other way. path = [ baoCli pkgs.coreutils @@ -1983,11 +2248,7 @@ in # regeneration guard below turns that into a decision. pkgs.openssl ]; - # Named but not placed is a legitimate state: all-local supplies the - # path as a default and the operator drops the file there after - # `bao operator init`. Skipping rather than failing is also what makes - # deleting the token at the end of that procedure safe. - unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; + environment = granterEnv; # A store that is up is not necessarily unsealed — under # `seal = "shamir"` an operator unseals BY HAND, so early attempts fail # for as long as that takes, which can be a day. @@ -2009,8 +2270,7 @@ in script = '' set -euo pipefail - BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" - export BAO_TOKEN + ${granterLogin} # Idempotent on purpose: a rebuild re-asserts the policy rather # than failing on one that already exists. @@ -2023,11 +2283,9 @@ in # controller's first credential write fails against a grant that # reads as correct. # - # Outside the client-CA block below on purpose: this mount is what - # the controller writes *through*, independent of who may log in. - # - # Asked rather than attempted, same as the auth mount: `secrets - # enable` errors on a path already in use. + # Asked rather than attempted, same as the auth mounts in + # `swarm-bao-granter-role`: `secrets enable` errors on a path + # already in use. mounts="$(bao secrets list -format=json)" case "$mounts" in *'"${credentialMountPath}/"'*) ;; @@ -2186,28 +2444,6 @@ in key_bits=4096 \ ttl=${servicesPkiLeafTtl} \ max_ttl=${servicesPkiLeafTtl} - '' - + lib.optionalString (baoDeploy.clientCaFile != null) '' - - # The policy above grants paths under `auth/cert/`, and nothing - # in this tree creates that mount. Without this, the grant names - # a location that does not exist and every certificate login - # fails — the controller's own, and the per-hive ones it later - # issues against the same mount. - # - # Asked rather than attempted: `auth enable` errors on a mount - # that already exists, and recognising that would tie a rebuild - # to an error string we have never seen this store emit. - mounted="$(bao auth list -format=json)" - case "$mounted" in - *'"cert/"'*) ;; - *) bao auth enable cert ;; - esac - - case "$mounted" in - *'"approle/"'*) ;; - *) bao auth enable approle ;; - esac # `certificate=` is the CA, so this role trusts every leaf that # CA signed and `allowed_common_names` is the whole narrowing — @@ -2230,23 +2466,25 @@ in # Widening it to two principals would make the name wrong, and renaming it # would make that instruction wrong. # - # `after` and not `requires`: the unit above creates the KV and cert-auth + # `after` and not `requires`: the unit above and the granter's create the # mounts this one writes into, but a failed oneshot still counts as # finished, so `requires` would neither wait for its success nor re-run # this one when its own retry eventually lands. Ordering plus this unit's # own retry is what actually converges. - systemd.services.swarm-bao-secret-publisher-policy = lib.mkIf haveBootstrapToken { + systemd.services.swarm-bao-secret-publisher-policy = lib.mkIf haveGranter { description = "write the swarm secret publisher's bao policy and cert-auth role"; after = [ "container@${cfg.machine}.service" "swarm-bao-controller-policy.service" - ]; + ] + ++ granterAfter; + requires = [ "swarm-bao-pki.service" ]; wantedBy = [ "multi-user.target" ]; path = [ baoCli pkgs.coreutils ]; - unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; + environment = granterEnv; # Same unseal wait as its sibling above, for the reason stated there: # under `seal = "shamir"` a human unseals by hand, which can take a day. startLimitBurst = 2880; @@ -2260,13 +2498,10 @@ in script = '' set -euo pipefail - BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" - export BAO_TOKEN + ${granterLogin} printf '%s' ${lib.escapeShellArg secretPublisherPolicyText} | bao policy write ${lib.escapeShellArg secretPublisherPolicyName} - - '' - + lib.optionalString (baoDeploy.clientCaFile != null) '' bao write auth/cert/certs/${lib.escapeShellArg secretPublisherPolicyName} \ certificate=@${tlsDir}/client-ca.pem \ @@ -2283,18 +2518,20 @@ in # creates the mounts this one writes into, but a failed oneshot still # counts as finished, so only ordering plus this unit's own retry # converges. - systemd.services.swarm-bao-matrix-ctl-policy = lib.mkIf haveBootstrapToken { + systemd.services.swarm-bao-matrix-ctl-policy = lib.mkIf haveGranter { description = "write swarm-matrix-ctl's bao policy and cert-auth role"; after = [ "container@${cfg.machine}.service" "swarm-bao-controller-policy.service" - ]; + ] + ++ granterAfter; + requires = [ "swarm-bao-pki.service" ]; wantedBy = [ "multi-user.target" ]; path = [ baoCli pkgs.coreutils ]; - unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; + environment = granterEnv; # Same unseal wait as its two siblings above, for the reason stated # there: under `seal = "shamir"` a human unseals by hand. startLimitBurst = 2880; @@ -2308,13 +2545,10 @@ in script = '' set -euo pipefail - BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" - export BAO_TOKEN + ${granterLogin} printf '%s' ${lib.escapeShellArg matrixCtlPolicyText} | bao policy write ${lib.escapeShellArg matrixCtlPolicyName} - - '' - + lib.optionalString (baoDeploy.clientCaFile != null) '' bao write auth/cert/certs/${lib.escapeShellArg matrixCtlPolicyName} \ certificate=@${tlsDir}/client-ca.pem \ @@ -2349,18 +2583,20 @@ in # The policy text moved here from the controller's unit, where it sat # while it attached to nothing — a policy and the role that carries it # belong in one place, and now there is a principal to put them with. - systemd.services.swarm-bao-services-issuer-policy = lib.mkIf haveBootstrapToken { + systemd.services.swarm-bao-services-issuer-policy = lib.mkIf haveGranter { description = "write the swarm services issuer's bao policy and cert-auth role"; after = [ "container@${cfg.machine}.service" "swarm-bao-controller-policy.service" - ]; + ] + ++ granterAfter; + requires = [ "swarm-bao-pki.service" ]; wantedBy = [ "multi-user.target" ]; path = [ baoCli pkgs.coreutils ]; - unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; + environment = granterEnv; # Same unseal wait as its three siblings above, for the reason stated # there: under `seal = "shamir"` a human unseals by hand. startLimitBurst = 2880; @@ -2374,13 +2610,10 @@ in script = '' set -euo pipefail - BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" - export BAO_TOKEN + ${granterLogin} printf '%s' ${lib.escapeShellArg servicesIssuerPolicyText} | bao policy write ${lib.escapeShellArg servicesIssuerPolicyName} - - '' - + lib.optionalString (baoDeploy.clientCaFile != null) '' bao write auth/cert/certs/${lib.escapeShellArg servicesIssuerPolicyName} \ certificate=@${tlsDir}/client-ca.pem \ @@ -2398,18 +2631,20 @@ in # The role narrows exactly as `swarm-services` does, to one name. It is # the queue's domain alone, since the same name reaches it from every # hive; no IP SANs, since nothing dials an address. - systemd.services.swarm-bao-nats-tls-policy = lib.mkIf haveBootstrapToken { + systemd.services.swarm-bao-nats-tls-policy = lib.mkIf haveGranter { description = "write the swarm queue's pki role, bao policy and cert-auth role"; after = [ "container@${cfg.machine}.service" "swarm-bao-controller-policy.service" - ]; + ] + ++ granterAfter; + requires = [ "swarm-bao-pki.service" ]; wantedBy = [ "multi-user.target" ]; path = [ baoCli pkgs.coreutils ]; - unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; + environment = granterEnv; # Same unseal wait as its siblings above. startLimitBurst = 2880; startLimitIntervalSec = 90000; @@ -2422,8 +2657,7 @@ in script = '' set -euo pipefail - BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" - export BAO_TOKEN + ${granterLogin} bao write ${lib.escapeShellArg "${servicesPkiMountPath}/roles/${natsPkiRoleName}"} \ allowed_domains=${lib.escapeShellArg hyperhiveCfg.swarm.nats.domain} \ @@ -2443,8 +2677,6 @@ in printf '%s' ${lib.escapeShellArg natsPolicyText} | bao policy write ${lib.escapeShellArg natsPolicyName} - - '' - + lib.optionalString (baoDeploy.clientCaFile != null) '' bao write auth/cert/certs/${lib.escapeShellArg natsPolicyName} \ certificate=@${tlsDir}/client-ca.pem \ diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index 1c992d09..b7ac396e 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -52,6 +52,7 @@ let deployCfg.bao.forwarderOidcCommonName deployCfg.bao.servicesIssuerCommonName deployCfg.bao.natsCommonName + deployCfg.bao.granterCommonName ] # The two per-hive readers' subjects, spelled out per hive rather than as the # prefix. The prefix alone would reserve the wrong string: the role for hive diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index c3c7cfa7..2a89ccf7 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -22,7 +22,7 @@ let ; # The store, plus a placed bootstrap token: the only shape in which the - # swarm's first grant can be written at all. + # granter's own role can be written at all. baoGrantHere = hive { deploy.bao.enable = true; deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; @@ -36,10 +36,31 @@ let deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; }; + # The store with no bootstrap token: the steady state once the granter is set + # up, and the state of a store host that has never named one. + baoGranterNoToken = hive { + deploy.bao.enable = true; + }; + + # The store with the granter's pair taken away: the deployment that writes + # its grants some other way. + baoGranterOptOut = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + deploy.bao.granterClientCertFile = lib.mkForce null; + deploy.bao.granterClientKeyFile = lib.mkForce null; + }; + + # A pki role the granter's `roles/swarm-*` does not reach. + baoGranterOddPkiRole = hive { + deploy.bao.enable = true; + deploy.bao.natsPkiRoleName = "queue"; + }; + # The store and the token, with no CA to trust. `mkForce` because the PKI # glue supplies one by default here — this is the deployment that brings its - # own certificates and has not named the authority yet, and it separates - # "the grant unit runs" from "cert auth can be set up". + # own certificates and has not named the authority yet, in which nothing can + # log in as the granter. baoGrantNoClientCa = hive { deploy.bao.enable = true; deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; @@ -102,38 +123,71 @@ let "swarm-bao-otel-oidc" ]; - # What the bootstrap token may do, read from the file the operator writes it - # from (../../docs/getting-started/setup.md points there), against what the - # units holding that token actually call. The units are found by the token - # path in their script rather than by name, so a new one is checked without - # anyone listing it here. + # Two credentials write grants, and each is checked against what the units + # holding it actually call. The bootstrap token's policy is read from the + # file the operator writes it from (../../docs/getting-started/setup.md + # points there); the granter's from the unit that writes it. Units are found + # by the credential they read rather than by name, so a new one is checked + # without anyone listing it here. bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + # The READ, not the path: every granting unit prints the path in the + # one-time step it shows when the granter is refused. bootstrapUnits = lib.filterAttrs ( - _: u: lib.hasInfix bootstrapTokenFile u.script + _: u: lib.hasInfix "cat ${lib.escapeShellArg bootstrapTokenFile}" u.script ) baoGrantWithConsumers.systemd.services; + # The pair ./glue-bao-tls.nix defaults on a store host. + granterCertFile = "/var/lib/swarm-bao-pki/granter.pem"; + granterKeyFile = "/var/lib/swarm-bao-pki/granter-key.pem"; + + granterUnits = lib.filterAttrs ( + _: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile + ) baoGrantWithConsumers.systemd.services; + + # The ten units that write a `swarm-*` grant, by name, for the discovery + # control below. + grantingUnitNames = [ + "swarm-bao-controller-policy" + "swarm-bao-secret-publisher-policy" + "swarm-bao-matrix-ctl-policy" + "swarm-bao-matrix-token-policy" + "swarm-bao-queue-agent-policy" + "swarm-bao-grafana-oidc-policy" + "swarm-bao-otel-oidc-policy" + "swarm-bao-forwarder-oidc-policy" + "swarm-bao-services-issuer-policy" + "swarm-bao-nats-tls-policy" + ]; + # Comment lines dropped first: both the HCL and the scripts explain # themselves in prose that names paths and `bao` commands. codeLines = text: lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" text); bootstrapPolicyText = lib.concatStringsSep "\n" ( - codeLines (builtins.readFile ../host-modules/swarm-bao-bootstrap-policy.hcl) + codeLines (builtins.readFile ../host-modules/bao-bootstrap-policy.hcl) ); + # The granter's HCL is the only policy text in the unit that writes it. + granterPolicyText = baoGrantHere.systemd.services.swarm-bao-granter-role.script; + matches = re: text: lib.filter lib.isList (builtins.split re text); - bootstrapGrants = + grantsIn = + text: map (m: { path = lib.elemAt m 0; caps = map lib.head (matches ''"([a-z]+)"'' (lib.elemAt m 1)); }) ( - matches ''path "([^"]+)"[[:space:]]*[{][[:space:]]*capabilities[[:space:]]*=[[:space:]]*[[]([a-z", ]*)'' bootstrapPolicyText + matches ''path "([^"]+)"[[:space:]]*[{][[:space:]]*capabilities[[:space:]]*=[[:space:]]*[[]([a-z", ]*)'' text ); + bootstrapGrants = grantsIn bootstrapPolicyText; + granterGrants = grantsIn granterPolicyText; + # One `bao …` invocation → the path and capabilities it needs, as # `bao -output-policy` reports them. Path-specific `sudo` (bao's # root-protected paths, e.g. `pki/root` for a delete) does not follow from @@ -154,7 +208,10 @@ let "update" ]; in - if a 0 == "policy" && a 1 == "write" then + # A login and a seal-status check are unauthenticated: no policy grants them. + if a 0 == "login" || a 0 == "status" then + null + else if a 0 == "policy" && a 1 == "write" then need "sys/policies/acl/${a 2}" cu else if a 0 == "secrets" && a 1 == "list" then need "sys/mounts" [ "read" ] @@ -179,25 +236,28 @@ let baoCalls = script: - map - ( - inv: - baoCallNeeds (lib.filter (w: w != "") (lib.splitString " " (lib.replaceStrings [ "'" ] [ "" ] inv))) - ) - ( - lib.concatMap (l: map (m: lib.elemAt m 1) (matches "(^[[:space:]]*|[$][(]|[)] )bao ([^|;)]*)" l)) ( - codeLines script + lib.filter (n: n != null) ( + map + ( + inv: + baoCallNeeds (lib.filter (w: w != "") (lib.splitString " " (lib.replaceStrings [ "'" ] [ "" ] inv))) ) - ); + ( + lib.concatMap (l: map (m: lib.elemAt m 1) (matches "(^[[:space:]]*|[$][(]|[)] )bao ([^|;)]*)" l)) ( + codeLines script + ) + ) + ); - # bao's own rule: an exact path wins, otherwise the longest glob prefix. - bootstrapGrantFor = - path: + # bao's own rule (vault/policy/acl.go): an exact path wins, otherwise the + # longest glob prefix, and a trailing `*` is a plain string prefix. + grantFor = + grants: path: let - exact = lib.filter (g: g.path == path) bootstrapGrants; + exact = lib.filter (g: g.path == path) grants; globs = lib.filter ( g: lib.hasSuffix "*" g.path && lib.hasPrefix (lib.removeSuffix "*" g.path) path - ) bootstrapGrants; + ) grants; in if exact != [ ] then lib.head exact @@ -206,33 +266,40 @@ let best: g: if best == null || lib.stringLength g.path > lib.stringLength best.path then g else best ) null globs; - bootstrapUngranted = lib.concatLists ( - lib.mapAttrsToList ( - unit: u: - map (n: "${unit}: `bao ${n.call}` needs ${n.path} [${toString n.caps}]") ( - lib.filter ( - n: - let - g = bootstrapGrantFor n.path; - in - g == null || !(lib.all (c: lib.elem c g.caps) n.caps) - ) (baoCalls u.script) - ) - ) bootstrapUnits - ); + ungranted = + grants: units: + lib.concatLists ( + lib.mapAttrsToList ( + unit: u: + map (n: "${unit}: `bao ${n.call}` needs ${n.path} [${toString n.caps}]") ( + lib.filter ( + n: + let + g = grantFor grants n.path; + in + g == null || !(lib.all (c: lib.elem c g.caps) n.caps) + ) (baoCalls u.script) + ) + ) units + ); + + bootstrapUngranted = ungranted bootstrapGrants bootstrapUnits; + granterUngranted = ungranted granterGrants granterUnits; cases = [ { # Reads the rendered unit on the HOST, which is where the write happens: # every API listener demands a client certificate, and the host is the # side that has one. - name = "a store host with a placed bootstrap token renders the granting unit on the host"; + name = "a store host renders the granting unit on the host, logging in as the granter"; ok = let u = baoGrantHere.systemd.services.swarm-bao-controller-policy; in - lib.hasInfix "/run/secrets/bao-bootstrap.token" u.script - && u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token"; + u.environment.BAO_CLIENT_CERT == granterCertFile + && u.environment.BAO_CLIENT_KEY == granterKeyFile + && lib.hasInfix "bao login -method=cert -token-only" u.script + && !(u.unitConfig ? ConditionPathExists); } { # The move is the fix, so pin the side it landed on: in the container it @@ -273,13 +340,12 @@ let { # Same host-side reasoning as the controller's granting unit above: the # write needs a client certificate and the host is the side that has one. - name = "a store host with a placed bootstrap token renders the publisher's granting unit too"; + name = "a store host renders the publisher's granting unit too, logging in as the granter"; ok = let u = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy; in - u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token" - && lib.hasInfix "swarm-secret-publisher" u.script; + u.environment.BAO_CLIENT_CERT == granterCertFile && lib.hasInfix "swarm-secret-publisher" u.script; } { # The control for the case above, and the same one the controller's unit @@ -590,29 +656,18 @@ let ]; } { - # The absence arm: with no client CA there is no trust anchor, so the - # login roles cannot be written — but the policies they would attach are - # still asserted, exactly as the three service principals above behave in - # this deployment. A unit that vanished here would take the policy with - # it and leave nothing to diagnose. - name = "with no client CA the five readers get policies but no login roles"; + # The absence arm: with no client CA there is no trust anchor, so no + # role can be written and nothing can log in as the granter. The units + # are gone, so the deployment has to say so itself. + name = "with no client CA no granting unit renders, and the deployment warns"; ok = let - units = [ - "swarm-bao-matrix-token-policy" - "swarm-bao-queue-agent-policy" - "swarm-bao-grafana-oidc-policy" - "swarm-bao-otel-oidc-policy" - "swarm-bao-forwarder-oidc-policy" - ]; - scriptOf = unit: baoGrantNoClientCa.systemd.services.${unit}.script; + s = baoGrantNoClientCa.systemd.services; in - lib.all ( - unit: - (baoGrantNoClientCa.systemd.services ? ${unit}) - && lib.hasInfix "bao policy write" (scriptOf unit) - && !(lib.hasInfix "auth/cert/certs" (scriptOf unit)) - ) units; + lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ]) + && lib.any (lib.hasInfix "services.hyperhive.deploy.bao.clientCaFile is null") baoGrantNoClientCa.warnings + # The control: a store with a CA does not warn. + && !(lib.any (lib.hasInfix "clientCaFile is null") baoGrantHere.warnings); } { # Same control the three service principals carry: the write needs a @@ -639,7 +694,8 @@ let { # The other end of those units: each reader logs in against the role its # own policy unit writes, so it has to wait for that unit. Ordering and - # never a requirement, since the policy unit skips once the token is gone. + # never a requirement: a failed policy unit still counts as done, and the + # reader's own retries carry it past that. # # The forwarder is listed apart from `policyReaders`: it renders wherever # the store does, so it is never absent on a store host and never present @@ -684,21 +740,237 @@ let lib.all unordered policyReaders; } { - # A store host that has not placed a bootstrap token can write no grant at - # all, so none of the four units may exist — the same claim - # `baoGrantNoStore` makes for the controller's, one file over. Without - # this arm `lib.mkIf haveBootstrapToken` could be dropped from the shared - # builder and every other case here would still pass. - name = "without a bootstrap token none of the five readers' granting units render"; + # A store host without the granter's pair writes its grants some other + # way, so none of the ten units may exist. Without this arm + # `lib.mkIf haveGranter` could be dropped from any of them and every other + # case here would still pass. + name = "without the granter's pair none of the ten granting units render"; ok = let - s = baoGrantNoStore.systemd.services; + s = baoGranterOptOut.systemd.services; in - !(s ? swarm-bao-matrix-token-policy) - && !(s ? swarm-bao-queue-agent-policy) - && !(s ? swarm-bao-grafana-oidc-policy) - && !(s ? swarm-bao-otel-oidc-policy) - && !(s ? swarm-bao-forwarder-oidc-policy); + lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ]) + # The control: the same store with the pair renders all ten. + && lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames; + } + { + # 🩸 What replaced the silent skip. With no bootstrap token the ten still + # render, and a refused granter fails them with the step that fixes it. + # A store host that never named a token is told to name one, since the + # unit that sets the granter up renders only where it has. + name = "a store host without a bootstrap token renders the ten, each failing loudly with the one-time step"; + ok = + let + s = baoGranterNoToken.systemd.services; + loud = + unit: + s ? ${unit} + && + lib.hasInfix "bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl" + s.${unit}.script + && lib.hasInfix "set services.hyperhive.deploy.bao.bootstrapTokenFile" s.${unit}.script + && lib.hasInfix "exit 1" s.${unit}.script; + in + lib.all loud grantingUnitNames && !(s ? swarm-bao-granter-role); + } + { + # Where the token is named, the step names the file to put it in and the + # unit to restart. + name = "with a bootstrap token named, the one-time step places it and restarts the granter's unit"; + ok = lib.all ( + unit: + let + sc = baoGrantHere.systemd.services.${unit}.script; + in + lib.hasInfix "install -D -m 0600 /dev/stdin /run/secrets/bao-bootstrap.token" sc + && lib.hasInfix "systemctl restart swarm-bao-granter-role" sc + ) grantingUnitNames; + } + { + # Every granting unit retries a sealed or late store for a day, in the + # `[Unit]` section systemd reads it from, and waits for the unit that + # mints the granter's leaf. + name = "each granting unit requires the PKI unit and retries 2880 times at 30s"; + ok = lib.all ( + unit: + let + u = baoGrantHere.systemd.services.${unit}; + in + lib.elem "swarm-bao-pki.service" u.requires + && lib.elem "swarm-bao-pki.service" u.after + && lib.elem "swarm-bao-granter-role.service" u.after + && !(lib.elem "swarm-bao-granter-role.service" (u.requires ++ u.wants)) + && toString u.unitConfig.StartLimitBurst == "2880" + && toString u.unitConfig.StartLimitIntervalSec == "90000" + && toString u.serviceConfig.RestartSec == "30" + && u.serviceConfig.Restart == "on-failure" + ) grantingUnitNames; + } + { + # The only unit left acting with the token, so the only one that may + # skip on it. + name = "no unit but the granter's role reads the bootstrap token or skips on it"; + ok = + lib.attrNames bootstrapUnits == [ "swarm-bao-granter-role" ] + && lib.all (u: !(u.unitConfig ? ConditionPathExists)) (lib.attrValues granterUnits) + && + baoGrantHere.systemd.services.swarm-bao-granter-role.unitConfig.ConditionPathExists + == bootstrapTokenFile; + } + { + # The granter's grants, whole. Pinned as the full list, because an added + # path or capability is exactly what a presence check misses. + name = "the granter's policy is exactly these eleven stanzas"; + ok = + let + cu = [ + "create" + "update" + ]; + in + granterGrants == [ + { + path = "sys/policies/acl/swarm-*"; + caps = cu; + } + { + path = "auth/cert/certs/swarm-*"; + caps = cu; + } + { + path = "pki/roles/swarm-*"; + caps = cu; + } + { + path = "sys/mounts"; + caps = [ "read" ]; + } + { + path = "sys/mounts/secret"; + caps = cu; + } + { + path = "sys/mounts/pki"; + caps = cu; + } + { + path = "sys/mounts/pki/tune"; + caps = cu; + } + { + path = "pki/issuers"; + caps = [ "list" ]; + } + { + path = "pki/cert/ca"; + caps = [ "read" ]; + } + { + path = "pki/root"; + caps = [ + "delete" + "sudo" + ]; + } + { + path = "pki/root/generate/internal"; + caps = cu; + } + ]; + } + { + # Neither its own policy and role nor the bootstrap policy may be + # reachable, or the granter could rewrite what constrains it and what the + # next bootstrap token carries. + name = "the granter cannot reach the policy or role that constrains it, nor the bootstrap policy"; + ok = lib.all (p: grantFor granterGrants p == null) [ + "sys/policies/acl/bao-granter" + "auth/cert/certs/bao-granter" + "sys/policies/acl/bao-bootstrap" + ]; + } + { + # Outside `swarm-*` and the store's own mounts it holds nothing: no + # hive's policy or role, no auth mount, no token, no secret. + name = "the granter grants nothing outside swarm-* and the store's own mounts"; + ok = + lib.all (p: grantFor granterGrants p == null) [ + "sys/policies/acl/hive-x" + "auth/cert/certs/hive-x" + "sys/auth" + "sys/auth/cert" + "sys/auth/x" + "auth/token/create" + "auth/token/create-orphan" + "secret/data/x" + "secret/data/swarm/agents/x/queue" + "sys/policies/acl/x" + "sys/policies/acl/root" + "pki/issue/swarm-services" + "pki/sign/swarm-services" + "*" + ] + && !(lib.any ( + g: + lib.elem g.path [ + "*" + "sys/policies/acl/*" + "auth/cert/certs/*" + "pki/roles/*" + ] + ) granterGrants); + } + { + # Its names sit outside both globs that write grants — its own + # `swarm-*` and the controller's `hive-*`. + name = "the granter's own names are outside swarm-* and hive-*"; + ok = + let + sc = baoGrantHere.systemd.services.swarm-bao-granter-role.script; + cn = baoGrantHere.services.hyperhive.deploy.bao.granterCommonName; + in + lib.hasInfix "bao policy write bao-granter -" sc + && lib.hasInfix "auth/cert/certs/bao-granter" sc + && lib.hasInfix "token_policies=bao-granter" sc + && lib.hasInfix "token_ttl=15m" sc + && !(lib.hasPrefix "swarm-" cn) + && !(lib.hasPrefix "hive-" cn); + } + { + # The other principals are what they were: no unit but the granter's own + # hands its policy to a role, and none of them logs in as it. + name = "no other principal gains the granter's policy"; + ok = + lib.all (u: !(lib.hasInfix "token_policies=bao-granter" u.script)) ( + lib.attrValues (lib.removeAttrs baoGrantWithConsumers.systemd.services [ "swarm-bao-granter-role" ]) + ) + && lib.all (u: (u.environment.BAO_CLIENT_CERT or null) != granterCertFile) ( + lib.attrValues (lib.removeAttrs baoGrantWithConsumers.systemd.services grantingUnitNames) + ); + } + { + # The minting side: a role matching a subject nothing signs is a + # granter that cannot log in. + name = "the PKI unit signs the granter's leaf under its own subject"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-pki.script; + in + lib.hasInfix "/granter.pem ]" s && lib.hasInfix "bao-granter \"\" clientAuth" s; + } + { + # The granter writes pki roles through `roles/swarm-*` only, so a role + # named otherwise is refused at eval rather than 403'd at deploy. + name = "a pki role name outside swarm-* is refused, naming both options"; + ok = + let + names = + a: + lib.hasInfix "services.hyperhive.deploy.bao.servicesPkiRoleName" a.message + && lib.hasInfix "services.hyperhive.deploy.bao.natsPkiRoleName" a.message; + in + lib.any (a: !a.assertion && names a) baoGranterOddPkiRole.assertions + && !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions); } { # 🩸 The refusal half of the forwarder's own leaf. It renders wherever the @@ -762,15 +1034,17 @@ let ok = lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/matrix/appservice-token\" {\n capabilities = [\"read\"]\n}" baoGrantHere.systemd.services.swarm-bao-controller-policy.script; } { - # The policy above grants paths under a mount nothing else creates, so - # the unit that writes the policy has to create it too — otherwise every - # certificate login fails against a path that is not there. - name = "the granting unit creates the cert auth mount and the controller's role"; + # Every role lives under a mount nothing else creates, and the granter + # holds no `sys/auth`, so the token-holding unit creates it — otherwise + # every certificate login fails against a path that is not there. + name = "the granter's role unit creates the cert auth mount, and the controller's unit writes its role"; ok = let s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + g = baoGrantHere.systemd.services.swarm-bao-granter-role.script; in - lib.hasInfix "bao auth enable cert" s + lib.hasInfix "bao auth enable cert" g + && !(lib.hasInfix "bao auth enable" s) && lib.hasInfix "auth/cert/certs/swarm-controller" s && lib.hasInfix "/var/lib/swarm-bao-tls/client-ca.pem" s; } @@ -790,28 +1064,6 @@ let in lib.hasInfix "bao secrets enable -path=secret kv-v2" s; } - { - # The arm that makes the one above mean something. A role's trust anchor - # is the CA, so with none named there is nothing to write — and the - # policy write, which needs no CA, must survive that. - # - # ⚠️ Matched on the COMMANDS, not on `auth/cert/certs`: the policy text is - # embedded in this same script and grants that very path, so the shorter - # infix is present either way and the arm could never fail. - name = "with no client CA the unit still writes the policy and skips the role"; - ok = - let - s = baoGrantNoClientCa.systemd.services.swarm-bao-controller-policy.script; - in - lib.hasInfix "bao policy write" s - && !(lib.hasInfix "bao auth enable cert" s) - && !(lib.hasInfix "client-ca.pem" s) - # The KV mount is NOT part of what a missing client CA switches off: - # the controller writes through it whether or not anything can log in - # by certificate. Asserted here rather than trusted, because both - # steps live in the same script and one indentation level decides it. - && lib.hasInfix "bao secrets enable -path=secret kv-v2" s; - } { # What makes the granting-unit cases mean something, and the property # the host-side half depends on: no store here, so no bind mount and no @@ -821,43 +1073,66 @@ let ok = !(baoGrantNoStore.systemd.services ? swarm-bao-bootstrap-dir); } { - # The drift this case exists to stop: setup.md's copy of the policy - # stayed at the controller's first six grants while seven more units - # started using the token. Failing names every ungranted call. + # The operator writes this policy by hand, so a call the token-holding + # unit makes and the file does not grant is a one-time step that fails. + # Failing names every ungranted call. name = - "every bao call a bootstrap-token unit makes is granted by swarm-bao-bootstrap-policy.hcl" + "every bao call the bootstrap-token unit makes is granted by bao-bootstrap-policy.hcl" + lib.optionalString (bootstrapUngranted != [ ]) ( ": " + lib.concatStringsSep "; " bootstrapUngranted ); ok = bootstrapUngranted == [ ]; } { - # What makes the case above mean something: discovery by token path - # reaches every unit that uses the token today, and each yields calls. - name = "the bootstrap-policy check sees all ten units that use the token, and parses calls from each"; + # The same check for the granter: a grant a unit writes outside its + # globs is a 403 on deploy. Failing names every ungranted call. + name = + "every bao call a granting unit makes is granted by the granter's policy" + + lib.optionalString (granterUngranted != [ ]) (": " + lib.concatStringsSep "; " granterUngranted); + ok = granterUngranted == [ ]; + } + { + # What makes the case above mean something: discovery by the granter's + # certificate reaches all ten units, and each yields calls. + name = "the granter-policy check sees all ten granting units, and parses calls from each"; ok = - lib.all (n: bootstrapUnits ? ${n}) [ - "swarm-bao-controller-policy" - "swarm-bao-secret-publisher-policy" - "swarm-bao-matrix-ctl-policy" - "swarm-bao-matrix-token-policy" - "swarm-bao-queue-agent-policy" - "swarm-bao-grafana-oidc-policy" - "swarm-bao-otel-oidc-policy" - "swarm-bao-forwarder-oidc-policy" - "swarm-bao-services-issuer-policy" - "swarm-bao-nats-tls-policy" - ] + lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames + && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits) && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues bootstrapUnits); } { # And the grants side: a stanza the parser skipped would read as a # grant that is not there. - name = "every path stanza in swarm-bao-bootstrap-policy.hcl parses"; + name = "every path stanza in bao-bootstrap-policy.hcl and the granter's policy parses"; ok = - bootstrapGrants != [ ] - && lib.length bootstrapGrants == lib.length (matches ''path "'' bootstrapPolicyText) - && lib.all (g: g.caps != [ ]) bootstrapGrants; + lib.all + ( + t: + let + grants = grantsIn t; + in + grants != [ ] + && lib.length grants == lib.length (matches ''path "'' t) + && lib.all (g: g.caps != [ ]) grants + ) + [ + bootstrapPolicyText + granterPolicyText + ]; + } + { + # The bootstrap policy, whole: the auth mounts and the granter's own two + # objects, and nothing a `swarm-*` grant lives at. + name = "the bootstrap policy is exactly the auth mounts and the granter's policy and role"; + ok = + lib.map (g: g.path) bootstrapGrants == [ + "sys/auth" + "sys/auth/cert" + "sys/auth/approle" + "sys/policies/acl/bao-granter" + "auth/cert/certs/bao-granter" + ] + && grantFor bootstrapGrants "sys/policies/acl/swarm-controller" == null; } ]; in diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index 31f60a9f..de2e8000 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -23,18 +23,16 @@ let runGroup ; - # Every service on one host, with a bootstrap token so the store's granting - # unit renders the role this leaf is issued through. + # Every service on one host, so the store's granting unit renders the role + # this leaf is issued through. allLocal = hive { deploy.singleHostSwarm = true; - deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; }; # The same host with the role's lifetime moved, so a threshold that is a # number of its own shows up as one that did not move with it. shortTtl = hive { deploy.singleHostSwarm = true; - deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; deploy.bao.servicesPkiLeafTtlHours = 48; }; diff --git a/nix/module-eval/name-guards.nix b/nix/module-eval/name-guards.nix index 9a4f85a9..5d60da37 100644 --- a/nix/module-eval/name-guards.nix +++ b/nix/module-eval/name-guards.nix @@ -83,6 +83,13 @@ let swarm.hives.fwctl.domain = "f.t.local"; }; + # The granter's, the one subject whose role may write every `swarm-*` grant. + hiveNamedAfterGranterSubject = hive { + deploy.swarm-otel.enable = false; + deploy.bao.granterCommonName = "grctl"; + swarm.hives.grctl.domain = "gr.t.local"; + }; + # 🩸 A different shape from every fixture above: the matrix-token and # queue-credential roles are written PER HIVE, so the subject a hive must not # be is `-` rather than the prefix itself. Reserving @@ -173,6 +180,16 @@ let a: !a.assertion && lib.hasInfix "'fwctl'" a.message ) hiveNamedAfterForwarderOidcSubject.assertions; } + { + # And the granter's, whose role is root-equivalent: a hive holding a leaf + # it accepts could grant itself anything. + name = "a hive named after the bao granter's subject is refused too"; + ok = + equalityGuardFired hiveNamedAfterGranterSubject + && lib.any ( + a: !a.assertion && lib.hasInfix "'grctl'" a.message + ) hiveNamedAfterGranterSubject.assertions; + } { # 🩸 The per-hive half, and the one a prefix-only reservation would miss: # the role is `-`, so the reserved string has to be composed diff --git a/nix/module-eval/nats-tls.nix b/nix/module-eval/nats-tls.nix index b68ae667..817837b2 100644 --- a/nix/module-eval/nats-tls.nix +++ b/nix/module-eval/nats-tls.nix @@ -26,12 +26,10 @@ let natsName = "nats.t.local"; natsUrl = "tls://${natsName}:4222"; - # Every service on one host, with a bootstrap token so the store's granting - # units render. The queue, the store and every in-tree client of the queue + # Every service on one host. The queue, the store and every in-tree client of the queue # are all here, so the scan below reads each of them. allLocal = hive { deploy.singleHostSwarm = true; - deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; }; # The same host on the mesh. @@ -232,8 +230,8 @@ let && !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts); } { - # Ordering, never a requirement: the policy unit skips once the bootstrap - # token is gone, and a skipped unit counts as done. + # Ordering, never a requirement: a policy unit that failed still counts + # as done, and the leaf unit's own retries carry it past that. name = "the leaf unit is ordered after its policy unit, with no requires"; ok = let diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 6192fe94..dbf0a70a 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -80,6 +80,7 @@ let "hive-tls-ca" "swarm-services-cert" "hive-gateway-self-signed-cert" + "swarm-bao-granter-role" "swarm-bao-controller-policy" "swarm-bao-secret-publisher-policy" "swarm-bao-matrix-ctl-policy"