From 2f7d3e02e9cc76a5da2b22a41f81fda4eee50356 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 7 Sep 2026 17:36:24 +0200 Subject: [PATCH] bao: write the swarm controller's policy from inside the store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cert auth answers a role, so nothing in the swarm can authenticate to the store until some role exists. Creating the first one therefore cannot itself use a certificate — the credential has to come from outside that cycle, and an operator places it. `deploy.bao.bootstrapTokenFile` names that token. A one-shot unit inside the store's container reads it and writes the `swarm-controller` policy; `local-defaults.nix` supplies the path (never the file) on an all-local deploy, since co-location makes only the location derivable. `bao operator init` stays an operator action in every shape. The unit runs in the container rather than on the host because writing the first grant is a store-side operation: it reaches the store locally and needs no client certificate. Gating it on the store being here is therefore not the co-location assumption glue-matrix-bao-token.nix warns about — a reader has to work from anywhere, the first write never does. Policy only, deliberately: a cert-auth role binds a certificate and the controller has no bao identity yet — it holds no leaf and contains no bao code at all. The two certificates that do exist are both wrong to bind. `clientCertFile` is the store host's own reader leaf rather than the controller's, and the CA that signed it also signs every other reader's, so binding that would let any reader authenticate as the controller. Whoever gives the controller an identity writes the role. The grants are scoped to `hive-*`. Without the prefix the controller could rewrite the policy that constrains it. Two things the module-eval arms pin: the unit renders inside the container with the token path in both its script and its ConditionPathExists, and a host that names a token while running no store gets neither the unit nor the host-side directory. The capabilities were derived with `bao write -output-policy` rather than written from memory, and the setup.md commands were run against a real binary for shape before being written down. Refs #3726. --- docs/getting-started/setup.md | 25 ++++++ nix/host-modules/local-defaults.nix | 18 ++++ nix/host-modules/swarm-bao.nix | 133 ++++++++++++++++++++++++++++ nix/module-eval.nix | 47 ++++++++++ 4 files changed, 223 insertions(+) diff --git a/docs/getting-started/setup.md b/docs/getting-started/setup.md index 4cbb3dc3..c5b6dedf 100644 --- a/docs/getting-started/setup.md +++ b/docs/getting-started/setup.md @@ -56,6 +56,31 @@ never initialised. Do this before anything is pointed at it. bao operator init # keep the keys it prints and the root token OFF this host ``` +While you still hold that root token, mint the one credential the swarm needs +to grant itself anything. Cert auth answers a _role_, so nothing can +authenticate until some role exists — this token is what breaks that cycle, +and it's the only step that needs the root token. + +```bash +# A policy that can write exactly one policy, and nothing else. +bao policy write swarm-bootstrap - <<'EOF' +path "sys/policies/acl/swarm-controller" { + capabilities = ["create", "update"] +} +EOF + +# A token holding it. `-orphan` so it outlives the session that made it. +bao token create -policy=swarm-bootstrap -ttl=24h -orphan -display-name=swarm-bootstrap +``` + +Put the token's value at `services.hyperhive.deploy.bao.bootstrapTokenFile` +(all-local names that path for you), then rebuild. A one-shot unit inside the +store's container reads it and writes the `swarm-controller` policy. + +**Delete the file once that has run.** The unit skips when it's absent, so a +host that has finished bootstrapping stops carrying the credential — and the +TTL above means a forgotten one expires rather than lingering. + Whether anything more is needed depends on `services.hyperhive.deploy.bao.seal`: diff --git a/nix/host-modules/local-defaults.nix b/nix/host-modules/local-defaults.nix index 7f23761e..55a9fd61 100644 --- a/nix/host-modules/local-defaults.nix +++ b/nix/host-modules/local-defaults.nix @@ -129,6 +129,24 @@ in # itself, and this is a different top-level path. config.services.hyperhive.deploy.swarm-controller.enable = lib.mkDefault cfg.deploy.singleHostSwarm; + # Where the operator drops the token that writes the swarm's first bao + # grants. All-local supplies the path, never the file: `bao operator init` + # stays a human step in every shape (see ./swarm-bao.nix), so what + # co-location makes derivable is only *where* it goes. The granting unit + # skips until the file appears, so naming the path early costs a boot + # nothing. + # + # ⚠️ Deliberately NOT under `/var/lib/swarm-bao-token`, which already holds + # the PKCS11 seal material. Those are both "a bao token" and neither can + # substitute for the other: one unseals the store, this one authorises the + # first write. + # + # Outside the `swarm` attrset above for the same reason the controller's + # options are — a path on this host is a `deploy.*` fact. + config.services.hyperhive.deploy.bao.bootstrapTokenFile = lib.mkIf cfg.deploy.singleHostSwarm ( + lib.mkDefault "/var/lib/swarm-bao-bootstrap/grant.token" + ); + # Same reason, one option later: the queue secret is a path on THIS host, # so it moved to `deploy.*` with the rest of the controller's credentials. # It has to sit out here rather than in the `swarm` attrset above — a bare diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index f1031f53..f48700b6 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -138,6 +138,50 @@ let # rather than as the missing setting it is. haveServerTls = baoDeploy.serverCertFile != null && baoDeploy.serverKeyFile != null; + # The credential that writes the swarm's first grant. A token and not a + # certificate: cert auth answers a *role*, so nothing can authenticate here + # until some role exists, and whatever creates the first one cannot itself + # use one. An operator places it — ../../docs/getting-started/setup.md. + haveBootstrapToken = baoDeploy.bootstrapTokenFile != null; + + # Bind the directory, never the file, for the reason `tokenStoreDir` gives + # above: nothing creates a `hostPath`, so naming a file an operator has not + # placed yet would stop the container from starting. + bootstrapTokenDir = + if haveBootstrapToken then builtins.dirOf baoDeploy.bootstrapTokenFile else null; + + # The name both ends must agree on: whoever later gives the controller a + # cert-auth role attaches this policy by spelling it the same way. + controllerPolicyName = "swarm-controller"; + + # Derived with `bao write -output-policy`, which short-circuits the request + # and needs no server, rather than written from memory. + # + # The `hive-` prefix is load-bearing — without it the controller could + # rewrite the policy that constrains it. + # + # Piped to `bao policy write -` as a shell-quoted argument, not a + # `writeText`: a store path puts the grants behind a hash where + # ../module-eval.nix cannot read them, and a heredoc would make the HCL's + # indentation a function of this file's. + controllerPolicyText = '' + path "auth/cert/certs/hive-*" { + capabilities = ["create", "update", "read", "delete"] + } + + path "auth/cert/certs" { + capabilities = ["list"] + } + + path "sys/policies/acl/hive-*" { + capabilities = ["create", "update", "read", "delete"] + } + + path "sys/policies/acl" { + capabilities = ["list"] + } + ''; + # Every listener serves the same identity: they differ in which address # they answer on, not in who they are. Client verification is separate and # optional — a store with no `clientCaFile` still serves TLS, it just does @@ -396,6 +440,29 @@ in ''; }; + bootstrapTokenFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-bootstrap/grant.token"; + description = '' + Token used **once per swarm** to write the first authorisation grants, + after which every client authenticates with a certificate instead. + + Cert auth answers a *role*, so no client can authenticate until some + role exists — and creating that first one is what this token is for. + It has to come from outside that cycle, which is why an operator places + it rather than the deployment minting it. + + Produce it from the root token `bao operator init` printed, scoped to + that one policy write and nothing else, then delete it once the swarm + has come up — {file}`docs/getting-started/setup.md` has the commands. + Setting this is what enables the granting unit; leaving it null means + the deployment writes those grants some other way. + + A path, never a value. + ''; + }; + clientKeyFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -692,6 +759,22 @@ in ''; }; + # Same shape and the same create-only reason as the token directory + # above: the operator owns what they drop here, so a rebuild must not + # re-impose ownership on it. + systemd.services.swarm-bao-bootstrap-dir = lib.mkIf haveBootstrapToken { + description = "create the directory the swarm's bao bootstrap token goes in"; + before = [ "container@${cfg.machine}.service" ]; + requiredBy = [ "container@${cfg.machine}.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + test -d ${bootstrapTokenDir} || install -d -m 0700 ${bootstrapTokenDir} + ''; + }; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -733,6 +816,14 @@ in hostPath = "/dev/tpmrm0"; isReadOnly = false; }; + } + // lib.optionalAttrs haveBootstrapToken { + # Read-only: the grant unit only reads it, and removing the file + # once the swarm is up is an operator action on the host. + ${bootstrapTokenDir} = { + hostPath = bootstrapTokenDir; + isReadOnly = true; + }; }; # The seal talks to the TPM through the kernel's resource manager, so @@ -903,6 +994,48 @@ in # `nixos-rebuild switch` — it lands in the config file and waits. # Restarting is an operator action with an unseal on the far side of # it, which is why nothing here tries to be clever about it. + + # The swarm's first grant, written from in here because this is + # where the store is reachable without a client certificate — which + # is the point, since no role exists yet to issue one against. + systemd.services.swarm-bao-controller-policy = lib.mkIf haveBootstrapToken { + description = "write the swarm controller's bao policy"; + after = [ "openbao.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ + baoDeploy.package + pkgs.coreutils + ]; + # Named but not placed is a legitimate state: all-local supplies + # the path as a default and the operator drops the file there + # after `bao operator init`. Skipping rather than failing is also + # what makes deleting the token at the end of that procedure safe. + unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + # A store that is up is not necessarily unsealed — under + # `seal = "shamir"` an operator unseals by hand after every + # restart — so early attempts legitimately fail. Bounded, + # because a token that is wrong rather than early would + # otherwise retry forever. + Restart = "on-failure"; + RestartSec = 30; + StartLimitBurst = 10; + }; + environment.BAO_ADDR = "https://${cfg.domain}:${toString cfg.port}"; + script = '' + set -euo pipefail + + BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" + export BAO_TOKEN + + # Idempotent on purpose: a rebuild re-asserts the policy rather + # than failing on one that already exists. + printf '%s' ${lib.escapeShellArg controllerPolicyText} | + bao policy write ${lib.escapeShellArg controllerPolicyName} - + ''; + }; }; }; }) diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 44b1561e..610ae2b8 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -203,6 +203,20 @@ let # is deployed" from "this host can authenticate to the store". matrixNoBaoIdentity = hive { deploy.matrix.enable = true; }; + # The store, plus a placed bootstrap token: the only shape in which the + # swarm's first grant can be written at all. + baoGrantHere = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + }; + # The credential without the store. Writing the first grant is a store-side + # operation, so a host holding only the token has nothing to do — and this + # is the arm that separates "an operator placed a token" from "this box can + # act on it". + baoGrantNoStore = hive { + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + }; + baoNames = machine: machine.services.hyperhive.gateway.localNames; baoTwoAddresses = hive { @@ -428,6 +442,39 @@ let lib.any (c: lib.hasInfix "/var/lib/swarm-otel-oidc/by-hand.secret" c) otelRemoteAuthelia.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential; } + { + # Reads the rendered unit INSIDE the container, which is where the write + # happens: reaching the store locally is what lets the grant be written + # without a client certificate at all. + name = "a store host with a placed bootstrap token renders the granting unit inside the container"; + ok = + let + u = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy; + in + lib.hasInfix "/run/secrets/bao-bootstrap.token" u.script + && u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token"; + } + { + # The grants themselves, and the `hive-` prefix is the whole point: + # without it the controller can rewrite the policy that constrains it, + # which is a privilege escalation that renders, deploys and looks fine. + # Readable here only because the HCL is piped as an argument rather than + # written to a store path. + name = "the controller's bao grants cannot reach the policy that constrains it"; + ok = + let + s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "sys/policies/acl/hive-*" s && !(lib.hasInfix "sys/policies/acl/*" s); + } + { + # The arm that makes the one above mean something, and the property the + # host-side half depends on: no store here, so no bind mount and no unit. + # Without it a hive that merely names a token would drag the store's + # container config into its evaluation. + name = "a bootstrap token on a host that runs no store grants nothing"; + ok = !(baoGrantNoStore.systemd.services ? swarm-bao-bootstrap-dir); + } { name = "a config written against the pre-rename authelia usersFile still reaches the bridge"; ok =