diff --git a/docs/getting-started/setup.md b/docs/getting-started/setup.md index 4cbb3dc3..c5b6dedf 100644 --- a/docs/getting-started/setup.md +++ b/docs/getting-started/setup.md @@ -56,6 +56,31 @@ never initialised. Do this before anything is pointed at it. bao operator init # keep the keys it prints and the root token OFF this host ``` +While you still hold that root token, mint the one credential the swarm needs +to grant itself anything. Cert auth answers a _role_, so nothing can +authenticate until some role exists — this token is what breaks that cycle, +and it's the only step that needs the root token. + +```bash +# A policy that can write exactly one policy, and nothing else. +bao policy write swarm-bootstrap - <<'EOF' +path "sys/policies/acl/swarm-controller" { + capabilities = ["create", "update"] +} +EOF + +# A token holding it. `-orphan` so it outlives the session that made it. +bao token create -policy=swarm-bootstrap -ttl=24h -orphan -display-name=swarm-bootstrap +``` + +Put the token's value at `services.hyperhive.deploy.bao.bootstrapTokenFile` +(all-local names that path for you), then rebuild. A one-shot unit inside the +store's container reads it and writes the `swarm-controller` policy. + +**Delete the file once that has run.** The unit skips when it's absent, so a +host that has finished bootstrapping stops carrying the credential — and the +TTL above means a forgotten one expires rather than lingering. + Whether anything more is needed depends on `services.hyperhive.deploy.bao.seal`: diff --git a/nix/host-modules/local-defaults.nix b/nix/host-modules/local-defaults.nix index 7f23761e..55a9fd61 100644 --- a/nix/host-modules/local-defaults.nix +++ b/nix/host-modules/local-defaults.nix @@ -129,6 +129,24 @@ in # itself, and this is a different top-level path. config.services.hyperhive.deploy.swarm-controller.enable = lib.mkDefault cfg.deploy.singleHostSwarm; + # Where the operator drops the token that writes the swarm's first bao + # grants. All-local supplies the path, never the file: `bao operator init` + # stays a human step in every shape (see ./swarm-bao.nix), so what + # co-location makes derivable is only *where* it goes. The granting unit + # skips until the file appears, so naming the path early costs a boot + # nothing. + # + # ⚠️ Deliberately NOT under `/var/lib/swarm-bao-token`, which already holds + # the PKCS11 seal material. Those are both "a bao token" and neither can + # substitute for the other: one unseals the store, this one authorises the + # first write. + # + # Outside the `swarm` attrset above for the same reason the controller's + # options are — a path on this host is a `deploy.*` fact. + config.services.hyperhive.deploy.bao.bootstrapTokenFile = lib.mkIf cfg.deploy.singleHostSwarm ( + lib.mkDefault "/var/lib/swarm-bao-bootstrap/grant.token" + ); + # Same reason, one option later: the queue secret is a path on THIS host, # so it moved to `deploy.*` with the rest of the controller's credentials. # It has to sit out here rather than in the `swarm` attrset above — a bare diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index f1031f53..f48700b6 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -138,6 +138,50 @@ let # rather than as the missing setting it is. haveServerTls = baoDeploy.serverCertFile != null && baoDeploy.serverKeyFile != null; + # The credential that writes the swarm's first grant. A token and not a + # certificate: cert auth answers a *role*, so nothing can authenticate here + # until some role exists, and whatever creates the first one cannot itself + # use one. An operator places it — ../../docs/getting-started/setup.md. + haveBootstrapToken = baoDeploy.bootstrapTokenFile != null; + + # Bind the directory, never the file, for the reason `tokenStoreDir` gives + # above: nothing creates a `hostPath`, so naming a file an operator has not + # placed yet would stop the container from starting. + bootstrapTokenDir = + if haveBootstrapToken then builtins.dirOf baoDeploy.bootstrapTokenFile else null; + + # The name both ends must agree on: whoever later gives the controller a + # cert-auth role attaches this policy by spelling it the same way. + controllerPolicyName = "swarm-controller"; + + # Derived with `bao write -output-policy`, which short-circuits the request + # and needs no server, rather than written from memory. + # + # The `hive-` prefix is load-bearing — without it the controller could + # rewrite the policy that constrains it. + # + # Piped to `bao policy write -` as a shell-quoted argument, not a + # `writeText`: a store path puts the grants behind a hash where + # ../module-eval.nix cannot read them, and a heredoc would make the HCL's + # indentation a function of this file's. + controllerPolicyText = '' + path "auth/cert/certs/hive-*" { + capabilities = ["create", "update", "read", "delete"] + } + + path "auth/cert/certs" { + capabilities = ["list"] + } + + path "sys/policies/acl/hive-*" { + capabilities = ["create", "update", "read", "delete"] + } + + path "sys/policies/acl" { + capabilities = ["list"] + } + ''; + # Every listener serves the same identity: they differ in which address # they answer on, not in who they are. Client verification is separate and # optional — a store with no `clientCaFile` still serves TLS, it just does @@ -396,6 +440,29 @@ in ''; }; + bootstrapTokenFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-bootstrap/grant.token"; + description = '' + Token used **once per swarm** to write the first authorisation grants, + after which every client authenticates with a certificate instead. + + Cert auth answers a *role*, so no client can authenticate until some + role exists — and creating that first one is what this token is for. + It has to come from outside that cycle, which is why an operator places + it rather than the deployment minting it. + + Produce it from the root token `bao operator init` printed, scoped to + that one policy write and nothing else, then delete it once the swarm + has come up — {file}`docs/getting-started/setup.md` has the commands. + Setting this is what enables the granting unit; leaving it null means + the deployment writes those grants some other way. + + A path, never a value. + ''; + }; + clientKeyFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -692,6 +759,22 @@ in ''; }; + # Same shape and the same create-only reason as the token directory + # above: the operator owns what they drop here, so a rebuild must not + # re-impose ownership on it. + systemd.services.swarm-bao-bootstrap-dir = lib.mkIf haveBootstrapToken { + description = "create the directory the swarm's bao bootstrap token goes in"; + before = [ "container@${cfg.machine}.service" ]; + requiredBy = [ "container@${cfg.machine}.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + test -d ${bootstrapTokenDir} || install -d -m 0700 ${bootstrapTokenDir} + ''; + }; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -733,6 +816,14 @@ in hostPath = "/dev/tpmrm0"; isReadOnly = false; }; + } + // lib.optionalAttrs haveBootstrapToken { + # Read-only: the grant unit only reads it, and removing the file + # once the swarm is up is an operator action on the host. + ${bootstrapTokenDir} = { + hostPath = bootstrapTokenDir; + isReadOnly = true; + }; }; # The seal talks to the TPM through the kernel's resource manager, so @@ -903,6 +994,48 @@ in # `nixos-rebuild switch` — it lands in the config file and waits. # Restarting is an operator action with an unseal on the far side of # it, which is why nothing here tries to be clever about it. + + # The swarm's first grant, written from in here because this is + # where the store is reachable without a client certificate — which + # is the point, since no role exists yet to issue one against. + systemd.services.swarm-bao-controller-policy = lib.mkIf haveBootstrapToken { + description = "write the swarm controller's bao policy"; + after = [ "openbao.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ + baoDeploy.package + pkgs.coreutils + ]; + # Named but not placed is a legitimate state: all-local supplies + # the path as a default and the operator drops the file there + # after `bao operator init`. Skipping rather than failing is also + # what makes deleting the token at the end of that procedure safe. + unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + # A store that is up is not necessarily unsealed — under + # `seal = "shamir"` an operator unseals by hand after every + # restart — so early attempts legitimately fail. Bounded, + # because a token that is wrong rather than early would + # otherwise retry forever. + Restart = "on-failure"; + RestartSec = 30; + StartLimitBurst = 10; + }; + environment.BAO_ADDR = "https://${cfg.domain}:${toString cfg.port}"; + script = '' + set -euo pipefail + + BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" + export BAO_TOKEN + + # Idempotent on purpose: a rebuild re-asserts the policy rather + # than failing on one that already exists. + printf '%s' ${lib.escapeShellArg controllerPolicyText} | + bao policy write ${lib.escapeShellArg controllerPolicyName} - + ''; + }; }; }; }) diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 44b1561e..610ae2b8 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -203,6 +203,20 @@ let # is deployed" from "this host can authenticate to the store". matrixNoBaoIdentity = hive { deploy.matrix.enable = true; }; + # The store, plus a placed bootstrap token: the only shape in which the + # swarm's first grant can be written at all. + baoGrantHere = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + }; + # The credential without the store. Writing the first grant is a store-side + # operation, so a host holding only the token has nothing to do — and this + # is the arm that separates "an operator placed a token" from "this box can + # act on it". + baoGrantNoStore = hive { + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + }; + baoNames = machine: machine.services.hyperhive.gateway.localNames; baoTwoAddresses = hive { @@ -428,6 +442,39 @@ let lib.any (c: lib.hasInfix "/var/lib/swarm-otel-oidc/by-hand.secret" c) otelRemoteAuthelia.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential; } + { + # Reads the rendered unit INSIDE the container, which is where the write + # happens: reaching the store locally is what lets the grant be written + # without a client certificate at all. + name = "a store host with a placed bootstrap token renders the granting unit inside the container"; + ok = + let + u = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy; + in + lib.hasInfix "/run/secrets/bao-bootstrap.token" u.script + && u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token"; + } + { + # The grants themselves, and the `hive-` prefix is the whole point: + # without it the controller can rewrite the policy that constrains it, + # which is a privilege escalation that renders, deploys and looks fine. + # Readable here only because the HCL is piped as an argument rather than + # written to a store path. + name = "the controller's bao grants cannot reach the policy that constrains it"; + ok = + let + s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "sys/policies/acl/hive-*" s && !(lib.hasInfix "sys/policies/acl/*" s); + } + { + # The arm that makes the one above mean something, and the property the + # host-side half depends on: no store here, so no bind mount and no unit. + # Without it a hive that merely names a token would drag the store's + # container config into its evaluation. + name = "a bootstrap token on a host that runs no store grants nothing"; + ok = !(baoGrantNoStore.systemd.services ? swarm-bao-bootstrap-dir); + } { name = "a config written against the pre-rename authelia usersFile still reaches the bridge"; ok =