diff --git a/docs/getting-started/setup.md b/docs/getting-started/setup.md index cd0bdacb..84fbabcc 100644 --- a/docs/getting-started/setup.md +++ b/docs/getting-started/setup.md @@ -103,17 +103,16 @@ bao token create -policy=swarm-bootstrap -ttl=24h -orphan -display-name=swarm-bo ``` Put the token's value at `services.hyperhive.deploy.bao.bootstrapTokenFile` -(all-local names that path for you), then rebuild. A one-shot unit **on the -host** reads it, writes the `swarm-controller` policy, enables the cert auth -method, and creates the `swarm-controller` role that attaches the two. It runs -there because every API listener demands a client certificate, and the host is -the side that has one. +(all-local names that path for you), then rebuild. A one-shot unit inside the +store's container reads it, writes the `swarm-controller` policy, enables the +cert auth method, and creates the `swarm-controller` role that attaches the two. -⚠️ **The grants above are derived, not proven.** They come from -`-output-policy`, not from a swarm that came up on them. The unit has run twice -against a live store and failed both times for reasons of its own — it could -not reach the store from where it then ran — so neither run exercised these -grants at all. +⚠️ **This has only run once, and it didn't go all the way through.** The +first real provision against a live store returned 403, so +treat the block above as derived, then only partly tested — not proven: the +grants still come from `-output-policy`, not from a swarm that came up on them. +What that 403 does _not_ tell you is whether the login or the write was +refused — `bao read auth/cert/certs/swarm-controller` separates the two. **Delete the file only once `bao read auth/cert/certs/swarm-controller` returns the role.** The unit skips when the token is absent, so a host that has finished diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 9d538a65..83f77383 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -856,88 +856,6 @@ in ''; }; - # The swarm's first grant, written from the HOST. Every API listener sets - # `tls_require_and_verify_client_cert`, so a client needs an identity - # wherever it runs — and only the host has one. The bootstrap token is a - # host path too; the container saw it through a bind mount. - systemd.services.swarm-bao-controller-policy = lib.mkIf haveBootstrapToken { - description = "write the swarm controller's bao policy and cert-auth role"; - after = [ "container@${cfg.machine}.service" ]; - wantedBy = [ "multi-user.target" ]; - # The wrapper rather than the package: it carries the address, the CA - # and this host's certificate, which is what makes running here cheaper - # than shipping an identity the other way. - path = [ - baoCli - pkgs.coreutils - ]; - # Named but not placed is a legitimate state: all-local supplies the - # path as a default and the operator drops the file there after - # `bao operator init`. Skipping rather than failing is also what makes - # deleting the token at the end of that procedure safe. - unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; - # A store that is up is not necessarily unsealed — under - # `seal = "shamir"` an operator unseals BY HAND, so early attempts fail - # for as long as that takes, which can be a day. - # - # `StartLimit*` are `[Unit]` settings that systemd ignores under - # `[Service]`, which is what these two options render to. The window has - # to exceed `RestartSec × burst`, or it closes between attempts and the - # burst is never reached: 2880 × 30s is 24h, inside a 25h window. - # Bounding it tighter would give up before a human could unseal, and - # `start-limit-hit` does not self-heal. - startLimitBurst = 2880; - startLimitIntervalSec = 90000; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - Restart = "on-failure"; - RestartSec = 30; - }; - script = '' - set -euo pipefail - - BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" - export BAO_TOKEN - - # Idempotent on purpose: a rebuild re-asserts the policy rather - # than failing on one that already exists. - printf '%s' ${lib.escapeShellArg controllerPolicyText} | - bao policy write ${lib.escapeShellArg controllerPolicyName} - - '' - + lib.optionalString (baoDeploy.clientCaFile != null) '' - - # The policy above grants paths under `auth/cert/`, and nothing - # in this tree creates that mount. Without this, the grant names - # a location that does not exist and every certificate login - # fails — the controller's own, and the per-hive ones it later - # issues against the same mount. - # - # Asked rather than attempted: `auth enable` errors on a mount - # that already exists, and recognising that would tie a rebuild - # to an error string we have never seen this store emit. - mounted="$(bao auth list -format=json)" - case "$mounted" in - *'"cert/"'*) ;; - *) bao auth enable cert ;; - esac - - # `certificate=` is the CA, so this role trusts every leaf that - # CA signed and `allowed_common_names` is the whole of what - # narrows it to one identity. ⚠️ The same CA signs each hive's - # reader leaf with CN = the hive's name, so a hive named - # `${controllerCn}` would satisfy this role. - # - # Named outside the `hive-*` namespace the policy grants, so the - # controller cannot rewrite the role that constrains it. - bao write auth/cert/certs/${lib.escapeShellArg controllerPolicyName} \ - certificate=@${tlsDir}/client-ca.pem \ - allowed_common_names=${lib.escapeShellArg controllerCn} \ - token_policies=${lib.escapeShellArg controllerPolicyName} \ - display_name=${lib.escapeShellArg controllerCn} - ''; - }; - containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -1158,6 +1076,78 @@ in # Restarting is an operator action with an unseal on the far side of # it, which is why nothing here tries to be clever about it. + # The swarm's first grant, written from in here because this is + # where the store is reachable without a client certificate — which + # is the point, since no role exists yet to issue one against. + systemd.services.swarm-bao-controller-policy = lib.mkIf haveBootstrapToken { + description = "write the swarm controller's bao policy and cert-auth role"; + after = [ "openbao.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ + baoDeploy.package + pkgs.coreutils + ]; + # Named but not placed is a legitimate state: all-local supplies + # the path as a default and the operator drops the file there + # after `bao operator init`. Skipping rather than failing is also + # what makes deleting the token at the end of that procedure safe. + unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + # A store that is up is not necessarily unsealed — under + # `seal = "shamir"` an operator unseals by hand after every + # restart — so early attempts legitimately fail. Bounded, + # because a token that is wrong rather than early would + # otherwise retry forever. + Restart = "on-failure"; + RestartSec = 30; + StartLimitBurst = 10; + }; + environment.BAO_ADDR = "https://${cfg.domain}:${toString cfg.port}"; + script = '' + set -euo pipefail + + BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" + export BAO_TOKEN + + # Idempotent on purpose: a rebuild re-asserts the policy rather + # than failing on one that already exists. + printf '%s' ${lib.escapeShellArg controllerPolicyText} | + bao policy write ${lib.escapeShellArg controllerPolicyName} - + '' + + lib.optionalString (baoDeploy.clientCaFile != null) '' + + # The policy above grants paths under `auth/cert/`, and nothing + # in this tree creates that mount. Without this, the grant names + # a location that does not exist and every certificate login + # fails — the controller's own, and the per-hive ones it later + # issues against the same mount. + # + # Asked rather than attempted: `auth enable` errors on a mount + # that already exists, and recognising that would tie a rebuild + # to an error string we have never seen this store emit. + mounted="$(bao auth list -format=json)" + case "$mounted" in + *'"cert/"'*) ;; + *) bao auth enable cert ;; + esac + + # `certificate=` is the CA, so this role trusts every leaf that + # CA signed and `allowed_common_names` is the whole of what + # narrows it to one identity. ⚠️ The same CA signs each hive's + # reader leaf with CN = the hive's name, so a hive named + # `${controllerCn}` would satisfy this role. + # + # Named outside the `hive-*` namespace the policy grants, so the + # controller cannot rewrite the role that constrains it. + bao write auth/cert/certs/${lib.escapeShellArg controllerPolicyName} \ + certificate=@${tlsDir}/client-ca.pem \ + allowed_common_names=${lib.escapeShellArg controllerCn} \ + token_policies=${lib.escapeShellArg controllerPolicyName} \ + display_name=${lib.escapeShellArg controllerCn} + ''; + }; }; }; }) diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 7a48f032..d51f92fe 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -525,40 +525,17 @@ let otelRemoteAuthelia.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential; } { - # Reads the rendered unit on the HOST, which is where the write happens: - # every API listener demands a client certificate, and the host is the - # side that has one. - name = "a store host with a placed bootstrap token renders the granting unit on the host"; + # Reads the rendered unit INSIDE the container, which is where the write + # happens: reaching the store locally is what lets the grant be written + # without a client certificate at all. + name = "a store host with a placed bootstrap token renders the granting unit inside the container"; ok = let - u = baoGrantHere.systemd.services.swarm-bao-controller-policy; + u = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy; in lib.hasInfix "/run/secrets/bao-bootstrap.token" u.script && u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token"; } - { - # The move is the fix, so pin the side it landed on: in the container it - # had no identity to open a connection with, and no address that resolved - # to the store from its own netns. - name = "the granting unit is not rendered inside the store's container"; - ok = !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-controller-policy); - } - { - # `StartLimit*` are `[Unit]` settings that systemd ignores under - # `[Service]`, so a bound written into `serviceConfig` renders, deploys - # and does nothing. Asserted where nixpkgs puts it rather than where it - # was written. The values are pinned because they are the bound: under - # `shamir` a human unseals by hand, and anything shorter than a day gives - # up first — `start-limit-hit` does not self-heal. - name = "the granting unit's start limit lands in [Unit], not [Service]"; - ok = - let - u = baoGrantHere.systemd.services.swarm-bao-controller-policy; - in - toString u.unitConfig.StartLimitBurst == "2880" - && toString u.unitConfig.StartLimitIntervalSec == "90000" - && !(u.serviceConfig ? StartLimitBurst); - } { # The grants themselves, and the `hive-` prefix is the whole point: # without it the controller can rewrite the policy that constrains it, @@ -568,7 +545,7 @@ let name = "the controller's bao grants cannot reach the policy that constrains it"; ok = let - s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "sys/policies/acl/hive-*" s && !(lib.hasInfix "sys/policies/acl/*" s); } @@ -582,7 +559,7 @@ let name = "the controller may write agent credentials, and only under the agent prefix"; ok = let - s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "secret/data/swarm/agents/*" s && !(lib.hasInfix "secret/data/*" s) @@ -596,7 +573,7 @@ let name = "the controller's grant on agent credentials is write-only"; ok = let - s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "path \"secret/data/swarm/agents/*\" {\n capabilities = [\"create\", \"update\"]" s; } @@ -607,7 +584,7 @@ let name = "the granting unit creates the cert auth mount and the controller's role"; ok = let - s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "bao auth enable cert" s && lib.hasInfix "auth/cert/certs/swarm-controller" s @@ -760,7 +737,7 @@ let ok = let m = baoControllerHere; - role = m.systemd.services.swarm-bao-controller-policy.script; + role = m.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; pki = m.systemd.services.swarm-bao-pki.script; in lib.hasInfix "cn-marker-not-a-default" role && lib.hasInfix "cn-marker-not-a-default" pki; @@ -776,7 +753,8 @@ let name = "with no client CA the unit still writes the policy and skips the role"; ok = let - s = baoGrantNoClientCa.systemd.services.swarm-bao-controller-policy.script; + s = + baoGrantNoClientCa.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "bao policy write" s && !(lib.hasInfix "bao auth enable cert" s)