diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 282f9687..9d538a65 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -856,6 +856,88 @@ in ''; }; + # The swarm's first grant, written from the HOST. Every API listener sets + # `tls_require_and_verify_client_cert`, so a client needs an identity + # wherever it runs — and only the host has one. The bootstrap token is a + # host path too; the container saw it through a bind mount. + systemd.services.swarm-bao-controller-policy = lib.mkIf haveBootstrapToken { + description = "write the swarm controller's bao policy and cert-auth role"; + after = [ "container@${cfg.machine}.service" ]; + wantedBy = [ "multi-user.target" ]; + # The wrapper rather than the package: it carries the address, the CA + # and this host's certificate, which is what makes running here cheaper + # than shipping an identity the other way. + path = [ + baoCli + pkgs.coreutils + ]; + # Named but not placed is a legitimate state: all-local supplies the + # path as a default and the operator drops the file there after + # `bao operator init`. Skipping rather than failing is also what makes + # deleting the token at the end of that procedure safe. + unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; + # A store that is up is not necessarily unsealed — under + # `seal = "shamir"` an operator unseals BY HAND, so early attempts fail + # for as long as that takes, which can be a day. + # + # `StartLimit*` are `[Unit]` settings that systemd ignores under + # `[Service]`, which is what these two options render to. The window has + # to exceed `RestartSec × burst`, or it closes between attempts and the + # burst is never reached: 2880 × 30s is 24h, inside a 25h window. + # Bounding it tighter would give up before a human could unseal, and + # `start-limit-hit` does not self-heal. + startLimitBurst = 2880; + startLimitIntervalSec = 90000; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + Restart = "on-failure"; + RestartSec = 30; + }; + script = '' + set -euo pipefail + + BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" + export BAO_TOKEN + + # Idempotent on purpose: a rebuild re-asserts the policy rather + # than failing on one that already exists. + printf '%s' ${lib.escapeShellArg controllerPolicyText} | + bao policy write ${lib.escapeShellArg controllerPolicyName} - + '' + + lib.optionalString (baoDeploy.clientCaFile != null) '' + + # The policy above grants paths under `auth/cert/`, and nothing + # in this tree creates that mount. Without this, the grant names + # a location that does not exist and every certificate login + # fails — the controller's own, and the per-hive ones it later + # issues against the same mount. + # + # Asked rather than attempted: `auth enable` errors on a mount + # that already exists, and recognising that would tie a rebuild + # to an error string we have never seen this store emit. + mounted="$(bao auth list -format=json)" + case "$mounted" in + *'"cert/"'*) ;; + *) bao auth enable cert ;; + esac + + # `certificate=` is the CA, so this role trusts every leaf that + # CA signed and `allowed_common_names` is the whole of what + # narrows it to one identity. ⚠️ The same CA signs each hive's + # reader leaf with CN = the hive's name, so a hive named + # `${controllerCn}` would satisfy this role. + # + # Named outside the `hive-*` namespace the policy grants, so the + # controller cannot rewrite the role that constrains it. + bao write auth/cert/certs/${lib.escapeShellArg controllerPolicyName} \ + certificate=@${tlsDir}/client-ca.pem \ + allowed_common_names=${lib.escapeShellArg controllerCn} \ + token_policies=${lib.escapeShellArg controllerPolicyName} \ + display_name=${lib.escapeShellArg controllerCn} + ''; + }; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -1076,84 +1158,6 @@ in # Restarting is an operator action with an unseal on the far side of # it, which is why nothing here tries to be clever about it. - # The swarm's first grant, written from in here because this is - # where the store is reachable without a client certificate — which - # is the point, since no role exists yet to issue one against. - systemd.services.swarm-bao-controller-policy = lib.mkIf haveBootstrapToken { - description = "write the swarm controller's bao policy and cert-auth role"; - after = [ "openbao.service" ]; - wantedBy = [ "multi-user.target" ]; - path = [ - baoDeploy.package - pkgs.coreutils - ]; - # Named but not placed is a legitimate state: all-local supplies - # the path as a default and the operator drops the file there - # after `bao operator init`. Skipping rather than failing is also - # what makes deleting the token at the end of that procedure safe. - unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile; - # A store that is up is not necessarily unsealed — under - # `seal = "shamir"` an operator unseals by hand after every - # restart — so early attempts legitimately fail. Bounded, - # because a token that is wrong rather than early would - # otherwise retry forever. - # - # `StartLimit*` are `[Unit]` settings that systemd ignores under - # `[Service]`, which is what these two options render to. The - # interval also has to exceed `RestartSec × burst`, or the window - # closes between attempts and the burst is never reached. - startLimitBurst = 10; - startLimitIntervalSec = 600; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - Restart = "on-failure"; - RestartSec = 30; - }; - environment.BAO_ADDR = "https://${cfg.domain}:${toString cfg.port}"; - script = '' - set -euo pipefail - - BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})" - export BAO_TOKEN - - # Idempotent on purpose: a rebuild re-asserts the policy rather - # than failing on one that already exists. - printf '%s' ${lib.escapeShellArg controllerPolicyText} | - bao policy write ${lib.escapeShellArg controllerPolicyName} - - '' - + lib.optionalString (baoDeploy.clientCaFile != null) '' - - # The policy above grants paths under `auth/cert/`, and nothing - # in this tree creates that mount. Without this, the grant names - # a location that does not exist and every certificate login - # fails — the controller's own, and the per-hive ones it later - # issues against the same mount. - # - # Asked rather than attempted: `auth enable` errors on a mount - # that already exists, and recognising that would tie a rebuild - # to an error string we have never seen this store emit. - mounted="$(bao auth list -format=json)" - case "$mounted" in - *'"cert/"'*) ;; - *) bao auth enable cert ;; - esac - - # `certificate=` is the CA, so this role trusts every leaf that - # CA signed and `allowed_common_names` is the whole of what - # narrows it to one identity. ⚠️ The same CA signs each hive's - # reader leaf with CN = the hive's name, so a hive named - # `${controllerCn}` would satisfy this role. - # - # Named outside the `hive-*` namespace the policy grants, so the - # controller cannot rewrite the role that constrains it. - bao write auth/cert/certs/${lib.escapeShellArg controllerPolicyName} \ - certificate=@${tlsDir}/client-ca.pem \ - allowed_common_names=${lib.escapeShellArg controllerCn} \ - token_policies=${lib.escapeShellArg controllerPolicyName} \ - display_name=${lib.escapeShellArg controllerCn} - ''; - }; }; }; }) diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 625fc875..7a48f032 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -525,30 +525,38 @@ let otelRemoteAuthelia.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential; } { - # Reads the rendered unit INSIDE the container, which is where the write - # happens: reaching the store locally is what lets the grant be written - # without a client certificate at all. - name = "a store host with a placed bootstrap token renders the granting unit inside the container"; + # Reads the rendered unit on the HOST, which is where the write happens: + # every API listener demands a client certificate, and the host is the + # side that has one. + name = "a store host with a placed bootstrap token renders the granting unit on the host"; ok = let - u = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy; + u = baoGrantHere.systemd.services.swarm-bao-controller-policy; in lib.hasInfix "/run/secrets/bao-bootstrap.token" u.script && u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token"; } + { + # The move is the fix, so pin the side it landed on: in the container it + # had no identity to open a connection with, and no address that resolved + # to the store from its own netns. + name = "the granting unit is not rendered inside the store's container"; + ok = !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-controller-policy); + } { # `StartLimit*` are `[Unit]` settings that systemd ignores under # `[Service]`, so a bound written into `serviceConfig` renders, deploys # and does nothing. Asserted where nixpkgs puts it rather than where it - # was written, and the interval is part of the bound: it has to exceed - # `RestartSec × burst` or the window closes between attempts. + # was written. The values are pinned because they are the bound: under + # `shamir` a human unseals by hand, and anything shorter than a day gives + # up first — `start-limit-hit` does not self-heal. name = "the granting unit's start limit lands in [Unit], not [Service]"; ok = let - u = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy; + u = baoGrantHere.systemd.services.swarm-bao-controller-policy; in - toString u.unitConfig.StartLimitBurst == "10" - && toString u.unitConfig.StartLimitIntervalSec == "600" + toString u.unitConfig.StartLimitBurst == "2880" + && toString u.unitConfig.StartLimitIntervalSec == "90000" && !(u.serviceConfig ? StartLimitBurst); } { @@ -560,7 +568,7 @@ let name = "the controller's bao grants cannot reach the policy that constrains it"; ok = let - s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "sys/policies/acl/hive-*" s && !(lib.hasInfix "sys/policies/acl/*" s); } @@ -574,7 +582,7 @@ let name = "the controller may write agent credentials, and only under the agent prefix"; ok = let - s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "secret/data/swarm/agents/*" s && !(lib.hasInfix "secret/data/*" s) @@ -588,7 +596,7 @@ let name = "the controller's grant on agent credentials is write-only"; ok = let - s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "path \"secret/data/swarm/agents/*\" {\n capabilities = [\"create\", \"update\"]" s; } @@ -599,7 +607,7 @@ let name = "the granting unit creates the cert auth mount and the controller's role"; ok = let - s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "bao auth enable cert" s && lib.hasInfix "auth/cert/certs/swarm-controller" s @@ -752,7 +760,7 @@ let ok = let m = baoControllerHere; - role = m.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; + role = m.systemd.services.swarm-bao-controller-policy.script; pki = m.systemd.services.swarm-bao-pki.script; in lib.hasInfix "cn-marker-not-a-default" role && lib.hasInfix "cn-marker-not-a-default" pki; @@ -768,8 +776,7 @@ let name = "with no client CA the unit still writes the policy and skips the role"; ok = let - s = - baoGrantNoClientCa.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script; + s = baoGrantNoClientCa.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "bao policy write" s && !(lib.hasInfix "bao auth enable cert" s)