diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index a0e68661..0725b7d4 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -118,6 +118,16 @@ in description = "mint the swarm secret store's own CA and leaves"; before = [ "swarm-bao-certs.service" ]; requiredBy = [ "swarm-bao-certs.service" ]; + # 🩸 A target wants this, not only `swarm-bao-certs`, and the reason is + # what happens when the LIST of leaves grows. `requiredBy` alone is + # satisfied by a unit that already ran: this one is `RemainAfterExit`, + # so an activation that adds a leaf here finds it active, pulls nothing, + # and the new leaf is never minted — while every sibling granting unit, + # each of which carries this line, was re-run by the same activation. + # That is how `services-issuer.pem` came to be missing on a host whose + # config named it, leaving `swarm-services-cert` to fail on an absent + # file until someone restarted this by hand. + wantedBy = [ "multi-user.target" ]; path = [ pkgs.openssl pkgs.coreutils diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 28b1f133..e6061327 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -386,14 +386,19 @@ in # to wait for, so the dependency is conditional rather than a unit # that exists and does nothing. # - # `swarm-services-cert` is ordered before this one rather than the - # other way round, and not because this unit needs the leaf — it - # never touches it. The trust bundle written at the end of this - # script anchors on every root a consumer must hold, and the - # services root is one of them, so the file has to be on disk - # before the `cat`. Get it wrong and the bundle is a boot behind: - # correct-looking, and missing the anchor for exactly the names the - # gateway serves with it. + # `swarm-services-cert` is NOT ordered against this unit in either + # direction, and the absence is load-bearing — see the cycle spelled + # out at that unit. It runs after the store's container, the container + # runs after this unit, so an edge back to here is a loop and systemd + # resolves a loop by dropping a job. + # + # What that costs: the trust bundle written at the end of this script + # anchors on every root a consumer must hold, and the services root is + # one of them, so on a first boot the `cat` below runs before that file + # exists and the bundle is one restart short of complete. The `if [ -s + # "$svcroot" ]` guard is what makes that state legal rather than fatal, + # and `swarm-services-cert` restarts this unit once the root it wrote + # is new — so the bundle converges instead of being a boot behind. after = lib.optionals swarmCaCfg.autoConfigure [ "swarm-ca.service" ]; requires = lib.optionals swarmCaCfg.autoConfigure [ "swarm-ca.service" ]; path = [ pkgs.openssl ]; @@ -647,18 +652,38 @@ in # coincidence, and `requiredBy` is what makes a store that never # comes up an outage that says so instead of a gateway serving the # wrong name. - before = [ - "hive-tls-ca.service" - "hive-gateway-self-signed-cert.service" - ]; + # + # 🩸 NOT `Before=hive-tls-ca.service`, however much the trust bundle + # wants it. That edge closed a cycle — the store's container is + # `After=hive-tls-ca` (it serves the material that unit writes), and + # this unit is `After=` the container — so systemd broke the loop the + # only way it can, by deleting a job: + # + # Found ordering cycle: container@swarm-bao.service/start after + # hive-tls-ca.service/start after swarm-services-cert.service/start + # Job swarm-services-cert.service/start deleted to break ordering cycle + # + # With the job gone the leaf was never issued, `Requires=` took the + # gateway's cert import down with it, and nginx never started at all — + # so the swarm served nothing rather than serving the wrong name. The + # bundle reaches the root the other way round, at the end of the script + # below: it restarts `hive-tls-ca` when the root CHANGED, which is the + # same path that already covered a store coming up hours late. + before = [ "hive-gateway-self-signed-cert.service" ]; requiredBy = [ "hive-gateway-self-signed-cert.service" ]; # The store's container, where it runs here. On a hive that reads a # store hosted elsewhere no such unit exists and systemd ignores # the name, which is the correct behaviour rather than a gap: what # this unit actually needs is the store reachable, and on a remote # store nothing local can order against that. + # + # `swarm-bao-pki` mints the services-issuer leaf the login below + # presents, and is absent on a hive holding an operator-placed one — + # ignored there for the same reason, rather than being a dependency + # only some hives may declare. after = [ "container@${baoCfg.machine}.service" + "swarm-bao-pki.service" "swarm-bao-services-issuer-policy.service" ]; wants = [ "container@${baoCfg.machine}.service" ]; @@ -723,6 +748,16 @@ in covers "$svcleaf" "$svcNames" || reissue=1 [ -s "$svcroot" ] || reissue=1 + # The ROOT's window, not the leaf's, and it is what keeps the two + # sides of this from needing to talk. The store replaces an issuer + # that has less than a leaf's window left (./swarm-bao.nix's + # regeneration guard) — so a hive testing its own copy of that same + # certificate against the same threshold asks for a new leaf on the + # same activation, and gets the replacement root back with it. + # Without this a leaf stays "valid" while the anchor it chains to no + # longer exists in the mount, which no expiry check would ever catch. + openssl x509 -in "$svcroot" -noout -checkend 2592000 >/dev/null 2>&1 || reissue=1 + if [ "$reissue" = 0 ]; then echo "swarm-services leaf valid and covering the configured names — leaving it alone" exit 0 @@ -857,11 +892,12 @@ in systemctl reload --no-block nginx.service fi - # The bundle is assembled by `hive-tls-ca`, which is ordered - # after this unit and therefore already holds the current root on - # any normal boot. On the retry path it ran a long time ago, so a - # CHANGED root has to reach it — and only a changed one, or every - # boot would bounce a unit with nothing to do. + # The bundle is assembled by `hive-tls-ca`, which runs BEFORE this + # unit — it has to, or the store's container cannot start and there + # is nothing to ask for a leaf. So on a first boot it wrote a bundle + # without the root below, and this restart is what completes it + # rather than a retry-path fallback. Only on a CHANGED root: every + # boot otherwise bounces a unit with nothing to do. if [ "$rootchanged" = 1 ] && systemctl is-active --quiet hive-tls-ca.service; then echo "the services root changed — rebuilding the trust bundle" systemctl restart --no-block hive-tls-ca.service diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 8abadefd..fcf48209 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -336,6 +336,18 @@ let # absorbed the churn; nothing sits above this one. servicesPkiRootTtl = "262800h"; + # The leaf's window, pinned on the role rather than left to the mount's + # default. Two reasons it cannot be implicit: the default is 768h, which is + # also what an untuned mount clamps the ROOT to, so the two collided at + # exactly the length that makes issuance impossible; and the guard below + # compares the issuer's remaining life against this number, which it can + # only do if this number exists. 720h matches + # `deploy.hive-controller.tls.leafValidityDays`, the 30 days ./hive-tls.nix + # documents for every other leaf it holds. + servicesPkiLeafTtlHours = 720; + servicesPkiLeafTtl = "${toString servicesPkiLeafTtlHours}h"; + servicesPkiLeafTtlSeconds = servicesPkiLeafTtlHours * 3600; + # The role every swarm-service certificate will be issued through, and the # whole narrowing of this mount. A role rather than a name-constrained # sub-CA: the engine enforces the same "these names and no others" at issue @@ -1788,6 +1800,9 @@ in path = [ baoCli pkgs.coreutils + # Reads the notAfter of the issuer already on the pki mount; the + # regeneration guard below turns that into a decision. + pkgs.openssl ]; # Named but not placed is a legitimate state: all-local supplies the # path as a default and the operator drops the file there after @@ -1849,6 +1864,21 @@ in *) bao secrets enable -path=${servicesPkiMountPath} pki ;; esac + # 🩸 `max_lease_ttl` is the root's real lifetime, not the `ttl=` the + # generation below asks for. A mount defaults to 768h and SILENTLY + # CLAMPS anything longer, so the 30-year root came out 32 days long + # — and then every issue call failed, because a leaf of the mount's + # own default length would outlive the CA that signs it: + # + # cannot satisfy request, as TTL would result in notAfter of + # that is beyond the expiration of the CA certificate + # + # Tuned on EVERY run rather than only at enable: the clamp already + # happened on stores provisioned before this line existed, and a + # `tune` is the only thing that lets the regeneration guard below + # replace what it produced. + bao secrets tune -max-lease-ttl=${servicesPkiRootTtl} ${servicesPkiMountPath} + # ⚠️ THE ROOT, AND THE ONE THING THIS UNIT MUST NEVER DO TWICE. # # `root/generate/internal` mints a new self-signed CA every time it @@ -1868,7 +1898,29 @@ in # status IS the question — no output parsing, no error string to # recognise. The default issuer is what `${servicesPkiMountPath}/issue/` # signs with, and there is exactly one. + # + # "Never twice" is the rule; "an issuer that cannot issue" is not a + # case it was written for. A root with less than a leaf's window left + # refuses every request rather than returning a shorter certificate, + # so leaving it alone preserves an anchor nothing can chain to — the + # outage the rule exists to prevent, arrived at by obeying it. Note + # the asymmetry: this replaces a root that is already useless, and + # still never touches one a leaf can be issued under. + regenerate=0 if ! bao list ${lib.escapeShellArg "${servicesPkiMountPath}/issuers"} >/dev/null 2>&1; then + regenerate=1 + elif ! bao read -field=certificate ${lib.escapeShellArg "${servicesPkiMountPath}/cert/ca"} \ + | openssl x509 -noout -checkend ${toString servicesPkiLeafTtlSeconds} >/dev/null 2>&1; then + echo "the ${servicesPkiMountPath} root cannot outlive a ${servicesPkiLeafTtl} leaf — replacing it" >&2 + # `root` rather than the named issuer: the replacement re-uses + # `issuer_name`, and generating into a name already in use is a + # refusal. Clears the mount's keys with it, which is the whole of + # what it holds — one issuer, by the design above. + bao delete ${lib.escapeShellArg "${servicesPkiMountPath}/root"} >/dev/null + regenerate=1 + fi + + if [ "$regenerate" = 1 ]; then echo "generating the swarm services root into the ${servicesPkiMountPath} mount" # `-field=issuing_ca` discards the rest of the response. The # private key is not in it under `internal` and cannot be — that @@ -1938,7 +1990,9 @@ in server_flag=true \ client_flag=false \ key_type=rsa \ - key_bits=4096 + key_bits=4096 \ + ttl=${servicesPkiLeafTtl} \ + max_ttl=${servicesPkiLeafTtl} '' + lib.optionalString (baoDeploy.clientCaFile != null) ''