diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index 6259e1b7..8cf0c637 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -98,9 +98,9 @@ The secret store's `pki` mount issues them a leaf of their own, through its this host's services-issuer certificate, calls `pki/issue/swarm-services`, and writes the result to `/var/lib/hive-tls/swarm-services.pem` — the path the gateway already -copies from. Every hive does this with its own identity, so holding the -swarm root's private key stopped being what decides whether a hive can -serve its swarm's names. +copies from. Every hive does this with its own identity, so the store's +role, not a key on the hive, decides whether a hive can serve its swarm's +names. `services.hyperhive.swarm.serviceDomains`, every swarm service name and the same list on every host, populates the role's @@ -137,8 +137,9 @@ and every browser taught to trust it. **A daily timer renews the leaf.** The store's role issues it for 720 hours. `swarm-services-cert-renew.timer` runs the same script as `swarm-services-cert.service` once a day, and that script asks the store -for a new leaf once the current one is past half that window or is -missing a configured name. A rotation re-imports the leaf into the +for a new leaf once the current one is past half that window, is +missing a configured name, or doesn't chain to the store's root +(`swarm-services-root.pem`). A rotation re-imports the leaf into the gateway and reloads nginx. A store that can't answer — still sealed, or out of reach — fails the unit, which leaves the current leaf in place, and the next day's run tries again: about fifteen tries before the leaf @@ -156,9 +157,15 @@ others. No host holds a CA or its key, except the store's mTLS CA (`glue-bao-tls.nix`), which signs only the store's server certificate and the certificates that log in to it. At boot and on every deploy, every host -deletes `/var/lib/swarm-ca` and the `ca.pem`, `ca-previous.pem` and -`ca-key.pem` in ``. The swarm distributes -only certificates. +deletes `/var/lib/swarm-ca` and the `ca.pem`, `ca-previous.pem`, +`ca-key.pem`, `ca.srl` and `.swarm-ca-adopted` in +``. The swarm distributes only +certificates. + +⚠️ **The deletion is irreversible.** Rolling back to a generation that still +has `swarm-ca.service` finds no root under `/var/lib/swarm-ca` and mints a +new one, unrelated to the deleted root. That's harmless: nothing issues +under it. A host whose `deploy.bao.clientCaFile`, `deploy.bao.serverCaFile` or `deploy.swarm-controller.hiveClientCaFile` names one of those paths fails diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 14e6f6b9..ef70e92d 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -692,6 +692,9 @@ in openssl x509 -in "$svcleaf" -noout -checkend "$renewat" >/dev/null 2>&1 || reissue=1 covers "$svcleaf" "$svcNames" || reissue=1 [ -s "$svcroot" ] || reissue=1 + # A leaf that does not chain to the root the trust bundle holds, + # whoever signed it, fails this and is re-issued. + openssl verify -CAfile "$svcroot" "$svcleaf" >/dev/null 2>&1 || reissue=1 # The ROOT's window, not the leaf's, and it is what keeps the two # sides of this from needing to talk. The store replaces an issuer diff --git a/nix/host-modules/host-ca-removed.nix b/nix/host-modules/host-ca-removed.nix index 390ceddb..cad80b1f 100644 --- a/nix/host-modules/host-ca-removed.nix +++ b/nix/host-modules/host-ca-removed.nix @@ -1,7 +1,7 @@ # Removal shim for the CAs a host held outside the secret store: the # swarm root under `/var/lib/swarm-ca` (and its `services.hyperhive.swarm.ca.*` -# options), and a hive CA's `ca.pem`, `ca-previous.pem` and `ca-key.pem` in -# `deploy.hive-controller.tls.stateDir`. +# options), and a hive CA's `ca.pem`, `ca-previous.pem`, `ca-key.pem`, +# `ca.srl` and `.swarm-ca-adopted` in `deploy.hive-controller.tls.stateDir`. # # The secret store's `pki` root is the only trust anchor: it issues every # hive's gateway leaf and the swarm's service names, and its key never leaves @@ -22,6 +22,12 @@ let "ca-key.pem" ]; deleted = p: p == swarmCaDir || lib.hasPrefix "${swarmCaDir}/" p || lib.elem p hiveCaFiles; + # The hive CA's serial file and its adoption marker: neither is a CA, so + # no CA-file option can name them, and nothing reads them. + hiveCaLeftovers = map (f: "${deployCfg.hive-controller.tls.stateDir}/${f}") [ + "ca.srl" + ".swarm-ca-adopted" + ]; # Every option that names a CA the store's mTLS checks against. One # pointing at a deleted file fails the unit that reads it at start. @@ -61,5 +67,8 @@ in # certificates beside them are anchors nothing issues under. The swarm # root at the option's old default path only: a relocated `stateDir` is # the operator's to delete. - systemd.tmpfiles.rules = [ "R ${swarmCaDir}" ] ++ map (f: "r ${f}") hiveCaFiles; + systemd.tmpfiles.rules = [ + "R ${swarmCaDir}" + ] + ++ map (f: "r ${f}") (hiveCaFiles ++ hiveCaLeftovers); } diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index e4e08aa5..ec07ea8f 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -131,6 +131,22 @@ let && lib.hasInfix ''-in "$svcleaf" -noout -checkend "$renewat"'' boot.script && lib.hasInfix "max_ttl=48h" shortTtl.systemd.services.swarm-bao-controller-policy.script; } + { + # Checked on the decision side of the exit, in the boot issuance and + # the renewal both. + name = "the services leaf is re-issued when it does not chain to the store's root"; + ok = + let + verify = ''openssl verify -CAfile "$svcroot" "$svcleaf" >/dev/null 2>&1 || reissue=1''; + decides = + u: + let + decision = lib.head (lib.splitString ''if [ "$reissue" = 0 ]'' u.script); + in + decision != u.script && lib.hasInfix verify decision; + in + decides boot && decides triggered; + } { # The store replaces its root once it has less than one leaf lifetime # left, and the hive asks for a fresh leaf, with the new root, at that @@ -335,6 +351,8 @@ let "ca.pem" "ca-previous.pem" "ca-key.pem" + "ca.srl" + ".swarm-ca-adopted" ]; relocated = hiveWithController { deploy.hive-controller.tls.stateDir = "/srv/hive-tls"; }; in @@ -342,6 +360,8 @@ let "/ca.pem" "ca-previous.pem" "ca-key.pem" + "ca.srl" + ".swarm-ca-adopted" ]) && reads allLocal "swarm-services-root.pem" && deletes allLocal "/var/lib/hive-tls"