hive TLS: re-issue a services leaf that does not chain to the store's root, delete the hive CA's leftovers

- swarm-services-cert (and its daily renewal) treats a services leaf that
  `openssl verify` rejects against swarm-services-root.pem as invalid and
  asks the store for a new one, as hive-tls-resign already does for the
  gateway leaf. The trust bundle is that root alone, so a leaf under any
  other issuer verifies for no consumer. With the store unreachable the
  unit fails, and where the gateway waits for the leaf, the import fails
  with it.
- Every host also deletes the old hive CA's ca.srl and .swarm-ca-adopted
  in deploy.hive-controller.tls.stateDir, with the same tmpfiles `r`
  rules as ca.pem/ca-previous.pem/ca-key.pem. Nothing reads either.
- docs/swarm/ca.md: the services leaf's re-issue conditions, the two
  extra deleted files, and that the deletion is irreversible: rolling back
  to a generation with swarm-ca.service mints a new, unrelated root that
  nothing issues under.

module-eval hive-tls: a case for the services-leaf check in the boot
issuance and the renewal (red with the line removed), and the deletion
case covers the two extra files.

Refs #4588

docs/swarm/ca.md

@@ -98,9 +98,9 @@ The secret store's `pki` mount issues them a leaf of their own, through its
 this host's services-issuer certificate, calls
 `pki/issue/swarm-services`, and writes the result to
 `/var/lib/hive-tls/swarm-services.pem` — the path the gateway already
-copies from. Every hive does this with its own identity, so holding the
+copies from. Every hive does this with its own identity, so the store's
-swarm root's private key stopped being what decides whether a hive can
+role, not a key on the hive, decides whether a hive can serve its swarm's
-serve its swarm's names.
+names.
 
 `services.hyperhive.swarm.serviceDomains`, every swarm service name and the
 same list on every host, populates the role's
@@ -137,8 +137,9 @@ and every browser taught to trust it.
 **A daily timer renews the leaf.** The store's role issues it for 720
 hours. `swarm-services-cert-renew.timer` runs the same script as
 `swarm-services-cert.service` once a day, and that script asks the store
-for a new leaf once the current one is past half that window or is
+for a new leaf once the current one is past half that window, is
-missing a configured name. A rotation re-imports the leaf into the
+missing a configured name, or doesn't chain to the store's root
+(`swarm-services-root.pem`). A rotation re-imports the leaf into the
 gateway and reloads nginx. A store that can't answer — still sealed, or
 out of reach — fails the unit, which leaves the current leaf in place,
 and the next day's run tries again: about fifteen tries before the leaf
@@ -156,9 +157,15 @@ others.
 No host holds a CA or its key, except the store's mTLS CA
 (`glue-bao-tls.nix`), which signs only the store's server certificate and the
 certificates that log in to it. At boot and on every deploy, every host
-deletes `/var/lib/swarm-ca` and the `ca.pem`, `ca-previous.pem` and
+deletes `/var/lib/swarm-ca` and the `ca.pem`, `ca-previous.pem`,
-`ca-key.pem` in `<deploy.hive-controller.tls.stateDir>`. The swarm distributes
+`ca-key.pem`, `ca.srl` and `.swarm-ca-adopted` in
-only certificates.
+`<deploy.hive-controller.tls.stateDir>`. The swarm distributes only
+certificates.
+
+⚠️ **The deletion is irreversible.** Rolling back to a generation that still
+has `swarm-ca.service` finds no root under `/var/lib/swarm-ca` and mints a
+new one, unrelated to the deleted root. That's harmless: nothing issues
+under it.
 
 A host whose `deploy.bao.clientCaFile`, `deploy.bao.serverCaFile` or
 `deploy.swarm-controller.hiveClientCaFile` names one of those paths fails

nix/host-modules/hive-tls.nix

@@ -692,6 +692,9 @@ in
         openssl x509 -in "$svcleaf" -noout -checkend "$renewat" >/dev/null 2>&1 || reissue=1
         covers "$svcleaf" "$svcNames" || reissue=1
         [ -s "$svcroot" ] || reissue=1
+        # A leaf that does not chain to the root the trust bundle holds,
+        # whoever signed it, fails this and is re-issued.
+        openssl verify -CAfile "$svcroot" "$svcleaf" >/dev/null 2>&1 || reissue=1
 
         # The ROOT's window, not the leaf's, and it is what keeps the two
         # sides of this from needing to talk. The store replaces an issuer

nix/host-modules/host-ca-removed.nix

@@ -1,7 +1,7 @@
 # Removal shim for the CAs a host held outside the secret store: the
 # swarm root under `/var/lib/swarm-ca` (and its `services.hyperhive.swarm.ca.*`
-# options), and a hive CA's `ca.pem`, `ca-previous.pem` and `ca-key.pem` in
+# options), and a hive CA's `ca.pem`, `ca-previous.pem`, `ca-key.pem`,
-# `deploy.hive-controller.tls.stateDir`.
+# `ca.srl` and `.swarm-ca-adopted` in `deploy.hive-controller.tls.stateDir`.
 #
 # The secret store's `pki` root is the only trust anchor: it issues every
 # hive's gateway leaf and the swarm's service names, and its key never leaves
@@ -22,6 +22,12 @@ let
     "ca-key.pem"
   ];
   deleted = p: p == swarmCaDir || lib.hasPrefix "${swarmCaDir}/" p || lib.elem p hiveCaFiles;
+  # The hive CA's serial file and its adoption marker: neither is a CA, so
+  # no CA-file option can name them, and nothing reads them.
+  hiveCaLeftovers = map (f: "${deployCfg.hive-controller.tls.stateDir}/${f}") [
+    "ca.srl"
+    ".swarm-ca-adopted"
+  ];
 
   # Every option that names a CA the store's mTLS checks against. One
   # pointing at a deleted file fails the unit that reads it at start.
@@ -61,5 +67,8 @@ in
   # certificates beside them are anchors nothing issues under. The swarm
   # root at the option's old default path only: a relocated `stateDir` is
   # the operator's to delete.
-  systemd.tmpfiles.rules = [ "R ${swarmCaDir}" ] ++ map (f: "r ${f}") hiveCaFiles;
+  systemd.tmpfiles.rules = [
+    "R ${swarmCaDir}"
+  ]
+  ++ map (f: "r ${f}") (hiveCaFiles ++ hiveCaLeftovers);
 }

nix/module-eval/hive-tls.nix

@@ -131,6 +131,22 @@ let
         && lib.hasInfix ''-in "$svcleaf" -noout -checkend "$renewat"'' boot.script
         && lib.hasInfix "max_ttl=48h" shortTtl.systemd.services.swarm-bao-controller-policy.script;
     }
+    {
+      # Checked on the decision side of the exit, in the boot issuance and
+      # the renewal both.
+      name = "the services leaf is re-issued when it does not chain to the store's root";
+      ok =
+        let
+          verify = ''openssl verify -CAfile "$svcroot" "$svcleaf" >/dev/null 2>&1 || reissue=1'';
+          decides =
+            u:
+            let
+              decision = lib.head (lib.splitString ''if [ "$reissue" = 0 ]'' u.script);
+            in
+            decision != u.script && lib.hasInfix verify decision;
+        in
+        decides boot && decides triggered;
+    }
     {
       # The store replaces its root once it has less than one leaf lifetime
       # left, and the hive asks for a fresh leaf, with the new root, at that
@@ -335,6 +351,8 @@ let
               "ca.pem"
               "ca-previous.pem"
               "ca-key.pem"
+              "ca.srl"
+              ".swarm-ca-adopted"
             ];
           relocated = hiveWithController { deploy.hive-controller.tls.stateDir = "/srv/hive-tls"; };
         in
@@ -342,6 +360,8 @@ let
           "/ca.pem"
           "ca-previous.pem"
           "ca-key.pem"
+          "ca.srl"
+          ".swarm-ca-adopted"
         ])
         && reads allLocal "swarm-services-root.pem"
         && deletes allLocal "/var/lib/hive-tls"