hive TLS: re-issue a services leaf that does not chain to the store's root, delete the hive CA's leftovers

- swarm-services-cert (and its daily renewal) treats a services leaf that
  `openssl verify` rejects against swarm-services-root.pem as invalid and
  asks the store for a new one, as hive-tls-resign already does for the
  gateway leaf. The trust bundle is that root alone, so a leaf under any
  other issuer verifies for no consumer. With the store unreachable the
  unit fails, and where the gateway waits for the leaf, the import fails
  with it.
- Every host also deletes the old hive CA's ca.srl and .swarm-ca-adopted
  in deploy.hive-controller.tls.stateDir, with the same tmpfiles `r`
  rules as ca.pem/ca-previous.pem/ca-key.pem. Nothing reads either.
- docs/swarm/ca.md: the services leaf's re-issue conditions, the two
  extra deleted files, and that the deletion is irreversible: rolling back
  to a generation with swarm-ca.service mints a new, unrelated root that
  nothing issues under.

module-eval hive-tls: a case for the services-leaf check in the boot
issuance and the renewal (red with the line removed), and the deletion
case covers the two extra files.

Refs #4588

nix/host-modules/hive-tls.nix

@@ -692,6 +692,9 @@ in
         openssl x509 -in "$svcleaf" -noout -checkend "$renewat" >/dev/null 2>&1 || reissue=1
         covers "$svcleaf" "$svcNames" || reissue=1
         [ -s "$svcroot" ] || reissue=1
+        # A leaf that does not chain to the root the trust bundle holds,
+        # whoever signed it, fails this and is re-issued.
+        openssl verify -CAfile "$svcroot" "$svcleaf" >/dev/null 2>&1 || reissue=1
 
         # The ROOT's window, not the leaf's, and it is what keeps the two
         # sides of this from needing to talk. The store replaces an issuer

nix/host-modules/host-ca-removed.nix

@@ -1,7 +1,7 @@
 # Removal shim for the CAs a host held outside the secret store: the
 # swarm root under `/var/lib/swarm-ca` (and its `services.hyperhive.swarm.ca.*`
-# options), and a hive CA's `ca.pem`, `ca-previous.pem` and `ca-key.pem` in
-# `deploy.hive-controller.tls.stateDir`.
+# options), and a hive CA's `ca.pem`, `ca-previous.pem`, `ca-key.pem`,
+# `ca.srl` and `.swarm-ca-adopted` in `deploy.hive-controller.tls.stateDir`.
 #
 # The secret store's `pki` root is the only trust anchor: it issues every
 # hive's gateway leaf and the swarm's service names, and its key never leaves
@@ -22,6 +22,12 @@ let
     "ca-key.pem"
   ];
   deleted = p: p == swarmCaDir || lib.hasPrefix "${swarmCaDir}/" p || lib.elem p hiveCaFiles;
+  # The hive CA's serial file and its adoption marker: neither is a CA, so
+  # no CA-file option can name them, and nothing reads them.
+  hiveCaLeftovers = map (f: "${deployCfg.hive-controller.tls.stateDir}/${f}") [
+    "ca.srl"
+    ".swarm-ca-adopted"
+  ];
 
   # Every option that names a CA the store's mTLS checks against. One
   # pointing at a deleted file fails the unit that reads it at start.
@@ -61,5 +67,8 @@ in
   # certificates beside them are anchors nothing issues under. The swarm
   # root at the option's old default path only: a relocated `stateDir` is
   # the operator's to delete.
-  systemd.tmpfiles.rules = [ "R ${swarmCaDir}" ] ++ map (f: "r ${f}") hiveCaFiles;
+  systemd.tmpfiles.rules = [
+    "R ${swarmCaDir}"
+  ]
+  ++ map (f: "r ${f}") (hiveCaFiles ++ hiveCaLeftovers);
 }

nix/module-eval/hive-tls.nix

@@ -131,6 +131,22 @@ let
         && lib.hasInfix ''-in "$svcleaf" -noout -checkend "$renewat"'' boot.script
         && lib.hasInfix "max_ttl=48h" shortTtl.systemd.services.swarm-bao-controller-policy.script;
     }
+    {
+      # Checked on the decision side of the exit, in the boot issuance and
+      # the renewal both.
+      name = "the services leaf is re-issued when it does not chain to the store's root";
+      ok =
+        let
+          verify = ''openssl verify -CAfile "$svcroot" "$svcleaf" >/dev/null 2>&1 || reissue=1'';
+          decides =
+            u:
+            let
+              decision = lib.head (lib.splitString ''if [ "$reissue" = 0 ]'' u.script);
+            in
+            decision != u.script && lib.hasInfix verify decision;
+        in
+        decides boot && decides triggered;
+    }
     {
       # The store replaces its root once it has less than one leaf lifetime
       # left, and the hive asks for a fresh leaf, with the new root, at that
@@ -335,6 +351,8 @@ let
               "ca.pem"
               "ca-previous.pem"
               "ca-key.pem"
+              "ca.srl"
+              ".swarm-ca-adopted"
             ];
           relocated = hiveWithController { deploy.hive-controller.tls.stateDir = "/srv/hive-tls"; };
         in
@@ -342,6 +360,8 @@ let
           "/ca.pem"
           "ca-previous.pem"
           "ca-key.pem"
+          "ca.srl"
+          ".swarm-ca-adopted"
         ])
         && reads allLocal "swarm-services-root.pem"
         && deletes allLocal "/var/lib/hive-tls"