swarm-bao-pki: sign every other hive's three store leaves

The store host's `swarm-bao-pki` now also signs, for every hive in
`swarm.hives` other than its own, the reader (`<hive>`), queue-agent
(`<queueAgentCommonNamePrefix>-<hive>`) and gateway-leaf
(`<hiveLeafCommonNamePrefix>-<hive>`) leaves, into
/var/lib/swarm-bao-pki/hives/<hive>/ with the basenames the own hive's
leaves use. Same CA, same `signLeaf`, idempotent on absence like every
other leaf in the unit. `signLeaf` now takes the CA from the top of the
PKI directory rather than from the leaf's directory; every existing
caller passed that directory, so their output is unchanged.

The operator still copies the files to the other hive's host and names
them in config; only the openssl step against the CA goes away
(mara on #5016, c95685: "ok mint").

Closes #5016

docs/swarm/bao.md

@@ -103,8 +103,23 @@ certificates come from a real internal CA; the store has no opinion about
 which. A host that does **not** deploy the store names the reader's three
 itself, plus a pair for every principal it runs — see
 [per-principal identities](secrets.md#per-principal-identities) for the list.
-The operator issues those leaves out of band; they're the credentials the
+They're the credentials the store can't hand you, being what opens it, so the
-store can't hand you, being what opens it.
+store's host signs them and the operator copies them to the host that needs
+them and names them in config. The service principals' leaves are at the top of
+`/var/lib/swarm-bao-pki/`, named after the principal.
+
+For every hive in `swarm.hives` other than its own, the store's host signs
+that hive's three leaves into `/var/lib/swarm-bao-pki/hives/<hive>/`:
+
+| file                                     | subject                                          | name it in                                                                      |
+| ---------------------------------------- | ------------------------------------------------ | ------------------------------------------------------------------------------- |
+| `client.pem`, `client-key.pem`           | `<hive>`                                         | `deploy.bao.clientCertFile`, `.clientKeyFile`                                   |
+| `queue-agent.pem`, `queue-agent-key.pem` | `<deploy.bao.queueAgentCommonNamePrefix>-<hive>` | `deploy.bao.queueAgentClientCertFile`, `.queueAgentClientKeyFile`               |
+| `hive-leaf.pem`, `hive-leaf-key.pem`     | `<deploy.bao.hiveLeafCommonNamePrefix>-<hive>`   | `deploy.hive-controller.tls.baoHiveIdentityCertFile`, `.baoHiveIdentityKeyFile` |
+
+Copy `/var/lib/swarm-bao-pki/ca.pem` with them and name it in
+`deploy.bao.serverCaFile`. A hive added to `swarm.hives` gets its directory on
+the store host's next deploy; a leaf already there is never re-signed.
 
 ⚠️ Not the gateway's HTTPS certificates and not the store's `pki` root — this is **mTLS
 between services and the store**, a separate trust domain, because a store

docs/swarm/ca.md

@@ -35,8 +35,10 @@ from the store's own client CA, with the subject
 
 - On the host running the store, `swarm-bao-pki.service` mints it at
   `/var/lib/swarm-bao-pki/hive-leaf.pem` and the two options default to it.
-- On every other host, copy a leaf with that subject from the store's host
+- On every other host, copy `hive-leaf.pem` and `hive-leaf-key.pem` from
-  and set both options, before the host's next deploy.
+  `/var/lib/swarm-bao-pki/hives/<hiveName>/` on the store's host, which
+  `swarm-bao-pki.service` mints for every other hive in `swarm.hives`, and
+  set both options, before the host's next deploy.
 
 Without it, `hive-tls-resign.service` fails with a message naming both
 options, and so does `swarm-services-cert.service` on a hive fronting no

docs/swarm/secrets.md

@@ -426,8 +426,11 @@ leaf two units share: the forge's host holds one store identity for the token it
 publishes and the OIDC client secret it reads.
 
 On a host that mints its own PKI, `glue-bao-tls.nix` signs all seven and defaults
-all fourteen options, and there is nothing to do. Elsewhere you issue each leaf from
+all fourteen options, and there is nothing to do. Elsewhere you copy each leaf from
-that CA out of band and name it here — one file per principal rather than one file
+the store's host and name it here: a service principal's from the top of
+`/var/lib/swarm-bao-pki/`, another hive's queue-agent pair from
+`/var/lib/swarm-bao-pki/hives/<hive>/` ([bao.md § TLS](bao.md#tls) lists that
+directory). That's one file per principal rather than one file
 shared by seven, which is the whole of what this buys. `swarm-ci-runner.nix`
 refuses the build wherever the runner runs without its pair, as
 `swarm-review-classifier.nix` does for the classifier and `swarm-grafana.nix`

nix/host-modules/glue-bao-tls.nix

@@ -49,7 +49,13 @@ let
   # thing.
   clientCn = hyperhiveCfg.hiveName;
 
+  # Every other hive in the swarm directory. This host's own hive is signed
+  # into the top of the directory, beside the leaves its units read.
+  otherHives = lib.filter (h: h != clientCn) (lib.attrNames hyperhiveCfg.swarm.hives);
+
   # $1 dir  $2 basename  $3 CN  $4 SAN or ""  $5 EKU
+  # The CA is always the one at the top of `pkiDir`, whichever dir the leaf
+  # lands in.
   signLeaf = pkgs.writeShellScript "swarm-bao-sign-leaf" ''
     set -euo pipefail
     d="$1"; base="$2"; cn="$3"; sans="$4"; eku="$5"
@@ -65,7 +71,7 @@ let
       printf 'keyUsage=critical,digitalSignature,keyEncipherment\n'
       printf 'extendedKeyUsage=%s\n' "$eku"
     } > "$ext"
-    openssl x509 -req -in "$csr" -CA "$d/ca.pem" -CAkey "$d/ca-key.pem" \
+    openssl x509 -req -in "$csr" -CA ${pkiDir}/ca.pem -CAkey ${pkiDir}/ca-key.pem \
       -CAcreateserial -days 3650 -sha256 -extfile "$ext" -out "$d/$base.pem"
     chmod 0600 "$d/$base-key.pem"
     chmod 0644 "$d/$base.pem"
@@ -162,9 +168,9 @@ in
         fi
 
         # The store's own identity, and the identity of a reader on this host.
-        # A reader elsewhere gets its leaf from this CA out of band — that is
+        # A reader elsewhere gets its leaf from this CA too — that is what
-        # what makes the store reachable from another machine at all, and why
+        # makes the store reachable from another machine at all, and why the
-        # the CA is a file rather than a service.
+        # CA is a file rather than a service.
         [ -s ${pkiDir}/server.pem ] || ${signLeaf} ${pkiDir} server \
           ${lib.escapeShellArg cfg.domain} ${lib.escapeShellArg "DNS:${cfg.domain}"} serverAuth
         [ -s ${pkiDir}/client.pem ] || ${signLeaf} ${pkiDir} client \
@@ -288,11 +294,30 @@ in
           # `pki/issue/<hiveLeafPkiRolePrefix>-<hive>` and nothing else. Its
           # subject carries this hive's name, like the queue-agent leaf
           # above, because ./swarm-bao.nix writes one role
-          # per hive. Only where a hive runs; hives on other hosts are issued
+          # per hive. Only where a hive runs; the other hives' are below.
-          # theirs out of band.
           [ -s ${pkiDir}/hive-leaf.pem ] || ${signLeaf} ${pkiDir} hive-leaf \
             ${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${clientCn}"} "" clientAuth
         ''}
+
+        # The reader, queue-agent and gateway-leaf leaves of every other hive in
+        # the swarm directory: the files an operator copies to that hive's
+        # host. One directory per hive, with the basenames this hive's leaves
+        # use, so a hive's set is one directory to copy.
+        ${lib.concatMapStrings (
+          h:
+          let
+            d = lib.escapeShellArg "${pkiDir}/hives/${h}";
+          in
+          ''
+            install -d -m 0700 ${d}
+            [ -s ${d}/client.pem ] || ${signLeaf} ${d} client \
+              ${lib.escapeShellArg h} "" clientAuth
+            [ -s ${d}/queue-agent.pem ] || ${signLeaf} ${d} queue-agent \
+              ${lib.escapeShellArg "${deployCfg.bao.queueAgentCommonNamePrefix}-${h}"} "" clientAuth
+            [ -s ${d}/hive-leaf.pem ] || ${signLeaf} ${d} hive-leaf \
+              ${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${h}"} "" clientAuth
+          ''
+        ) otherHives}
       '';
     };
   };

nix/host-modules/glue-queue-agent-credential.nix

@@ -147,8 +147,9 @@ in
             would evaluate, deploy and log in — and undo the split.
 
             On a hive that runs the store, glue-bao-tls.nix supplies both as
-            defaults and there is nothing to do. Elsewhere the leaf is issued
+            defaults and there is nothing to do. Elsewhere copy the pair from
-            from that CA out of band and named here — see docs/swarm/secrets.md.
+            /var/lib/swarm-bao-pki/hives/<hive>/ on the store's host and name it
+            here — see docs/swarm/bao.md.
           '';
         }
       ];

nix/module-eval/bao-grants.nix

@@ -1289,6 +1289,30 @@ let
           t.baoHiveIdentityCertFile
           != baoGrantHere.services.hyperhive.deploy.bao.servicesIssuerClientCertFile;
     }
+    {
+      # The other hives' side: every hive in the directory but this one gets
+      # the same three leaves, under its own name, in a directory of its own
+      # that the operator copies to its host.
+      name = "the PKI unit signs every other hive's reader, queue-agent and gateway-leaf identities";
+      ok =
+        let
+          s = baoGrantTwoHives.systemd.services.swarm-bao-pki.script;
+          b = baoGrantTwoHives.services.hyperhive.deploy.bao;
+          signs =
+            file: cn: lib.hasInfix "/hives/h2/${file}.pem ]" s && lib.hasInfix " ${cn} \"\" clientAuth" s;
+        in
+        signs "client" "h2"
+        && signs "queue-agent" "${b.queueAgentCommonNamePrefix}-h2"
+        && signs "hive-leaf" "${b.hiveLeafCommonNamePrefix}-h2"
+        # This hive's own leaves stay at the top of the directory.
+        && !(lib.hasInfix "/hives/h1/" s);
+    }
+    {
+      # What makes the one above mean something: a hive outside the directory
+      # gets nothing, so a store with only its own hive signs no other set.
+      name = "the PKI unit signs no other hive's leaves when the directory holds only its own";
+      ok = !(lib.hasInfix "swarm-bao-pki/hives/" baoGrantHere.systemd.services.swarm-bao-pki.script);
+    }
     {
       name = "a gateway-leaf role prefix outside swarm-* is refused, naming the option";
       ok =