swarm-bao-pki: enabling a service mints its client leaf on the same switch

`swarm-bao-pki` was a `RemainAfterExit` oneshot. Once active, nothing
re-ran it: a switch restarts it only when its definition changes, and a
`Requires=`/`Wants=` on an active unit is satisfied without running it.

On muede-lpt2 the 10:57Z switch that added the builder line did change
the unit, so switch-to-configuration stopped it in its stop phase. A
start of `container@swarm-bao` 4 s later (before the daemon-reload at
10:57:35Z) pulled it back in through `swarm-bao-certs` with the OLD
definition, which had no builder line: 48 ms, nothing missing. The start
phase then found it active and did nothing. The 20:14Z switch that
enabled the builder left the unit unchanged, so it was never re-run, and
`container@swarm-builder` crash-looped on the absent `builder.pem`.

The unit now goes inactive after each run. Every switch starts
`multi-user.target`, which runs it again post-reload with the current
definition, and every unit that wants or requires it runs it on its own
start. Each line is idempotent on presence, so an existing leaf is never
re-signed.

The three consumers started from `machines.target` rather than
`multi-user.target` also want it, since `After=` alone does not order
against a job in another transaction: the builder's and the review
classifier's containers (nspawn `--load-credential`) and
`swarm-nats-auth-secrets` (pulled in by the queue's container).

Fixes #5284

docs/swarm/bao.md

@@ -91,12 +91,15 @@ a store that was never initialised.
 
 ## TLS
 
-The store serves TLS, and on a host that deploys it you need do nothing: a
-first-boot unit mints a CA of the store's own plus the leaves it signs — the
+The store serves TLS, and on a host that deploys it you need do nothing:
+`swarm-bao-pki` mints a CA of the store's own plus the leaves it signs — the
 store's server certificate, this host's client certificate, and one per service
 principal — and points `deploy.bao.serverCertFile`, `.serverKeyFile` and
 `.clientCaFile` at the store's half, `.clientCertFile`, `.clientKeyFile` and
 `.serverCaFile` at the reader's, and each principal's own pair at its own leaf.
+It runs at every boot and every deploy, and before each start of a container
+that loads one of its leaves, and signs only the files that are missing, so a
+leaf a deploy adds, or a service a deploy enables, has its file on that deploy.
 
 Those are `mkDefault`s, so naming your own paths wins. Do that when your
 certificates come from a real internal CA; the store has no opinion about

nix/host-modules/glue-bao-tls.nix

@@ -133,28 +133,25 @@ in
     # every client certificate already trusting it, so a rebuild that
     # "refreshed" it would lock every reader in the swarm out at once — the
     # same rule the store's TPM PIN unit follows, for a sharper reason.
+    #
+    # ⚠️ Not `RemainAfterExit`: the unit goes inactive after every run, so
+    # each start of a unit that wants or requires it, and each switch's start
+    # of `multi-user.target`, runs it again with the definition loaded at that
+    # moment. That is what mints a leaf a deploy adds, or a consumer it
+    # enables, on that deploy. An active `RemainAfterExit` unit satisfies all
+    # of those without running, and a switch only restarts it when its
+    # definition changes. A run where nothing is missing is one `[ -s ]` test
+    # per leaf.
     systemd.services.swarm-bao-pki = {
       description = "mint the swarm secret store's own CA and leaves";
       before = [ "swarm-bao-certs.service" ];
       requiredBy = [ "swarm-bao-certs.service" ];
-      # 🩸 A target wants this, not only `swarm-bao-certs`, and the reason is
-      # what happens when the LIST of leaves grows. `requiredBy` alone is
-      # satisfied by a unit that already ran: this one is `RemainAfterExit`,
-      # so an activation that adds a leaf here finds it active, pulls nothing,
-      # and the new leaf is never minted — while every sibling granting unit,
-      # each of which carries this line, was re-run by the same activation.
-      # That is how `services-issuer.pem` came to be missing on a host whose
-      # config named it, leaving `swarm-services-cert` to fail on an absent
-      # file until someone restarted this by hand.
       wantedBy = [ "multi-user.target" ];
       path = [
         pkgs.openssl
         pkgs.coreutils
       ];
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
+      serviceConfig.Type = "oneshot";
       script = ''
         set -euo pipefail
         install -d -m 0700 ${pkiDir}

nix/host-modules/swarm-builder.nix

@@ -131,10 +131,17 @@ in
     services.hyperhive.gateway.dns.enable = lib.mkDefault true;
 
     # nspawn fails to start the container when a credential source is
-    # missing, so where this host mints the leaf the container waits for it.
+    # missing, so where this host mints the leaf the container pulls the
+    # minting unit in and waits for it. Ordering alone is not enough: the
+    # container starts from `machines.target`, outside the transaction that
+    # runs the minting unit from `multi-user.target`. Ignored elsewhere, where
+    # the unit does not exist.
     systemd.services."container@${machine}" = lib.mkMerge [
       caTrust.containerOrdering
-      { after = [ "swarm-bao-pki.service" ]; }
+      {
+        after = [ "swarm-bao-pki.service" ];
+        wants = [ "swarm-bao-pki.service" ];
+      }
     ];
 
     containers.${machine} = {

nix/host-modules/swarm-nats.nix

@@ -877,8 +877,13 @@ in
       before = [ "container@swarm-nats.service" ];
       wantedBy = [ "container@swarm-nats.service" ];
       # Where the store's PKI is minted on this host, the responder's leaf is
-      # one of its files. Ordering only: elsewhere the unit does not exist.
+      # one of its files, and the minting unit runs before every copy. Ordering
+      # alone is not enough: the queue's container, which pulls this unit in,
+      # starts from `machines.target`, outside the transaction that runs the
+      # minting unit from `multi-user.target`. Ignored elsewhere, where the unit
+      # does not exist.
       after = [ "swarm-bao-pki.service" ];
+      wants = [ "swarm-bao-pki.service" ];
       serviceConfig = {
         Type = "oneshot";
         RemainAfterExit = true;

nix/host-modules/swarm-review-classifier.nix

@@ -137,10 +137,17 @@ in
     services.hyperhive.gateway.dns.enable = lib.mkDefault true;
 
     # nspawn fails to start the container when a credential source is
-    # missing, so where this host mints the leaf the container waits for it.
+    # missing, so where this host mints the leaf the container pulls the
+    # minting unit in and waits for it. Ordering alone is not enough: the
+    # container starts from `machines.target`, outside the transaction that
+    # runs the minting unit from `multi-user.target`. Ignored elsewhere, where
+    # the unit does not exist.
     systemd.services."container@${machine}" = lib.mkMerge [
       caTrust.containerOrdering
-      { after = [ "swarm-bao-pki.service" ]; }
+      {
+        after = [ "swarm-bao-pki.service" ];
+        wants = [ "swarm-bao-pki.service" ];
+      }
     ];
 
     containers.${machine} = {

nix/module-eval/builder.nix

@@ -94,6 +94,22 @@ let
         && svc.serviceConfig.DynamicUser
         && !(lib.any (lib.hasSuffix "TOKEN") (lib.attrNames env));
     }
+    {
+      # The minting unit goes inactive after each run, so a start that pulls it
+      # runs it again and mints a leaf that is missing; an active
+      # `RemainAfterExit` unit would satisfy the pull without running.
+      name = "the builder's container runs the leaf-minting unit before every start";
+      ok =
+        let
+          pki = withBuilder.systemd.services.swarm-bao-pki;
+          c = withBuilder.systemd.services."container@swarm-builder";
+        in
+        pki.serviceConfig.Type == "oneshot"
+        && !(pki.serviceConfig.RemainAfterExit or false)
+        && lib.elem "multi-user.target" pki.wantedBy
+        && lib.elem "swarm-bao-pki.service" c.wants
+        && lib.elem "swarm-bao-pki.service" c.after;
+    }
     {
       # The check builds on the sources a hive on this host would render into
       # its meta flake, so a green check means the same base.

nix/module-eval/core-toggle.nix

@@ -417,6 +417,15 @@ let
           lib.attrValues withClassifier.systemd.services
         ));
     }
+    {
+      # See the same case in ./builder.nix.
+      name = "the review classifier's container runs the leaf-minting unit before every start";
+      ok =
+        let
+          c = withClassifier.systemd.services."container@swarm-review-classifier";
+        in
+        lib.elem "swarm-bao-pki.service" c.wants && lib.elem "swarm-bao-pki.service" c.after;
+    }
     {
       # The collector reaches these routes through the gateway now, so each
       # store needs an ingest location of its own. Without one the write rides

nix/module-eval/nats-authelia.nix

@@ -253,6 +253,16 @@ let
         && waitsFor "swarm-bao-secret-publisher-policy"
         && natsOnStoreHost.systemd.services ? swarm-bao-secret-publisher-policy;
     }
+    {
+      # The queue's container pulls the copy in, outside the switch's
+      # `multi-user.target` transaction; see the same case in ./builder.nix.
+      name = "the responder's leaf copy runs the leaf-minting unit first";
+      ok =
+        let
+          u = natsOnStoreHost.systemd.services.swarm-nats-auth-secrets;
+        in
+        lib.elem "swarm-bao-pki.service" u.wants && lib.elem "swarm-bao-pki.service" u.after;
+    }
     {
       # The control is the fixture above the store: off the store's host
       # there is no local policy unit to wait for.