hive-tls: stage bao pki root in every hive's trust bundle

A self-signed hive fronting no swarm service name never filled
`$svcroot` (/var/lib/hive-tls/swarm-services-root.pem): the only writer
was `swarm-services-cert`'s issuance, which exits early on
`want_svc=0`. So those hives' trust-bundle.pem lacked the store's `pki`
root, even though every other hive's swarm-service names chain to it.

On `want_svc=0`, `swarm-services-cert` now reads the root from the
store's unauthenticated `GET /v1/pki/ca/pem`, over the hive's existing
BAO_ADDR, `deploy.bao.serverCaFile` and reader identity
(`deploy.bao.clientCertFile`/`clientKeyFile`) at the TLS layer only; no
login. It writes `$svcroot` and restarts `hive-tls-ca` when the root
changed, as the issuing path already does. The daily
`swarm-services-cert-renew` timer runs the same script, so the copy
follows a root rotation.

The gateway no longer `Requires=` `swarm-services-cert` on such a host:
nginx reads nothing that unit writes there, and the fetch retries
against a sealed store.

`hivectl peer-config` prints the root as step 2 of the self-signed
instructions: the swarm's service names are issued under it, not the
swarm root.

No new option. Refs #4588.

nix/host-modules/hive-tls.nix

@@ -42,7 +42,12 @@ let
   # path down whenever it is sealed. There the gateway starts on the hive
   # leaf fallback, and the issuing script re-imports the services leaf into
   # the running nginx once it lands.
-  gatewayWaitsForServicesLeaf = localServiceDomains != [ baoCfg.ui.domain ];
+  #
+  # Nor on a host fronting no swarm name: it asks the store for the root
+  # alone, which nginx never reads, and a sealed store would otherwise hold
+  # its gateway down.
+  gatewayWaitsForServicesLeaf =
+    localServiceDomains != [ ] && localServiceDomains != [ baoCfg.ui.domain ];
 
   # The host-managed hive CA is the trust anchor for self-signed mode.
   # It is only stood up when the gateway actually serves a self-signed
@@ -283,6 +288,8 @@ let
     pkgs.diffutils
     # `flock`, for the lock at the top of the script.
     pkgs.util-linux
+    # The root-only fetch on a host fronting no swarm name.
+    pkgs.curl
   ];
   servicesCertEnvironment = {
     BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
@@ -299,6 +306,21 @@ let
   // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
     BAO_CACERT = baoDeploy.serverCaFile;
   };
+  # The root-only fetch. Same store and server CA as `servicesCertEnvironment`,
+  # but the hive's reader identity: a host fronting no swarm name holds no
+  # services-issuer pair.
+  storeRootCurlTls = lib.escapeShellArgs (
+    lib.optionals (baoDeploy.serverCaFile != null) [
+      "--cacert"
+      baoDeploy.serverCaFile
+    ]
+    ++ lib.optionals (baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null) [
+      "--cert"
+      baoDeploy.clientCertFile
+      "--key"
+      baoDeploy.clientKeyFile
+    ]
+  );
   storeRetry = import ./lib/store-retry.nix { };
 in
 {
@@ -768,8 +790,42 @@ in
 
         ${leafCoverage}
 
+        # No leaf to ask for, but the root still belongs in the trust
+        # bundle: every other hive's swarm-service names chain to it.
+        # `ca/pem` takes no token; the TLS client certificate is the
+        # store's listener's demand, not bao's, so this presents the
+        # hive's reader identity and never logs in with it.
         if [ "$want_svc" = 0 ]; then
-          echo "no swarm service domains configured — nothing to issue"
+          echo "no swarm service domains configured — fetching only the store's pki root"
+          err="$(mktemp)"
+          raw="$(mktemp "$d/swarm-services-root.XXXXXX")"
+          trap 'rm -f "$err" "$raw"' EXIT
+          if ! curl --fail --silent --show-error \
+              ${storeRootCurlTls} --output "$raw" \
+              "$BAO_ADDR/v1/${baoServicesPkiMount}/ca/pem" 2>"$err"; then
+            echo "could not read the store's ${baoServicesPkiMount} root." >&2
+            cat "$err" >&2
+            exit 1
+          fi
+          # Re-encoded rather than copied: it fails on a body that does not
+          # parse as a certificate, and ends the file in the newline the
+          # bundle's `cat` relies on.
+          if ! openssl x509 -in "$raw" -out "$svcroot.new" 2>"$err"; then
+            echo "the store's ${baoServicesPkiMount} root does not parse as a certificate." >&2
+            cat "$err" >&2
+            exit 1
+          fi
+          if cmp -s "$svcroot.new" "$svcroot"; then
+            rm -f "$svcroot.new"
+            echo "the store's pki root is unchanged"
+            exit 0
+          fi
+          mv -f "$svcroot.new" "$svcroot"
+          chmod 0644 "$svcroot"
+          if systemctl is-active --quiet hive-tls-ca.service; then
+            echo "the services root changed — rebuilding the trust bundle"
+            systemctl restart --no-block hive-tls-ca.service
+          fi
           exit 0
         fi
 

nix/module-eval/hive-tls.nix

@@ -2,7 +2,8 @@
 # this suite exists, naming convention, "evaluates not executes").
 #
 # The swarm-services leaf's renewal: a timer that really re-runs the
-# issuance, at a threshold read off the store role's lifetime.
+# issuance, at a threshold read off the store role's lifetime. And the
+# store's root on a hive that asks it for no leaf.
 {
   pkgs,
   lib,
@@ -36,6 +37,15 @@ let
     deploy.bao.servicesPkiLeafTtlHours = 48;
   };
 
+  # Fronting none of the swarm's names, reading a store it does not run,
+  # with the reader identity placed by hand.
+  noService = hive {
+    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
+    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
+    deploy.bao.serverCaFile = "/etc/pki/bao-ca.pem";
+  };
+  noSvcFetch = noService.systemd.services.swarm-services-cert.script;
+
   units = allLocal.systemd.services;
   boot = units.swarm-services-cert;
   timer = allLocal.systemd.timers.swarm-services-cert-renew;
@@ -117,6 +127,40 @@ let
         && lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (48 * 3600)} '' shortBoot
         && lib.hasInfix "-checkend ${toString (48 * 3600)} <<<\"$root_ca\"" shortStore;
     }
+    {
+      # The slot the bundle reads the store's root from, filled on a hive
+      # that asks the store for no leaf.
+      name = "a hive fronting no swarm name puts the store's pki root in its trust bundle";
+      ok =
+        noService.services.hyperhive.swarm.localServiceDomains == [ ]
+        && lib.hasInfix ''"$BAO_ADDR/v1/pki/ca/pem"'' noSvcFetch
+        && lib.hasInfix ''mv -f "$svcroot.new" "$svcroot"'' noSvcFetch
+        && lib.hasInfix ''svcroot="$d/swarm-services-root.pem"'' noService.systemd.services.hive-tls-ca.script
+        && lib.hasInfix ''if [ -s "$svcroot" ]; then anchors+=("$svcroot"); fi'' noService.systemd.services.hive-tls-ca.script;
+    }
+    {
+      name = "the root fetch reaches the store with the hive's own reader identity and server CA";
+      ok =
+        lib.hasInfix "--cacert ${lib.escapeShellArg "/etc/pki/bao-ca.pem"}" noSvcFetch
+        && lib.hasInfix (
+          "--cert ${lib.escapeShellArg "/etc/pki/bao-client.pem"}"
+          + " --key ${lib.escapeShellArg "/etc/pki/bao-client-key.pem"}"
+        ) noSvcFetch;
+    }
+    {
+      # The fetch retries against a sealed store; a gateway `Requires=` on it
+      # would go down with each attempt. The control is a host fronting
+      # swarm names, whose gateway does wait for its leaf.
+      name = "a hive fronting no swarm name does not hold its gateway on the store";
+      ok =
+        let
+          holds =
+            m:
+            lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy
+            || lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.before;
+        in
+        !(holds noService) && holds allLocal;
+    }
   ];
 in
 runGroup "hive-tls" cases