hive-tls: issue the gateway leaf from bao with a per-hive identity

Every hive gets its own store identity (CN
<deploy.bao.hiveLeafCommonNamePrefix>-<hive>) and its own role on the
existing pki mount (<deploy.bao.hiveLeafPkiRolePrefix>-<hive>), whose
allowed_domains is the hive's domain and *.<domain>. The policy grants
update on that one issue path. swarm-bao-hive-leaf-policy writes role,
policy and cert-auth role for every hive in swarm.hives; the store's host
mints its own hive's identity in glue-bao-tls.nix and a new glue file
points deploy.hive-controller.tls.baoHiveIdentity{Cert,Key}File at it.
Hives on other hosts place it by hand. swarm.nix reserves the composed CN.

hive-tls-ca no longer generates a hive CA or signs a leaf: caGenScript,
signLeafScript/signHiveLeaf and the adoption block are gone, ca-key.pem
is deleted where it exists, and the unit only assembles trust-bundle.pem.
The retired ca.pem/ca-previous.pem stay in the bundle so leaves they
signed keep verifying.

hive-tls-resign becomes the bao issuance: at boot, on deploy and from a
daily timer it re-issues when the leaf is missing, past half the role's
lifetime, missing a name, or not chained to the store's pki root. Nothing
orders the gateway after it, so a valid leaf on disk never waits on the
store; a failed renewal leaves the leaf in place. A missing identity
fails the unit naming both options, as does the root fetch on a hive
fronting no swarm name, which now presents this identity instead of the
reader cert.

caValidityDays and leafValidityDays are removed (both spellings), since
nothing reads them.

Refs #4588

nix/module-eval/hive-tls.nix

@@ -3,7 +3,8 @@
 #
 # The swarm-services leaf's renewal: a timer that really re-runs the
 # issuance, at a threshold read off the store role's lifetime. And the
-# store's root on a hive that asks it for no leaf.
+# store's root on a hive that asks it for no leaf. And the gateway leaf,
+# issued by the store through the hive's own role, with no hive CA key.
 {
   pkgs,
   lib,
@@ -38,15 +39,17 @@ let
   };
 
   # Fronting none of the swarm's names, reading a store it does not run,
-  # with the reader identity placed by hand.
+  # with the reader identity and the hive's own identity placed by hand.
   noService = hive {
     deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
     deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
     deploy.bao.serverCaFile = "/etc/pki/bao-ca.pem";
+    deploy.hive-controller.tls.baoHiveIdentityCertFile = "/etc/pki/bao-hive.pem";
+    deploy.hive-controller.tls.baoHiveIdentityKeyFile = "/etc/pki/bao-hive-key.pem";
   };
   noSvcFetch = noService.systemd.services.swarm-services-cert.script;
 
-  # The same hive with no reader identity at all.
+  # The same hive with no store identity at all.
   noIdentity = hive { };
   noIdFetch = noIdentity.systemd.services.swarm-services-cert.script;
 
@@ -60,6 +63,8 @@ let
 
   units = allLocal.systemd.services;
   boot = units.swarm-services-cert;
+  caUnit = units.hive-tls-ca;
+  leafUnit = units.hive-tls-resign;
   timer = allLocal.systemd.timers.swarm-services-cert-renew;
 
   # What the timer starts, read off the timer rather than assumed from its
@@ -151,13 +156,16 @@ let
         && lib.hasInfix ''if [ -s "$svcroot" ]; then anchors+=("$svcroot"); fi'' noService.systemd.services.hive-tls-ca.script;
     }
     {
-      name = "the root fetch reaches the store with the hive's own reader identity and server CA";
+      # The hive's own identity, not its reader identity: the reader's policy
+      # reads every secret the hive is entitled to.
+      name = "the root fetch reaches the store with the hive's own identity and server CA";
       ok =
         lib.hasInfix "--cacert ${lib.escapeShellArg "/etc/pki/bao-ca.pem"}" noSvcFetch
         && lib.hasInfix (
-          "--cert ${lib.escapeShellArg "/etc/pki/bao-client.pem"}"
-          + " --key ${lib.escapeShellArg "/etc/pki/bao-client-key.pem"}"
-        ) noSvcFetch;
+          "--cert ${lib.escapeShellArg "/etc/pki/bao-hive.pem"}"
+          + " --key ${lib.escapeShellArg "/etc/pki/bao-hive-key.pem"}"
+        ) noSvcFetch
+        && !(lib.hasInfix "/etc/pki/bao-client.pem" (noSvcBranch noSvcFetch));
     }
     {
       # No identity to present: the unit fails, naming both options, rather
@@ -166,11 +174,11 @@ let
       name = "a hive fronting no swarm name with no store identity fails the unit, naming both options";
       ok =
         noIdentity.services.hyperhive.swarm.localServiceDomains == [ ]
-        && noIdentity.services.hyperhive.deploy.bao.clientCertFile == null
+        && noIdentity.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null
         && !(lib.hasInfix "/v1/pki/ca/pem" noIdFetch)
-        && lib.hasInfix "services.hyperhive.deploy.bao.clientCertFile" (noSvcBranch noIdFetch)
-        && lib.hasInfix "services.hyperhive.deploy.bao.clientKeyFile" (noSvcBranch noIdFetch)
-        && lib.hasSuffix "exit 1" (noSvcBranch noIdFetch)
+        && lib.hasInfix "baoHiveIdentityCertFile (unset)" (noSvcBranch noIdFetch)
+        && lib.hasInfix "baoHiveIdentityKeyFile (unset)" (noSvcBranch noIdFetch)
+        && lib.hasSuffix "exit 1" (lib.removeSuffix "\n" (noSvcBranch noIdFetch))
         && !(lib.hasInfix "exit 0" (noSvcBranch noIdFetch))
         && !(lib.hasInfix "not staged" noIdFetch)
         && lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noSvcFetch);
@@ -179,13 +187,11 @@ let
       # An identity named but not on disk fails before curl, naming the paths.
       name = "a hive fronting no swarm name whose identity files are absent fails the unit, naming the paths";
       ok =
-        lib.hasInfix "for f in /etc/pki/bao-client.pem /etc/pki/bao-client-key.pem; do" (
-          noSvcBranch noSvcFetch
-        )
+        lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (noSvcBranch noSvcFetch)
         && lib.hasInfix ''[ -r "$f" ] || missing="$missing $f"'' (noSvcBranch noSvcFetch)
         && lib.hasInfix "cannot read$missing" (noSvcBranch noSvcFetch)
-        && lib.hasInfix "clientCertFile (/etc/pki/bao-client.pem)" (noSvcBranch noSvcFetch)
-        && lib.hasInfix "clientKeyFile (/etc/pki/bao-client-key.pem)" (noSvcBranch noSvcFetch);
+        && lib.hasInfix "baoHiveIdentityCertFile (/etc/pki/bao-hive.pem)" (noSvcBranch noSvcFetch)
+        && lib.hasInfix "baoHiveIdentityKeyFile (/etc/pki/bao-hive-key.pem)" (noSvcBranch noSvcFetch);
     }
     {
       # The fetch retries against a sealed store; a gateway `Requires=` on it
@@ -201,6 +207,113 @@ let
         in
         !(holds noService) && holds allLocal;
     }
+    {
+      # The controls are the store's PKI unit, which still mints its own CA
+      # with `openssl req`, and the swarm root's unit, which still reads
+      # `root-key.pem`: absence here is not a matcher that cannot match.
+      name = "no hive TLS unit generates a CA, signs with a CA key or reads the swarm root key";
+      ok =
+        let
+          hasCaWork =
+            s:
+            lib.any (p: lib.hasInfix p s) [
+              "openssl req"
+              "-CAkey"
+              "root-key.pem"
+            ];
+          pkiScript = units.swarm-bao-pki.script;
+          rootScript = units.swarm-ca.script;
+        in
+        !(hasCaWork caUnit.script)
+        && !(hasCaWork leafUnit.script)
+        && !(hasCaWork noService.systemd.services.hive-tls-ca.script)
+        && lib.hasInfix "openssl req" pkiScript
+        && lib.hasInfix "root-key.pem" rootScript;
+    }
+    {
+      name = "a retired hive CA key is deleted, and its public certificate stays in the trust bundle";
+      ok =
+        lib.hasInfix ''rm -f "$d/ca-key.pem"'' caUnit.script
+        && lib.hasInfix ''ca="$d/ca.pem"'' caUnit.script
+        && lib.hasInfix ''if [ -s "$ca" ]; then anchors+=("$ca"); fi'' caUnit.script
+        && lib.hasInfix ''if [ -s "$prev" ]; then anchors+=("$prev"); fi'' caUnit.script
+        && lib.hasInfix ''cat "''${anchors[@]}" > "$bundle"'' caUnit.script;
+    }
+    {
+      # The store host's hive presents the leaf ./glue-bao-tls.nix mints for
+      # it; not the reader's, not the services issuer's.
+      name = "the gateway leaf is issued through this hive's own role, presenting its own identity";
+      ok =
+        let
+          t = allLocal.services.hyperhive.deploy.hive-controller.tls;
+        in
+        lib.hasInfix "pki/issue/swarm-hive-leaf-h1" leafUnit.script
+        && lib.hasInfix "common_name=h1.t.local" leafUnit.script
+        && lib.hasInfix "alt_names='*.h1.t.local'" leafUnit.script
+        && lib.hasInfix "bao login -method=cert -token-only" leafUnit.script
+        && leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/hive-leaf.pem"
+        && leafUnit.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/hive-leaf-key.pem"
+        && t.baoHiveIdentityCertFile == leafUnit.environment.BAO_CLIENT_CERT
+        && leafUnit.environment.BAO_CLIENT_CERT != t.baoClientCertFile
+        && leafUnit.environment.BAO_CLIENT_CERT != allLocal.services.hyperhive.deploy.bao.clientCertFile;
+    }
+    {
+      # Checked before the leaf, so the unit is red even while a valid leaf
+      # keeps the gateway up. The control is a hive with the identity named,
+      # whose script checks the files instead.
+      name = "a hive with no store identity fails the gateway-leaf unit, naming both options";
+      ok =
+        let
+          s = noIdentity.systemd.services.hive-tls-resign.script;
+          before = lib.head (lib.splitString "reissue=0" s);
+          ok' = noService.systemd.services.hive-tls-resign.script;
+        in
+        lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile (unset)" before
+        && lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityKeyFile (unset)" before
+        && lib.hasInfix "exit 1" before
+        && lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (
+          lib.head (lib.splitString "reissue=0" ok')
+        )
+        && !(lib.hasInfix "(unset)" ok');
+    }
+    {
+      # A leaf the retired hive CA signed is re-issued from the store on the
+      # first run that reaches it; one that renews in time never is.
+      name = "the gateway leaf is re-issued when not chained to the store's root, and at half its role lifetime";
+      ok =
+        lib.hasInfix ''openssl verify -CAfile "$svcroot" "$leaf"'' leafUnit.script
+        && lib.hasInfix (renewAt 720) leafUnit.script
+        && lib.hasInfix (renewAt 48) shortTtl.systemd.services.hive-tls-resign.script
+        && lib.hasInfix ''-in "$leaf" -noout -checkend "$renewat"'' leafUnit.script;
+    }
+    {
+      # A valid leaf on disk starts the gateway with the store sealed: nothing
+      # requires or waits on the issuing unit. The control is that unit
+      # itself, which does wait on the store's container.
+      name = "the gateway does not wait on the gateway-leaf unit, which waits on the store";
+      ok =
+        (leafUnit.requiredBy or [ ]) == [ ]
+        && (leafUnit.before or [ ]) == [ ]
+        && !(lib.any (u: lib.elem "hive-tls-resign.service" ((u.requires or [ ]) ++ (u.after or [ ]))) (
+          lib.attrValues (removeAttrs units [ "hive-tls-resign" ])
+        ))
+        && !(lib.elem "container@swarm-bao.service" ((caUnit.requires or [ ]) ++ (caUnit.after or [ ])))
+        && lib.elem "container@swarm-bao.service" leafUnit.after
+        && lib.elem "multi-user.target" leafUnit.wantedBy;
+    }
+    {
+      # Not `RemainAfterExit`, or the timer would start an active unit and
+      # renew nothing.
+      name = "a daily timer re-runs the gateway-leaf unit";
+      ok =
+        let
+          t = allLocal.systemd.timers.hive-tls-resign;
+        in
+        lib.elem "timers.target" t.wantedBy
+        && t.timerConfig.OnCalendar == "daily"
+        && (t.timerConfig.Unit or "hive-tls-resign.service") == "hive-tls-resign.service"
+        && !(remainsAfterExit leafUnit);
+    }
   ];
 in
 runGroup "hive-tls" cases