glue: consumers' start-after-bao ordering moves into one glue module

Each store reader (forge's OIDC fetch and token publisher, matrix's OIDC
fetch, the hive collector's fetch, Grafana's OIDC fetch and admin password,
the queue's callout-xkey unit, the CI runner's token fetch, the swarm
collector's OIDC fetch) used to read `deploy.bao.enable` in its own module
to add `after`/`wants` on `container@<store machine>` and
`after`/`requires` on `swarm-bao-pki.service`. Per the #3742 ruling, that
cross-service read moves into glue-bao-readers-store-order.nix, which sets
the same edges when the store and the reader are both on this host and
restates each reader's own gate so no stub unit is rendered.

Rendered `after`/`wants`/`requires` are identical as sets to main on 16
fixtures, store here and elsewhere. New module-eval cases in existing
fixtures pin both arms.

Part of #3742.

nix/host-modules/default.nix

@@ -26,6 +26,7 @@
     ./hive-tls.nix
     ./otel.nix
     ./glue-bao-readers-policy-order.nix
+    ./glue-bao-readers-store-order.nix
     ./glue-bao-tls.nix
     ./glue-bao-ui-oidc-client.nix
     ./glue-controller-bao-identity.nix

nix/host-modules/glue-bao-readers-store-order.nix

@@ -0,0 +1,74 @@
+# Glue: where the store and one of its readers share a host, the reader waits
+# for the store's container and requires `swarm-bao-pki`, which mints the leaf
+# the reader logs in with.
+#
+# ONE PAIRING PER FILE — the store's container and PKI unit ← the readers that
+# log in to it, and nothing else. The waits on the store's policy units are
+# ./glue-bao-readers-policy-order.nix's.
+#
+# ⚠️ Gated on BOTH the store and that reader being here. Off the store's host
+# neither unit exists, and `Requires=` on an absent unit fails the job, so
+# there the reader's own start timeout bounds the attempt instead. On the
+# store's host without the reader, setting `systemd.services.<reader>.after`
+# would define a unit with no ExecStart, so each gate below restates the one
+# the reader's own module puts on it. A reader whose gate changes must change
+# here too.
+{
+  lib,
+  config,
+  ...
+}:
+let
+  hyperhiveCfg = config.services.hyperhive;
+  deployCfg = hyperhiveCfg.deploy;
+  baoDeploy = deployCfg.bao;
+  baoCfg = hyperhiveCfg.swarm.bao;
+  otel = hyperhiveCfg.otel;
+
+  havePair = cert: key: cert != null && key != null;
+
+  # Reader unit → the gate its own module defines it under.
+  readersHere = {
+    # ./hive-forge/default.nix
+    swarm-bao-forge-oidc = deployCfg.forgejo.enable;
+    swarm-bao-forge-publisher = deployCfg.forgejo.enable;
+    # ./hive-matrix.nix
+    swarm-bao-matrix-oidc = deployCfg.matrix.enable;
+    # ./otel.nix: `fetches`, the collector loading the file this unit writes
+    # under the hive's own leaf.
+    swarm-bao-hive-otel-oidc =
+      otel.enable
+      && hyperhiveCfg.hiveName != null
+      && havePair baoDeploy.clientCertFile baoDeploy.clientKeyFile
+      &&
+        otel.clientSecretFile
+        == "/var/lib/hive-otel-oidc/${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${toString hyperhiveCfg.hiveName}.secret";
+    # ./swarm-grafana.nix
+    swarm-bao-grafana-oidc = deployCfg.grafana.enable;
+    swarm-bao-grafana-admin = deployCfg.grafana.enable;
+    # ./swarm-nats.nix
+    swarm-bao-nats-callout-xkey = deployCfg.nats.enable;
+    # ./swarm-ci-runner.nix
+    swarm-bao-ci-runner = deployCfg.ci-runner.enable;
+    # ./swarm-otel.nix
+    swarm-bao-otel-oidc =
+      deployCfg.swarm-otel.enable
+      && havePair baoDeploy.otelOidcClientCertFile baoDeploy.otelOidcClientKeyFile;
+  };
+
+  orderAfterStore =
+    reader: here:
+    lib.mkIf (baoDeploy.enable && here) {
+      systemd.services.${reader} = {
+        after = [
+          "swarm-bao-pki.service"
+          "container@${baoCfg.machine}.service"
+        ];
+        wants = [ "container@${baoCfg.machine}.service" ];
+        requires = [ "swarm-bao-pki.service" ];
+      };
+    };
+in
+{
+  config = lib.mkMerge (lib.mapAttrsToList orderAfterStore readersHere);
+}

nix/host-modules/hive-forge/default.nix

@@ -1011,15 +1011,9 @@ in
     # rendering no unit.
     systemd.services.swarm-bao-forge-oidc = {
       description = "fetch the forge's OIDC client secret from the swarm secret store";
-      # The store's units exist only where the store runs; `Requires=` on an
+      # Ordered after the store where it runs here
-      # absent unit fails the job, so off-host the timeout below bounds the
+      # (../glue-bao-readers-store-order.nix); elsewhere the timeout below
-      # attempt instead.
+      # bounds the attempt.
-      after = lib.optionals deployCfg.bao.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
       before = [ "container@hive-forge.service" ];
       wantedBy = [
         "multi-user.target"
@@ -1141,17 +1135,11 @@ in
     # here, naming the options, rather than rendering no unit.
     systemd.services.swarm-bao-forge-publisher = {
       description = "publish the swarm-controller's forge token to the swarm secret store";
-      # The store's units exist only where the store runs; `Requires=` on an
+      # Also ordered after the store where it runs here:
-      # absent unit fails the job.
+      # ../glue-bao-readers-store-order.nix.
       after = [
         "container@hive-forge.service"
-      ]
-      ++ lib.optionals deployCfg.bao.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
       ];
-      wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
       wantedBy = [
         "multi-user.target"
         "container@hive-forge.service"

nix/host-modules/hive-matrix.nix

@@ -595,15 +595,9 @@ in
     # no unit.
     systemd.services.swarm-bao-matrix-oidc = {
       description = "fetch the homeserver's OIDC client secret from the swarm secret store";
-      # The store's units exist only where the store runs; `Requires=` on an
+      # Ordered after the store where it runs here
-      # absent unit fails the job, so off-host the timeout below bounds the
+      # (./glue-bao-readers-store-order.nix); elsewhere the timeout below
-      # attempt instead.
+      # bounds the attempt.
-      after = lib.optionals deployCfg.bao.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
       before = [ "container@hive-matrix.service" ];
       wantedBy = [
         "multi-user.target"

nix/host-modules/otel.nix

@@ -337,6 +337,7 @@ in
       # Only when the collector reads the file the unit writes: a secret
       # delivered some other way is the operator's, and a unit restarting the
       # collector on a file it never loads would be noise.
+      # ./glue-bao-readers-store-order.nix restates this gate.
       fetches = senderAuth && hiveLeaf && otel.clientSecretFile == fetchedSecretPath;
       # An operator-delivered file has no unit that restarts the collector
       # once it lands, so the collector waits for it instead.
@@ -793,15 +794,9 @@ in
       # secret reaches it only by a restart, which `refresh_unit` does.
       systemd.services.${fetchUnit} = lib.mkIf fetches {
         description = "fetch this hive's telemetry OIDC client secret from the swarm secret store";
-        # The store's units exist only where the store runs; `Requires=` on an
+        # Ordered after the store where it runs here
-        # absent unit fails the job, so off-host the timeout below bounds the
+        # (./glue-bao-readers-store-order.nix, which restates `fetches`);
-        # attempt instead.
+        # elsewhere the timeout below bounds the attempt.
-        after = lib.optionals deployCfg.bao.enable [
-          "swarm-bao-pki.service"
-          "container@${baoCfg.machine}.service"
-        ];
-        wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
-        requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
         before = [ "opentelemetry-collector.service" ];
         wantedBy = [
           "multi-user.target"

nix/host-modules/swarm-ci-runner.nix

@@ -189,12 +189,8 @@ in
     # every five minutes, so a retry is what turns "no value there" into one.
     systemd.services.swarm-bao-ci-runner = {
       description = "fetch the CI runner's registration token from the swarm secret store";
-      after = lib.optionals baoDeploy.enable [
+      # Ordered after the store where it runs here:
-        "swarm-bao-pki.service"
+      # ./glue-bao-readers-store-order.nix.
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
       before = [ "container@${machine}.service" ];
       wantedBy = [
         "multi-user.target"

nix/host-modules/swarm-grafana.nix

@@ -558,16 +558,9 @@ in
     # the only thing separating that from a mystery.
     systemd.services.swarm-bao-grafana-oidc = {
       description = "fetch Grafana's OIDC client secret from the swarm secret store";
-      # Every one of these names a unit that exists only where the store runs.
+      # Ordered after the store where it runs here
-      # `Requires=` on an absent unit fails the job outright, so the ordering
+      # (./glue-bao-readers-store-order.nix); elsewhere the timeout below
-      # is conditional even though the read is not: off-host there is nothing
+      # bounds the attempt.
-      # local to wait for, and the timeout below bounds the attempt instead.
-      after = lib.optionals baoDeploy.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
       before = [ "container@${cfg.machine}.service" ];
       wantedBy = [
         "multi-user.target"
@@ -686,12 +679,8 @@ in
     # Grafana down rather than up with the upstream default.
     systemd.services.swarm-bao-grafana-admin = {
       description = "create Grafana's admin password in the swarm secret store if absent and fetch it";
-      after = lib.optionals baoDeploy.enable [
+      # Ordered after the store where it runs here:
-        "swarm-bao-pki.service"
+      # ./glue-bao-readers-store-order.nix.
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
       before = [ "container@${cfg.machine}.service" ];
       wantedBy = [
         "multi-user.target"

nix/host-modules/swarm-nats.nix

@@ -830,15 +830,9 @@ in
     # responder's own leaf, the one principal that may create the keys' paths.
     systemd.services.swarm-bao-nats-callout-xkey = {
       description = "create the swarm queue's callout keys if absent and fetch their public halves for the server";
-      # The store's units exist only where the store runs; `Requires=` on an
+      # Ordered after the store where it runs here
-      # absent unit fails the job, so off-host the timeout below bounds the
+      # (./glue-bao-readers-store-order.nix); elsewhere the timeout below
-      # attempt instead.
+      # bounds the attempt.
-      after = lib.optionals baoDeploy.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
       before = [ "container@${machine}.service" ];
       wantedBy = [
         "multi-user.target"

nix/host-modules/swarm-otel.nix

@@ -429,16 +429,9 @@ in
     # regression rather than that degrade — `hiveReaderIdentity` above.
     systemd.services.swarm-bao-otel-oidc = lib.mkIf haveClientIdentity {
       description = "fetch the swarm collector's OIDC client secret from the swarm secret store";
-      # Every one of these names a unit that exists only where the store runs.
+      # Ordered after the store where it runs here
-      # `Requires=` on an absent unit fails the job outright, so the ordering
+      # (./glue-bao-readers-store-order.nix); elsewhere the timeout below
-      # is conditional even though the read is not: off-host there is nothing
+      # bounds the attempt.
-      # local to wait for, and the timeout below bounds the attempt instead.
-      after = lib.optionals baoDeploy.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
       before = [ "container@${cfg.machine}.service" ];
       wantedBy = [
         "multi-user.target"

nix/module-eval/forge-placement.nix

@@ -187,6 +187,21 @@ let
           ciAlone.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT
           == "/etc/pki/bao-ci-runner.pem";
     }
+    {
+      # The store's units exist only on its host; `Requires=` on an absent one
+      # fails the fetch's start. The arm where the store is here is in
+      # ./swarm-services-switch.nix.
+      name = "the CI runner's token fetch names none of the store's units on a host without the store";
+      ok =
+        let
+          u = ciAlone.systemd.services.swarm-bao-ci-runner;
+          storeUnits = [
+            "swarm-bao-pki.service"
+            "container@${ciAlone.services.hyperhive.swarm.bao.machine}.service"
+          ];
+        in
+        !(lib.any (n: lib.elem n (u.after ++ u.wants ++ u.requires)) storeUnits);
+    }
     {
       # Without the identity the token never arrives; the control is the
       # placed pair above.

nix/module-eval/hive-otel.nix

@@ -278,6 +278,26 @@ let
         in
         lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish;
     }
+    {
+      # The store's units exist only on its host; `Requires=` on an absent one
+      # fails the fetch's start.
+      name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here";
+      ok =
+        let
+          storeUnits = m: [
+            "swarm-bao-pki.service"
+            "container@${m.services.hyperhive.swarm.bao.machine}.service"
+          ];
+          here = hiveOtelFetch hiveOtelAllLocal;
+          elsewhere = hiveOtelFetch hiveOtelLeaf;
+        in
+        lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal)
+        && lib.elem "swarm-bao-pki.service" here.requires
+        && lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants
+        && !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) (
+          storeUnits hiveOtelLeaf
+        ));
+    }
     {
       # A secret delivered some other way is the operator's: no unit fetches
       # into a file the collector does not load, or restarts it on one. Nothing

nix/module-eval/swarm-otel-identity.nix

@@ -217,6 +217,21 @@ let
         !(refusedOver otelReaderNamedInFull "otelOidcClientCertFile")
         && !(refusedOver otelRemoteAuthelia "otelOidcClientCertFile");
     }
+    {
+      # The store's units exist only on its host; `Requires=` on an absent one
+      # fails the fetch's start. The arm where the store is here is in
+      # ./swarm-services-switch.nix.
+      name = "the remote reader's fetch names none of the store's units";
+      ok =
+        let
+          u = otelReaderNamedInFull.systemd.services.swarm-bao-otel-oidc;
+          storeUnits = [
+            "swarm-bao-pki.service"
+            "container@${otelReaderNamedInFull.services.hyperhive.swarm.bao.machine}.service"
+          ];
+        in
+        !(lib.any (n: lib.elem n (u.after ++ u.wants ++ u.requires)) storeUnits);
+    }
   ];
 in
 runGroup "swarm-otel-identity" cases

nix/module-eval/swarm-services-switch.nix

@@ -575,6 +575,54 @@ let
         == "/etc/marker-avatar.png"
         && !(controllerWithMirrors.systemd.services.hive-c0re.environment ? HIVE_ORG_AVATAR_PNG);
     }
+    {
+      # Where the store runs here, each reader waits for its container and
+      # requires the PKI unit minting the leaf it logs in with. Elsewhere
+      # neither unit exists, and `Requires=` on an absent unit fails the
+      # reader's start. On the store's host without the readers, none of them
+      # is rendered as a stub. Rosters are counted: an empty one holds
+      # vacuously.
+      name = "a store reader waits for the store's container and PKI unit only where the store runs here";
+      ok =
+        let
+          machine = serviceContainersWithCi.services.hyperhive.swarm.bao.machine;
+          storeUnits = [
+            "swarm-bao-pki.service"
+            "container@${machine}.service"
+          ];
+          readers = [
+            "swarm-bao-forge-oidc"
+            "swarm-bao-forge-publisher"
+            "swarm-bao-matrix-oidc"
+            "swarm-bao-hive-otel-oidc"
+            "swarm-bao-grafana-oidc"
+            "swarm-bao-grafana-admin"
+            "swarm-bao-nats-callout-xkey"
+            "swarm-bao-ci-runner"
+            "swarm-bao-otel-oidc"
+          ];
+          here = serviceContainersWithCi.systemd.services;
+          elsewhere = swarmServicesBaoElsewhere.systemd.services;
+          readersElsewhere = lib.filter (u: elsewhere ? ${u}) readers;
+          waits =
+            u:
+            here ? ${u}
+            && lib.elem "swarm-bao-pki.service" here.${u}.after
+            && lib.elem "swarm-bao-pki.service" here.${u}.requires
+            && lib.elem "container@${machine}.service" here.${u}.after
+            && lib.elem "container@${machine}.service" here.${u}.wants;
+          namesNoStoreUnit =
+            u:
+            let
+              s = elsewhere.${u};
+            in
+            !(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
+        in
+        lib.all waits readers
+        && lib.length readersElsewhere == 6
+        && lib.all namesNoStoreUnit readersElsewhere
+        && !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);
+    }
   ];
 in
 runGroup "swarm-services-switch" cases