From 9407db69b5681e7a59fedba0748210764f868e05 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Tue, 13 Oct 2026 09:38:33 +0200
Subject: [PATCH] glue: consumers' start-after-bao ordering moves into one glue
 module

Each store reader (forge's OIDC fetch and token publisher, matrix's OIDC
fetch, the hive collector's fetch, Grafana's OIDC fetch and admin password,
the queue's callout-xkey unit, the CI runner's token fetch, the swarm
collector's OIDC fetch) used to read `deploy.bao.enable` in its own module
to add `after`/`wants` on `container@<store machine>` and
`after`/`requires` on `swarm-bao-pki.service`. Per the #3742 ruling, that
cross-service read moves into glue-bao-readers-store-order.nix, which sets
the same edges when the store and the reader are both on this host and
restates each reader's own gate so no stub unit is rendered.

Rendered `after`/`wants`/`requires` are identical as sets to main on 16
fixtures, store here and elsewhere. New module-eval cases in existing
fixtures pin both arms.

Part of #3742.
---
 nix/host-modules/default.nix                  |  1 +
 .../glue-bao-readers-store-order.nix          | 74 +++++++++++++++++++
 nix/host-modules/hive-forge/default.nix       | 22 ++----
 nix/host-modules/hive-matrix.nix              | 12 +--
 nix/host-modules/otel.nix                     | 13 +---
 nix/host-modules/swarm-ci-runner.nix          |  8 +-
 nix/host-modules/swarm-grafana.nix            | 21 ++----
 nix/host-modules/swarm-nats.nix               | 12 +--
 nix/host-modules/swarm-otel.nix               | 13 +---
 nix/module-eval/forge-placement.nix           | 15 ++++
 nix/module-eval/hive-otel.nix                 | 20 +++++
 nix/module-eval/swarm-otel-identity.nix       | 15 ++++
 nix/module-eval/swarm-services-switch.nix     | 48 ++++++++++++
 13 files changed, 198 insertions(+), 76 deletions(-)
 create mode 100644 nix/host-modules/glue-bao-readers-store-order.nix

diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix
index 5d9a7f20..99e988e6 100644
--- a/nix/host-modules/default.nix
+++ b/nix/host-modules/default.nix
@@ -26,6 +26,7 @@
     ./hive-tls.nix
     ./otel.nix
     ./glue-bao-readers-policy-order.nix
+    ./glue-bao-readers-store-order.nix
     ./glue-bao-tls.nix
     ./glue-bao-ui-oidc-client.nix
     ./glue-controller-bao-identity.nix
diff --git a/nix/host-modules/glue-bao-readers-store-order.nix b/nix/host-modules/glue-bao-readers-store-order.nix
new file mode 100644
index 00000000..6a4776e6
--- /dev/null
+++ b/nix/host-modules/glue-bao-readers-store-order.nix
@@ -0,0 +1,74 @@
+# Glue: where the store and one of its readers share a host, the reader waits
+# for the store's container and requires `swarm-bao-pki`, which mints the leaf
+# the reader logs in with.
+#
+# ONE PAIRING PER FILE — the store's container and PKI unit ← the readers that
+# log in to it, and nothing else. The waits on the store's policy units are
+# ./glue-bao-readers-policy-order.nix's.
+#
+# ⚠️ Gated on BOTH the store and that reader being here. Off the store's host
+# neither unit exists, and `Requires=` on an absent unit fails the job, so
+# there the reader's own start timeout bounds the attempt instead. On the
+# store's host without the reader, setting `systemd.services.<reader>.after`
+# would define a unit with no ExecStart, so each gate below restates the one
+# the reader's own module puts on it. A reader whose gate changes must change
+# here too.
+{
+  lib,
+  config,
+  ...
+}:
+let
+  hyperhiveCfg = config.services.hyperhive;
+  deployCfg = hyperhiveCfg.deploy;
+  baoDeploy = deployCfg.bao;
+  baoCfg = hyperhiveCfg.swarm.bao;
+  otel = hyperhiveCfg.otel;
+
+  havePair = cert: key: cert != null && key != null;
+
+  # Reader unit → the gate its own module defines it under.
+  readersHere = {
+    # ./hive-forge/default.nix
+    swarm-bao-forge-oidc = deployCfg.forgejo.enable;
+    swarm-bao-forge-publisher = deployCfg.forgejo.enable;
+    # ./hive-matrix.nix
+    swarm-bao-matrix-oidc = deployCfg.matrix.enable;
+    # ./otel.nix: `fetches`, the collector loading the file this unit writes
+    # under the hive's own leaf.
+    swarm-bao-hive-otel-oidc =
+      otel.enable
+      && hyperhiveCfg.hiveName != null
+      && havePair baoDeploy.clientCertFile baoDeploy.clientKeyFile
+      &&
+        otel.clientSecretFile
+        == "/var/lib/hive-otel-oidc/${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${toString hyperhiveCfg.hiveName}.secret";
+    # ./swarm-grafana.nix
+    swarm-bao-grafana-oidc = deployCfg.grafana.enable;
+    swarm-bao-grafana-admin = deployCfg.grafana.enable;
+    # ./swarm-nats.nix
+    swarm-bao-nats-callout-xkey = deployCfg.nats.enable;
+    # ./swarm-ci-runner.nix
+    swarm-bao-ci-runner = deployCfg.ci-runner.enable;
+    # ./swarm-otel.nix
+    swarm-bao-otel-oidc =
+      deployCfg.swarm-otel.enable
+      && havePair baoDeploy.otelOidcClientCertFile baoDeploy.otelOidcClientKeyFile;
+  };
+
+  orderAfterStore =
+    reader: here:
+    lib.mkIf (baoDeploy.enable && here) {
+      systemd.services.${reader} = {
+        after = [
+          "swarm-bao-pki.service"
+          "container@${baoCfg.machine}.service"
+        ];
+        wants = [ "container@${baoCfg.machine}.service" ];
+        requires = [ "swarm-bao-pki.service" ];
+      };
+    };
+in
+{
+  config = lib.mkMerge (lib.mapAttrsToList orderAfterStore readersHere);
+}
diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix
index 684f5415..754be86d 100644
--- a/nix/host-modules/hive-forge/default.nix
+++ b/nix/host-modules/hive-forge/default.nix
@@ -1011,15 +1011,9 @@ in
     # rendering no unit.
     systemd.services.swarm-bao-forge-oidc = {
       description = "fetch the forge's OIDC client secret from the swarm secret store";
-      # The store's units exist only where the store runs; `Requires=` on an
-      # absent unit fails the job, so off-host the timeout below bounds the
-      # attempt instead.
-      after = lib.optionals deployCfg.bao.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
+      # Ordered after the store where it runs here
+      # (../glue-bao-readers-store-order.nix); elsewhere the timeout below
+      # bounds the attempt.
       before = [ "container@hive-forge.service" ];
       wantedBy = [
         "multi-user.target"
@@ -1141,17 +1135,11 @@ in
     # here, naming the options, rather than rendering no unit.
     systemd.services.swarm-bao-forge-publisher = {
       description = "publish the swarm-controller's forge token to the swarm secret store";
-      # The store's units exist only where the store runs; `Requires=` on an
-      # absent unit fails the job.
+      # Also ordered after the store where it runs here:
+      # ../glue-bao-readers-store-order.nix.
       after = [
         "container@hive-forge.service"
-      ]
-      ++ lib.optionals deployCfg.bao.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
       ];
-      wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
       wantedBy = [
         "multi-user.target"
         "container@hive-forge.service"
diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix
index ee260bbf..f5b16fdd 100644
--- a/nix/host-modules/hive-matrix.nix
+++ b/nix/host-modules/hive-matrix.nix
@@ -595,15 +595,9 @@ in
     # no unit.
     systemd.services.swarm-bao-matrix-oidc = {
       description = "fetch the homeserver's OIDC client secret from the swarm secret store";
-      # The store's units exist only where the store runs; `Requires=` on an
-      # absent unit fails the job, so off-host the timeout below bounds the
-      # attempt instead.
-      after = lib.optionals deployCfg.bao.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
+      # Ordered after the store where it runs here
+      # (./glue-bao-readers-store-order.nix); elsewhere the timeout below
+      # bounds the attempt.
       before = [ "container@hive-matrix.service" ];
       wantedBy = [
         "multi-user.target"
diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix
index ba83b510..bea8748c 100644
--- a/nix/host-modules/otel.nix
+++ b/nix/host-modules/otel.nix
@@ -337,6 +337,7 @@ in
       # Only when the collector reads the file the unit writes: a secret
       # delivered some other way is the operator's, and a unit restarting the
       # collector on a file it never loads would be noise.
+      # ./glue-bao-readers-store-order.nix restates this gate.
       fetches = senderAuth && hiveLeaf && otel.clientSecretFile == fetchedSecretPath;
       # An operator-delivered file has no unit that restarts the collector
       # once it lands, so the collector waits for it instead.
@@ -793,15 +794,9 @@ in
       # secret reaches it only by a restart, which `refresh_unit` does.
       systemd.services.${fetchUnit} = lib.mkIf fetches {
         description = "fetch this hive's telemetry OIDC client secret from the swarm secret store";
-        # The store's units exist only where the store runs; `Requires=` on an
-        # absent unit fails the job, so off-host the timeout below bounds the
-        # attempt instead.
-        after = lib.optionals deployCfg.bao.enable [
-          "swarm-bao-pki.service"
-          "container@${baoCfg.machine}.service"
-        ];
-        wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
-        requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
+        # Ordered after the store where it runs here
+        # (./glue-bao-readers-store-order.nix, which restates `fetches`);
+        # elsewhere the timeout below bounds the attempt.
         before = [ "opentelemetry-collector.service" ];
         wantedBy = [
           "multi-user.target"
diff --git a/nix/host-modules/swarm-ci-runner.nix b/nix/host-modules/swarm-ci-runner.nix
index d8b0d747..4c19f19c 100644
--- a/nix/host-modules/swarm-ci-runner.nix
+++ b/nix/host-modules/swarm-ci-runner.nix
@@ -189,12 +189,8 @@ in
     # every five minutes, so a retry is what turns "no value there" into one.
     systemd.services.swarm-bao-ci-runner = {
       description = "fetch the CI runner's registration token from the swarm secret store";
-      after = lib.optionals baoDeploy.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
+      # Ordered after the store where it runs here:
+      # ./glue-bao-readers-store-order.nix.
       before = [ "container@${machine}.service" ];
       wantedBy = [
         "multi-user.target"
diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix
index d8fcc75f..d4743f25 100644
--- a/nix/host-modules/swarm-grafana.nix
+++ b/nix/host-modules/swarm-grafana.nix
@@ -558,16 +558,9 @@ in
     # the only thing separating that from a mystery.
     systemd.services.swarm-bao-grafana-oidc = {
       description = "fetch Grafana's OIDC client secret from the swarm secret store";
-      # Every one of these names a unit that exists only where the store runs.
-      # `Requires=` on an absent unit fails the job outright, so the ordering
-      # is conditional even though the read is not: off-host there is nothing
-      # local to wait for, and the timeout below bounds the attempt instead.
-      after = lib.optionals baoDeploy.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
+      # Ordered after the store where it runs here
+      # (./glue-bao-readers-store-order.nix); elsewhere the timeout below
+      # bounds the attempt.
       before = [ "container@${cfg.machine}.service" ];
       wantedBy = [
         "multi-user.target"
@@ -686,12 +679,8 @@ in
     # Grafana down rather than up with the upstream default.
     systemd.services.swarm-bao-grafana-admin = {
       description = "create Grafana's admin password in the swarm secret store if absent and fetch it";
-      after = lib.optionals baoDeploy.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
+      # Ordered after the store where it runs here:
+      # ./glue-bao-readers-store-order.nix.
       before = [ "container@${cfg.machine}.service" ];
       wantedBy = [
         "multi-user.target"
diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix
index fc384ae7..c2ff6938 100644
--- a/nix/host-modules/swarm-nats.nix
+++ b/nix/host-modules/swarm-nats.nix
@@ -830,15 +830,9 @@ in
     # responder's own leaf, the one principal that may create the keys' paths.
     systemd.services.swarm-bao-nats-callout-xkey = {
       description = "create the swarm queue's callout keys if absent and fetch their public halves for the server";
-      # The store's units exist only where the store runs; `Requires=` on an
-      # absent unit fails the job, so off-host the timeout below bounds the
-      # attempt instead.
-      after = lib.optionals baoDeploy.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
+      # Ordered after the store where it runs here
+      # (./glue-bao-readers-store-order.nix); elsewhere the timeout below
+      # bounds the attempt.
       before = [ "container@${machine}.service" ];
       wantedBy = [
         "multi-user.target"
diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
index ce07b7e9..a81c0075 100644
--- a/nix/host-modules/swarm-otel.nix
+++ b/nix/host-modules/swarm-otel.nix
@@ -429,16 +429,9 @@ in
     # regression rather than that degrade — `hiveReaderIdentity` above.
     systemd.services.swarm-bao-otel-oidc = lib.mkIf haveClientIdentity {
       description = "fetch the swarm collector's OIDC client secret from the swarm secret store";
-      # Every one of these names a unit that exists only where the store runs.
-      # `Requires=` on an absent unit fails the job outright, so the ordering
-      # is conditional even though the read is not: off-host there is nothing
-      # local to wait for, and the timeout below bounds the attempt instead.
-      after = lib.optionals baoDeploy.enable [
-        "swarm-bao-pki.service"
-        "container@${baoCfg.machine}.service"
-      ];
-      wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
-      requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
+      # Ordered after the store where it runs here
+      # (./glue-bao-readers-store-order.nix); elsewhere the timeout below
+      # bounds the attempt.
       before = [ "container@${cfg.machine}.service" ];
       wantedBy = [
         "multi-user.target"
diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix
index 12ee4e1f..3ab329f7 100644
--- a/nix/module-eval/forge-placement.nix
+++ b/nix/module-eval/forge-placement.nix
@@ -187,6 +187,21 @@ let
           ciAlone.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT
           == "/etc/pki/bao-ci-runner.pem";
     }
+    {
+      # The store's units exist only on its host; `Requires=` on an absent one
+      # fails the fetch's start. The arm where the store is here is in
+      # ./swarm-services-switch.nix.
+      name = "the CI runner's token fetch names none of the store's units on a host without the store";
+      ok =
+        let
+          u = ciAlone.systemd.services.swarm-bao-ci-runner;
+          storeUnits = [
+            "swarm-bao-pki.service"
+            "container@${ciAlone.services.hyperhive.swarm.bao.machine}.service"
+          ];
+        in
+        !(lib.any (n: lib.elem n (u.after ++ u.wants ++ u.requires)) storeUnits);
+    }
     {
       # Without the identity the token never arrives; the control is the
       # placed pair above.
diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix
index e4e5a5b4..f773cc88 100644
--- a/nix/module-eval/hive-otel.nix
+++ b/nix/module-eval/hive-otel.nix
@@ -278,6 +278,26 @@ let
         in
         lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish;
     }
+    {
+      # The store's units exist only on its host; `Requires=` on an absent one
+      # fails the fetch's start.
+      name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here";
+      ok =
+        let
+          storeUnits = m: [
+            "swarm-bao-pki.service"
+            "container@${m.services.hyperhive.swarm.bao.machine}.service"
+          ];
+          here = hiveOtelFetch hiveOtelAllLocal;
+          elsewhere = hiveOtelFetch hiveOtelLeaf;
+        in
+        lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal)
+        && lib.elem "swarm-bao-pki.service" here.requires
+        && lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants
+        && !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) (
+          storeUnits hiveOtelLeaf
+        ));
+    }
     {
       # A secret delivered some other way is the operator's: no unit fetches
       # into a file the collector does not load, or restarts it on one. Nothing
diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix
index 58b34d1c..d23f0074 100644
--- a/nix/module-eval/swarm-otel-identity.nix
+++ b/nix/module-eval/swarm-otel-identity.nix
@@ -217,6 +217,21 @@ let
         !(refusedOver otelReaderNamedInFull "otelOidcClientCertFile")
         && !(refusedOver otelRemoteAuthelia "otelOidcClientCertFile");
     }
+    {
+      # The store's units exist only on its host; `Requires=` on an absent one
+      # fails the fetch's start. The arm where the store is here is in
+      # ./swarm-services-switch.nix.
+      name = "the remote reader's fetch names none of the store's units";
+      ok =
+        let
+          u = otelReaderNamedInFull.systemd.services.swarm-bao-otel-oidc;
+          storeUnits = [
+            "swarm-bao-pki.service"
+            "container@${otelReaderNamedInFull.services.hyperhive.swarm.bao.machine}.service"
+          ];
+        in
+        !(lib.any (n: lib.elem n (u.after ++ u.wants ++ u.requires)) storeUnits);
+    }
   ];
 in
 runGroup "swarm-otel-identity" cases
diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix
index 96182176..edbe577c 100644
--- a/nix/module-eval/swarm-services-switch.nix
+++ b/nix/module-eval/swarm-services-switch.nix
@@ -575,6 +575,54 @@ let
         == "/etc/marker-avatar.png"
         && !(controllerWithMirrors.systemd.services.hive-c0re.environment ? HIVE_ORG_AVATAR_PNG);
     }
+    {
+      # Where the store runs here, each reader waits for its container and
+      # requires the PKI unit minting the leaf it logs in with. Elsewhere
+      # neither unit exists, and `Requires=` on an absent unit fails the
+      # reader's start. On the store's host without the readers, none of them
+      # is rendered as a stub. Rosters are counted: an empty one holds
+      # vacuously.
+      name = "a store reader waits for the store's container and PKI unit only where the store runs here";
+      ok =
+        let
+          machine = serviceContainersWithCi.services.hyperhive.swarm.bao.machine;
+          storeUnits = [
+            "swarm-bao-pki.service"
+            "container@${machine}.service"
+          ];
+          readers = [
+            "swarm-bao-forge-oidc"
+            "swarm-bao-forge-publisher"
+            "swarm-bao-matrix-oidc"
+            "swarm-bao-hive-otel-oidc"
+            "swarm-bao-grafana-oidc"
+            "swarm-bao-grafana-admin"
+            "swarm-bao-nats-callout-xkey"
+            "swarm-bao-ci-runner"
+            "swarm-bao-otel-oidc"
+          ];
+          here = serviceContainersWithCi.systemd.services;
+          elsewhere = swarmServicesBaoElsewhere.systemd.services;
+          readersElsewhere = lib.filter (u: elsewhere ? ${u}) readers;
+          waits =
+            u:
+            here ? ${u}
+            && lib.elem "swarm-bao-pki.service" here.${u}.after
+            && lib.elem "swarm-bao-pki.service" here.${u}.requires
+            && lib.elem "container@${machine}.service" here.${u}.after
+            && lib.elem "container@${machine}.service" here.${u}.wants;
+          namesNoStoreUnit =
+            u:
+            let
+              s = elsewhere.${u};
+            in
+            !(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
+        in
+        lib.all waits readers
+        && lib.length readersElsewhere == 6
+        && lib.all namesNoStoreUnit readersElsewhere
+        && !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);
+    }
   ];
 in
 runGroup "swarm-services-switch" cases