diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 5d9a7f20..99e988e6 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -26,6 +26,7 @@ ./hive-tls.nix ./otel.nix ./glue-bao-readers-policy-order.nix + ./glue-bao-readers-store-order.nix ./glue-bao-tls.nix ./glue-bao-ui-oidc-client.nix ./glue-controller-bao-identity.nix diff --git a/nix/host-modules/glue-bao-readers-store-order.nix b/nix/host-modules/glue-bao-readers-store-order.nix new file mode 100644 index 00000000..6a4776e6 --- /dev/null +++ b/nix/host-modules/glue-bao-readers-store-order.nix @@ -0,0 +1,74 @@ +# Glue: where the store and one of its readers share a host, the reader waits +# for the store's container and requires `swarm-bao-pki`, which mints the leaf +# the reader logs in with. +# +# ONE PAIRING PER FILE — the store's container and PKI unit ← the readers that +# log in to it, and nothing else. The waits on the store's policy units are +# ./glue-bao-readers-policy-order.nix's. +# +# ⚠️ Gated on BOTH the store and that reader being here. Off the store's host +# neither unit exists, and `Requires=` on an absent unit fails the job, so +# there the reader's own start timeout bounds the attempt instead. On the +# store's host without the reader, setting `systemd.services..after` +# would define a unit with no ExecStart, so each gate below restates the one +# the reader's own module puts on it. A reader whose gate changes must change +# here too. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + baoDeploy = deployCfg.bao; + baoCfg = hyperhiveCfg.swarm.bao; + otel = hyperhiveCfg.otel; + + havePair = cert: key: cert != null && key != null; + + # Reader unit → the gate its own module defines it under. + readersHere = { + # ./hive-forge/default.nix + swarm-bao-forge-oidc = deployCfg.forgejo.enable; + swarm-bao-forge-publisher = deployCfg.forgejo.enable; + # ./hive-matrix.nix + swarm-bao-matrix-oidc = deployCfg.matrix.enable; + # ./otel.nix: `fetches`, the collector loading the file this unit writes + # under the hive's own leaf. + swarm-bao-hive-otel-oidc = + otel.enable + && hyperhiveCfg.hiveName != null + && havePair baoDeploy.clientCertFile baoDeploy.clientKeyFile + && + otel.clientSecretFile + == "/var/lib/hive-otel-oidc/${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${toString hyperhiveCfg.hiveName}.secret"; + # ./swarm-grafana.nix + swarm-bao-grafana-oidc = deployCfg.grafana.enable; + swarm-bao-grafana-admin = deployCfg.grafana.enable; + # ./swarm-nats.nix + swarm-bao-nats-callout-xkey = deployCfg.nats.enable; + # ./swarm-ci-runner.nix + swarm-bao-ci-runner = deployCfg.ci-runner.enable; + # ./swarm-otel.nix + swarm-bao-otel-oidc = + deployCfg.swarm-otel.enable + && havePair baoDeploy.otelOidcClientCertFile baoDeploy.otelOidcClientKeyFile; + }; + + orderAfterStore = + reader: here: + lib.mkIf (baoDeploy.enable && here) { + systemd.services.${reader} = { + after = [ + "swarm-bao-pki.service" + "container@${baoCfg.machine}.service" + ]; + wants = [ "container@${baoCfg.machine}.service" ]; + requires = [ "swarm-bao-pki.service" ]; + }; + }; +in +{ + config = lib.mkMerge (lib.mapAttrsToList orderAfterStore readersHere); +} diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index 684f5415..754be86d 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -1011,15 +1011,9 @@ in # rendering no unit. systemd.services.swarm-bao-forge-oidc = { description = "fetch the forge's OIDC client secret from the swarm secret store"; - # The store's units exist only where the store runs; `Requires=` on an - # absent unit fails the job, so off-host the timeout below bounds the - # attempt instead. - after = lib.optionals deployCfg.bao.enable [ - "swarm-bao-pki.service" - "container@${baoCfg.machine}.service" - ]; - wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ]; - requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ]; + # Ordered after the store where it runs here + # (../glue-bao-readers-store-order.nix); elsewhere the timeout below + # bounds the attempt. before = [ "container@hive-forge.service" ]; wantedBy = [ "multi-user.target" @@ -1141,17 +1135,11 @@ in # here, naming the options, rather than rendering no unit. systemd.services.swarm-bao-forge-publisher = { description = "publish the swarm-controller's forge token to the swarm secret store"; - # The store's units exist only where the store runs; `Requires=` on an - # absent unit fails the job. + # Also ordered after the store where it runs here: + # ../glue-bao-readers-store-order.nix. after = [ "container@hive-forge.service" - ] - ++ lib.optionals deployCfg.bao.enable [ - "swarm-bao-pki.service" - "container@${baoCfg.machine}.service" ]; - wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ]; - requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ]; wantedBy = [ "multi-user.target" "container@hive-forge.service" diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index ee260bbf..f5b16fdd 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -595,15 +595,9 @@ in # no unit. systemd.services.swarm-bao-matrix-oidc = { description = "fetch the homeserver's OIDC client secret from the swarm secret store"; - # The store's units exist only where the store runs; `Requires=` on an - # absent unit fails the job, so off-host the timeout below bounds the - # attempt instead. - after = lib.optionals deployCfg.bao.enable [ - "swarm-bao-pki.service" - "container@${baoCfg.machine}.service" - ]; - wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ]; - requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ]; + # Ordered after the store where it runs here + # (./glue-bao-readers-store-order.nix); elsewhere the timeout below + # bounds the attempt. before = [ "container@hive-matrix.service" ]; wantedBy = [ "multi-user.target" diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index ba83b510..bea8748c 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -337,6 +337,7 @@ in # Only when the collector reads the file the unit writes: a secret # delivered some other way is the operator's, and a unit restarting the # collector on a file it never loads would be noise. + # ./glue-bao-readers-store-order.nix restates this gate. fetches = senderAuth && hiveLeaf && otel.clientSecretFile == fetchedSecretPath; # An operator-delivered file has no unit that restarts the collector # once it lands, so the collector waits for it instead. @@ -793,15 +794,9 @@ in # secret reaches it only by a restart, which `refresh_unit` does. systemd.services.${fetchUnit} = lib.mkIf fetches { description = "fetch this hive's telemetry OIDC client secret from the swarm secret store"; - # The store's units exist only where the store runs; `Requires=` on an - # absent unit fails the job, so off-host the timeout below bounds the - # attempt instead. - after = lib.optionals deployCfg.bao.enable [ - "swarm-bao-pki.service" - "container@${baoCfg.machine}.service" - ]; - wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ]; - requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ]; + # Ordered after the store where it runs here + # (./glue-bao-readers-store-order.nix, which restates `fetches`); + # elsewhere the timeout below bounds the attempt. before = [ "opentelemetry-collector.service" ]; wantedBy = [ "multi-user.target" diff --git a/nix/host-modules/swarm-ci-runner.nix b/nix/host-modules/swarm-ci-runner.nix index d8b0d747..4c19f19c 100644 --- a/nix/host-modules/swarm-ci-runner.nix +++ b/nix/host-modules/swarm-ci-runner.nix @@ -189,12 +189,8 @@ in # every five minutes, so a retry is what turns "no value there" into one. systemd.services.swarm-bao-ci-runner = { description = "fetch the CI runner's registration token from the swarm secret store"; - after = lib.optionals baoDeploy.enable [ - "swarm-bao-pki.service" - "container@${baoCfg.machine}.service" - ]; - wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; - requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; + # Ordered after the store where it runs here: + # ./glue-bao-readers-store-order.nix. before = [ "container@${machine}.service" ]; wantedBy = [ "multi-user.target" diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index d8fcc75f..d4743f25 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -558,16 +558,9 @@ in # the only thing separating that from a mystery. systemd.services.swarm-bao-grafana-oidc = { description = "fetch Grafana's OIDC client secret from the swarm secret store"; - # Every one of these names a unit that exists only where the store runs. - # `Requires=` on an absent unit fails the job outright, so the ordering - # is conditional even though the read is not: off-host there is nothing - # local to wait for, and the timeout below bounds the attempt instead. - after = lib.optionals baoDeploy.enable [ - "swarm-bao-pki.service" - "container@${baoCfg.machine}.service" - ]; - wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; - requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; + # Ordered after the store where it runs here + # (./glue-bao-readers-store-order.nix); elsewhere the timeout below + # bounds the attempt. before = [ "container@${cfg.machine}.service" ]; wantedBy = [ "multi-user.target" @@ -686,12 +679,8 @@ in # Grafana down rather than up with the upstream default. systemd.services.swarm-bao-grafana-admin = { description = "create Grafana's admin password in the swarm secret store if absent and fetch it"; - after = lib.optionals baoDeploy.enable [ - "swarm-bao-pki.service" - "container@${baoCfg.machine}.service" - ]; - wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; - requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; + # Ordered after the store where it runs here: + # ./glue-bao-readers-store-order.nix. before = [ "container@${cfg.machine}.service" ]; wantedBy = [ "multi-user.target" diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index fc384ae7..c2ff6938 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -830,15 +830,9 @@ in # responder's own leaf, the one principal that may create the keys' paths. systemd.services.swarm-bao-nats-callout-xkey = { description = "create the swarm queue's callout keys if absent and fetch their public halves for the server"; - # The store's units exist only where the store runs; `Requires=` on an - # absent unit fails the job, so off-host the timeout below bounds the - # attempt instead. - after = lib.optionals baoDeploy.enable [ - "swarm-bao-pki.service" - "container@${baoCfg.machine}.service" - ]; - wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; - requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; + # Ordered after the store where it runs here + # (./glue-bao-readers-store-order.nix); elsewhere the timeout below + # bounds the attempt. before = [ "container@${machine}.service" ]; wantedBy = [ "multi-user.target" diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index ce07b7e9..a81c0075 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -429,16 +429,9 @@ in # regression rather than that degrade — `hiveReaderIdentity` above. systemd.services.swarm-bao-otel-oidc = lib.mkIf haveClientIdentity { description = "fetch the swarm collector's OIDC client secret from the swarm secret store"; - # Every one of these names a unit that exists only where the store runs. - # `Requires=` on an absent unit fails the job outright, so the ordering - # is conditional even though the read is not: off-host there is nothing - # local to wait for, and the timeout below bounds the attempt instead. - after = lib.optionals baoDeploy.enable [ - "swarm-bao-pki.service" - "container@${baoCfg.machine}.service" - ]; - wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; - requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; + # Ordered after the store where it runs here + # (./glue-bao-readers-store-order.nix); elsewhere the timeout below + # bounds the attempt. before = [ "container@${cfg.machine}.service" ]; wantedBy = [ "multi-user.target" diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index 12ee4e1f..3ab329f7 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -187,6 +187,21 @@ let ciAlone.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT == "/etc/pki/bao-ci-runner.pem"; } + { + # The store's units exist only on its host; `Requires=` on an absent one + # fails the fetch's start. The arm where the store is here is in + # ./swarm-services-switch.nix. + name = "the CI runner's token fetch names none of the store's units on a host without the store"; + ok = + let + u = ciAlone.systemd.services.swarm-bao-ci-runner; + storeUnits = [ + "swarm-bao-pki.service" + "container@${ciAlone.services.hyperhive.swarm.bao.machine}.service" + ]; + in + !(lib.any (n: lib.elem n (u.after ++ u.wants ++ u.requires)) storeUnits); + } { # Without the identity the token never arrives; the control is the # placed pair above. diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index e4e5a5b4..f773cc88 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -278,6 +278,26 @@ let in lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish; } + { + # The store's units exist only on its host; `Requires=` on an absent one + # fails the fetch's start. + name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here"; + ok = + let + storeUnits = m: [ + "swarm-bao-pki.service" + "container@${m.services.hyperhive.swarm.bao.machine}.service" + ]; + here = hiveOtelFetch hiveOtelAllLocal; + elsewhere = hiveOtelFetch hiveOtelLeaf; + in + lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal) + && lib.elem "swarm-bao-pki.service" here.requires + && lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants + && !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) ( + storeUnits hiveOtelLeaf + )); + } { # A secret delivered some other way is the operator's: no unit fetches # into a file the collector does not load, or restarts it on one. Nothing diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 58b34d1c..d23f0074 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -217,6 +217,21 @@ let !(refusedOver otelReaderNamedInFull "otelOidcClientCertFile") && !(refusedOver otelRemoteAuthelia "otelOidcClientCertFile"); } + { + # The store's units exist only on its host; `Requires=` on an absent one + # fails the fetch's start. The arm where the store is here is in + # ./swarm-services-switch.nix. + name = "the remote reader's fetch names none of the store's units"; + ok = + let + u = otelReaderNamedInFull.systemd.services.swarm-bao-otel-oidc; + storeUnits = [ + "swarm-bao-pki.service" + "container@${otelReaderNamedInFull.services.hyperhive.swarm.bao.machine}.service" + ]; + in + !(lib.any (n: lib.elem n (u.after ++ u.wants ++ u.requires)) storeUnits); + } ]; in runGroup "swarm-otel-identity" cases diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 96182176..edbe577c 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -575,6 +575,54 @@ let == "/etc/marker-avatar.png" && !(controllerWithMirrors.systemd.services.hive-c0re.environment ? HIVE_ORG_AVATAR_PNG); } + { + # Where the store runs here, each reader waits for its container and + # requires the PKI unit minting the leaf it logs in with. Elsewhere + # neither unit exists, and `Requires=` on an absent unit fails the + # reader's start. On the store's host without the readers, none of them + # is rendered as a stub. Rosters are counted: an empty one holds + # vacuously. + name = "a store reader waits for the store's container and PKI unit only where the store runs here"; + ok = + let + machine = serviceContainersWithCi.services.hyperhive.swarm.bao.machine; + storeUnits = [ + "swarm-bao-pki.service" + "container@${machine}.service" + ]; + readers = [ + "swarm-bao-forge-oidc" + "swarm-bao-forge-publisher" + "swarm-bao-matrix-oidc" + "swarm-bao-hive-otel-oidc" + "swarm-bao-grafana-oidc" + "swarm-bao-grafana-admin" + "swarm-bao-nats-callout-xkey" + "swarm-bao-ci-runner" + "swarm-bao-otel-oidc" + ]; + here = serviceContainersWithCi.systemd.services; + elsewhere = swarmServicesBaoElsewhere.systemd.services; + readersElsewhere = lib.filter (u: elsewhere ? ${u}) readers; + waits = + u: + here ? ${u} + && lib.elem "swarm-bao-pki.service" here.${u}.after + && lib.elem "swarm-bao-pki.service" here.${u}.requires + && lib.elem "container@${machine}.service" here.${u}.after + && lib.elem "container@${machine}.service" here.${u}.wants; + namesNoStoreUnit = + u: + let + s = elsewhere.${u}; + in + !(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits); + in + lib.all waits readers + && lib.length readersElsewhere == 6 + && lib.all namesNoStoreUnit readersElsewhere + && !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers); + } ]; in runGroup "swarm-services-switch" cases