From 868c57321c7c85ecdeb93ae9bb0f5f34fa426850 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 29 Oct 2026 14:31:30 +0100 Subject: [PATCH] order the store's readers after its container without wanting it --- .../glue-bao-readers-store-order.nix | 11 +++++++---- nix/host-modules/hive-priv-cache.nix | 6 +++--- nix/host-modules/hive-squid-credential.nix | 6 +++--- nix/host-modules/hive-tls.nix | 2 -- nix/host-modules/swarm-bao.nix | 8 +++----- nix/host-modules/swarm-nats.nix | 1 - nix/module-eval/hive-otel.nix | 4 +++- nix/module-eval/hive-tls.nix | 18 +++++++++++++----- nix/module-eval/swarm-services-switch.nix | 9 +++++---- 9 files changed, 37 insertions(+), 28 deletions(-) diff --git a/nix/host-modules/glue-bao-readers-store-order.nix b/nix/host-modules/glue-bao-readers-store-order.nix index e385f556..ae9a702d 100644 --- a/nix/host-modules/glue-bao-readers-store-order.nix +++ b/nix/host-modules/glue-bao-readers-store-order.nix @@ -6,6 +6,12 @@ # log in to it, and nothing else. The waits on the store's policy units are # ./glue-bao-readers-policy-order.nix's. # +# No `Wants=` on the container: it is started by `machines.target` and the +# switch's start phase, and a reader's `Wants=` would pull it up mid-switch, +# before `activate` has replaced its configuration, on the OLD closure. A +# reader that starts before the store costs one failed attempt, retried by +# ./lib/store-retry.nix. +# # `Wants=` on `swarm-bao-pki`, not `Requires=`: a pki start that fails or is # refused fails each requirer's job with `dependency`, which leaves the reader # inactive rather than failed, so its `Restart=on-failure` never fires. Here @@ -77,10 +83,7 @@ let "swarm-bao-pki.service" "container@${baoCfg.machine}.service" ]; - wants = [ - "swarm-bao-pki.service" - "container@${baoCfg.machine}.service" - ]; + wants = [ "swarm-bao-pki.service" ]; }; }; in diff --git a/nix/host-modules/hive-priv-cache.nix b/nix/host-modules/hive-priv-cache.nix index ca523831..be38c926 100644 --- a/nix/host-modules/hive-priv-cache.nix +++ b/nix/host-modules/hive-priv-cache.nix @@ -43,8 +43,8 @@ let dir = "/run/hyperhive-priv/attic"; - # Where the store runs on this host, its container is up before the first - # login attempt instead of failing against a store that is still starting. + # Where the store runs on this host, the first login waits for its container. + # No `wants`: a pull would start the container mid-switch on the old closure. storeContainer = lib.optional baoDeploy.enable "container@${baoCfg.machine}.service"; storeRetry = import ./lib/store-retry.nix { }; @@ -62,7 +62,7 @@ in systemd.services.hive-priv-cache-facts = { description = "Read the swarm cache's public key and pull token for hive-priv"; wantedBy = [ "multi-user.target" ]; - wants = [ "network-online.target" ] ++ storeContainer; + wants = [ "network-online.target" ]; after = [ "network-online.target" "systemd-tmpfiles-setup.service" diff --git a/nix/host-modules/hive-squid-credential.nix b/nix/host-modules/hive-squid-credential.nix index f1932918..7485bc5c 100644 --- a/nix/host-modules/hive-squid-credential.nix +++ b/nix/host-modules/hive-squid-credential.nix @@ -30,8 +30,8 @@ let credentialFile = "/run/credstore/${hyperhiveCfg.deploy.hive-controller.squid.selfcheck.credentialName}"; - # Where the store runs on this host, its container is up before the first - # login attempt instead of failing against a store that is still starting. + # Where the store runs on this host, the first login waits for its container. + # No `wants`: a pull would start the container mid-switch on the old closure. storeContainer = lib.optional baoDeploy.enable "container@${baoCfg.machine}.service"; storeRetry = import ./lib/store-retry.nix { }; @@ -43,7 +43,7 @@ in description = "Read the squid self-check's password from the swarm secret store"; wantedBy = [ "multi-user.target" ]; before = [ "squid-selfcheck.service" ]; - wants = [ "network-online.target" ] ++ storeContainer; + wants = [ "network-online.target" ]; after = [ "network-online.target" "systemd-tmpfiles-setup.service" diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index af763e5c..d2ff9ba2 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -488,7 +488,6 @@ in "swarm-bao-hive-leaf-policy.service" "swarm-services-cert.service" ]; - wants = [ "container@${baoCfg.machine}.service" ]; path = [ baoDeploy.package pkgs.jq @@ -661,7 +660,6 @@ in "swarm-bao-pki.service" "swarm-bao-services-issuer-policy.service" ]; - wants = [ "container@${baoCfg.machine}.service" ]; path = servicesCertPath; # ./lib/store-retry.nix: the login below fails for as long as the # store is sealed or down. diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index c53fccd4..64f17c38 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -3122,11 +3122,9 @@ in "swarm-bao-pki.service" "container@${cfg.machine}.service" ]; - # `Wants=` pki, for ./glue-bao-readers-store-order.nix's reason. - wants = [ - "swarm-bao-pki.service" - "container@${cfg.machine}.service" - ]; + # `Wants=` pki and none on the container, for + # ./glue-bao-readers-store-order.nix's reasons. + wants = [ "swarm-bao-pki.service" ]; wantedBy = [ "multi-user.target" ]; path = [ baoDeploy.package diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index f5fb9c46..3737b445 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -1183,7 +1183,6 @@ in "container@${baoCfg.machine}.service" "swarm-bao-pki.service" ]; - wants = [ "container@${baoCfg.machine}.service" ]; path = [ baoDeploy.package pkgs.jq diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 658f5fed..2ad614bf 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -365,6 +365,8 @@ let { # The store's units exist only on its host. The PKI unit is wanted, never # required: a refused pki start fails a requirer's job with `dependency`. + # The container is waited for, not wanted: a `Wants=` would start it + # mid-switch on the old closure. name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here"; ok = let @@ -378,7 +380,7 @@ let lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal) && lib.elem "swarm-bao-pki.service" here.wants && !(lib.elem "swarm-bao-pki.service" here.requires) - && lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants + && !(lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants) && !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) ( storeUnits hiveOtelLeaf )); diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index 71a14d5b..ca857561 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -824,19 +824,27 @@ let } { # Both units log in to the store, so where its container runs here they - # start after it. Controls: the units exist on `noService`, which reads - # a store it does not run, and carry no container there. - name = "the cache-facts and squid credential units wait for the store's container only where it runs here"; + # start after it, and none of the store's readers wants it: a `Wants=` + # would start it mid-switch on the old closure. Controls: the units exist + # on `noService`, which reads a store it does not run, and carry no + # container there. + name = "the cache-facts and squid credential units wait for the store's container, without wanting it, only where it runs here"; ok = let ref = "container@swarm-bao.service"; - waits = u: lib.elem ref u.after && lib.elem ref u.wants; + waits = u: lib.elem ref u.after && !(lib.elem ref u.wants); names = [ "hive-priv-cache-facts" "squid-selfcheck-credential" ]; in - lib.all (n: waits allLocal.systemd.services.${n}) names + lib.all (n: waits allLocal.systemd.services.${n}) ( + names + ++ [ + "hive-tls-resign" + "swarm-services-cert" + ] + ) && lib.all ( n: noService.systemd.services ? ${n} diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index fc534688..84db09a7 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -1239,13 +1239,14 @@ let } { # Where the store runs here, each reader waits for its container and - # the PKI unit minting the leaf it logs in with, and wants both. It - # never requires the PKI unit: a refused pki start fails a requirer's + # the PKI unit minting the leaf it logs in with, and wants the PKI unit only: + # a `Wants=` on the container would start it mid-switch on the old closure. + # It never requires the PKI unit: a refused pki start fails a requirer's # job with `dependency`, which its `Restart=on-failure` never retries. # Elsewhere neither unit exists. On the store's host without the # readers, none of them is rendered as a stub. Rosters are counted: an # empty one holds vacuously. - name = "a store reader waits for and wants the store's container and PKI unit only where the store runs here, and requires neither"; + name = "a store reader waits for the store's container and PKI unit only where the store runs here, wants only the PKI unit, and requires neither"; ok = let machine = serviceContainersWithCi.services.hyperhive.swarm.bao.machine; @@ -1276,7 +1277,7 @@ let && lib.elem "swarm-bao-pki.service" here.${u}.wants && !(lib.any (n: lib.elem n here.${u}.requires) storeUnits) && lib.elem "container@${machine}.service" here.${u}.after - && lib.elem "container@${machine}.service" here.${u}.wants; + && !(lib.elem "container@${machine}.service" here.${u}.wants); namesNoStoreUnit = u: let