diff --git a/nix/host-modules/hive-priv-cache.nix b/nix/host-modules/hive-priv-cache.nix index 3e7b701b..ca523831 100644 --- a/nix/host-modules/hive-priv-cache.nix +++ b/nix/host-modules/hive-priv-cache.nix @@ -43,6 +43,10 @@ let dir = "/run/hyperhive-priv/attic"; + # Where the store runs on this host, its container is up before the first + # login attempt instead of failing against a store that is still starting. + storeContainer = lib.optional baoDeploy.enable "container@${baoCfg.machine}.service"; + storeRetry = import ./lib/store-retry.nix { }; storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in @@ -58,11 +62,12 @@ in systemd.services.hive-priv-cache-facts = { description = "Read the swarm cache's public key and pull token for hive-priv"; wantedBy = [ "multi-user.target" ]; - wants = [ "network-online.target" ]; + wants = [ "network-online.target" ] ++ storeContainer; after = [ "network-online.target" "systemd-tmpfiles-setup.service" - ]; + ] + ++ storeContainer; path = [ baoDeploy.package pkgs.coreutils diff --git a/nix/host-modules/hive-squid-credential.nix b/nix/host-modules/hive-squid-credential.nix index 36587477..f1932918 100644 --- a/nix/host-modules/hive-squid-credential.nix +++ b/nix/host-modules/hive-squid-credential.nix @@ -30,6 +30,10 @@ let credentialFile = "/run/credstore/${hyperhiveCfg.deploy.hive-controller.squid.selfcheck.credentialName}"; + # Where the store runs on this host, its container is up before the first + # login attempt instead of failing against a store that is still starting. + storeContainer = lib.optional baoDeploy.enable "container@${baoCfg.machine}.service"; + storeRetry = import ./lib/store-retry.nix { }; storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in @@ -39,11 +43,12 @@ in description = "Read the squid self-check's password from the swarm secret store"; wantedBy = [ "multi-user.target" ]; before = [ "squid-selfcheck.service" ]; - wants = [ "network-online.target" ]; + wants = [ "network-online.target" ] ++ storeContainer; after = [ "network-online.target" "systemd-tmpfiles-setup.service" - ]; + ] + ++ storeContainer; path = [ baoDeploy.package pkgs.coreutils diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 600c4390..c53fccd4 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -4136,6 +4136,12 @@ in containers.${cfg.machine} = { autoStart = true; ephemeral = false; + + # The container is `Type=notify`, so its start job waits for OpenBao, + # whose first start after a restart takes 60-144 s opening the raft + # storage. The 1min default kills it mid-open and every retry is cold. + timeoutStartSec = "5min"; + inherit privateNetwork; # Only material an operator has to be able to back up crosses the diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index c59cbdbe..71a14d5b 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -822,6 +822,34 @@ let && !(noIdentity.systemd.services ? squid-selfcheck-credential) && !(noIdentity.systemd.timers ? squid-selfcheck-credential); } + { + # Both units log in to the store, so where its container runs here they + # start after it. Controls: the units exist on `noService`, which reads + # a store it does not run, and carry no container there. + name = "the cache-facts and squid credential units wait for the store's container only where it runs here"; + ok = + let + ref = "container@swarm-bao.service"; + waits = u: lib.elem ref u.after && lib.elem ref u.wants; + names = [ + "hive-priv-cache-facts" + "squid-selfcheck-credential" + ]; + in + lib.all (n: waits allLocal.systemd.services.${n}) names + && lib.all ( + n: + noService.systemd.services ? ${n} + && !(lib.elem ref noService.systemd.services.${n}.after) + && !(lib.elem ref noService.systemd.services.${n}.wants) + ) names; + } + { + # The container is `Type=notify`: its start job lasts as long as + # OpenBao's first start, which a 1min limit cuts off mid-open. + name = "the store's container is given five minutes to start"; + ok = lib.hasInfix "TimeoutStartSec=5min" allLocal.systemd.units."container@swarm-bao.service".text; + } { name = "hive-priv is told the cache's address by the host's configuration"; ok =