diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index b3a62ce1..8db3b95e 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -229,6 +229,10 @@ let # The root-only fetch on a host fronting no swarm name. pkgs.curl ]; + # One attempt of the issuance: up to 10 minutes on the lock at the top of + # the script, then the store calls. A oneshot has no start timeout of its + # own, and an attempt that never ends never fails into a retry. + servicesCertTimeoutStartSec = "20min"; servicesCertEnvironment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; } @@ -455,6 +459,9 @@ in Type = "oneshot"; UMask = "0077"; SyslogIdentifier = "hive-tls-resign"; + # One attempt. A oneshot has no start timeout of its own, and an + # attempt that never ends never fails into a retry. + TimeoutStartSec = "10min"; }; script = '' set -euo pipefail @@ -619,6 +626,7 @@ in RemainAfterExit = true; UMask = "0077"; SyslogIdentifier = "swarm-services-cert"; + TimeoutStartSec = servicesCertTimeoutStartSec; }; environment = servicesCertEnvironment; script = '' @@ -654,6 +662,7 @@ in raw="$(mktemp "$d/swarm-services-root.XXXXXX")" trap 'rm -f "$err" "$raw"' EXIT if ! curl --fail --silent --show-error \ + --connect-timeout 10 --max-time 30 \ ${storeRootCurlTls} --output "$raw" \ "$BAO_ADDR/v1/${baoServicesPkiMount}/ca/pem" 2>"$err"; then echo "could not read the store's ${baoServicesPkiMount} root." >&2 @@ -824,10 +833,10 @@ in # `Before=` both of these, so a blocking `systemctl restart` or # `start` enqueues a job that systemd will not start until this # unit is active — and this unit is not active until its ExecStart - # returns, which is waiting on that job. A deadlock, held until - # the 24h retry window's `TimeoutStartSec` fires. Queueing the - # job and letting it run once we exit is the only ordering that - # terminates. + # returns, which is waiting on that job. A deadlock, broken only + # when `TimeoutStartSec` fails the attempt, and entered again by + # every retry. Queueing the job and letting it run once we exit is + # the only ordering that terminates. # # The cost is real and is the reason `hive-tls-resign` does NOT # do this: a propagation that fails after we return does not fail @@ -885,6 +894,7 @@ in Type = "oneshot"; UMask = "0077"; SyslogIdentifier = "swarm-services-cert-renew"; + TimeoutStartSec = servicesCertTimeoutStartSec; }; }; diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index cbb2d5c6..051bc27e 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -684,6 +684,18 @@ let && (t.timerConfig.Unit or "hive-tls-resign.service") == "hive-tls-resign.service" && !(remainsAfterExit leafUnit); } + { + # A oneshot has no start timeout by default, so an attempt that never + # ends never fails into a retry. + name = "every unit that calls the store bounds its attempt, and the root fetch bounds its curl"; + ok = + lib.all (u: (u.serviceConfig.TimeoutStartSec or "infinity") != "infinity") [ + boot + triggered + leafUnit + ] + && lib.hasInfix "--connect-timeout 10 --max-time 30" (noSvcBranch noSvcFetch); + } ]; in runGroup "hive-tls" cases