diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index a129778f..de97be3e 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -1057,12 +1057,13 @@ in # reason `hive-forge-oidc-secret` gives, and under the forge's own store # identity, so nothing here asks where the store or the controller is. # - # It can only depend on the container being up, not on the in-container - # oneshot that mints the token (containers run their own systemd instance, - # invisible to this one by unit name), so it polls the same bounded way - # `hive-forge-oidc-secret` does. It runs whenever the token can have - # changed: each time the container starts (`wantedBy`), and each time the - # file is written (`systemd.paths.swarm-bao-forge-publisher` below). + # It runs at boot and on a switch that adds or changes it + # (`multi-user.target`), each time the container starts, and each time the + # mint writes the file + # (`systemd.paths.swarm-bao-forge-publisher` below). The mint is a oneshot in + # the container's own systemd, invisible to this one by unit name, so a + # missing file is not an error: the path unit runs this again when the mint + # writes it, the shape of `swarm-secret-publish`. # # ⚠️ No `RemainAfterExit`. A oneshot that stays `active (exited)` turns # every later start into a no-op, so neither trigger would run it again. @@ -1085,21 +1086,22 @@ in ]; wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ]; - wantedBy = [ "container@hive-forge.service" ]; + wantedBy = [ + "multi-user.target" + "container@hive-forge.service" + ]; path = [ deployCfg.bao.package pkgs.coreutils ]; - # ../lib/store-retry.nix: a sealed or unreachable store is retried, and - # so is a token the forge has not minted yet. + # ../lib/store-retry.nix: a sealed or unreachable store is retried. inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { Type = "oneshot"; SyslogIdentifier = "swarm-bao-forge-publisher"; - # Longer than the 120s bounded wait below. `DefaultTimeoutStartSec` - # is 90s, so without this systemd kills the unit at 90 — before it - # can emit the message naming the file it was waiting for. - TimeoutStartSec = "180s"; + # Bounded here so the number a switch waits on is in this file. A sealed + # store answers on the port and never answers the write. + TimeoutStartSec = 60; }; environment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; @@ -1123,13 +1125,9 @@ in src=${lib.escapeShellArg swarmControllerTokenHostPath} - for _ in $(seq 1 60); do - [ -s "$src" ] && break - sleep 2 - done if [ ! -s "$src" ]; then - echo "forge has not minted $src after 120s" >&2 - exit 1 + echo "the forge has not minted $src yet; the path unit publishes it when it does" >&2 + exit 0 fi err="$(mktemp)" diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 4fe6ad6a..ec5e8f82 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -718,12 +718,7 @@ in systemd.services.swarm-controller = { description = "hyperhive swarm-level controller daemon"; wantedBy = [ "multi-user.target" ]; - # Where the forge runs here too, the first read waits out one publish - # attempt. `After=` on a unit this host does not define is a no-op. - after = [ - "network.target" - "swarm-bao-forge-publisher.service" - ]; + after = [ "network.target" ]; serviceConfig = { ExecStart = "${deployCfg.swarm-controller.package}/bin/swarm-controller"; diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index 501d52bf..a392f28c 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -292,12 +292,16 @@ let && lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/forge/token value=@\"$src\"" u.script && lib.hasInfix "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token" u.script && lib.elem "container@hive-forge.service" u.wantedBy + # The deploy that adds the unit starts it only through a target. + && lib.elem "multi-user.target" u.wantedBy && !(allLocal.systemd.services ? hive-forge-swarm-controller-token); } { # What makes it re-run. A oneshot left `active (exited)` makes every later - # start a no-op, so `RemainAfterExit` would silence both triggers: the - # container's start (`wantedBy`, above) and a write of the token file. + # start a no-op, so `RemainAfterExit` would silence every trigger: the + # targets and the container's start (`wantedBy`, above) and a write of + # the token file. A missing file exits 0, so a switch with the container + # down does not fail. name = "the forge publisher re-runs on each container start and each write of the token file"; ok = let @@ -308,6 +312,8 @@ let p.pathConfig.PathChanged == "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token" && p.pathConfig.Unit == "swarm-bao-forge-publisher.service" + && lib.hasInfix "the path unit publishes it when it does" (publisher allLocal).script + && lib.hasInfix "exit 0" (publisher allLocal).script && lib.elem "multi-user.target" p.wantedBy && !(bare.systemd.paths ? swarm-bao-forge-publisher); } diff --git a/swarm-controller/src/forge.rs b/swarm-controller/src/forge.rs index 0942d67e..658936f1 100644 --- a/swarm-controller/src/forge.rs +++ b/swarm-controller/src/forge.rs @@ -183,21 +183,31 @@ pub fn url_from_env() -> Result> { .map(Some) } +/// The longest the background re-read waits between attempts. +const BACKGROUND_CAP: std::time::Duration = std::time::Duration::from_mins(5); + /// How long the background re-read waits before its `n`-th attempt: 30s, -/// doubling, capped at 5 minutes. +/// doubling, capped at [`BACKGROUND_CAP`]. fn background_delay(n: u32) -> std::time::Duration { const FIRST_S: u64 = 30; - const CAP_S: u64 = 300; std::time::Duration::from_secs( FIRST_S .saturating_mul(1u64.checked_shl(n).unwrap_or(u64::MAX)) - .min(CAP_S), + .min(BACKGROUND_CAP.as_secs()), ) } +/// Whether the `n`-th failed attempt is logged at `info` rather than `debug`: +/// once the backoff is capped, so a controller that never gets its token says +/// so once per [`BACKGROUND_CAP`] without a line per early retry. +fn failure_logs_at_info(n: u32) -> bool { + background_delay(n) >= BACKGROUND_CAP +} + /// Fill `slot` with the first value `attempt` yields, sleeping -/// [`background_delay`] before each attempt. Never gives up, and logs no -/// failure: the caller logs the first one and the recovery. +/// [`background_delay`] before each attempt. Never gives up. A failure is logged +/// as [`failure_logs_at_info`] says; the caller logs the first one and the +/// recovery. async fn fill_when_ready(slot: &OnceLock, mut attempt: F) -> &T where F: FnMut() -> Fut, @@ -208,6 +218,10 @@ where tokio::time::sleep(background_delay(n)).await; match attempt().await { Ok(value) => return slot.get_or_init(|| value), + Err(e) if failure_logs_at_info(n) => tracing::info!( + error = %format!("{e:#}"), + "still no forge access: the controller's forge token is unavailable" + ), Err(e) => tracing::debug!( error = %format!("{e:#}"), "the controller's forge token is still unavailable" @@ -1402,6 +1416,13 @@ mod tests { } } + #[test] + fn a_failed_re_read_logs_at_info_only_once_the_backoff_is_capped() { + assert!((0..4).all(|n| !failure_logs_at_info(n))); + assert!(failure_logs_at_info(4)); + assert!(failure_logs_at_info(40)); + } + #[test] fn the_background_re_read_backs_off_from_30s_to_a_5_minute_cap() { let s = |n| background_delay(n).as_secs();