From 597fdebaf39f64d4eb1746b5b989f828bb65aaba Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 8 Oct 2026 13:58:11 +0200 Subject: [PATCH] swarm-bao-forge-publisher starts on the deploy that adds it The publisher was wanted only by container@hive-forge.service. A switch starts a new unit only through the targets that want it, and the container is already running, so the deploy that introduced the publisher left it dead until the next container restart or boot; the path unit cannot help, because the mint's only-once guard never rewrites the file. - The publisher is also wanted by multi-user.target, like swarm-secret-publish. - A token file that is not there yet exits 0 instead of waiting 120s and failing: the path unit runs the publisher when the mint writes it, so a switch with the forge container down does not fail. TimeoutStartSec is 60. - swarm-controller loses its After= on the publisher: a switch issues the two jobs separately, so it ordered nothing there, and the background re-read covers the gap. - The controller's background re-read logs at info once the backoff is capped, one line per five minutes, so a forge host that never got its publisher leaf stays visible; earlier retries stay at debug. Refs #4998 --- nix/host-modules/hive-forge/default.nix | 36 ++++++++++++------------- nix/host-modules/swarm-controller.nix | 7 +---- nix/module-eval/forge-placement.nix | 10 +++++-- swarm-controller/src/forge.rs | 31 +++++++++++++++++---- 4 files changed, 52 insertions(+), 32 deletions(-) diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index a129778f..de97be3e 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -1057,12 +1057,13 @@ in # reason `hive-forge-oidc-secret` gives, and under the forge's own store # identity, so nothing here asks where the store or the controller is. # - # It can only depend on the container being up, not on the in-container - # oneshot that mints the token (containers run their own systemd instance, - # invisible to this one by unit name), so it polls the same bounded way - # `hive-forge-oidc-secret` does. It runs whenever the token can have - # changed: each time the container starts (`wantedBy`), and each time the - # file is written (`systemd.paths.swarm-bao-forge-publisher` below). + # It runs at boot and on a switch that adds or changes it + # (`multi-user.target`), each time the container starts, and each time the + # mint writes the file + # (`systemd.paths.swarm-bao-forge-publisher` below). The mint is a oneshot in + # the container's own systemd, invisible to this one by unit name, so a + # missing file is not an error: the path unit runs this again when the mint + # writes it, the shape of `swarm-secret-publish`. # # ⚠️ No `RemainAfterExit`. A oneshot that stays `active (exited)` turns # every later start into a no-op, so neither trigger would run it again. @@ -1085,21 +1086,22 @@ in ]; wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ]; - wantedBy = [ "container@hive-forge.service" ]; + wantedBy = [ + "multi-user.target" + "container@hive-forge.service" + ]; path = [ deployCfg.bao.package pkgs.coreutils ]; - # ../lib/store-retry.nix: a sealed or unreachable store is retried, and - # so is a token the forge has not minted yet. + # ../lib/store-retry.nix: a sealed or unreachable store is retried. inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { Type = "oneshot"; SyslogIdentifier = "swarm-bao-forge-publisher"; - # Longer than the 120s bounded wait below. `DefaultTimeoutStartSec` - # is 90s, so without this systemd kills the unit at 90 — before it - # can emit the message naming the file it was waiting for. - TimeoutStartSec = "180s"; + # Bounded here so the number a switch waits on is in this file. A sealed + # store answers on the port and never answers the write. + TimeoutStartSec = 60; }; environment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; @@ -1123,13 +1125,9 @@ in src=${lib.escapeShellArg swarmControllerTokenHostPath} - for _ in $(seq 1 60); do - [ -s "$src" ] && break - sleep 2 - done if [ ! -s "$src" ]; then - echo "forge has not minted $src after 120s" >&2 - exit 1 + echo "the forge has not minted $src yet; the path unit publishes it when it does" >&2 + exit 0 fi err="$(mktemp)" diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 4fe6ad6a..ec5e8f82 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -718,12 +718,7 @@ in systemd.services.swarm-controller = { description = "hyperhive swarm-level controller daemon"; wantedBy = [ "multi-user.target" ]; - # Where the forge runs here too, the first read waits out one publish - # attempt. `After=` on a unit this host does not define is a no-op. - after = [ - "network.target" - "swarm-bao-forge-publisher.service" - ]; + after = [ "network.target" ]; serviceConfig = { ExecStart = "${deployCfg.swarm-controller.package}/bin/swarm-controller"; diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index 501d52bf..a392f28c 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -292,12 +292,16 @@ let && lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/forge/token value=@\"$src\"" u.script && lib.hasInfix "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token" u.script && lib.elem "container@hive-forge.service" u.wantedBy + # The deploy that adds the unit starts it only through a target. + && lib.elem "multi-user.target" u.wantedBy && !(allLocal.systemd.services ? hive-forge-swarm-controller-token); } { # What makes it re-run. A oneshot left `active (exited)` makes every later - # start a no-op, so `RemainAfterExit` would silence both triggers: the - # container's start (`wantedBy`, above) and a write of the token file. + # start a no-op, so `RemainAfterExit` would silence every trigger: the + # targets and the container's start (`wantedBy`, above) and a write of + # the token file. A missing file exits 0, so a switch with the container + # down does not fail. name = "the forge publisher re-runs on each container start and each write of the token file"; ok = let @@ -308,6 +312,8 @@ let p.pathConfig.PathChanged == "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token" && p.pathConfig.Unit == "swarm-bao-forge-publisher.service" + && lib.hasInfix "the path unit publishes it when it does" (publisher allLocal).script + && lib.hasInfix "exit 0" (publisher allLocal).script && lib.elem "multi-user.target" p.wantedBy && !(bare.systemd.paths ? swarm-bao-forge-publisher); } diff --git a/swarm-controller/src/forge.rs b/swarm-controller/src/forge.rs index 0942d67e..658936f1 100644 --- a/swarm-controller/src/forge.rs +++ b/swarm-controller/src/forge.rs @@ -183,21 +183,31 @@ pub fn url_from_env() -> Result> { .map(Some) } +/// The longest the background re-read waits between attempts. +const BACKGROUND_CAP: std::time::Duration = std::time::Duration::from_mins(5); + /// How long the background re-read waits before its `n`-th attempt: 30s, -/// doubling, capped at 5 minutes. +/// doubling, capped at [`BACKGROUND_CAP`]. fn background_delay(n: u32) -> std::time::Duration { const FIRST_S: u64 = 30; - const CAP_S: u64 = 300; std::time::Duration::from_secs( FIRST_S .saturating_mul(1u64.checked_shl(n).unwrap_or(u64::MAX)) - .min(CAP_S), + .min(BACKGROUND_CAP.as_secs()), ) } +/// Whether the `n`-th failed attempt is logged at `info` rather than `debug`: +/// once the backoff is capped, so a controller that never gets its token says +/// so once per [`BACKGROUND_CAP`] without a line per early retry. +fn failure_logs_at_info(n: u32) -> bool { + background_delay(n) >= BACKGROUND_CAP +} + /// Fill `slot` with the first value `attempt` yields, sleeping -/// [`background_delay`] before each attempt. Never gives up, and logs no -/// failure: the caller logs the first one and the recovery. +/// [`background_delay`] before each attempt. Never gives up. A failure is logged +/// as [`failure_logs_at_info`] says; the caller logs the first one and the +/// recovery. async fn fill_when_ready(slot: &OnceLock, mut attempt: F) -> &T where F: FnMut() -> Fut, @@ -208,6 +218,10 @@ where tokio::time::sleep(background_delay(n)).await; match attempt().await { Ok(value) => return slot.get_or_init(|| value), + Err(e) if failure_logs_at_info(n) => tracing::info!( + error = %format!("{e:#}"), + "still no forge access: the controller's forge token is unavailable" + ), Err(e) => tracing::debug!( error = %format!("{e:#}"), "the controller's forge token is still unavailable" @@ -1402,6 +1416,13 @@ mod tests { } } + #[test] + fn a_failed_re_read_logs_at_info_only_once_the_backoff_is_capped() { + assert!((0..4).all(|n| !failure_logs_at_info(n))); + assert!(failure_logs_at_info(4)); + assert!(failure_logs_at_info(40)); + } + #[test] fn the_background_re_read_backs_off_from_30s_to_a_5_minute_cap() { let s = |n| background_delay(n).as_secs();