diff --git a/nix/modules/hive-ci.nix b/nix/modules/hive-ci.nix index 9a5b5c81..f78282cc 100644 --- a/nix/modules/hive-ci.nix +++ b/nix/modules/hive-ci.nix @@ -20,7 +20,7 @@ let # without entering the container. containerRoot = "/var/lib/nixos-containers/hive-ci"; - # Host-side oneshot. Runs before `nixos-container@hive-ci.service`. + # Host-side oneshot. Runs before `container@hive-ci.service`. # # The core token stays on the host. The container only ever sees the # TOKEN= env-file populated here, never the core token itself. @@ -111,10 +111,33 @@ let continue fi CORE_TOKEN=$(cat ${coreTokenPath}) - REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \ + # Capture the HTTP status so a stale/invalid core token (401/403) is + # distinguished from a transient forge hiccup. With the old bare + # `curl -sf | jq`, a forge-core-token that's stale for the current + # forge (e.g. after a forge rebuild) 401s and fails silently every + # attempt for the full 60s loop, then exits with a misleading + # "core token absent or forge unreachable" — masking the real cause. + # Fail fast + loudly on 401/403 so the failure mode is legible and + # the operator/hive-c0re knows to re-mint forge-core-token (#1475). + RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \ "$FORGE_URL/api/v1/admin/runners/registration-token" \ - -H "Authorization: token $CORE_TOKEN" \ - | ${pkgs.jq}/bin/jq -r .token) && break + -H "Authorization: token $CORE_TOKEN" || printf '\n000') + HTTP=$(printf '%s' "$RESP" | tail -n1) + BODY=$(printf '%s' "$RESP" | sed '$d') + case "$HTTP" in + 2*) + REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token) + if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi + echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2 + ;; + 401 | 403) + echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s. (#1475)" >&2 + exit 1 + ;; + *) + echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2 + ;; + esac sleep 1 done @@ -246,8 +269,8 @@ in "systemd-tmpfiles-setup.service" "hive-c0re.service" ]; - before = [ "nixos-container@hive-ci.service" ]; - wantedBy = [ "nixos-container@hive-ci.service" ]; + before = [ "container@hive-ci.service" ]; + wantedBy = [ "container@hive-ci.service" ]; # partOf binds this oneshot's lifecycle to the container: when the # container is stopped or restarted, systemd propagates that to this # unit so it re-runs on the NEXT container start. Without this, the @@ -258,7 +281,16 @@ in # is never refreshed. The in-container register service then fails # with "runner registration token not found". partOf guarantees a # fresh token is fetched before every container (re)start. - partOf = [ "nixos-container@hive-ci.service" ]; + # + # Unit name: a declarative `containers.` is the host unit + # `container@.service` (the nspawn template), NOT + # `nixos-container@…`. The earlier `nixos-container@hive-ci.service` + # matched no real unit, so before/wantedBy/partOf were silent + # no-ops — the partOf never bound, the oneshot stayed + # `active (exited)`, and the token was never refreshed on restart + # (a contributor to #1475). Confirmed against the live + # `container@hive-matrix.service` unit during the #1465 incident. + partOf = [ "container@hive-ci.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true;