From d9c7b7fcc1c48ec7f6cbf58cf486d575e9f7a13f Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 6 Jun 2026 11:57:10 +0200 Subject: [PATCH] =?UTF-8?q?fix(ci):=20hive-ci=20prefetch=20=E2=80=94=20cor?= =?UTF-8?q?rect=20partOf=20unit=20name=20+=20401-harden=20token=20fetch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs that together kept the runner registration token from refreshing (#1475): 1. Unit name: the prefetch's before/wantedBy/partOf targeted nixos-container@hive-ci.service, but a declarative containers. is the host unit container@.service (confirmed against the live container@hive-matrix.service during the #1465 incident). The wrong name made all three silent no-ops, so the partOf never bound — the RemainAfterExit oneshot stayed 'active (exited)' and never re-ran on nixos-container restart, leaving the stale token in place. Corrected to container@hive-ci.service. 2. 401-hardening: the registration-token fetch used a bare curl -sf | jq, so a forge-core-token that is stale/invalid for the current forge (e.g. after a forge rebuild) 401s and fails silently every attempt for the full 60s loop, then exits with a misleading 'core token absent or forge unreachable'. Now capture the HTTP status and fail fast + loudly on 401/403 with a clear message pointing at re-minting the core token. --- nix/modules/hive-ci.nix | 46 ++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/nix/modules/hive-ci.nix b/nix/modules/hive-ci.nix index 9a5b5c81..f78282cc 100644 --- a/nix/modules/hive-ci.nix +++ b/nix/modules/hive-ci.nix @@ -20,7 +20,7 @@ let # without entering the container. containerRoot = "/var/lib/nixos-containers/hive-ci"; - # Host-side oneshot. Runs before `nixos-container@hive-ci.service`. + # Host-side oneshot. Runs before `container@hive-ci.service`. # # The core token stays on the host. The container only ever sees the # TOKEN= env-file populated here, never the core token itself. @@ -111,10 +111,33 @@ let continue fi CORE_TOKEN=$(cat ${coreTokenPath}) - REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \ + # Capture the HTTP status so a stale/invalid core token (401/403) is + # distinguished from a transient forge hiccup. With the old bare + # `curl -sf | jq`, a forge-core-token that's stale for the current + # forge (e.g. after a forge rebuild) 401s and fails silently every + # attempt for the full 60s loop, then exits with a misleading + # "core token absent or forge unreachable" — masking the real cause. + # Fail fast + loudly on 401/403 so the failure mode is legible and + # the operator/hive-c0re knows to re-mint forge-core-token (#1475). + RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \ "$FORGE_URL/api/v1/admin/runners/registration-token" \ - -H "Authorization: token $CORE_TOKEN" \ - | ${pkgs.jq}/bin/jq -r .token) && break + -H "Authorization: token $CORE_TOKEN" || printf '\n000') + HTTP=$(printf '%s' "$RESP" | tail -n1) + BODY=$(printf '%s' "$RESP" | sed '$d') + case "$HTTP" in + 2*) + REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token) + if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi + echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2 + ;; + 401 | 403) + echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s. (#1475)" >&2 + exit 1 + ;; + *) + echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2 + ;; + esac sleep 1 done @@ -246,8 +269,8 @@ in "systemd-tmpfiles-setup.service" "hive-c0re.service" ]; - before = [ "nixos-container@hive-ci.service" ]; - wantedBy = [ "nixos-container@hive-ci.service" ]; + before = [ "container@hive-ci.service" ]; + wantedBy = [ "container@hive-ci.service" ]; # partOf binds this oneshot's lifecycle to the container: when the # container is stopped or restarted, systemd propagates that to this # unit so it re-runs on the NEXT container start. Without this, the @@ -258,7 +281,16 @@ in # is never refreshed. The in-container register service then fails # with "runner registration token not found". partOf guarantees a # fresh token is fetched before every container (re)start. - partOf = [ "nixos-container@hive-ci.service" ]; + # + # Unit name: a declarative `containers.` is the host unit + # `container@.service` (the nspawn template), NOT + # `nixos-container@…`. The earlier `nixos-container@hive-ci.service` + # matched no real unit, so before/wantedBy/partOf were silent + # no-ops — the partOf never bound, the oneshot stayed + # `active (exited)`, and the token was never refreshed on restart + # (a contributor to #1475). Confirmed against the live + # `container@hive-matrix.service` unit during the #1465 incident. + partOf = [ "container@hive-ci.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true;