vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(ci): hive-ci prefetch — correct partOf unit name + 401-harden token fetch

Browse source 
Two bugs that together kept the runner registration token from
refreshing (#1475):

1. Unit name: the prefetch's before/wantedBy/partOf targeted
   nixos-container@hive-ci.service, but a declarative containers.<n> is
   the host unit container@<n>.service (confirmed against the live
   container@hive-matrix.service during the #1465 incident). The wrong
   name made all three silent no-ops, so the partOf never bound — the
   RemainAfterExit oneshot stayed 'active (exited)' and never re-ran on
   nixos-container restart, leaving the stale token in place. Corrected
   to container@hive-ci.service.

2. 401-hardening: the registration-token fetch used a bare curl -sf | jq,
   so a forge-core-token that is stale/invalid for the current forge
   (e.g. after a forge rebuild) 401s and fails silently every attempt for
   the full 60s loop, then exits with a misleading 'core token absent or
   forge unreachable'. Now capture the HTTP status and fail fast + loudly
   on 401/403 with a clear message pointing at re-minting the core token.
This commit is contained in:
atlas 2026-06-06 11:57:10 +02:00 • committed by mara
commit d9c7b7fcc1
1 changed files with 39 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

46
nix/modules/hive-ci.nix
View file

@ -20,7 +20,7 @@ let
# without entering the container.
containerRoot = "/var/lib/nixos-containers/hive-ci";
# Host-side oneshot. Runs before `nixos-container@hive-ci.service`.
# Host-side oneshot. Runs before `container@hive-ci.service`.
#
# The core token stays on the host. The container only ever sees the
# TOKEN= env-file populated here, never the core token itself.
@ -111,10 +111,33 @@ let
continue
fi
CORE_TOKEN=$(cat ${coreTokenPath})
REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \
# Capture the HTTP status so a stale/invalid core token (401/403) is
# distinguished from a transient forge hiccup. With the old bare
# `curl -sf | jq`, a forge-core-token that's stale for the current
# forge (e.g. after a forge rebuild) 401s and fails silently every
# attempt for the full 60s loop, then exits with a misleading
# "core token absent or forge unreachable" — masking the real cause.
# Fail fast + loudly on 401/403 so the failure mode is legible and
# the operator/hive-c0re knows to re-mint forge-core-token (#1475).
RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \
"$FORGE_URL/api/v1/admin/runners/registration-token" \
-H "Authorization: token $CORE_TOKEN" \
| ${pkgs.jq}/bin/jq -r .token) && break
-H "Authorization: token $CORE_TOKEN" || printf '\n000')
HTTP=$(printf '%s' "$RESP" | tail -n1)
BODY=$(printf '%s' "$RESP" | sed '$d')
case "$HTTP" in
2*)
REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token)
if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi
echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2
;;
401 | 403)
echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s. (#1475)" >&2
exit 1
;;
*)
echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2
;;
esac
sleep 1
done
@ -246,8 +269,8 @@ in
"systemd-tmpfiles-setup.service"
"hive-c0re.service"
];
before = [ "nixos-container@hive-ci.service" ];
wantedBy = [ "nixos-container@hive-ci.service" ];
before = [ "container@hive-ci.service" ];
wantedBy = [ "container@hive-ci.service" ];
# partOf binds this oneshot's lifecycle to the container: when the
# container is stopped or restarted, systemd propagates that to this
# unit so it re-runs on the NEXT container start. Without this, the
@ -258,7 +281,16 @@ in
# is never refreshed. The in-container register service then fails
# with "runner registration token not found". partOf guarantees a
# fresh token is fetched before every container (re)start.
partOf = [ "nixos-container@hive-ci.service" ];
#
# Unit name: a declarative `containers.<name>` is the host unit
# `container@<name>.service` (the nspawn template), NOT
# `nixos-container@…`. The earlier `nixos-container@hive-ci.service`
# matched no real unit, so before/wantedBy/partOf were silent
# no-ops — the partOf never bound, the oneshot stayed
# `active (exited)`, and the token was never refreshed on restart
# (a contributor to #1475). Confirmed against the live
# `container@hive-matrix.service` unit during the #1465 incident.
partOf = [ "container@hive-ci.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;