From dacd83f2781742092235639d5591b483399ecc71 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 3 Jun 2026 22:32:53 +0200 Subject: [PATCH] fix: wait for forge-core-token in prefetch instead of bailing early MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On first boot (or after a wipe) hive-c0re writes forge-core-token only after the forge container starts and the admin is provisioned. This lags hive-c0re.service becoming active. The previous code bailed immediately with TOKEN=placeholder if the token file was absent, causing the runner to fail registration with 'token not found'. Fix: merge both waits (core-token file appearance + forge API ready) into a single 60s retry loop. The early-bail path is removed; the script only exits cleanly if .runner is valid (writes placeholder) or a fresh registration token is obtained. When .runner exists but core-token is absent after 60s, we keep the existing credentials (safe — the runner holds valid creds; next boot will validate properly). Closes #1224 (which tracks #1221). --- nix/modules/hive-ci.nix | 88 +++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/nix/modules/hive-ci.nix b/nix/modules/hive-ci.nix index 0a3f18cc..0c589e0a 100644 --- a/nix/modules/hive-ci.nix +++ b/nix/modules/hive-ci.nix @@ -26,53 +26,67 @@ let # TOKEN= env-file populated here, never the core token itself. # # Flow: - # 1. If forge-core-token doesn't exist yet (forge still initialising): - # write TOKEN=placeholder and exit — the container starts but the - # runner will fail to register. systemd will restart on next boot. - # 2. If .runner exists: validate the runner ID against forge. + # 1. If .runner exists: validate the runner ID against forge. + # Waits up to 60s for forge-core-token (hive-c0re writes it after + # forge container starts and admin is provisioned — this lags + # hive-c0re.service becoming active on first boot). # 404 → purge .runner (re-registration needed). # 000 (forge unreachable) → keep credentials, write placeholder. # other non-200 → purge .runner. - # 3. If .runner absent or just purged: fetch a fresh registration token - # (retry 30s for forge to start) and write TOKEN=. + # Token absent after 60s → keep credentials (safe — the runner + # holds valid creds; next boot will validate properly). + # 2. If .runner absent or just purged: wait up to 60s for both + # forge-core-token to appear AND forge API to respond, then + # fetch a fresh registration token and write TOKEN=. + # Exits non-zero if both timeout — systemd logs the failure; + # the placeholder from tmpfiles means the container still starts + # but the runner will error. Operator restarts the service once + # forge is healthy. prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" '' set -euo pipefail TOKEN_FILE=/run/hive-ci/runner-token FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}" RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner" - # Guard: if the core token hasn't been written yet (forge still - # initialising) write a placeholder and exit cleanly. The runner - # service will fail gracefully; the next boot will re-run this - # service and pick up the real token. - if [ ! -f "${coreTokenPath}" ]; then - echo "hive-ci-prefetch: core token absent, deferring registration" >&2 - echo "TOKEN=placeholder" > "$TOKEN_FILE" - chmod 600 "$TOKEN_FILE" - exit 0 - fi - CORE_TOKEN=$(cat ${coreTokenPath}) - # Validate existing .runner credentials against forge. Purge if # the runner was deleted (404) or the file is malformed. Keep if # forge is unreachable (000) — transient outage shouldn't wipe creds. + # + # We wait up to 60s for forge-core-token to appear (hive-c0re writes + # it after provisioning the forge admin, which requires the forge + # container to start and become ready — this can lag behind + # hive-c0re.service becoming "active" on first boot). The same loop + # doubles as a wait for forge's API to become ready. if [ -f "$RUNNER_FILE" ]; then RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true) if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2 rm -f "$RUNNER_FILE" else - HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \ - -H "Authorization: token $CORE_TOKEN" \ - "$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000") - if [ "$HTTP" = "404" ]; then - echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2 - rm -f "$RUNNER_FILE" - elif [ "$HTTP" = "000" ]; then - echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2 - elif [ "$HTTP" != "200" ]; then - echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2 - rm -f "$RUNNER_FILE" + # Wait for the core token before validating (same timeout as below). + CORE_TOKEN="" + for i in $(seq 1 60); do + if [ -f "${coreTokenPath}" ]; then + CORE_TOKEN=$(cat ${coreTokenPath}) + break + fi + sleep 1 + done + if [ -z "''${CORE_TOKEN:-}" ]; then + echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2 + else + HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: token $CORE_TOKEN" \ + "$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000") + if [ "$HTTP" = "404" ]; then + echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2 + rm -f "$RUNNER_FILE" + elif [ "$HTTP" = "000" ]; then + echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2 + elif [ "$HTTP" != "200" ]; then + echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2 + rm -f "$RUNNER_FILE" + fi fi fi fi @@ -85,10 +99,18 @@ let exit 0 fi - # First boot or stale creds purged — fetch a fresh registration token. - # Retry up to 30s for forge to come up (containers autoStart in parallel). + # First boot or stale creds purged — wait for forge-core-token, then + # fetch a fresh registration token. Single retry loop covers both: + # waiting for hive-c0re to write the token file AND for forge's API + # to become responsive (they race on first boot). REG_TOKEN="" - for i in $(seq 1 30); do + for i in $(seq 1 60); do + if [ ! -f "${coreTokenPath}" ]; then + echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2 + sleep 1 + continue + fi + CORE_TOKEN=$(cat ${coreTokenPath}) REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \ "$FORGE_URL/api/v1/admin/runners/registration-token" \ -H "Authorization: token $CORE_TOKEN" \ @@ -97,7 +119,7 @@ let done if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then - echo "hive-ci-prefetch: failed to fetch runner registration token from forge" >&2 + echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2 exit 1 fi