fix: wait for forge-core-token in prefetch instead of bailing early
On first boot (or after a wipe) hive-c0re writes forge-core-token only after the forge container starts and the admin is provisioned. This lags hive-c0re.service becoming active. The previous code bailed immediately with TOKEN=placeholder if the token file was absent, causing the runner to fail registration with 'token not found'. Fix: merge both waits (core-token file appearance + forge API ready) into a single 60s retry loop. The early-bail path is removed; the script only exits cleanly if .runner is valid (writes placeholder) or a fresh registration token is obtained. When .runner exists but core-token is absent after 60s, we keep the existing credentials (safe — the runner holds valid creds; next boot will validate properly). Closes #1224 (which tracks #1221).
This commit is contained in:
atlas
mara
parent
280ae23c0c
commit
dacd83f278
1 changed files with 55 additions and 33 deletions
|
|
@ -26,53 +26,67 @@ let
|
|||
# TOKEN= env-file populated here, never the core token itself.
|
||||
#
|
||||
# Flow:
|
||||
# 1. If forge-core-token doesn't exist yet (forge still initialising):
|
||||
# write TOKEN=placeholder and exit — the container starts but the
|
||||
# runner will fail to register. systemd will restart on next boot.
|
||||
# 2. If .runner exists: validate the runner ID against forge.
|
||||
# 1. If .runner exists: validate the runner ID against forge.
|
||||
# Waits up to 60s for forge-core-token (hive-c0re writes it after
|
||||
# forge container starts and admin is provisioned — this lags
|
||||
# hive-c0re.service becoming active on first boot).
|
||||
# 404 → purge .runner (re-registration needed).
|
||||
# 000 (forge unreachable) → keep credentials, write placeholder.
|
||||
# other non-200 → purge .runner.
|
||||
# 3. If .runner absent or just purged: fetch a fresh registration token
|
||||
# (retry 30s for forge to start) and write TOKEN=<real>.
|
||||
# Token absent after 60s → keep credentials (safe — the runner
|
||||
# holds valid creds; next boot will validate properly).
|
||||
# 2. If .runner absent or just purged: wait up to 60s for both
|
||||
# forge-core-token to appear AND forge API to respond, then
|
||||
# fetch a fresh registration token and write TOKEN=<real>.
|
||||
# Exits non-zero if both timeout — systemd logs the failure;
|
||||
# the placeholder from tmpfiles means the container still starts
|
||||
# but the runner will error. Operator restarts the service once
|
||||
# forge is healthy.
|
||||
prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
|
||||
set -euo pipefail
|
||||
TOKEN_FILE=/run/hive-ci/runner-token
|
||||
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
||||
RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
|
||||
|
||||
# Guard: if the core token hasn't been written yet (forge still
|
||||
# initialising) write a placeholder and exit cleanly. The runner
|
||||
# service will fail gracefully; the next boot will re-run this
|
||||
# service and pick up the real token.
|
||||
if [ ! -f "${coreTokenPath}" ]; then
|
||||
echo "hive-ci-prefetch: core token absent, deferring registration" >&2
|
||||
echo "TOKEN=placeholder" > "$TOKEN_FILE"
|
||||
chmod 600 "$TOKEN_FILE"
|
||||
exit 0
|
||||
fi
|
||||
CORE_TOKEN=$(cat ${coreTokenPath})
|
||||
|
||||
# Validate existing .runner credentials against forge. Purge if
|
||||
# the runner was deleted (404) or the file is malformed. Keep if
|
||||
# forge is unreachable (000) — transient outage shouldn't wipe creds.
|
||||
#
|
||||
# We wait up to 60s for forge-core-token to appear (hive-c0re writes
|
||||
# it after provisioning the forge admin, which requires the forge
|
||||
# container to start and become ready — this can lag behind
|
||||
# hive-c0re.service becoming "active" on first boot). The same loop
|
||||
# doubles as a wait for forge's API to become ready.
|
||||
if [ -f "$RUNNER_FILE" ]; then
|
||||
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
|
||||
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
|
||||
echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
|
||||
rm -f "$RUNNER_FILE"
|
||||
else
|
||||
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token $CORE_TOKEN" \
|
||||
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
|
||||
if [ "$HTTP" = "404" ]; then
|
||||
echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
|
||||
rm -f "$RUNNER_FILE"
|
||||
elif [ "$HTTP" = "000" ]; then
|
||||
echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
|
||||
elif [ "$HTTP" != "200" ]; then
|
||||
echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
|
||||
rm -f "$RUNNER_FILE"
|
||||
# Wait for the core token before validating (same timeout as below).
|
||||
CORE_TOKEN=""
|
||||
for i in $(seq 1 60); do
|
||||
if [ -f "${coreTokenPath}" ]; then
|
||||
CORE_TOKEN=$(cat ${coreTokenPath})
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ -z "''${CORE_TOKEN:-}" ]; then
|
||||
echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
|
||||
else
|
||||
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token $CORE_TOKEN" \
|
||||
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
|
||||
if [ "$HTTP" = "404" ]; then
|
||||
echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
|
||||
rm -f "$RUNNER_FILE"
|
||||
elif [ "$HTTP" = "000" ]; then
|
||||
echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
|
||||
elif [ "$HTTP" != "200" ]; then
|
||||
echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
|
||||
rm -f "$RUNNER_FILE"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
|
@ -85,10 +99,18 @@ let
|
|||
exit 0
|
||||
fi
|
||||
|
||||
# First boot or stale creds purged — fetch a fresh registration token.
|
||||
# Retry up to 30s for forge to come up (containers autoStart in parallel).
|
||||
# First boot or stale creds purged — wait for forge-core-token, then
|
||||
# fetch a fresh registration token. Single retry loop covers both:
|
||||
# waiting for hive-c0re to write the token file AND for forge's API
|
||||
# to become responsive (they race on first boot).
|
||||
REG_TOKEN=""
|
||||
for i in $(seq 1 30); do
|
||||
for i in $(seq 1 60); do
|
||||
if [ ! -f "${coreTokenPath}" ]; then
|
||||
echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
|
||||
sleep 1
|
||||
continue
|
||||
fi
|
||||
CORE_TOKEN=$(cat ${coreTokenPath})
|
||||
REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \
|
||||
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
||||
-H "Authorization: token $CORE_TOKEN" \
|
||||
|
|
@ -97,7 +119,7 @@ let
|
|||
done
|
||||
|
||||
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
|
||||
echo "hive-ci-prefetch: failed to fetch runner registration token from forge" >&2
|
||||
echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue