fix: wait for forge-core-token in prefetch instead of bailing early

On first boot (or after a wipe) hive-c0re writes forge-core-token only
after the forge container starts and the admin is provisioned. This
lags hive-c0re.service becoming active. The previous code bailed
immediately with TOKEN=placeholder if the token file was absent,
causing the runner to fail registration with 'token not found'.

Fix: merge both waits (core-token file appearance + forge API ready)
into a single 60s retry loop. The early-bail path is removed; the
script only exits cleanly if .runner is valid (writes placeholder)
or a fresh registration token is obtained.

When .runner exists but core-token is absent after 60s, we keep the
existing credentials (safe — the runner holds valid creds; next boot
will validate properly).

Closes #1224 (which tracks #1221).

nix/modules/hive-ci.nix

@@ -26,53 +26,67 @@ let
   # TOKEN= env-file populated here, never the core token itself.
   #
   # Flow:
-  #   1. If forge-core-token doesn't exist yet (forge still initialising):
+  #   1. If .runner exists: validate the runner ID against forge.
-  #      write TOKEN=placeholder and exit — the container starts but the
+  #      Waits up to 60s for forge-core-token (hive-c0re writes it after
-  #      runner will fail to register. systemd will restart on next boot.
+  #      forge container starts and admin is provisioned — this lags
-  #   2. If .runner exists: validate the runner ID against forge.
+  #      hive-c0re.service becoming active on first boot).
   #      404 → purge .runner (re-registration needed).
   #      000 (forge unreachable) → keep credentials, write placeholder.
   #      other non-200 → purge .runner.
-  #   3. If .runner absent or just purged: fetch a fresh registration token
+  #      Token absent after 60s → keep credentials (safe — the runner
-  #      (retry 30s for forge to start) and write TOKEN=<real>.
+  #      holds valid creds; next boot will validate properly).
+  #   2. If .runner absent or just purged: wait up to 60s for both
+  #      forge-core-token to appear AND forge API to respond, then
+  #      fetch a fresh registration token and write TOKEN=<real>.
+  #      Exits non-zero if both timeout — systemd logs the failure;
+  #      the placeholder from tmpfiles means the container still starts
+  #      but the runner will error. Operator restarts the service once
+  #      forge is healthy.
   prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
     set -euo pipefail
     TOKEN_FILE=/run/hive-ci/runner-token
     FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
     RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
 
-    # Guard: if the core token hasn't been written yet (forge still
-    # initialising) write a placeholder and exit cleanly. The runner
-    # service will fail gracefully; the next boot will re-run this
-    # service and pick up the real token.
-    if [ ! -f "${coreTokenPath}" ]; then
-      echo "hive-ci-prefetch: core token absent, deferring registration" >&2
-      echo "TOKEN=placeholder" > "$TOKEN_FILE"
-      chmod 600 "$TOKEN_FILE"
-      exit 0
-    fi
-    CORE_TOKEN=$(cat ${coreTokenPath})
-
     # Validate existing .runner credentials against forge. Purge if
     # the runner was deleted (404) or the file is malformed. Keep if
     # forge is unreachable (000) — transient outage shouldn't wipe creds.
+    #
+    # We wait up to 60s for forge-core-token to appear (hive-c0re writes
+    # it after provisioning the forge admin, which requires the forge
+    # container to start and become ready — this can lag behind
+    # hive-c0re.service becoming "active" on first boot). The same loop
+    # doubles as a wait for forge's API to become ready.
     if [ -f "$RUNNER_FILE" ]; then
       RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
       if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
         echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
         rm -f "$RUNNER_FILE"
       else
-        HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
+        # Wait for the core token before validating (same timeout as below).
-          -H "Authorization: token $CORE_TOKEN" \
+        CORE_TOKEN=""
-          "$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
+        for i in $(seq 1 60); do
-        if [ "$HTTP" = "404" ]; then
+          if [ -f "${coreTokenPath}" ]; then
-          echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
+            CORE_TOKEN=$(cat ${coreTokenPath})
-          rm -f "$RUNNER_FILE"
+            break
-        elif [ "$HTTP" = "000" ]; then
+          fi
-          echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
+          sleep 1
-        elif [ "$HTTP" != "200" ]; then
+        done
-          echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
+        if [ -z "''${CORE_TOKEN:-}" ]; then
-          rm -f "$RUNNER_FILE"
+          echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
+        else
+          HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
+            -H "Authorization: token $CORE_TOKEN" \
+            "$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
+          if [ "$HTTP" = "404" ]; then
+            echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
+            rm -f "$RUNNER_FILE"
+          elif [ "$HTTP" = "000" ]; then
+            echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
+          elif [ "$HTTP" != "200" ]; then
+            echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
+            rm -f "$RUNNER_FILE"
+          fi
         fi
       fi
     fi
@@ -85,10 +99,18 @@ let
       exit 0
     fi
 
-    # First boot or stale creds purged — fetch a fresh registration token.
+    # First boot or stale creds purged — wait for forge-core-token, then
-    # Retry up to 30s for forge to come up (containers autoStart in parallel).
+    # fetch a fresh registration token. Single retry loop covers both:
+    # waiting for hive-c0re to write the token file AND for forge's API
+    # to become responsive (they race on first boot).
     REG_TOKEN=""
-    for i in $(seq 1 30); do
+    for i in $(seq 1 60); do
+      if [ ! -f "${coreTokenPath}" ]; then
+        echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
+        sleep 1
+        continue
+      fi
+      CORE_TOKEN=$(cat ${coreTokenPath})
       REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \
         "$FORGE_URL/api/v1/admin/runners/registration-token" \
         -H "Authorization: token $CORE_TOKEN" \
@@ -97,7 +119,7 @@ let
     done
 
     if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
-      echo "hive-ci-prefetch: failed to fetch runner registration token from forge" >&2
+      echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
       exit 1
     fi