feat(#2415): move hive-ci runner registration off the boot-critical path (nix)

Completes #2415. Registration no longer gates container@hive-ci start:
- Retire the host-side hive-ci-prefetch.service (+ its 100-line script and
  the now-dead coreTokenPath/containerRoot let-bindings) — the forge round-trip
  it did on the boot path now lives in hive-c0re (forge/ci_runner.rs), run out
  of band during the startup sweep.
- Drop the container@hive-ci TimeoutStartSec = mkForce 180s band-aid that
  papered over that boot-path wait tripping the nspawn start timeout (the ~60s
  restart loop of #2410).
- gitea-runner-hive gains an ExecStartPre precond (ahead of the nix-daemon
  wait) that fails fast unless it is already registered (.runner present) or a
  real, non-placeholder token is in place — so missing creds just hold the
  runner down instead of blocking the container. Restart=on-failure (no start
  limit) self-heals it once hive-c0re writes the token; c0re's explicit restart
  is the primary path.
- Set HYPERHIVE_FORGE_CI_ENABLED=1 on hive-c0re.service so the sweep registers.
The tmpfiles TOKEN=placeholder seed + read-only bind-mount are unchanged; the
core admin token still never enters the container.

nix/host-modules/hive-ci.nix

@@ -24,147 +24,6 @@ let
   useSelfSigned = caTrust.useSelfSigned;
   caContainerPath = caTrust.caContainerPath;
 
-  # hive-c0re writes its own admin token here on first forge startup.
-  # The token has read:admin + write:admin scopes — sufficient to call
-  # the runner registration-token API endpoint.
-  # This path is HOST-ONLY. It is never bind-mounted into hive-ci.
-  coreTokenPath = "/var/lib/hyperhive/forge-core-token";
-
-  # Container state root on the host. Non-ephemeral containers keep
-  # their filesystem here across reboots. The prefetch service accesses
-  # the runner's .runner file via this path to validate credentials
-  # without entering the container.
-  containerRoot = "/var/lib/nixos-containers/hive-ci";
-
-  # Host-side oneshot. Runs before `container@hive-ci.service`.
-  #
-  # The core token stays on the host. The container only ever sees the
-  # TOKEN= env-file populated here, never the core token itself.
-  #
-  # Flow:
-  #   1. If .runner exists: validate the runner ID against forge.
-  #      Waits up to 60s for forge-core-token (hive-c0re writes it after
-  #      forge container starts and admin is provisioned — this lags
-  #      hive-c0re.service becoming active on first boot).
-  #      404 → purge .runner (re-registration needed).
-  #      000 (forge unreachable) → keep credentials, write placeholder.
-  #      other non-200 → purge .runner.
-  #      Token absent after 60s → keep credentials (safe — the runner
-  #      holds valid creds; next boot will validate properly).
-  #   2. If .runner absent or just purged: wait up to 60s for both
-  #      forge-core-token to appear AND forge API to respond, then
-  #      fetch a fresh registration token and write TOKEN=<real>.
-  #      Exits non-zero if both timeout — systemd logs the failure;
-  #      the placeholder from tmpfiles means the container still starts
-  #      but the runner will error. Operator restarts the service once
-  #      forge is healthy.
-  prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
-    set -euo pipefail
-    TOKEN_FILE=/run/hive-ci/runner-token
-    FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
-    RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
-
-    # Validate existing .runner credentials against forge. Purge if
-    # the runner was deleted (404) or the file is malformed. Keep if
-    # forge is unreachable (000) — transient outage shouldn't wipe creds.
-    #
-    # We wait up to 60s for forge-core-token to appear (hive-c0re writes
-    # it after provisioning the forge admin, which requires the forge
-    # container to start and become ready — this can lag behind
-    # hive-c0re.service becoming "active" on first boot). The same loop
-    # doubles as a wait for forge's API to become ready.
-    if [ -f "$RUNNER_FILE" ]; then
-      RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
-      if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
-        echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
-        rm -f "$RUNNER_FILE"
-      else
-        # Wait for the core token before validating (same timeout as below).
-        CORE_TOKEN=""
-        for i in $(seq 1 60); do
-          if [ -f "${coreTokenPath}" ]; then
-            CORE_TOKEN=$(cat ${coreTokenPath})
-            break
-          fi
-          sleep 1
-        done
-        if [ -z "''${CORE_TOKEN:-}" ]; then
-          echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
-        else
-          HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
-            -H "Authorization: token $CORE_TOKEN" \
-            "$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
-          if [ "$HTTP" = "404" ]; then
-            echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
-            rm -f "$RUNNER_FILE"
-          elif [ "$HTTP" = "000" ]; then
-            echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
-          elif [ "$HTTP" != "200" ]; then
-            echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
-            rm -f "$RUNNER_FILE"
-          fi
-        fi
-      fi
-    fi
-
-    # .runner valid → write placeholder; gitea-actions-runner skips
-    # re-registration when .runner exists (TOKEN value is irrelevant).
-    if [ -f "$RUNNER_FILE" ]; then
-      echo "TOKEN=placeholder" > "$TOKEN_FILE"
-      chmod 600 "$TOKEN_FILE"
-      exit 0
-    fi
-
-    # First boot or stale creds purged — wait for forge-core-token, then
-    # fetch a fresh registration token. Single retry loop covers both:
-    # waiting for hive-c0re to write the token file AND for forge's API
-    # to become responsive (they race on first boot).
-    REG_TOKEN=""
-    for i in $(seq 1 60); do
-      if [ ! -f "${coreTokenPath}" ]; then
-        echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
-        sleep 1
-        continue
-      fi
-      CORE_TOKEN=$(cat ${coreTokenPath})
-      # Capture the HTTP status so a stale/invalid core token (401/403) is
-      # distinguished from a transient forge hiccup. A bare
-      # `curl -sf | jq` would let a forge-core-token that is stale for
-      # the current forge (e.g. after a forge rebuild) 401 silently on
-      # every attempt for the full 60s loop and exit with a misleading
-      # "core token absent or forge unreachable" — masking the real cause.
-      # Fail fast + loudly on 401/403 so the failure mode is legible and
-      # the operator/hive-c0re knows to re-mint forge-core-token.
-      RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \
-        "$FORGE_URL/api/v1/admin/runners/registration-token" \
-        -H "Authorization: token $CORE_TOKEN" || printf '\n000')
-      HTTP=$(printf '%s' "$RESP" | tail -n1)
-      BODY=$(printf '%s' "$RESP" | sed '$d')
-      case "$HTTP" in
-        2*)
-          REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token)
-          if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi
-          echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2
-          ;;
-        401 | 403)
-          echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s." >&2
-          exit 1
-          ;;
-        *)
-          echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2
-          ;;
-      esac
-      sleep 1
-    done
-
-    if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
-      echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
-      exit 1
-    fi
-
-    echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
-    chmod 600 "$TOKEN_FILE"
-  '';
 in
 {
   # Forgejo Actions runner in a `hive-ci` nixos-container.
@@ -181,11 +40,12 @@ in
   # on first registration and reuses them on every subsequent start).
   #
   # Credential isolation: the forge admin token (`forge-core-token`)
-  # never enters the hive-ci container. A host-side oneshot service
+  # never enters the hive-ci container. hive-c0re holds it and performs
-  # (`hive-ci-prefetch.service`) performs all forge API calls before
+  # all forge API calls (runner validation + registration-token mint,
-  # the container starts and writes only the runner registration token
+  # in `forge/ci_runner.rs`); via hive-priv it writes only the runner
-  # to `/run/hive-ci/runner-token`. The container bind-mounts this
+  # registration token to the host env-file `/run/hive-ci/runner-token`,
-  # file read-only and never has access to the wider admin token.
+  # which the container bind-mounts read-only. The container never has
+  # access to the wider admin token.
   #
   # Nix builds inside the container use the shared /nix/store (standard
   # nixos-container behaviour) with sandbox-fallback = true, because
@@ -293,83 +153,34 @@ in
     ];
 
     # Create /run/hive-ci/ on the host and seed runner-token with a
-    # placeholder. hive-ci-prefetch.service overwrites it with the real
+    # placeholder. The container bind-mounts this file read-only; hive-c0re
-    # token (or a fresh placeholder) before the container starts. The
+    # (via hive-priv's RegisterCiRunner) overwrites it with the real
-    # placeholder ensures the EnvironmentFile is always present even if
+    # registration token when it registers the runner out of band. The
-    # the prefetch service hasn't run yet (e.g. tmpfiles-setup timing).
+    # placeholder keeps the runner's EnvironmentFile present from first boot,
+    # before c0re has registered — the runner's ExecStartPre precond (below)
+    # distinguishes the placeholder from a real token.
     systemd.tmpfiles.rules = [
       "d /run/hive-ci 0700 root root -"
       "f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
     ];
 
-    # Host-side oneshot: validates/refreshes runner credentials before
+    # Tell hive-c0re that CI is enabled so its startup sweep registers the
-    # the container starts. The core admin token stays on the host and
+    # runner. Registration moved OFF the container's boot-critical path into
-    # is never bind-mounted into the container. Runs on every boot so
+    # hive-c0re (`forge/ci_runner.rs`): it validates the persisted `.runner`
-    # stale .runner credentials (runner deleted from forge) are detected
+    # against the forge and, when absent/stale, mints a registration token and
-    # and the container re-registers on the next start.
+    # hands it to hive-priv to write `/run/hive-ci/runner-token` + restart the
-    systemd.services.hive-ci-prefetch = {
+    # runner. The forge admin token stays in hive-c0re; only the registration
-      description = "Pre-fetch hive-ci runner registration token (host-side)";
+    # token reaches the host env-file the container mounts read-only.
-      # Run before the container starts but after tmpfiles so the token
+    systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_CI_ENABLED = "1";
-      # file directory exists. After hive-c0re so the forge token is
-      # likely written (best-effort — the script handles the absent case).
-      after = [
-        "systemd-tmpfiles-setup.service"
-        "hive-c0re.service"
-      ];
-      before = [ "container@hive-ci.service" ];
-      wantedBy = [ "container@hive-ci.service" ];
-      # partOf binds this oneshot's lifecycle to the container: when the
-      # container is stopped or restarted, systemd propagates that to this
-      # unit so it re-runs on the NEXT container start. Without this, the
-      # RemainAfterExit=true oneshot stays "active (exited)" forever after
-      # its first run — so a container restart skips it and the stale
-      # runner-token file (a placeholder from a boot where the forge token
-      # wasn't ready yet, or a registration token consumed/rotated since)
-      # is never refreshed. The in-container register service then fails
-      # with "runner registration token not found". partOf guarantees a
-      # fresh token is fetched before every container (re)start.
-      #
-      # Unit name: a declarative `containers.<name>` is the host unit
-      # `container@<name>.service` (the nspawn template), NOT
-      # `nixos-container@…`. The earlier `nixos-container@hive-ci.service`
-      # matched no real unit, so before/wantedBy/partOf were silent
-      # no-ops — the partOf never bound, the oneshot stayed
-      # `active (exited)`, and the token was never refreshed on restart.
-      # Confirmed against the live `container@hive-matrix.service` unit
-      # during the matrix-outage incident.
-      partOf = [ "container@hive-ci.service" ];
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-        ExecStart = prefetchScript;
-        # Pin the journal identity; ExecStart is a writeShellScript whose
-        # store-path basename would otherwise be the journal identifier.
-        SyslogIdentifier = "hive-ci-prefetch";
-      };
-    };
 
     # `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
-    # in self-signed mode (see the hive-ca-trust helper); merged with the
+    # in self-signed mode (see the hive-ca-trust helper). Runner registration
-    # hive-ci-specific start-timeout bump below.
+    # is no longer on the boot-critical path — hive-c0re owns it out of band —
-    systemd.services."container@hive-ci" = lib.mkMerge [
+    # so the container boots immediately and needs no start-timeout bump. The
-      caTrust.containerOrdering
+    # former `TimeoutStartSec = mkForce "180s"` band-aid (which papered over a
-      {
+    # boot-path forge + core-token round-trip that could trip the nspawn start
-        # gitea-runner registration (hive-ci-prefetch, host-side)
+    # timeout into a ~60s restart loop) is gone with that move.
-        # sits on the boot-critical path — the container's nspawn readiness
+    systemd.services."container@hive-ci" = caTrust.containerOrdering;
-        # is gated on the runner registering, a forge + core-token round
-        # trip that itself waits up to 60s for the core token. The default
-        # nspawn `TimeoutStartSec` (~60s, systemd's `DefaultTimeoutStartSec`)
-        # can therefore trip mid-register, especially right after a
-        # `.runner`-purge (every boot re-registers from scratch — see
-        # hive-ci-prefetch above) or during a hive-c0re restart storm that
-        # delays the core token. systemd then kills the half-started
-        # container and reschedules, producing an observed ~60s restart
-        # loop until the forge/token settle. Bumping the timeout well past
-        # the prefetch's own 60s wait gives one register attempt room to
-        # actually finish instead of being killed mid-flight.
-        serviceConfig.TimeoutStartSec = lib.mkForce "180s";
-      }
-    ];
 
     containers.hive-ci = {
       autoStart = true;
@@ -383,9 +194,10 @@ in
       hostBridge = networkCfg.bridgeName;
 
       bindMounts = {
-        # Pre-filled by hive-ci-prefetch.service (host-side) before the
+        # Pre-filled by hive-c0re (via hive-priv) with the runner
-        # container starts. Read-only: the container reads TOKEN= from
+        # registration token; tmpfiles seeds a `TOKEN=placeholder` before
-        # here; the core admin token never enters the container.
+        # that. Read-only: the container reads TOKEN= from here; the core
+        # admin token never enters the container.
         "/run/hive-ci/runner-token" = {
           hostPath = "/run/hive-ci/runner-token";
           isReadOnly = true;
@@ -446,10 +258,11 @@ in
             # the gateway vhost `forgeCfg.domain` proxies to the forge
             # on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect).
             url = "http://${forgeCfg.domain}";
-            # EnvironmentFile providing TOKEN= — pre-filled by the
+            # EnvironmentFile providing TOKEN= — pre-filled by hive-c0re
-            # host-side hive-ci-prefetch.service before the container
+            # (via hive-priv) with the runner registration token;
-            # starts; bind-mounted read-only from /run/hive-ci/runner-token
+            # bind-mounted read-only from /run/hive-ci/runner-token on the
-            # on the host.
+            # host. The runner's ExecStartPre precond gates on this being a
+            # real (non-placeholder) token or an already-registered .runner.
             tokenFile = "/run/hive-ci/runner-token";
             labels = cfg.labels;
             settings = {
@@ -465,9 +278,9 @@ in
           # automatically before launching the container's init.
 
           # No hive-ci-register.service inside the container: all forge
-          # API calls (runner validation, token fetch) moved to the
+          # API calls (runner validation, token fetch) live in hive-c0re
-          # host-side hive-ci-prefetch.service. The core admin token
+          # (forge/ci_runner.rs), out of band. The core admin token never
-          # never enters this container.
+          # enters this container.
 
           # git is already in the gitea-actions-runner service PATH (the
           # nixpkgs module builds it from the package's runtime deps).
@@ -516,6 +329,28 @@ in
             # daemon is in-container or a shared host socket). `mkBefore` so
             # this runs ahead of any pre-steps the upstream module adds.
             serviceConfig.ExecStartPre = lib.mkBefore [
+              # Fail fast unless the runner can actually come up. Registration
+              # is done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints
+              # a token and, via hive-priv, writes /run/hive-ci/runner-token +
+              # restarts this unit — so the runner never does the forge
+              # round-trip on the boot path (that was the old prefetch's
+              # boot-critical wait that could trip the nspawn start timeout).
+              # Pass iff already registered (.runner present) OR a real
+              # (non-placeholder) token is in place; otherwise exit 1 so the
+              # runner just stays down (Restart=on-failure retries) until c0re
+              # registers it, instead of blocking the container's start. Runs
+              # ahead of the nix-daemon wait below — no point waiting for the
+              # daemon if there are no credentials to come up with.
+              (pkgs.writeShellScript "hive-ci-runner-precond" ''
+                RUNNER=/var/lib/gitea-runner/hive/.runner
+                TOKEN_FILE=/run/hive-ci/runner-token
+                if [ -f "$RUNNER" ]; then exit 0; fi
+                if [ -s "$TOKEN_FILE" ] && ! ${pkgs.gnugrep}/bin/grep -q '^TOKEN=placeholder$' "$TOKEN_FILE"; then
+                  exit 0
+                fi
+                echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2
+                exit 1
+              '')
               (pkgs.writeShellScript "wait-nix-daemon" ''
                 # Up to ~180s; the daemon is normally up within seconds, this
                 # only bites in the post-restart cold window. Non-fatal shape:
@@ -547,6 +382,15 @@ in
                 exit 1
               '')
             ];
+            # Registration is out of band: hive-c0re explicitly restarts this
+            # unit once it writes the real token, which is the primary path. As
+            # a safety net, if the precond above fires first (no creds yet at
+            # boot) retry until c0re registers, rather than staying down — with
+            # no start-limit rate cap so it keeps retrying however long forge /
+            # the core token take to settle.
+            serviceConfig.Restart = lib.mkForce "on-failure";
+            serviceConfig.RestartSec = 15;
+            startLimitIntervalSec = 0;
           };
         };
     };