From 9d816431dc4676cd4540d0a27611b7a7b6006980 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 2 Jun 2026 00:10:15 +0200 Subject: [PATCH] fix(#981): validate runner credentials on every boot, purge stale .runner hive-ci-register.service now runs unconditionally on every boot (not just when .runner is absent). Before fetching a registration token it validates existing .runner credentials via the forge admin API: - 200: runner still registered, write dummy token and exit - 404: runner deleted from forge, purge .runner and re-register - 000: forge unreachable, keep credentials (runner surfaces the error) - other non-200 or malformed .runner: purge and re-register Removes ConditionPathExists so stale credentials from a wiped forge no longer block the runner indefinitely. Updates docs/ci.md to match. --- docs/ci.md | 22 ++++++----- nix/modules/hive-ci.nix | 82 +++++++++++++++++++++++++++++------------ 2 files changed, 72 insertions(+), 32 deletions(-) diff --git a/docs/ci.md b/docs/ci.md index d2669fe1..4b306deb 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -18,15 +18,19 @@ Set `services.hyperhive.ci.enable = true` in the host NixOS config. That's it ## Auto-registration flow -1. On first container boot, `systemd` runs the `ExecStartPre` script (from `hive-ci.nix:autoRegisterScript`). -2. Script checks for existing runner credentials (`/var/lib/gitea-runner/hive/.runner`): - - If found: container was previously registered — write dummy token to satisfy module's path check, exit. - - If not found: first boot — proceed to registration. -3. Script reads hive-c0re's admin token from `/run/hive-ci/core-token` (bind-mounted from `/var/lib/hyperhive/forge-core-token`). -4. Calls `POST /api/v1/admin/runners/registration-token` on the forge using `${pkgs.curl}/bin/curl` and `${pkgs.jq}/bin/jq` (absolute nix store paths — the container doesn't need these in systemPackages). Retries for 30s in case forge is still starting. -5. Writes real token to `/run/hive-ci/runner-token`. -6. `gitea-actions-runner` reads the token and registers itself, persisting credentials (`.runner` file) to stateDir. -7. On subsequent boots, runner reuses the `.runner` credentials — the preStart script detects the existing `.runner` file and writes a dummy token instead. The runner ignores the token file when `.runner` already exists. +`hive-ci-register.service` is a oneshot that runs on **every boot** before `gitea-runner-hive.service`. It handles both first-run registration and stale-credential detection. + +### Every boot + +1. Read the core admin token from `/run/hive-ci/core-token` (bind-mounted from `/var/lib/hyperhive/forge-core-token`). +2. If `.runner` exists: validate the runner ID against `GET /api/v1/admin/runners/{id}` using the core token: + - **200**: runner still registered — write dummy `TOKEN=placeholder` and exit; runner reuses `.runner` credentials. + - **404**: runner was deleted from forge (e.g. after a wipe) — delete `.runner`, proceed to re-registration below. + - **000** (forge unreachable): keep existing `.runner`; the runner itself will surface the connectivity error. + - **other non-200**: treat as stale, delete `.runner` and re-register. + - **malformed `.runner`** (no `id` field): delete and re-register. +3. If `.runner` is absent (first boot or purged above): fetch a fresh registration token from `GET /api/v1/admin/runners/registration-token`. Retries for 30s in case forge is still starting. Writes `TOKEN=` to `/run/hive-ci/runner-token`. +4. `gitea-actions-runner` reads the token, registers itself, and persists credentials to `.runner`. On subsequent boots step 2 validates these credentials and fast-paths past registration. ## CI workflow diff --git a/nix/modules/hive-ci.nix b/nix/modules/hive-ci.nix index af1ed595..66d8e510 100644 --- a/nix/modules/hive-ci.nix +++ b/nix/modules/hive-ci.nix @@ -13,27 +13,64 @@ let # the runner registration-token API endpoint. coreTokenPath = "/var/lib/hyperhive/forge-core-token"; - # Oneshot run before gitea-runner-hive.service starts. + # Oneshot run before gitea-runner-hive.service on every boot. # # The nixpkgs gitea-actions-runner module uses tokenFile as a systemd # EnvironmentFile (sets TOKEN= in the environment). EnvironmentFile is # loaded before any ExecStartPre, so the file MUST exist at service start — # an ExecStartPre override is too late. We solve this with: # 1. tmpfiles: pre-create /run/hive-ci/runner-token with TOKEN=placeholder - # 2. hive-ci-register.service (this script): on first boot only, - # fetch the real token from forge and overwrite with TOKEN=. + # 2. hive-ci-register.service (this script): runs on every boot: + # a. If .runner exists: validate the runner ID against the forge admin + # API. If the runner was deleted from forge (404), delete .runner so + # the next step re-registers. If forge is unreachable (000), keep + # the existing credentials (the runner itself will surface the error). + # If valid, write a dummy token and exit — runner reuses .runner creds. + # b. If .runner absent (first boot or stale creds were purged): fetch a + # fresh registration token and write TOKEN= to the EnvironmentFile. # 3. gitea-runner-hive.service: After=hive-ci-register.service - # - # On subsequent boots: .runner credentials file exists so the nixpkgs - # register step exits early — TOKEN value in the EnvironmentFile is - # irrelevant (placeholder is fine). registerScript = pkgs.writeShellScript "hive-ci-register" '' set -euo pipefail TOKEN_FILE=/run/hive-ci/runner-token CORE_TOKEN=$(cat /run/hive-ci/core-token) FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}" + RUNNER_FILE=/var/lib/gitea-runner/hive/.runner + # If .runner exists, validate runner credentials against forge. + # Delete .runner if the runner entry is gone (404) — this triggers + # fresh re-registration below. Skip validation if forge is unreachable + # (000) so a transient forge outage doesn't wipe valid credentials. + if [ -f "$RUNNER_FILE" ]; then + RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true) + if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then + echo "hive-ci: .runner malformed (no id), purging for re-registration" >&2 + rm -f "$RUNNER_FILE" + else + HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: token $CORE_TOKEN" \ + "$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000") + if [ "$HTTP" = "404" ]; then + echo "hive-ci: runner $RUNNER_ID not found in forge, purging .runner" >&2 + rm -f "$RUNNER_FILE" + elif [ "$HTTP" = "000" ]; then + echo "hive-ci: forge unreachable, keeping existing .runner credentials" >&2 + elif [ "$HTTP" != "200" ]; then + echo "hive-ci: runner validation returned HTTP $HTTP, purging .runner" >&2 + rm -f "$RUNNER_FILE" + fi + fi + fi + + # If .runner still valid, write dummy token and exit — nixpkgs register + # step exits early when .runner exists; TOKEN value is irrelevant. + if [ -f "$RUNNER_FILE" ]; then + echo "TOKEN=placeholder" > "$TOKEN_FILE" + exit 0 + fi + + # First boot or stale creds purged — fetch a fresh registration token. # Retry up to 30s for forge to come up (containers autoStart in parallel). + REG_TOKEN="" for i in $(seq 1 30); do REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \ "$FORGE_URL/api/v1/admin/runners/registration-token" \ @@ -60,11 +97,13 @@ in # survive restarts (gitea-actions-runner writes them to its stateDir # on first registration and reuses them on every subsequent start). # - # Auto-registration: on first boot a oneshot service fetches a runner - # registration token from the forge's admin API using the core token - # hive-c0re writes to /var/lib/hyperhive/forge-core-token. No manual - # token handling needed — `forge.ci.enable = true` is the full operator - # bootstrap. + # Auto-registration: on every boot a oneshot service validates existing + # runner credentials against forge (purging stale/.runner if the runner + # was deleted from forge). On first boot (or after credential purge) it + # fetches a fresh registration token via the forge admin API, using the + # core token hive-c0re writes to /var/lib/hyperhive/forge-core-token. + # No manual token handling needed — `forge.ci.enable = true` is the + # full operator bootstrap. # # Nix builds inside the container use the shared /nix/store (standard # nixos-container behaviour) with sandbox-fallback = true, because @@ -197,10 +236,9 @@ in # Pre-create the EnvironmentFile so gitea-runner-hive.service can # always load it. The nixpkgs module sets EnvironmentFile=tokenFile # which systemd loads before any ExecStartPre — the file must exist - # at service-start time. On first boot hive-ci-register.service - # overwrites this placeholder with the real token before the runner - # starts. On subsequent boots the placeholder is harmless because - # the nixpkgs register step exits early when .runner already exists. + # at service-start time. hive-ci-register.service overwrites this + # placeholder before the runner starts (real token on first/re-reg, + # dummy placeholder when .runner credentials are still valid). # # Note: `f` doesn't create parent directories, but /run/hive-ci/ # is guaranteed to exist by the time container systemd starts: @@ -211,19 +249,17 @@ in "f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder" ]; - # Oneshot that fetches a runner registration token from the forge - # admin API and writes it to the EnvironmentFile. Runs only on - # first boot (ConditionPathExists gates on .runner absence) and - # before gitea-runner-hive.service via the After= dependency below. + # Oneshot that validates/refreshes runner credentials on every boot: + # validates existing .runner against forge (purges if stale/404), + # then fetches a fresh registration token if .runner is absent. + # Runs before gitea-runner-hive.service via the After= dependency. systemd.services.hive-ci-register = { - description = "Register hive-ci runner against Forgejo (first boot)"; + description = "Validate/register hive-ci runner against Forgejo"; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = registerScript; }; - # Only needed before .runner credentials exist. - unitConfig.ConditionPathExists = "!/var/lib/gitea-runner/hive/.runner"; }; # Runner must start after the register oneshot so the EnvironmentFile