From f4bcc591524de9917ae6d4f8e5b5906aed5417aa Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 16 Jul 2026 11:48:56 +0200 Subject: [PATCH] feat(#2415): move hive-ci runner registration off the boot-critical path (nix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes #2415. Registration no longer gates container@hive-ci start: - Retire the host-side hive-ci-prefetch.service (+ its 100-line script and the now-dead coreTokenPath/containerRoot let-bindings) — the forge round-trip it did on the boot path now lives in hive-c0re (forge/ci_runner.rs), run out of band during the startup sweep. - Drop the container@hive-ci TimeoutStartSec = mkForce 180s band-aid that papered over that boot-path wait tripping the nspawn start timeout (the ~60s restart loop of #2410). - gitea-runner-hive gains an ExecStartPre precond (ahead of the nix-daemon wait) that fails fast unless it is already registered (.runner present) or a real, non-placeholder token is in place — so missing creds just hold the runner down instead of blocking the container. Restart=on-failure (no start limit) self-heals it once hive-c0re writes the token; c0re's explicit restart is the primary path. - Set HYPERHIVE_FORGE_CI_ENABLED=1 on hive-c0re.service so the sweep registers. The tmpfiles TOKEN=placeholder seed + read-only bind-mount are unchanged; the core admin token still never enters the container. --- nix/host-modules/hive-ci.nix | 296 +++++++++-------------------------- 1 file changed, 70 insertions(+), 226 deletions(-) diff --git a/nix/host-modules/hive-ci.nix b/nix/host-modules/hive-ci.nix index d973e953..9cd41ed5 100644 --- a/nix/host-modules/hive-ci.nix +++ b/nix/host-modules/hive-ci.nix @@ -24,147 +24,6 @@ let useSelfSigned = caTrust.useSelfSigned; caContainerPath = caTrust.caContainerPath; - # hive-c0re writes its own admin token here on first forge startup. - # The token has read:admin + write:admin scopes — sufficient to call - # the runner registration-token API endpoint. - # This path is HOST-ONLY. It is never bind-mounted into hive-ci. - coreTokenPath = "/var/lib/hyperhive/forge-core-token"; - - # Container state root on the host. Non-ephemeral containers keep - # their filesystem here across reboots. The prefetch service accesses - # the runner's .runner file via this path to validate credentials - # without entering the container. - containerRoot = "/var/lib/nixos-containers/hive-ci"; - - # Host-side oneshot. Runs before `container@hive-ci.service`. - # - # The core token stays on the host. The container only ever sees the - # TOKEN= env-file populated here, never the core token itself. - # - # Flow: - # 1. If .runner exists: validate the runner ID against forge. - # Waits up to 60s for forge-core-token (hive-c0re writes it after - # forge container starts and admin is provisioned — this lags - # hive-c0re.service becoming active on first boot). - # 404 → purge .runner (re-registration needed). - # 000 (forge unreachable) → keep credentials, write placeholder. - # other non-200 → purge .runner. - # Token absent after 60s → keep credentials (safe — the runner - # holds valid creds; next boot will validate properly). - # 2. If .runner absent or just purged: wait up to 60s for both - # forge-core-token to appear AND forge API to respond, then - # fetch a fresh registration token and write TOKEN=. - # Exits non-zero if both timeout — systemd logs the failure; - # the placeholder from tmpfiles means the container still starts - # but the runner will error. Operator restarts the service once - # forge is healthy. - prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" '' - set -euo pipefail - TOKEN_FILE=/run/hive-ci/runner-token - FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}" - RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner" - - # Validate existing .runner credentials against forge. Purge if - # the runner was deleted (404) or the file is malformed. Keep if - # forge is unreachable (000) — transient outage shouldn't wipe creds. - # - # We wait up to 60s for forge-core-token to appear (hive-c0re writes - # it after provisioning the forge admin, which requires the forge - # container to start and become ready — this can lag behind - # hive-c0re.service becoming "active" on first boot). The same loop - # doubles as a wait for forge's API to become ready. - if [ -f "$RUNNER_FILE" ]; then - RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true) - if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then - echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2 - rm -f "$RUNNER_FILE" - else - # Wait for the core token before validating (same timeout as below). - CORE_TOKEN="" - for i in $(seq 1 60); do - if [ -f "${coreTokenPath}" ]; then - CORE_TOKEN=$(cat ${coreTokenPath}) - break - fi - sleep 1 - done - if [ -z "''${CORE_TOKEN:-}" ]; then - echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2 - else - HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \ - -H "Authorization: token $CORE_TOKEN" \ - "$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000") - if [ "$HTTP" = "404" ]; then - echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2 - rm -f "$RUNNER_FILE" - elif [ "$HTTP" = "000" ]; then - echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2 - elif [ "$HTTP" != "200" ]; then - echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2 - rm -f "$RUNNER_FILE" - fi - fi - fi - fi - - # .runner valid → write placeholder; gitea-actions-runner skips - # re-registration when .runner exists (TOKEN value is irrelevant). - if [ -f "$RUNNER_FILE" ]; then - echo "TOKEN=placeholder" > "$TOKEN_FILE" - chmod 600 "$TOKEN_FILE" - exit 0 - fi - - # First boot or stale creds purged — wait for forge-core-token, then - # fetch a fresh registration token. Single retry loop covers both: - # waiting for hive-c0re to write the token file AND for forge's API - # to become responsive (they race on first boot). - REG_TOKEN="" - for i in $(seq 1 60); do - if [ ! -f "${coreTokenPath}" ]; then - echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2 - sleep 1 - continue - fi - CORE_TOKEN=$(cat ${coreTokenPath}) - # Capture the HTTP status so a stale/invalid core token (401/403) is - # distinguished from a transient forge hiccup. A bare - # `curl -sf | jq` would let a forge-core-token that is stale for - # the current forge (e.g. after a forge rebuild) 401 silently on - # every attempt for the full 60s loop and exit with a misleading - # "core token absent or forge unreachable" — masking the real cause. - # Fail fast + loudly on 401/403 so the failure mode is legible and - # the operator/hive-c0re knows to re-mint forge-core-token. - RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \ - "$FORGE_URL/api/v1/admin/runners/registration-token" \ - -H "Authorization: token $CORE_TOKEN" || printf '\n000') - HTTP=$(printf '%s' "$RESP" | tail -n1) - BODY=$(printf '%s' "$RESP" | sed '$d') - case "$HTTP" in - 2*) - REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token) - if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi - echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2 - ;; - 401 | 403) - echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s." >&2 - exit 1 - ;; - *) - echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2 - ;; - esac - sleep 1 - done - - if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then - echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2 - exit 1 - fi - - echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE" - chmod 600 "$TOKEN_FILE" - ''; in { # Forgejo Actions runner in a `hive-ci` nixos-container. @@ -181,11 +40,12 @@ in # on first registration and reuses them on every subsequent start). # # Credential isolation: the forge admin token (`forge-core-token`) - # never enters the hive-ci container. A host-side oneshot service - # (`hive-ci-prefetch.service`) performs all forge API calls before - # the container starts and writes only the runner registration token - # to `/run/hive-ci/runner-token`. The container bind-mounts this - # file read-only and never has access to the wider admin token. + # never enters the hive-ci container. hive-c0re holds it and performs + # all forge API calls (runner validation + registration-token mint, + # in `forge/ci_runner.rs`); via hive-priv it writes only the runner + # registration token to the host env-file `/run/hive-ci/runner-token`, + # which the container bind-mounts read-only. The container never has + # access to the wider admin token. # # Nix builds inside the container use the shared /nix/store (standard # nixos-container behaviour) with sandbox-fallback = true, because @@ -293,83 +153,34 @@ in ]; # Create /run/hive-ci/ on the host and seed runner-token with a - # placeholder. hive-ci-prefetch.service overwrites it with the real - # token (or a fresh placeholder) before the container starts. The - # placeholder ensures the EnvironmentFile is always present even if - # the prefetch service hasn't run yet (e.g. tmpfiles-setup timing). + # placeholder. The container bind-mounts this file read-only; hive-c0re + # (via hive-priv's RegisterCiRunner) overwrites it with the real + # registration token when it registers the runner out of band. The + # placeholder keeps the runner's EnvironmentFile present from first boot, + # before c0re has registered — the runner's ExecStartPre precond (below) + # distinguishes the placeholder from a real token. systemd.tmpfiles.rules = [ "d /run/hive-ci 0700 root root -" "f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder" ]; - # Host-side oneshot: validates/refreshes runner credentials before - # the container starts. The core admin token stays on the host and - # is never bind-mounted into the container. Runs on every boot so - # stale .runner credentials (runner deleted from forge) are detected - # and the container re-registers on the next start. - systemd.services.hive-ci-prefetch = { - description = "Pre-fetch hive-ci runner registration token (host-side)"; - # Run before the container starts but after tmpfiles so the token - # file directory exists. After hive-c0re so the forge token is - # likely written (best-effort — the script handles the absent case). - after = [ - "systemd-tmpfiles-setup.service" - "hive-c0re.service" - ]; - before = [ "container@hive-ci.service" ]; - wantedBy = [ "container@hive-ci.service" ]; - # partOf binds this oneshot's lifecycle to the container: when the - # container is stopped or restarted, systemd propagates that to this - # unit so it re-runs on the NEXT container start. Without this, the - # RemainAfterExit=true oneshot stays "active (exited)" forever after - # its first run — so a container restart skips it and the stale - # runner-token file (a placeholder from a boot where the forge token - # wasn't ready yet, or a registration token consumed/rotated since) - # is never refreshed. The in-container register service then fails - # with "runner registration token not found". partOf guarantees a - # fresh token is fetched before every container (re)start. - # - # Unit name: a declarative `containers.` is the host unit - # `container@.service` (the nspawn template), NOT - # `nixos-container@…`. The earlier `nixos-container@hive-ci.service` - # matched no real unit, so before/wantedBy/partOf were silent - # no-ops — the partOf never bound, the oneshot stayed - # `active (exited)`, and the token was never refreshed on restart. - # Confirmed against the live `container@hive-matrix.service` unit - # during the matrix-outage incident. - partOf = [ "container@hive-ci.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - ExecStart = prefetchScript; - # Pin the journal identity; ExecStart is a writeShellScript whose - # store-path basename would otherwise be the journal identifier. - SyslogIdentifier = "hive-ci-prefetch"; - }; - }; + # Tell hive-c0re that CI is enabled so its startup sweep registers the + # runner. Registration moved OFF the container's boot-critical path into + # hive-c0re (`forge/ci_runner.rs`): it validates the persisted `.runner` + # against the forge and, when absent/stale, mints a registration token and + # hands it to hive-priv to write `/run/hive-ci/runner-token` + restart the + # runner. The forge admin token stays in hive-c0re; only the registration + # token reaches the host env-file the container mounts read-only. + systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_CI_ENABLED = "1"; # `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service` - # in self-signed mode (see the hive-ca-trust helper); merged with the - # hive-ci-specific start-timeout bump below. - systemd.services."container@hive-ci" = lib.mkMerge [ - caTrust.containerOrdering - { - # gitea-runner registration (hive-ci-prefetch, host-side) - # sits on the boot-critical path — the container's nspawn readiness - # is gated on the runner registering, a forge + core-token round - # trip that itself waits up to 60s for the core token. The default - # nspawn `TimeoutStartSec` (~60s, systemd's `DefaultTimeoutStartSec`) - # can therefore trip mid-register, especially right after a - # `.runner`-purge (every boot re-registers from scratch — see - # hive-ci-prefetch above) or during a hive-c0re restart storm that - # delays the core token. systemd then kills the half-started - # container and reschedules, producing an observed ~60s restart - # loop until the forge/token settle. Bumping the timeout well past - # the prefetch's own 60s wait gives one register attempt room to - # actually finish instead of being killed mid-flight. - serviceConfig.TimeoutStartSec = lib.mkForce "180s"; - } - ]; + # in self-signed mode (see the hive-ca-trust helper). Runner registration + # is no longer on the boot-critical path — hive-c0re owns it out of band — + # so the container boots immediately and needs no start-timeout bump. The + # former `TimeoutStartSec = mkForce "180s"` band-aid (which papered over a + # boot-path forge + core-token round-trip that could trip the nspawn start + # timeout into a ~60s restart loop) is gone with that move. + systemd.services."container@hive-ci" = caTrust.containerOrdering; containers.hive-ci = { autoStart = true; @@ -383,9 +194,10 @@ in hostBridge = networkCfg.bridgeName; bindMounts = { - # Pre-filled by hive-ci-prefetch.service (host-side) before the - # container starts. Read-only: the container reads TOKEN= from - # here; the core admin token never enters the container. + # Pre-filled by hive-c0re (via hive-priv) with the runner + # registration token; tmpfiles seeds a `TOKEN=placeholder` before + # that. Read-only: the container reads TOKEN= from here; the core + # admin token never enters the container. "/run/hive-ci/runner-token" = { hostPath = "/run/hive-ci/runner-token"; isReadOnly = true; @@ -446,10 +258,11 @@ in # the gateway vhost `forgeCfg.domain` proxies to the forge # on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect). url = "http://${forgeCfg.domain}"; - # EnvironmentFile providing TOKEN= — pre-filled by the - # host-side hive-ci-prefetch.service before the container - # starts; bind-mounted read-only from /run/hive-ci/runner-token - # on the host. + # EnvironmentFile providing TOKEN= — pre-filled by hive-c0re + # (via hive-priv) with the runner registration token; + # bind-mounted read-only from /run/hive-ci/runner-token on the + # host. The runner's ExecStartPre precond gates on this being a + # real (non-placeholder) token or an already-registered .runner. tokenFile = "/run/hive-ci/runner-token"; labels = cfg.labels; settings = { @@ -465,9 +278,9 @@ in # automatically before launching the container's init. # No hive-ci-register.service inside the container: all forge - # API calls (runner validation, token fetch) moved to the - # host-side hive-ci-prefetch.service. The core admin token - # never enters this container. + # API calls (runner validation, token fetch) live in hive-c0re + # (forge/ci_runner.rs), out of band. The core admin token never + # enters this container. # git is already in the gitea-actions-runner service PATH (the # nixpkgs module builds it from the package's runtime deps). @@ -516,6 +329,28 @@ in # daemon is in-container or a shared host socket). `mkBefore` so # this runs ahead of any pre-steps the upstream module adds. serviceConfig.ExecStartPre = lib.mkBefore [ + # Fail fast unless the runner can actually come up. Registration + # is done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints + # a token and, via hive-priv, writes /run/hive-ci/runner-token + + # restarts this unit — so the runner never does the forge + # round-trip on the boot path (that was the old prefetch's + # boot-critical wait that could trip the nspawn start timeout). + # Pass iff already registered (.runner present) OR a real + # (non-placeholder) token is in place; otherwise exit 1 so the + # runner just stays down (Restart=on-failure retries) until c0re + # registers it, instead of blocking the container's start. Runs + # ahead of the nix-daemon wait below — no point waiting for the + # daemon if there are no credentials to come up with. + (pkgs.writeShellScript "hive-ci-runner-precond" '' + RUNNER=/var/lib/gitea-runner/hive/.runner + TOKEN_FILE=/run/hive-ci/runner-token + if [ -f "$RUNNER" ]; then exit 0; fi + if [ -s "$TOKEN_FILE" ] && ! ${pkgs.gnugrep}/bin/grep -q '^TOKEN=placeholder$' "$TOKEN_FILE"; then + exit 0 + fi + echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2 + exit 1 + '') (pkgs.writeShellScript "wait-nix-daemon" '' # Up to ~180s; the daemon is normally up within seconds, this # only bites in the post-restart cold window. Non-fatal shape: @@ -547,6 +382,15 @@ in exit 1 '') ]; + # Registration is out of band: hive-c0re explicitly restarts this + # unit once it writes the real token, which is the primary path. As + # a safety net, if the precond above fires first (no creds yet at + # boot) retry until c0re registers, rather than staying down — with + # no start-limit rate cap so it keeps retrying however long forge / + # the core token take to settle. + serviceConfig.Restart = lib.mkForce "on-failure"; + serviceConfig.RestartSec = 15; + startLimitIntervalSec = 0; }; }; };