vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(#2415): move hive-ci runner registration off the boot-critical path (nix)

Browse source 
Completes #2415. Registration no longer gates container@hive-ci start:
- Retire the host-side hive-ci-prefetch.service (+ its 100-line script and
  the now-dead coreTokenPath/containerRoot let-bindings) — the forge round-trip
  it did on the boot path now lives in hive-c0re (forge/ci_runner.rs), run out
  of band during the startup sweep.
- Drop the container@hive-ci TimeoutStartSec = mkForce 180s band-aid that
  papered over that boot-path wait tripping the nspawn start timeout (the ~60s
  restart loop of #2410).
- gitea-runner-hive gains an ExecStartPre precond (ahead of the nix-daemon
  wait) that fails fast unless it is already registered (.runner present) or a
  real, non-placeholder token is in place — so missing creds just hold the
  runner down instead of blocking the container. Restart=on-failure (no start
  limit) self-heals it once hive-c0re writes the token; c0re's explicit restart
  is the primary path.
- Set HYPERHIVE_FORGE_CI_ENABLED=1 on hive-c0re.service so the sweep registers.
The tmpfiles TOKEN=placeholder seed + read-only bind-mount are unchanged; the
core admin token still never enters the container.
This commit is contained in:
atlas 2026-07-16 11:48:56 +02:00 • committed by mara
commit f4bcc59152
1 changed files with 70 additions and 226 deletions
Download patch file Download diff file Expand all files Collapse all files

296
nix/host-modules/hive-ci.nix
View file

@ -24,147 +24,6 @@ let
useSelfSigned = caTrust.useSelfSigned;
caContainerPath = caTrust.caContainerPath;
# hive-c0re writes its own admin token here on first forge startup.
# The token has read:admin + write:admin scopes — sufficient to call
# the runner registration-token API endpoint.
# This path is HOST-ONLY. It is never bind-mounted into hive-ci.
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
# Container state root on the host. Non-ephemeral containers keep
# their filesystem here across reboots. The prefetch service accesses
# the runner's .runner file via this path to validate credentials
# without entering the container.
containerRoot = "/var/lib/nixos-containers/hive-ci";
# Host-side oneshot. Runs before `container@hive-ci.service`.
#
# The core token stays on the host. The container only ever sees the
# TOKEN= env-file populated here, never the core token itself.
#
# Flow:
# 1. If .runner exists: validate the runner ID against forge.
# Waits up to 60s for forge-core-token (hive-c0re writes it after
# forge container starts and admin is provisioned — this lags
# hive-c0re.service becoming active on first boot).
# 404 → purge .runner (re-registration needed).
# 000 (forge unreachable) → keep credentials, write placeholder.
# other non-200 → purge .runner.
# Token absent after 60s → keep credentials (safe — the runner
# holds valid creds; next boot will validate properly).
# 2. If .runner absent or just purged: wait up to 60s for both
# forge-core-token to appear AND forge API to respond, then
# fetch a fresh registration token and write TOKEN=<real>.
# Exits non-zero if both timeout — systemd logs the failure;
# the placeholder from tmpfiles means the container still starts
# but the runner will error. Operator restarts the service once
# forge is healthy.
prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
set -euo pipefail
TOKEN_FILE=/run/hive-ci/runner-token
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
# Validate existing .runner credentials against forge. Purge if
# the runner was deleted (404) or the file is malformed. Keep if
# forge is unreachable (000) — transient outage shouldn't wipe creds.
#
# We wait up to 60s for forge-core-token to appear (hive-c0re writes
# it after provisioning the forge admin, which requires the forge
# container to start and become ready — this can lag behind
# hive-c0re.service becoming "active" on first boot). The same loop
# doubles as a wait for forge's API to become ready.
if [ -f "$RUNNER_FILE" ]; then
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
rm -f "$RUNNER_FILE"
else
# Wait for the core token before validating (same timeout as below).
CORE_TOKEN=""
for i in $(seq 1 60); do
if [ -f "${coreTokenPath}" ]; then
CORE_TOKEN=$(cat ${coreTokenPath})
break
fi
sleep 1
done
if [ -z "''${CORE_TOKEN:-}" ]; then
echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
else
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
-H "Authorization: token $CORE_TOKEN" \
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
if [ "$HTTP" = "404" ]; then
echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
rm -f "$RUNNER_FILE"
elif [ "$HTTP" = "000" ]; then
echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
elif [ "$HTTP" != "200" ]; then
echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
rm -f "$RUNNER_FILE"
fi
fi
fi
fi
# .runner valid → write placeholder; gitea-actions-runner skips
# re-registration when .runner exists (TOKEN value is irrelevant).
if [ -f "$RUNNER_FILE" ]; then
echo "TOKEN=placeholder" > "$TOKEN_FILE"
chmod 600 "$TOKEN_FILE"
exit 0
fi
# First boot or stale creds purged — wait for forge-core-token, then
# fetch a fresh registration token. Single retry loop covers both:
# waiting for hive-c0re to write the token file AND for forge's API
# to become responsive (they race on first boot).
REG_TOKEN=""
for i in $(seq 1 60); do
if [ ! -f "${coreTokenPath}" ]; then
echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
sleep 1
continue
fi
CORE_TOKEN=$(cat ${coreTokenPath})
# Capture the HTTP status so a stale/invalid core token (401/403) is
# distinguished from a transient forge hiccup. A bare
# `curl -sf | jq` would let a forge-core-token that is stale for
# the current forge (e.g. after a forge rebuild) 401 silently on
# every attempt for the full 60s loop and exit with a misleading
# "core token absent or forge unreachable" — masking the real cause.
# Fail fast + loudly on 401/403 so the failure mode is legible and
# the operator/hive-c0re knows to re-mint forge-core-token.
RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \
"$FORGE_URL/api/v1/admin/runners/registration-token" \
-H "Authorization: token $CORE_TOKEN" || printf '\n000')
HTTP=$(printf '%s' "$RESP" | tail -n1)
BODY=$(printf '%s' "$RESP" | sed '$d')
case "$HTTP" in
2*)
REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token)
if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi
echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2
;;
401 | 403)
echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s." >&2
exit 1
;;
*)
echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2
;;
esac
sleep 1
done
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
exit 1
fi
echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
chmod 600 "$TOKEN_FILE"
'';
in
{
# Forgejo Actions runner in a `hive-ci` nixos-container.
@ -181,11 +40,12 @@ in
# on first registration and reuses them on every subsequent start).
#
# Credential isolation: the forge admin token (`forge-core-token`)
# never enters the hive-ci container. A host-side oneshot service
# (`hive-ci-prefetch.service`) performs all forge API calls before
# the container starts and writes only the runner registration token
# to `/run/hive-ci/runner-token`. The container bind-mounts this
# file read-only and never has access to the wider admin token.
# never enters the hive-ci container. hive-c0re holds it and performs
# all forge API calls (runner validation + registration-token mint,
# in `forge/ci_runner.rs`); via hive-priv it writes only the runner
# registration token to the host env-file `/run/hive-ci/runner-token`,
# which the container bind-mounts read-only. The container never has
# access to the wider admin token.
#
# Nix builds inside the container use the shared /nix/store (standard
# nixos-container behaviour) with sandbox-fallback = true, because
@ -293,83 +153,34 @@ in
];
# Create /run/hive-ci/ on the host and seed runner-token with a
# placeholder. hive-ci-prefetch.service overwrites it with the real
# token (or a fresh placeholder) before the container starts. The
# placeholder ensures the EnvironmentFile is always present even if
# the prefetch service hasn't run yet (e.g. tmpfiles-setup timing).
# placeholder. The container bind-mounts this file read-only; hive-c0re
# (via hive-priv's RegisterCiRunner) overwrites it with the real
# registration token when it registers the runner out of band. The
# placeholder keeps the runner's EnvironmentFile present from first boot,
# before c0re has registered — the runner's ExecStartPre precond (below)
# distinguishes the placeholder from a real token.
systemd.tmpfiles.rules = [
"d /run/hive-ci 0700 root root -"
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
];
# Host-side oneshot: validates/refreshes runner credentials before
# the container starts. The core admin token stays on the host and
# is never bind-mounted into the container. Runs on every boot so
# stale .runner credentials (runner deleted from forge) are detected
# and the container re-registers on the next start.
systemd.services.hive-ci-prefetch = {
description = "Pre-fetch hive-ci runner registration token (host-side)";
# Run before the container starts but after tmpfiles so the token
# file directory exists. After hive-c0re so the forge token is
# likely written (best-effort — the script handles the absent case).
after = [
"systemd-tmpfiles-setup.service"
"hive-c0re.service"
];
before = [ "container@hive-ci.service" ];
wantedBy = [ "container@hive-ci.service" ];
# partOf binds this oneshot's lifecycle to the container: when the
# container is stopped or restarted, systemd propagates that to this
# unit so it re-runs on the NEXT container start. Without this, the
# RemainAfterExit=true oneshot stays "active (exited)" forever after
# its first run — so a container restart skips it and the stale
# runner-token file (a placeholder from a boot where the forge token
# wasn't ready yet, or a registration token consumed/rotated since)
# is never refreshed. The in-container register service then fails
# with "runner registration token not found". partOf guarantees a
# fresh token is fetched before every container (re)start.
#
# Unit name: a declarative `containers.<name>` is the host unit
# `container@<name>.service` (the nspawn template), NOT
# `nixos-container@…`. The earlier `nixos-container@hive-ci.service`
# matched no real unit, so before/wantedBy/partOf were silent
# no-ops — the partOf never bound, the oneshot stayed
# `active (exited)`, and the token was never refreshed on restart.
# Confirmed against the live `container@hive-matrix.service` unit
# during the matrix-outage incident.
partOf = [ "container@hive-ci.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = prefetchScript;
# Pin the journal identity; ExecStart is a writeShellScript whose
# store-path basename would otherwise be the journal identifier.
SyslogIdentifier = "hive-ci-prefetch";
};
};
# Tell hive-c0re that CI is enabled so its startup sweep registers the
# runner. Registration moved OFF the container's boot-critical path into
# hive-c0re (`forge/ci_runner.rs`): it validates the persisted `.runner`
# against the forge and, when absent/stale, mints a registration token and
# hands it to hive-priv to write `/run/hive-ci/runner-token` + restart the
# runner. The forge admin token stays in hive-c0re; only the registration
# token reaches the host env-file the container mounts read-only.
systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_CI_ENABLED = "1";
# `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
# in self-signed mode (see the hive-ca-trust helper); merged with the
# hive-ci-specific start-timeout bump below.
systemd.services."container@hive-ci" = lib.mkMerge [
caTrust.containerOrdering
{
# gitea-runner registration (hive-ci-prefetch, host-side)
# sits on the boot-critical path — the container's nspawn readiness
# is gated on the runner registering, a forge + core-token round
# trip that itself waits up to 60s for the core token. The default
# nspawn `TimeoutStartSec` (~60s, systemd's `DefaultTimeoutStartSec`)
# can therefore trip mid-register, especially right after a
# `.runner`-purge (every boot re-registers from scratch — see
# hive-ci-prefetch above) or during a hive-c0re restart storm that
# delays the core token. systemd then kills the half-started
# container and reschedules, producing an observed ~60s restart
# loop until the forge/token settle. Bumping the timeout well past
# the prefetch's own 60s wait gives one register attempt room to
# actually finish instead of being killed mid-flight.
serviceConfig.TimeoutStartSec = lib.mkForce "180s";
}
];
# in self-signed mode (see the hive-ca-trust helper). Runner registration
# is no longer on the boot-critical path — hive-c0re owns it out of band —
# so the container boots immediately and needs no start-timeout bump. The
# former `TimeoutStartSec = mkForce "180s"` band-aid (which papered over a
# boot-path forge + core-token round-trip that could trip the nspawn start
# timeout into a ~60s restart loop) is gone with that move.
systemd.services."container@hive-ci" = caTrust.containerOrdering;
containers.hive-ci = {
autoStart = true;
@ -383,9 +194,10 @@ in
hostBridge = networkCfg.bridgeName;
bindMounts = {
# Pre-filled by hive-ci-prefetch.service (host-side) before the
# container starts. Read-only: the container reads TOKEN= from
# here; the core admin token never enters the container.
# Pre-filled by hive-c0re (via hive-priv) with the runner
# registration token; tmpfiles seeds a `TOKEN=placeholder` before
# that. Read-only: the container reads TOKEN= from here; the core
# admin token never enters the container.
"/run/hive-ci/runner-token" = {
hostPath = "/run/hive-ci/runner-token";
isReadOnly = true;
@ -446,10 +258,11 @@ in
# the gateway vhost `forgeCfg.domain` proxies to the forge
# on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect).
url = "http://${forgeCfg.domain}";
# EnvironmentFile providing TOKEN= — pre-filled by the
# host-side hive-ci-prefetch.service before the container
# starts; bind-mounted read-only from /run/hive-ci/runner-token
# on the host.
# EnvironmentFile providing TOKEN= — pre-filled by hive-c0re
# (via hive-priv) with the runner registration token;
# bind-mounted read-only from /run/hive-ci/runner-token on the
# host. The runner's ExecStartPre precond gates on this being a
# real (non-placeholder) token or an already-registered .runner.
tokenFile = "/run/hive-ci/runner-token";
labels = cfg.labels;
settings = {
@ -465,9 +278,9 @@ in
# automatically before launching the container's init.
# No hive-ci-register.service inside the container: all forge
# API calls (runner validation, token fetch) moved to the
# host-side hive-ci-prefetch.service. The core admin token
# never enters this container.
# API calls (runner validation, token fetch) live in hive-c0re
# (forge/ci_runner.rs), out of band. The core admin token never
# enters this container.
# git is already in the gitea-actions-runner service PATH (the
# nixpkgs module builds it from the package's runtime deps).
@ -516,6 +329,28 @@ in
# daemon is in-container or a shared host socket). `mkBefore` so
# this runs ahead of any pre-steps the upstream module adds.
serviceConfig.ExecStartPre = lib.mkBefore [
# Fail fast unless the runner can actually come up. Registration
# is done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints
# a token and, via hive-priv, writes /run/hive-ci/runner-token +
# restarts this unit — so the runner never does the forge
# round-trip on the boot path (that was the old prefetch's
# boot-critical wait that could trip the nspawn start timeout).
# Pass iff already registered (.runner present) OR a real
# (non-placeholder) token is in place; otherwise exit 1 so the
# runner just stays down (Restart=on-failure retries) until c0re
# registers it, instead of blocking the container's start. Runs
# ahead of the nix-daemon wait below — no point waiting for the
# daemon if there are no credentials to come up with.
(pkgs.writeShellScript "hive-ci-runner-precond" ''
RUNNER=/var/lib/gitea-runner/hive/.runner
TOKEN_FILE=/run/hive-ci/runner-token
if [ -f "$RUNNER" ]; then exit 0; fi
if [ -s "$TOKEN_FILE" ] && ! ${pkgs.gnugrep}/bin/grep -q '^TOKEN=placeholder$' "$TOKEN_FILE"; then
exit 0
fi
echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2
exit 1
'')
(pkgs.writeShellScript "wait-nix-daemon" ''
# Up to ~180s; the daemon is normally up within seconds, this
# only bites in the post-restart cold window. Non-fatal shape:
@ -547,6 +382,15 @@ in
exit 1
'')
];
# Registration is out of band: hive-c0re explicitly restarts this
# unit once it writes the real token, which is the primary path. As
# a safety net, if the precond above fires first (no creds yet at
# boot) retry until c0re registers, rather than staying down — with
# no start-limit rate cap so it keeps retrying however long forge /
# the core token take to settle.
serviceConfig.Restart = lib.mkForce "on-failure";
serviceConfig.RestartSec = 15;
startLimitIntervalSec = 0;
};
};
};