fix(#981): validate runner credentials on every boot, purge stale .runner
hive-ci-register.service now runs unconditionally on every boot (not just when .runner is absent). Before fetching a registration token it validates existing .runner credentials via the forge admin API: - 200: runner still registered, write dummy token and exit - 404: runner deleted from forge, purge .runner and re-register - 000: forge unreachable, keep credentials (runner surfaces the error) - other non-200 or malformed .runner: purge and re-register Removes ConditionPathExists so stale credentials from a wiped forge no longer block the runner indefinitely. Updates docs/ci.md to match.
This commit is contained in:
atlas
mara
parent
4bd0228de0
commit
9d816431dc
2 changed files with 72 additions and 32 deletions
|
|
@ -13,27 +13,64 @@ let
|
|||
# the runner registration-token API endpoint.
|
||||
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
|
||||
|
||||
# Oneshot run before gitea-runner-hive.service starts.
|
||||
# Oneshot run before gitea-runner-hive.service on every boot.
|
||||
#
|
||||
# The nixpkgs gitea-actions-runner module uses tokenFile as a systemd
|
||||
# EnvironmentFile (sets TOKEN= in the environment). EnvironmentFile is
|
||||
# loaded before any ExecStartPre, so the file MUST exist at service start —
|
||||
# an ExecStartPre override is too late. We solve this with:
|
||||
# 1. tmpfiles: pre-create /run/hive-ci/runner-token with TOKEN=placeholder
|
||||
# 2. hive-ci-register.service (this script): on first boot only,
|
||||
# fetch the real token from forge and overwrite with TOKEN=<real>.
|
||||
# 2. hive-ci-register.service (this script): runs on every boot:
|
||||
# a. If .runner exists: validate the runner ID against the forge admin
|
||||
# API. If the runner was deleted from forge (404), delete .runner so
|
||||
# the next step re-registers. If forge is unreachable (000), keep
|
||||
# the existing credentials (the runner itself will surface the error).
|
||||
# If valid, write a dummy token and exit — runner reuses .runner creds.
|
||||
# b. If .runner absent (first boot or stale creds were purged): fetch a
|
||||
# fresh registration token and write TOKEN=<real> to the EnvironmentFile.
|
||||
# 3. gitea-runner-hive.service: After=hive-ci-register.service
|
||||
#
|
||||
# On subsequent boots: .runner credentials file exists so the nixpkgs
|
||||
# register step exits early — TOKEN value in the EnvironmentFile is
|
||||
# irrelevant (placeholder is fine).
|
||||
registerScript = pkgs.writeShellScript "hive-ci-register" ''
|
||||
set -euo pipefail
|
||||
TOKEN_FILE=/run/hive-ci/runner-token
|
||||
CORE_TOKEN=$(cat /run/hive-ci/core-token)
|
||||
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
||||
RUNNER_FILE=/var/lib/gitea-runner/hive/.runner
|
||||
|
||||
# If .runner exists, validate runner credentials against forge.
|
||||
# Delete .runner if the runner entry is gone (404) — this triggers
|
||||
# fresh re-registration below. Skip validation if forge is unreachable
|
||||
# (000) so a transient forge outage doesn't wipe valid credentials.
|
||||
if [ -f "$RUNNER_FILE" ]; then
|
||||
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
|
||||
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
|
||||
echo "hive-ci: .runner malformed (no id), purging for re-registration" >&2
|
||||
rm -f "$RUNNER_FILE"
|
||||
else
|
||||
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token $CORE_TOKEN" \
|
||||
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
|
||||
if [ "$HTTP" = "404" ]; then
|
||||
echo "hive-ci: runner $RUNNER_ID not found in forge, purging .runner" >&2
|
||||
rm -f "$RUNNER_FILE"
|
||||
elif [ "$HTTP" = "000" ]; then
|
||||
echo "hive-ci: forge unreachable, keeping existing .runner credentials" >&2
|
||||
elif [ "$HTTP" != "200" ]; then
|
||||
echo "hive-ci: runner validation returned HTTP $HTTP, purging .runner" >&2
|
||||
rm -f "$RUNNER_FILE"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# If .runner still valid, write dummy token and exit — nixpkgs register
|
||||
# step exits early when .runner exists; TOKEN value is irrelevant.
|
||||
if [ -f "$RUNNER_FILE" ]; then
|
||||
echo "TOKEN=placeholder" > "$TOKEN_FILE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# First boot or stale creds purged — fetch a fresh registration token.
|
||||
# Retry up to 30s for forge to come up (containers autoStart in parallel).
|
||||
REG_TOKEN=""
|
||||
for i in $(seq 1 30); do
|
||||
REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \
|
||||
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
||||
|
|
@ -60,11 +97,13 @@ in
|
|||
# survive restarts (gitea-actions-runner writes them to its stateDir
|
||||
# on first registration and reuses them on every subsequent start).
|
||||
#
|
||||
# Auto-registration: on first boot a oneshot service fetches a runner
|
||||
# registration token from the forge's admin API using the core token
|
||||
# hive-c0re writes to /var/lib/hyperhive/forge-core-token. No manual
|
||||
# token handling needed — `forge.ci.enable = true` is the full operator
|
||||
# bootstrap.
|
||||
# Auto-registration: on every boot a oneshot service validates existing
|
||||
# runner credentials against forge (purging stale/.runner if the runner
|
||||
# was deleted from forge). On first boot (or after credential purge) it
|
||||
# fetches a fresh registration token via the forge admin API, using the
|
||||
# core token hive-c0re writes to /var/lib/hyperhive/forge-core-token.
|
||||
# No manual token handling needed — `forge.ci.enable = true` is the
|
||||
# full operator bootstrap.
|
||||
#
|
||||
# Nix builds inside the container use the shared /nix/store (standard
|
||||
# nixos-container behaviour) with sandbox-fallback = true, because
|
||||
|
|
@ -197,10 +236,9 @@ in
|
|||
# Pre-create the EnvironmentFile so gitea-runner-hive.service can
|
||||
# always load it. The nixpkgs module sets EnvironmentFile=tokenFile
|
||||
# which systemd loads before any ExecStartPre — the file must exist
|
||||
# at service-start time. On first boot hive-ci-register.service
|
||||
# overwrites this placeholder with the real token before the runner
|
||||
# starts. On subsequent boots the placeholder is harmless because
|
||||
# the nixpkgs register step exits early when .runner already exists.
|
||||
# at service-start time. hive-ci-register.service overwrites this
|
||||
# placeholder before the runner starts (real token on first/re-reg,
|
||||
# dummy placeholder when .runner credentials are still valid).
|
||||
#
|
||||
# Note: `f` doesn't create parent directories, but /run/hive-ci/
|
||||
# is guaranteed to exist by the time container systemd starts:
|
||||
|
|
@ -211,19 +249,17 @@ in
|
|||
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
||||
];
|
||||
|
||||
# Oneshot that fetches a runner registration token from the forge
|
||||
# admin API and writes it to the EnvironmentFile. Runs only on
|
||||
# first boot (ConditionPathExists gates on .runner absence) and
|
||||
# before gitea-runner-hive.service via the After= dependency below.
|
||||
# Oneshot that validates/refreshes runner credentials on every boot:
|
||||
# validates existing .runner against forge (purges if stale/404),
|
||||
# then fetches a fresh registration token if .runner is absent.
|
||||
# Runs before gitea-runner-hive.service via the After= dependency.
|
||||
systemd.services.hive-ci-register = {
|
||||
description = "Register hive-ci runner against Forgejo (first boot)";
|
||||
description = "Validate/register hive-ci runner against Forgejo";
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
ExecStart = registerScript;
|
||||
};
|
||||
# Only needed before .runner credentials exist.
|
||||
unitConfig.ConditionPathExists = "!/var/lib/gitea-runner/hive/.runner";
|
||||
};
|
||||
|
||||
# Runner must start after the register oneshot so the EnvironmentFile
|
||||
|
|
|
|||
Loading…
Reference in a new issue