4f446269ae
The hive-ci-prefetch oneshot has RemainAfterExit=true and is wired to the container only via wantedBy + before. Once it runs successfully it stays 'active (exited)' indefinitely, so systemd skips it on subsequent container restarts. The runner-token file it wrote is never refreshed. This breaks the runner after its first registration: the token written on the first successful boot is either a placeholder (forge-core-token wasn't ready yet) or a registration token that has since been consumed/rotated. On the next container restart prefetch does not re-run, the stale token persists, and the in-container register service fails with 'invalid_argument: runner registration token not found' — exactly the symptom in the field (worked briefly, two orphan runners registered, then permanently offline). Add partOf = [ nixos-container@hive-ci.service ] so a container stop/restart propagates to the prefetch unit, forcing it to re-run and fetch a fresh registration token before the container comes back up. before= still orders it ahead of the container start within the same transaction.
346 lines
14 KiB
Nix
346 lines
14 KiB
Nix
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.forge.ci;
|
|
forgeCfg = config.services.hyperhive.forge;
|
|
|
|
# hive-c0re writes its own admin token here on first forge startup.
|
|
# The token has read:admin + write:admin scopes — sufficient to call
|
|
# the runner registration-token API endpoint.
|
|
# This path is HOST-ONLY. It is never bind-mounted into hive-ci.
|
|
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
|
|
|
|
# Container state root on the host. Non-ephemeral containers keep
|
|
# their filesystem here across reboots. The prefetch service accesses
|
|
# the runner's .runner file via this path to validate credentials
|
|
# without entering the container.
|
|
containerRoot = "/var/lib/nixos-containers/hive-ci";
|
|
|
|
# Host-side oneshot. Runs before `nixos-container@hive-ci.service`.
|
|
#
|
|
# The core token stays on the host. The container only ever sees the
|
|
# TOKEN= env-file populated here, never the core token itself.
|
|
#
|
|
# Flow:
|
|
# 1. If .runner exists: validate the runner ID against forge.
|
|
# Waits up to 60s for forge-core-token (hive-c0re writes it after
|
|
# forge container starts and admin is provisioned — this lags
|
|
# hive-c0re.service becoming active on first boot).
|
|
# 404 → purge .runner (re-registration needed).
|
|
# 000 (forge unreachable) → keep credentials, write placeholder.
|
|
# other non-200 → purge .runner.
|
|
# Token absent after 60s → keep credentials (safe — the runner
|
|
# holds valid creds; next boot will validate properly).
|
|
# 2. If .runner absent or just purged: wait up to 60s for both
|
|
# forge-core-token to appear AND forge API to respond, then
|
|
# fetch a fresh registration token and write TOKEN=<real>.
|
|
# Exits non-zero if both timeout — systemd logs the failure;
|
|
# the placeholder from tmpfiles means the container still starts
|
|
# but the runner will error. Operator restarts the service once
|
|
# forge is healthy.
|
|
prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
|
|
set -euo pipefail
|
|
TOKEN_FILE=/run/hive-ci/runner-token
|
|
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
|
RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
|
|
|
|
# Validate existing .runner credentials against forge. Purge if
|
|
# the runner was deleted (404) or the file is malformed. Keep if
|
|
# forge is unreachable (000) — transient outage shouldn't wipe creds.
|
|
#
|
|
# We wait up to 60s for forge-core-token to appear (hive-c0re writes
|
|
# it after provisioning the forge admin, which requires the forge
|
|
# container to start and become ready — this can lag behind
|
|
# hive-c0re.service becoming "active" on first boot). The same loop
|
|
# doubles as a wait for forge's API to become ready.
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
|
|
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
|
|
echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
else
|
|
# Wait for the core token before validating (same timeout as below).
|
|
CORE_TOKEN=""
|
|
for i in $(seq 1 60); do
|
|
if [ -f "${coreTokenPath}" ]; then
|
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
if [ -z "''${CORE_TOKEN:-}" ]; then
|
|
echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
|
|
else
|
|
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
|
|
-H "Authorization: token $CORE_TOKEN" \
|
|
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
|
|
if [ "$HTTP" = "404" ]; then
|
|
echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
elif [ "$HTTP" = "000" ]; then
|
|
echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
|
|
elif [ "$HTTP" != "200" ]; then
|
|
echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# .runner valid → write placeholder; gitea-actions-runner skips
|
|
# re-registration when .runner exists (TOKEN value is irrelevant).
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
echo "TOKEN=placeholder" > "$TOKEN_FILE"
|
|
chmod 600 "$TOKEN_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
# First boot or stale creds purged — wait for forge-core-token, then
|
|
# fetch a fresh registration token. Single retry loop covers both:
|
|
# waiting for hive-c0re to write the token file AND for forge's API
|
|
# to become responsive (they race on first boot).
|
|
REG_TOKEN=""
|
|
for i in $(seq 1 60); do
|
|
if [ ! -f "${coreTokenPath}" ]; then
|
|
echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
|
|
sleep 1
|
|
continue
|
|
fi
|
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
|
REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \
|
|
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
|
-H "Authorization: token $CORE_TOKEN" \
|
|
| ${pkgs.jq}/bin/jq -r .token) && break
|
|
sleep 1
|
|
done
|
|
|
|
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
|
|
echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
|
|
chmod 600 "$TOKEN_FILE"
|
|
'';
|
|
in
|
|
{
|
|
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
# Shares host netns (same as hive-forge), so the runner reaches
|
|
# the forge at `http://127.0.0.1:<httpPort>` without extra plumbing.
|
|
# Container is non-ephemeral: the runner's registered credentials
|
|
# survive restarts (gitea-actions-runner writes them to its stateDir
|
|
# on first registration and reuses them on every subsequent start).
|
|
#
|
|
# Credential isolation: the forge admin token (`forge-core-token`)
|
|
# never enters the hive-ci container. A host-side oneshot service
|
|
# (`hive-ci-prefetch.service`) performs all forge API calls before
|
|
# the container starts and writes only the runner registration token
|
|
# to `/run/hive-ci/runner-token`. The container bind-mounts this
|
|
# file read-only and never has access to the wider admin token.
|
|
#
|
|
# Nix builds inside the container use the shared /nix/store (standard
|
|
# nixos-container behaviour) with sandbox-fallback = true, because
|
|
# nspawn containers can't create the user-namespaces that nix sandboxing
|
|
# requires. See docs/gotchas.md.
|
|
|
|
options.services.hyperhive.forge.ci = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
example = true;
|
|
description = ''
|
|
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
Grouped under `services.hyperhive.forge` because the runner is
|
|
tightly coupled to the forge instance it registers against.
|
|
Disabled by default; `services.hyperhive.forge.enable = true` is
|
|
a prerequisite (enforced by assertion).
|
|
|
|
On first start the container auto-registers against hive-forge using
|
|
hive-c0re's admin token — no manual token provisioning needed.
|
|
Runner credentials are persisted in the container's state dir and
|
|
reused on every subsequent boot.
|
|
'';
|
|
};
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-ci";
|
|
example = "prod-hive";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel. Defaults to
|
|
"hive-ci"; override when multiple hives share a Forgejo instance.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; multiple parallel
|
|
jobs share the container's nix store and cargo registry cache.
|
|
Higher values trade memory + CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). Workflow
|
|
files target this runner with `runs-on: [hive-ci]`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = forgeCfg.enable;
|
|
message = ''
|
|
services.hyperhive.forge.ci.enable = true requires
|
|
services.hyperhive.forge.enable = true — the runner registers
|
|
against the hive-forge Forgejo instance.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Create /run/hive-ci/ on the host and seed runner-token with a
|
|
# placeholder. hive-ci-prefetch.service overwrites it with the real
|
|
# token (or a fresh placeholder) before the container starts. The
|
|
# placeholder ensures the EnvironmentFile is always present even if
|
|
# the prefetch service hasn't run yet (e.g. tmpfiles-setup timing).
|
|
systemd.tmpfiles.rules = [
|
|
"d /run/hive-ci 0700 root root -"
|
|
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
|
];
|
|
|
|
# Host-side oneshot: validates/refreshes runner credentials before
|
|
# the container starts. The core admin token stays on the host and
|
|
# is never bind-mounted into the container. Runs on every boot so
|
|
# stale .runner credentials (runner deleted from forge) are detected
|
|
# and the container re-registers on the next start.
|
|
systemd.services.hive-ci-prefetch = {
|
|
description = "Pre-fetch hive-ci runner registration token (host-side)";
|
|
# Run before the container starts but after tmpfiles so the token
|
|
# file directory exists. After hive-c0re so the forge token is
|
|
# likely written (best-effort — the script handles the absent case).
|
|
after = [
|
|
"systemd-tmpfiles-setup.service"
|
|
"hive-c0re.service"
|
|
];
|
|
before = [ "nixos-container@hive-ci.service" ];
|
|
wantedBy = [ "nixos-container@hive-ci.service" ];
|
|
# partOf binds this oneshot's lifecycle to the container: when the
|
|
# container is stopped or restarted, systemd propagates that to this
|
|
# unit so it re-runs on the NEXT container start. Without this, the
|
|
# RemainAfterExit=true oneshot stays "active (exited)" forever after
|
|
# its first run — so a container restart skips it and the stale
|
|
# runner-token file (a placeholder from a boot where the forge token
|
|
# wasn't ready yet, or a registration token consumed/rotated since)
|
|
# is never refreshed. The in-container register service then fails
|
|
# with "runner registration token not found". partOf guarantees a
|
|
# fresh token is fetched before every container (re)start.
|
|
partOf = [ "nixos-container@hive-ci.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = prefetchScript;
|
|
};
|
|
};
|
|
|
|
containers.hive-ci = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Shared host netns: runner reaches hive-forge at localhost.
|
|
privateNetwork = false;
|
|
|
|
bindMounts = {
|
|
# Pre-filled by hive-ci-prefetch.service (host-side) before the
|
|
# container starts. Read-only: the container reads TOKEN= from
|
|
# here; the core admin token never enters the container.
|
|
"/run/hive-ci/runner-token" = {
|
|
hostPath = "/run/hive-ci/runner-token";
|
|
isReadOnly = true;
|
|
};
|
|
};
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# nspawn containers can't create user-namespaces, so nix
|
|
# sandboxing always fails. Fall back to unsandboxed builds.
|
|
# See docs/gotchas.md.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# package is top-level on gitea-actions-runner, not per-instance.
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.hive = {
|
|
enable = true;
|
|
name = cfg.name;
|
|
url = "http://127.0.0.1:${toString forgeCfg.httpPort}";
|
|
# EnvironmentFile providing TOKEN= — pre-filled by the
|
|
# host-side hive-ci-prefetch.service before the container
|
|
# starts; bind-mounted read-only from /run/hive-ci/runner-token
|
|
# on the host.
|
|
tokenFile = "/run/hive-ci/runner-token";
|
|
labels = cfg.labels;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
# Generous timeout for cold-cache nix builds.
|
|
runner.timeout = "3h";
|
|
};
|
|
};
|
|
|
|
# No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted
|
|
# read-only from the host (pre-filled before container start).
|
|
# nspawn creates the /run/hive-ci/ mount-point directory
|
|
# automatically before launching the container's init.
|
|
|
|
# No hive-ci-register.service inside the container: all forge
|
|
# API calls (runner validation, token fetch) moved to the
|
|
# host-side hive-ci-prefetch.service. The core admin token
|
|
# never enters this container.
|
|
|
|
# git is already in the gitea-actions-runner service PATH (the
|
|
# nixpkgs module builds it from the package's runtime deps).
|
|
# nix is required for `nix flake check` / `nix build` workflow
|
|
# steps — it's not included by the upstream module.
|
|
# Use the `path` service attribute (generates ExecSearchPath=)
|
|
# to prepend nix's bin dir to PATH without touching the
|
|
# environment.PATH the nixpkgs module sets — overriding that
|
|
# would lose git, curl, nodejs, and other runner deps.
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
];
|
|
|
|
systemd.services."gitea-runner-hive".path = [ pkgs.nix ];
|
|
};
|
|
};
|
|
};
|
|
}
|