eb61660d35
Part of the tracker-tag cleanup: the hive convention is prose, not issue-tracker tags, in code. Reword the 21 tags in the nix tree (flake.nix + the hive-c0re/ci/gateway/network modules) to describe the thing they pointed at, preserving the context without the tag. Comment-only — no eval or logic change. Validated with nix fmt (no reformatting) and nix flake check --no-build (all checks evaluate clean); the full build check was skipped locally because the shared remote builder is degraded, so CI will exercise the build derivations once the runner recovers.
378 lines
16 KiB
Nix
378 lines
16 KiB
Nix
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.forge.ci;
|
|
forgeCfg = config.services.hyperhive.forge;
|
|
|
|
# hive-c0re writes its own admin token here on first forge startup.
|
|
# The token has read:admin + write:admin scopes — sufficient to call
|
|
# the runner registration-token API endpoint.
|
|
# This path is HOST-ONLY. It is never bind-mounted into hive-ci.
|
|
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
|
|
|
|
# Container state root on the host. Non-ephemeral containers keep
|
|
# their filesystem here across reboots. The prefetch service accesses
|
|
# the runner's .runner file via this path to validate credentials
|
|
# without entering the container.
|
|
containerRoot = "/var/lib/nixos-containers/hive-ci";
|
|
|
|
# Host-side oneshot. Runs before `container@hive-ci.service`.
|
|
#
|
|
# The core token stays on the host. The container only ever sees the
|
|
# TOKEN= env-file populated here, never the core token itself.
|
|
#
|
|
# Flow:
|
|
# 1. If .runner exists: validate the runner ID against forge.
|
|
# Waits up to 60s for forge-core-token (hive-c0re writes it after
|
|
# forge container starts and admin is provisioned — this lags
|
|
# hive-c0re.service becoming active on first boot).
|
|
# 404 → purge .runner (re-registration needed).
|
|
# 000 (forge unreachable) → keep credentials, write placeholder.
|
|
# other non-200 → purge .runner.
|
|
# Token absent after 60s → keep credentials (safe — the runner
|
|
# holds valid creds; next boot will validate properly).
|
|
# 2. If .runner absent or just purged: wait up to 60s for both
|
|
# forge-core-token to appear AND forge API to respond, then
|
|
# fetch a fresh registration token and write TOKEN=<real>.
|
|
# Exits non-zero if both timeout — systemd logs the failure;
|
|
# the placeholder from tmpfiles means the container still starts
|
|
# but the runner will error. Operator restarts the service once
|
|
# forge is healthy.
|
|
prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
|
|
set -euo pipefail
|
|
TOKEN_FILE=/run/hive-ci/runner-token
|
|
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
|
RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
|
|
|
|
# Validate existing .runner credentials against forge. Purge if
|
|
# the runner was deleted (404) or the file is malformed. Keep if
|
|
# forge is unreachable (000) — transient outage shouldn't wipe creds.
|
|
#
|
|
# We wait up to 60s for forge-core-token to appear (hive-c0re writes
|
|
# it after provisioning the forge admin, which requires the forge
|
|
# container to start and become ready — this can lag behind
|
|
# hive-c0re.service becoming "active" on first boot). The same loop
|
|
# doubles as a wait for forge's API to become ready.
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
|
|
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
|
|
echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
else
|
|
# Wait for the core token before validating (same timeout as below).
|
|
CORE_TOKEN=""
|
|
for i in $(seq 1 60); do
|
|
if [ -f "${coreTokenPath}" ]; then
|
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
if [ -z "''${CORE_TOKEN:-}" ]; then
|
|
echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
|
|
else
|
|
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
|
|
-H "Authorization: token $CORE_TOKEN" \
|
|
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
|
|
if [ "$HTTP" = "404" ]; then
|
|
echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
elif [ "$HTTP" = "000" ]; then
|
|
echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
|
|
elif [ "$HTTP" != "200" ]; then
|
|
echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# .runner valid → write placeholder; gitea-actions-runner skips
|
|
# re-registration when .runner exists (TOKEN value is irrelevant).
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
echo "TOKEN=placeholder" > "$TOKEN_FILE"
|
|
chmod 600 "$TOKEN_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
# First boot or stale creds purged — wait for forge-core-token, then
|
|
# fetch a fresh registration token. Single retry loop covers both:
|
|
# waiting for hive-c0re to write the token file AND for forge's API
|
|
# to become responsive (they race on first boot).
|
|
REG_TOKEN=""
|
|
for i in $(seq 1 60); do
|
|
if [ ! -f "${coreTokenPath}" ]; then
|
|
echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
|
|
sleep 1
|
|
continue
|
|
fi
|
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
|
# Capture the HTTP status so a stale/invalid core token (401/403) is
|
|
# distinguished from a transient forge hiccup. With the old bare
|
|
# `curl -sf | jq`, a forge-core-token that's stale for the current
|
|
# forge (e.g. after a forge rebuild) 401s and fails silently every
|
|
# attempt for the full 60s loop, then exits with a misleading
|
|
# "core token absent or forge unreachable" — masking the real cause.
|
|
# Fail fast + loudly on 401/403 so the failure mode is legible and
|
|
# the operator/hive-c0re knows to re-mint forge-core-token.
|
|
RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \
|
|
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
|
-H "Authorization: token $CORE_TOKEN" || printf '\n000')
|
|
HTTP=$(printf '%s' "$RESP" | tail -n1)
|
|
BODY=$(printf '%s' "$RESP" | sed '$d')
|
|
case "$HTTP" in
|
|
2*)
|
|
REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token)
|
|
if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi
|
|
echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2
|
|
;;
|
|
401 | 403)
|
|
echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2
|
|
;;
|
|
esac
|
|
sleep 1
|
|
done
|
|
|
|
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
|
|
echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
|
|
chmod 600 "$TOKEN_FILE"
|
|
'';
|
|
in
|
|
{
|
|
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
# Shares host netns (same as hive-forge), so the runner reaches
|
|
# the forge at `http://127.0.0.1:<httpPort>` without extra plumbing.
|
|
# Container is non-ephemeral: the runner's registered credentials
|
|
# survive restarts (gitea-actions-runner writes them to its stateDir
|
|
# on first registration and reuses them on every subsequent start).
|
|
#
|
|
# Credential isolation: the forge admin token (`forge-core-token`)
|
|
# never enters the hive-ci container. A host-side oneshot service
|
|
# (`hive-ci-prefetch.service`) performs all forge API calls before
|
|
# the container starts and writes only the runner registration token
|
|
# to `/run/hive-ci/runner-token`. The container bind-mounts this
|
|
# file read-only and never has access to the wider admin token.
|
|
#
|
|
# Nix builds inside the container use the shared /nix/store (standard
|
|
# nixos-container behaviour) with sandbox-fallback = true, because
|
|
# nspawn containers can't create the user-namespaces that nix sandboxing
|
|
# requires. See docs/gotchas.md.
|
|
|
|
options.services.hyperhive.forge.ci = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
example = true;
|
|
description = ''
|
|
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
Grouped under `services.hyperhive.forge` because the runner is
|
|
tightly coupled to the forge instance it registers against.
|
|
Disabled by default; `services.hyperhive.forge.enable = true` is
|
|
a prerequisite (enforced by assertion).
|
|
|
|
On first start the container auto-registers against hive-forge using
|
|
hive-c0re's admin token — no manual token provisioning needed.
|
|
Runner credentials are persisted in the container's state dir and
|
|
reused on every subsequent boot.
|
|
'';
|
|
};
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-ci";
|
|
example = "prod-hive";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel. Defaults to
|
|
"hive-ci"; override when multiple hives share a Forgejo instance.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; multiple parallel
|
|
jobs share the container's nix store and cargo registry cache.
|
|
Higher values trade memory + CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). Workflow
|
|
files target this runner with `runs-on: [hive-ci]`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = forgeCfg.enable;
|
|
message = ''
|
|
services.hyperhive.forge.ci.enable = true requires
|
|
services.hyperhive.forge.enable = true — the runner registers
|
|
against the hive-forge Forgejo instance.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Create /run/hive-ci/ on the host and seed runner-token with a
|
|
# placeholder. hive-ci-prefetch.service overwrites it with the real
|
|
# token (or a fresh placeholder) before the container starts. The
|
|
# placeholder ensures the EnvironmentFile is always present even if
|
|
# the prefetch service hasn't run yet (e.g. tmpfiles-setup timing).
|
|
systemd.tmpfiles.rules = [
|
|
"d /run/hive-ci 0700 root root -"
|
|
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
|
];
|
|
|
|
# Host-side oneshot: validates/refreshes runner credentials before
|
|
# the container starts. The core admin token stays on the host and
|
|
# is never bind-mounted into the container. Runs on every boot so
|
|
# stale .runner credentials (runner deleted from forge) are detected
|
|
# and the container re-registers on the next start.
|
|
systemd.services.hive-ci-prefetch = {
|
|
description = "Pre-fetch hive-ci runner registration token (host-side)";
|
|
# Run before the container starts but after tmpfiles so the token
|
|
# file directory exists. After hive-c0re so the forge token is
|
|
# likely written (best-effort — the script handles the absent case).
|
|
after = [
|
|
"systemd-tmpfiles-setup.service"
|
|
"hive-c0re.service"
|
|
];
|
|
before = [ "container@hive-ci.service" ];
|
|
wantedBy = [ "container@hive-ci.service" ];
|
|
# partOf binds this oneshot's lifecycle to the container: when the
|
|
# container is stopped or restarted, systemd propagates that to this
|
|
# unit so it re-runs on the NEXT container start. Without this, the
|
|
# RemainAfterExit=true oneshot stays "active (exited)" forever after
|
|
# its first run — so a container restart skips it and the stale
|
|
# runner-token file (a placeholder from a boot where the forge token
|
|
# wasn't ready yet, or a registration token consumed/rotated since)
|
|
# is never refreshed. The in-container register service then fails
|
|
# with "runner registration token not found". partOf guarantees a
|
|
# fresh token is fetched before every container (re)start.
|
|
#
|
|
# Unit name: a declarative `containers.<name>` is the host unit
|
|
# `container@<name>.service` (the nspawn template), NOT
|
|
# `nixos-container@…`. The earlier `nixos-container@hive-ci.service`
|
|
# matched no real unit, so before/wantedBy/partOf were silent
|
|
# no-ops — the partOf never bound, the oneshot stayed
|
|
# `active (exited)`, and the token was never refreshed on restart.
|
|
# Confirmed against the live `container@hive-matrix.service` unit
|
|
# during the matrix-outage incident.
|
|
partOf = [ "container@hive-ci.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = prefetchScript;
|
|
};
|
|
};
|
|
|
|
containers.hive-ci = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Shared host netns: runner reaches hive-forge at localhost.
|
|
privateNetwork = false;
|
|
|
|
bindMounts = {
|
|
# Pre-filled by hive-ci-prefetch.service (host-side) before the
|
|
# container starts. Read-only: the container reads TOKEN= from
|
|
# here; the core admin token never enters the container.
|
|
"/run/hive-ci/runner-token" = {
|
|
hostPath = "/run/hive-ci/runner-token";
|
|
isReadOnly = true;
|
|
};
|
|
};
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# nspawn containers can't create user-namespaces, so nix
|
|
# sandboxing always fails. Fall back to unsandboxed builds.
|
|
# See docs/gotchas.md.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# package is top-level on gitea-actions-runner, not per-instance.
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.hive = {
|
|
enable = true;
|
|
name = cfg.name;
|
|
url = "http://127.0.0.1:${toString forgeCfg.httpPort}";
|
|
# EnvironmentFile providing TOKEN= — pre-filled by the
|
|
# host-side hive-ci-prefetch.service before the container
|
|
# starts; bind-mounted read-only from /run/hive-ci/runner-token
|
|
# on the host.
|
|
tokenFile = "/run/hive-ci/runner-token";
|
|
labels = cfg.labels;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
# Generous timeout for cold-cache nix builds.
|
|
runner.timeout = "3h";
|
|
};
|
|
};
|
|
|
|
# No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted
|
|
# read-only from the host (pre-filled before container start).
|
|
# nspawn creates the /run/hive-ci/ mount-point directory
|
|
# automatically before launching the container's init.
|
|
|
|
# No hive-ci-register.service inside the container: all forge
|
|
# API calls (runner validation, token fetch) moved to the
|
|
# host-side hive-ci-prefetch.service. The core admin token
|
|
# never enters this container.
|
|
|
|
# git is already in the gitea-actions-runner service PATH (the
|
|
# nixpkgs module builds it from the package's runtime deps).
|
|
# nix is required for `nix flake check` / `nix build` workflow
|
|
# steps — it's not included by the upstream module.
|
|
# Use the `path` service attribute (generates ExecSearchPath=)
|
|
# to prepend nix's bin dir to PATH without touching the
|
|
# environment.PATH the nixpkgs module sets — overriding that
|
|
# would lose git, curl, nodejs, and other runner deps.
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
];
|
|
|
|
systemd.services."gitea-runner-hive".path = [ pkgs.nix ];
|
|
};
|
|
};
|
|
};
|
|
}
|