9902d9d9a5
Forgejo's Actions artifact API hands the runner an upload URL built from forgejo's ROOT_URL — the public forge domain (forge.<domain>), not the runner's 127.0.0.1:<httpPort> registration URL. The hive-ci container shares host netns but has no resolver entry for the hive's own domains (the gateway's localHostsEntry only touches the host), so actions/upload-artifact fails with 'getaddrinfo ENOTFOUND forge.<domain>' while normal CI (checkout via the localhost registration URL) is unaffected. Add a networking.extraHosts entry mapping the forge domain to loopback so the upload reaches the local forge via the gateway.
413 lines
18 KiB
Nix
413 lines
18 KiB
Nix
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.forge.ci;
|
|
forgeCfg = config.services.hyperhive.forge;
|
|
|
|
# hive-c0re writes its own admin token here on first forge startup.
|
|
# The token has read:admin + write:admin scopes — sufficient to call
|
|
# the runner registration-token API endpoint.
|
|
# This path is HOST-ONLY. It is never bind-mounted into hive-ci.
|
|
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
|
|
|
|
# Container state root on the host. Non-ephemeral containers keep
|
|
# their filesystem here across reboots. The prefetch service accesses
|
|
# the runner's .runner file via this path to validate credentials
|
|
# without entering the container.
|
|
containerRoot = "/var/lib/nixos-containers/hive-ci";
|
|
|
|
# Host-side oneshot. Runs before `container@hive-ci.service`.
|
|
#
|
|
# The core token stays on the host. The container only ever sees the
|
|
# TOKEN= env-file populated here, never the core token itself.
|
|
#
|
|
# Flow:
|
|
# 1. If .runner exists: validate the runner ID against forge.
|
|
# Waits up to 60s for forge-core-token (hive-c0re writes it after
|
|
# forge container starts and admin is provisioned — this lags
|
|
# hive-c0re.service becoming active on first boot).
|
|
# 404 → purge .runner (re-registration needed).
|
|
# 000 (forge unreachable) → keep credentials, write placeholder.
|
|
# other non-200 → purge .runner.
|
|
# Token absent after 60s → keep credentials (safe — the runner
|
|
# holds valid creds; next boot will validate properly).
|
|
# 2. If .runner absent or just purged: wait up to 60s for both
|
|
# forge-core-token to appear AND forge API to respond, then
|
|
# fetch a fresh registration token and write TOKEN=<real>.
|
|
# Exits non-zero if both timeout — systemd logs the failure;
|
|
# the placeholder from tmpfiles means the container still starts
|
|
# but the runner will error. Operator restarts the service once
|
|
# forge is healthy.
|
|
prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
|
|
set -euo pipefail
|
|
TOKEN_FILE=/run/hive-ci/runner-token
|
|
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
|
RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
|
|
|
|
# Validate existing .runner credentials against forge. Purge if
|
|
# the runner was deleted (404) or the file is malformed. Keep if
|
|
# forge is unreachable (000) — transient outage shouldn't wipe creds.
|
|
#
|
|
# We wait up to 60s for forge-core-token to appear (hive-c0re writes
|
|
# it after provisioning the forge admin, which requires the forge
|
|
# container to start and become ready — this can lag behind
|
|
# hive-c0re.service becoming "active" on first boot). The same loop
|
|
# doubles as a wait for forge's API to become ready.
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
|
|
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
|
|
echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
else
|
|
# Wait for the core token before validating (same timeout as below).
|
|
CORE_TOKEN=""
|
|
for i in $(seq 1 60); do
|
|
if [ -f "${coreTokenPath}" ]; then
|
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
if [ -z "''${CORE_TOKEN:-}" ]; then
|
|
echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
|
|
else
|
|
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
|
|
-H "Authorization: token $CORE_TOKEN" \
|
|
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
|
|
if [ "$HTTP" = "404" ]; then
|
|
echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
elif [ "$HTTP" = "000" ]; then
|
|
echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
|
|
elif [ "$HTTP" != "200" ]; then
|
|
echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# .runner valid → write placeholder; gitea-actions-runner skips
|
|
# re-registration when .runner exists (TOKEN value is irrelevant).
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
echo "TOKEN=placeholder" > "$TOKEN_FILE"
|
|
chmod 600 "$TOKEN_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
# First boot or stale creds purged — wait for forge-core-token, then
|
|
# fetch a fresh registration token. Single retry loop covers both:
|
|
# waiting for hive-c0re to write the token file AND for forge's API
|
|
# to become responsive (they race on first boot).
|
|
REG_TOKEN=""
|
|
for i in $(seq 1 60); do
|
|
if [ ! -f "${coreTokenPath}" ]; then
|
|
echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
|
|
sleep 1
|
|
continue
|
|
fi
|
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
|
# Capture the HTTP status so a stale/invalid core token (401/403) is
|
|
# distinguished from a transient forge hiccup. With the old bare
|
|
# `curl -sf | jq`, a forge-core-token that's stale for the current
|
|
# forge (e.g. after a forge rebuild) 401s and fails silently every
|
|
# attempt for the full 60s loop, then exits with a misleading
|
|
# "core token absent or forge unreachable" — masking the real cause.
|
|
# Fail fast + loudly on 401/403 so the failure mode is legible and
|
|
# the operator/hive-c0re knows to re-mint forge-core-token.
|
|
RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \
|
|
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
|
-H "Authorization: token $CORE_TOKEN" || printf '\n000')
|
|
HTTP=$(printf '%s' "$RESP" | tail -n1)
|
|
BODY=$(printf '%s' "$RESP" | sed '$d')
|
|
case "$HTTP" in
|
|
2*)
|
|
REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token)
|
|
if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi
|
|
echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2
|
|
;;
|
|
401 | 403)
|
|
echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2
|
|
;;
|
|
esac
|
|
sleep 1
|
|
done
|
|
|
|
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
|
|
echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
|
|
chmod 600 "$TOKEN_FILE"
|
|
'';
|
|
in
|
|
{
|
|
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
# Shares host netns (same as hive-forge), so the runner reaches
|
|
# the forge at `http://127.0.0.1:<httpPort>` without extra plumbing.
|
|
# Container is non-ephemeral: the runner's registered credentials
|
|
# survive restarts (gitea-actions-runner writes them to its stateDir
|
|
# on first registration and reuses them on every subsequent start).
|
|
#
|
|
# Credential isolation: the forge admin token (`forge-core-token`)
|
|
# never enters the hive-ci container. A host-side oneshot service
|
|
# (`hive-ci-prefetch.service`) performs all forge API calls before
|
|
# the container starts and writes only the runner registration token
|
|
# to `/run/hive-ci/runner-token`. The container bind-mounts this
|
|
# file read-only and never has access to the wider admin token.
|
|
#
|
|
# Nix builds inside the container use the shared /nix/store (standard
|
|
# nixos-container behaviour) with sandbox-fallback = true, because
|
|
# nspawn containers can't create the user-namespaces that nix sandboxing
|
|
# requires. See docs/gotchas.md.
|
|
|
|
options.services.hyperhive.forge.ci = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
example = true;
|
|
description = ''
|
|
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
Grouped under `services.hyperhive.forge` because the runner is
|
|
tightly coupled to the forge instance it registers against.
|
|
Disabled by default; `services.hyperhive.forge.enable = true` is
|
|
a prerequisite (enforced by assertion).
|
|
|
|
On first start the container auto-registers against hive-forge using
|
|
hive-c0re's admin token — no manual token provisioning needed.
|
|
Runner credentials are persisted in the container's state dir and
|
|
reused on every subsequent boot.
|
|
'';
|
|
};
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-ci";
|
|
example = "prod-hive";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel. Defaults to
|
|
"hive-ci"; override when multiple hives share a Forgejo instance.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; multiple parallel
|
|
jobs share the container's nix store and cargo registry cache.
|
|
Higher values trade memory + CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). Workflow
|
|
files target this runner with `runs-on: [hive-ci]`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
|
|
jobTimeout = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "1h";
|
|
example = "3h";
|
|
description = ''
|
|
Per-job wall-clock timeout the runner enforces (act_runner's
|
|
`runner.timeout`). A job that exceeds it is killed, so a hung or
|
|
runaway build is bounded instead of holding the runner's single
|
|
slot indefinitely. Default `1h` comfortably covers a cold-cache
|
|
nix build while still bounding a stuck job; raise it (e.g.
|
|
`"3h"`) if you legitimately run jobs longer than that. Accepts a
|
|
Go duration string (`30m`, `1h`, `2h30m`). Note: this is
|
|
enforced by the runner process, so it only fires while that
|
|
process is itself healthy.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = forgeCfg.enable;
|
|
message = ''
|
|
services.hyperhive.forge.ci.enable = true requires
|
|
services.hyperhive.forge.enable = true — the runner registers
|
|
against the hive-forge Forgejo instance.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Create /run/hive-ci/ on the host and seed runner-token with a
|
|
# placeholder. hive-ci-prefetch.service overwrites it with the real
|
|
# token (or a fresh placeholder) before the container starts. The
|
|
# placeholder ensures the EnvironmentFile is always present even if
|
|
# the prefetch service hasn't run yet (e.g. tmpfiles-setup timing).
|
|
systemd.tmpfiles.rules = [
|
|
"d /run/hive-ci 0700 root root -"
|
|
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
|
];
|
|
|
|
# Host-side oneshot: validates/refreshes runner credentials before
|
|
# the container starts. The core admin token stays on the host and
|
|
# is never bind-mounted into the container. Runs on every boot so
|
|
# stale .runner credentials (runner deleted from forge) are detected
|
|
# and the container re-registers on the next start.
|
|
systemd.services.hive-ci-prefetch = {
|
|
description = "Pre-fetch hive-ci runner registration token (host-side)";
|
|
# Run before the container starts but after tmpfiles so the token
|
|
# file directory exists. After hive-c0re so the forge token is
|
|
# likely written (best-effort — the script handles the absent case).
|
|
after = [
|
|
"systemd-tmpfiles-setup.service"
|
|
"hive-c0re.service"
|
|
];
|
|
before = [ "container@hive-ci.service" ];
|
|
wantedBy = [ "container@hive-ci.service" ];
|
|
# partOf binds this oneshot's lifecycle to the container: when the
|
|
# container is stopped or restarted, systemd propagates that to this
|
|
# unit so it re-runs on the NEXT container start. Without this, the
|
|
# RemainAfterExit=true oneshot stays "active (exited)" forever after
|
|
# its first run — so a container restart skips it and the stale
|
|
# runner-token file (a placeholder from a boot where the forge token
|
|
# wasn't ready yet, or a registration token consumed/rotated since)
|
|
# is never refreshed. The in-container register service then fails
|
|
# with "runner registration token not found". partOf guarantees a
|
|
# fresh token is fetched before every container (re)start.
|
|
#
|
|
# Unit name: a declarative `containers.<name>` is the host unit
|
|
# `container@<name>.service` (the nspawn template), NOT
|
|
# `nixos-container@…`. The earlier `nixos-container@hive-ci.service`
|
|
# matched no real unit, so before/wantedBy/partOf were silent
|
|
# no-ops — the partOf never bound, the oneshot stayed
|
|
# `active (exited)`, and the token was never refreshed on restart.
|
|
# Confirmed against the live `container@hive-matrix.service` unit
|
|
# during the matrix-outage incident.
|
|
partOf = [ "container@hive-ci.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = prefetchScript;
|
|
};
|
|
};
|
|
|
|
containers.hive-ci = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Shared host netns: runner reaches hive-forge at localhost.
|
|
privateNetwork = false;
|
|
|
|
bindMounts = {
|
|
# Pre-filled by hive-ci-prefetch.service (host-side) before the
|
|
# container starts. Read-only: the container reads TOKEN= from
|
|
# here; the core admin token never enters the container.
|
|
"/run/hive-ci/runner-token" = {
|
|
hostPath = "/run/hive-ci/runner-token";
|
|
isReadOnly = true;
|
|
};
|
|
};
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# Resolve the hive's own forge domain to loopback inside the
|
|
# runner. The Forgejo Actions artifact API hands the runner an
|
|
# upload URL built from forgejo's ROOT_URL — the *public* forge
|
|
# domain (`forge.<domain>`), not the runner's
|
|
# `127.0.0.1:<httpPort>` registration URL. This container shares
|
|
# host netns but carries no resolver entry for the hive's own
|
|
# domains (the gateway's `localHostsEntry` only touches the host),
|
|
# so `actions/upload-artifact` dies with
|
|
# Error: getaddrinfo ENOTFOUND forge.<domain>
|
|
# while normal CI is unaffected (checkout uses the localhost
|
|
# registration URL). Map the forge domain to loopback so the
|
|
# upload reaches the local forge — via the gateway on :80 when
|
|
# behindGateway, or forge directly on :httpPort otherwise. The
|
|
# gateway proxies all of `/` (artifact endpoints included), and a
|
|
# `localhost` domain (no hive-domain set) is already loopback, so
|
|
# this is a harmless no-op in that case.
|
|
networking.extraHosts = "127.0.0.1 ${forgeCfg.domain}";
|
|
|
|
# nspawn containers can't create user-namespaces, so nix
|
|
# sandboxing always fails. Fall back to unsandboxed builds.
|
|
# See docs/gotchas.md.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# package is top-level on gitea-actions-runner, not per-instance.
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.hive = {
|
|
enable = true;
|
|
name = cfg.name;
|
|
url = "http://127.0.0.1:${toString forgeCfg.httpPort}";
|
|
# EnvironmentFile providing TOKEN= — pre-filled by the
|
|
# host-side hive-ci-prefetch.service before the container
|
|
# starts; bind-mounted read-only from /run/hive-ci/runner-token
|
|
# on the host.
|
|
tokenFile = "/run/hive-ci/runner-token";
|
|
labels = cfg.labels;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
# Per-job wall-clock cap — see the `jobTimeout` option.
|
|
runner.timeout = cfg.jobTimeout;
|
|
};
|
|
};
|
|
|
|
# No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted
|
|
# read-only from the host (pre-filled before container start).
|
|
# nspawn creates the /run/hive-ci/ mount-point directory
|
|
# automatically before launching the container's init.
|
|
|
|
# No hive-ci-register.service inside the container: all forge
|
|
# API calls (runner validation, token fetch) moved to the
|
|
# host-side hive-ci-prefetch.service. The core admin token
|
|
# never enters this container.
|
|
|
|
# git is already in the gitea-actions-runner service PATH (the
|
|
# nixpkgs module builds it from the package's runtime deps).
|
|
# nix is required for `nix flake check` / `nix build` workflow
|
|
# steps — it's not included by the upstream module.
|
|
# Use the `path` service attribute (generates ExecSearchPath=)
|
|
# to prepend nix's bin dir to PATH without touching the
|
|
# environment.PATH the nixpkgs module sets — overriding that
|
|
# would lose git, curl, nodejs, and other runner deps.
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
];
|
|
|
|
systemd.services."gitea-runner-hive".path = [ pkgs.nix ];
|
|
};
|
|
};
|
|
};
|
|
}
|