vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

refactor: nix/host-modules + nix/agent-modules layout, update doc paths

Browse source
This commit is contained in:
müde 2026-07-13 22:05:49 +02:00
commit 4a48ce5024
52 changed files with 48 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

546
nix/modules/hive-ci.nix
View file

@ -1,546 +0,0 @@
{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.forge.ci;
forgeCfg = config.services.hyperhive.forge;
gatewayCfg = config.services.hyperhive.gateway;
networkCfg = config.services.hyperhive.network;
tlsCfg = config.services.hyperhive.tls;
# Self-signed TLS is the gateway default (no operator cert / ACME). When
# active, forgejo's ROOT_URL is `https://forge.<domain>` and the leaf is
# signed by the host hive CA — so the runner's Node-based actions (e.g.
# `upload-artifact`, which POSTs to the ROOT_URL-derived artifact endpoint)
# reject the chain, since Node trusts only its bundled CA bundle, not the
# system store. Trust the hive CA explicitly via NODE_EXTRA_CA_CERTS below.
# `gateway.useSelfSigned` is the gateway module's single source of truth
# for the self-signed condition (no duplicated derivation here).
useSelfSigned = gatewayCfg.useSelfSigned;
caHostPath = "${tlsCfg.stateDir}/ca.pem";
caContainerPath = "/run/hive-ca/ca.pem";
# hive-c0re writes its own admin token here on first forge startup.
# The token has read:admin + write:admin scopes — sufficient to call
# the runner registration-token API endpoint.
# This path is HOST-ONLY. It is never bind-mounted into hive-ci.
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
# Container state root on the host. Non-ephemeral containers keep
# their filesystem here across reboots. The prefetch service accesses
# the runner's .runner file via this path to validate credentials
# without entering the container.
containerRoot = "/var/lib/nixos-containers/hive-ci";
# Host-side oneshot. Runs before `container@hive-ci.service`.
#
# The core token stays on the host. The container only ever sees the
# TOKEN= env-file populated here, never the core token itself.
#
# Flow:
# 1. If .runner exists: validate the runner ID against forge.
# Waits up to 60s for forge-core-token (hive-c0re writes it after
# forge container starts and admin is provisioned — this lags
# hive-c0re.service becoming active on first boot).
# 404 → purge .runner (re-registration needed).
# 000 (forge unreachable) → keep credentials, write placeholder.
# other non-200 → purge .runner.
# Token absent after 60s → keep credentials (safe — the runner
# holds valid creds; next boot will validate properly).
# 2. If .runner absent or just purged: wait up to 60s for both
# forge-core-token to appear AND forge API to respond, then
# fetch a fresh registration token and write TOKEN=<real>.
# Exits non-zero if both timeout — systemd logs the failure;
# the placeholder from tmpfiles means the container still starts
# but the runner will error. Operator restarts the service once
# forge is healthy.
prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
set -euo pipefail
TOKEN_FILE=/run/hive-ci/runner-token
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
# Validate existing .runner credentials against forge. Purge if
# the runner was deleted (404) or the file is malformed. Keep if
# forge is unreachable (000) — transient outage shouldn't wipe creds.
#
# We wait up to 60s for forge-core-token to appear (hive-c0re writes
# it after provisioning the forge admin, which requires the forge
# container to start and become ready — this can lag behind
# hive-c0re.service becoming "active" on first boot). The same loop
# doubles as a wait for forge's API to become ready.
if [ -f "$RUNNER_FILE" ]; then
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
rm -f "$RUNNER_FILE"
else
# Wait for the core token before validating (same timeout as below).
CORE_TOKEN=""
for i in $(seq 1 60); do
if [ -f "${coreTokenPath}" ]; then
CORE_TOKEN=$(cat ${coreTokenPath})
break
fi
sleep 1
done
if [ -z "''${CORE_TOKEN:-}" ]; then
echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
else
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
-H "Authorization: token $CORE_TOKEN" \
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
if [ "$HTTP" = "404" ]; then
echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
rm -f "$RUNNER_FILE"
elif [ "$HTTP" = "000" ]; then
echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
elif [ "$HTTP" != "200" ]; then
echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
rm -f "$RUNNER_FILE"
fi
fi
fi
fi
# .runner valid → write placeholder; gitea-actions-runner skips
# re-registration when .runner exists (TOKEN value is irrelevant).
if [ -f "$RUNNER_FILE" ]; then
echo "TOKEN=placeholder" > "$TOKEN_FILE"
chmod 600 "$TOKEN_FILE"
exit 0
fi
# First boot or stale creds purged — wait for forge-core-token, then
# fetch a fresh registration token. Single retry loop covers both:
# waiting for hive-c0re to write the token file AND for forge's API
# to become responsive (they race on first boot).
REG_TOKEN=""
for i in $(seq 1 60); do
if [ ! -f "${coreTokenPath}" ]; then
echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
sleep 1
continue
fi
CORE_TOKEN=$(cat ${coreTokenPath})
# Capture the HTTP status so a stale/invalid core token (401/403) is
# distinguished from a transient forge hiccup. With the old bare
# `curl -sf | jq`, a forge-core-token that's stale for the current
# forge (e.g. after a forge rebuild) 401s and fails silently every
# attempt for the full 60s loop, then exits with a misleading
# "core token absent or forge unreachable" — masking the real cause.
# Fail fast + loudly on 401/403 so the failure mode is legible and
# the operator/hive-c0re knows to re-mint forge-core-token.
RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \
"$FORGE_URL/api/v1/admin/runners/registration-token" \
-H "Authorization: token $CORE_TOKEN" || printf '\n000')
HTTP=$(printf '%s' "$RESP" | tail -n1)
BODY=$(printf '%s' "$RESP" | sed '$d')
case "$HTTP" in
2*)
REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token)
if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi
echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2
;;
401 | 403)
echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s." >&2
exit 1
;;
*)
echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2
;;
esac
sleep 1
done
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
exit 1
fi
echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
chmod 600 "$TOKEN_FILE"
'';
in
{
# Forgejo Actions runner in a `hive-ci` nixos-container.
# Uses a private network namespace (bridge-connected, not host netns)
# so CI build scripts cannot reach host-loopback services (dashboard,
# forge internal port, etc.) — a key defence against prompt-injection
# via PR nix builds. The runner reaches the forge via the
# gateway at `http://${forgeCfg.domain}` (resolved to the bridge IP
# via `networking.extraHosts`; gateway port 80 is always open on the
# bridge; `addSSL = true` means HTTP is served alongside HTTPS without
# a redirect). See docs/network.md.
# Container is non-ephemeral: the runner's registered credentials
# survive restarts (gitea-actions-runner writes them to its stateDir
# on first registration and reuses them on every subsequent start).
#
# Credential isolation: the forge admin token (`forge-core-token`)
# never enters the hive-ci container. A host-side oneshot service
# (`hive-ci-prefetch.service`) performs all forge API calls before
# the container starts and writes only the runner registration token
# to `/run/hive-ci/runner-token`. The container bind-mounts this
# file read-only and never has access to the wider admin token.
#
# Nix builds inside the container use the shared /nix/store (standard
# nixos-container behaviour) with sandbox-fallback = true, because
# nspawn containers can't create the user-namespaces that nix sandboxing
# requires. See docs/gotchas.md.
options.services.hyperhive.forge.ci = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = ''
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
Grouped under `services.hyperhive.forge` because the runner is
tightly coupled to the forge instance it registers against.
Disabled by default; the internal forge it registers against is
always present (mandatory), so enabling this is all that's needed.
On first start the container auto-registers against hive-forge using
hive-c0re's admin token — no manual token provisioning needed.
Runner credentials are persisted in the container's state dir and
reused on every subsequent boot.
'';
};
name = lib.mkOption {
type = lib.types.str;
default = "hive-ci";
example = "prod-hive";
description = ''
Runner name as shown in the Forgejo admin panel. Defaults to
"hive-ci"; override when multiple hives share a Forgejo instance.
'';
};
concurrency = lib.mkOption {
type = lib.types.ints.positive;
default = 1;
example = 4;
description = ''
Maximum number of workflow jobs the runner executes in parallel.
Each job gets its own temporary working directory; multiple parallel
jobs share the container's nix store and cargo registry cache.
Higher values trade memory + CPU headroom for throughput.
'';
};
labels = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ "hive-ci:host" ];
example = [
"hive-ci:host"
"nix:host"
];
description = ''
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
commands directly in the container (no docker/podman). Workflow
files target this runner with `runs-on: [hive-ci]`.
'';
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.gitea-actions-runner;
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
description = "gitea-actions-runner package.";
};
jobTimeout = lib.mkOption {
type = lib.types.str;
default = "1h";
example = "3h";
description = ''
Per-job wall-clock timeout the runner enforces (act_runner's
`runner.timeout`). A job that exceeds it is killed, so a hung or
runaway build is bounded instead of holding the runner's single
slot indefinitely. Default `1h` comfortably covers a cold-cache
nix build while still bounding a stuck job; raise it (e.g.
`"3h"`) if you legitimately run jobs longer than that. Accepts a
Go duration string (`30m`, `1h`, `2h30m`). Note: this is
enforced by the runner process, so it only fires while that
process is itself healthy.
'';
};
};
config = lib.mkIf cfg.enable {
# `forge.behindGateway = true` (the default) is required because the
# CI container uses private networking and reaches the forge through
# the gateway vhost. Without the gateway vhost there is no HTTP
# listener for `forgeCfg.domain` on the bridge that the runner can
# connect to.
assertions = [
{
assertion = forgeCfg.behindGateway;
message = ''
services.hyperhive.forge.ci.enable requires
services.hyperhive.forge.behindGateway = true.
The CI container runs with a private network namespace and
reaches the forge through the gateway vhost on the bridge IP.
Set behindGateway = true (it defaults to true alongside
services.hyperhive.enable).
'';
}
];
# Create /run/hive-ci/ on the host and seed runner-token with a
# placeholder. hive-ci-prefetch.service overwrites it with the real
# token (or a fresh placeholder) before the container starts. The
# placeholder ensures the EnvironmentFile is always present even if
# the prefetch service hasn't run yet (e.g. tmpfiles-setup timing).
systemd.tmpfiles.rules = [
"d /run/hive-ci 0700 root root -"
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
];
# Host-side oneshot: validates/refreshes runner credentials before
# the container starts. The core admin token stays on the host and
# is never bind-mounted into the container. Runs on every boot so
# stale .runner credentials (runner deleted from forge) are detected
# and the container re-registers on the next start.
systemd.services.hive-ci-prefetch = {
description = "Pre-fetch hive-ci runner registration token (host-side)";
# Run before the container starts but after tmpfiles so the token
# file directory exists. After hive-c0re so the forge token is
# likely written (best-effort — the script handles the absent case).
after = [
"systemd-tmpfiles-setup.service"
"hive-c0re.service"
];
before = [ "container@hive-ci.service" ];
wantedBy = [ "container@hive-ci.service" ];
# partOf binds this oneshot's lifecycle to the container: when the
# container is stopped or restarted, systemd propagates that to this
# unit so it re-runs on the NEXT container start. Without this, the
# RemainAfterExit=true oneshot stays "active (exited)" forever after
# its first run — so a container restart skips it and the stale
# runner-token file (a placeholder from a boot where the forge token
# wasn't ready yet, or a registration token consumed/rotated since)
# is never refreshed. The in-container register service then fails
# with "runner registration token not found". partOf guarantees a
# fresh token is fetched before every container (re)start.
#
# Unit name: a declarative `containers.<name>` is the host unit
# `container@<name>.service` (the nspawn template), NOT
# `nixos-container@…`. The earlier `nixos-container@hive-ci.service`
# matched no real unit, so before/wantedBy/partOf were silent
# no-ops — the partOf never bound, the oneshot stayed
# `active (exited)`, and the token was never refreshed on restart.
# Confirmed against the live `container@hive-matrix.service` unit
# during the matrix-outage incident.
partOf = [ "container@hive-ci.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = prefetchScript;
# Pin the journal identity; ExecStart is a writeShellScript whose
# store-path basename would otherwise be the journal identifier.
SyslogIdentifier = "hive-ci-prefetch";
};
};
# Self-signed mode: the CA cert the bind-mount above sources is
# generated by the host `hive-tls-ca` service. Order the container after
# it so the bind source exists before nspawn sets the mount up (a
# condition-skipped/late CA would otherwise fail the container start).
systemd.services."container@hive-ci" = lib.mkIf useSelfSigned {
after = [ "hive-tls-ca.service" ];
requires = [ "hive-tls-ca.service" ];
};
containers.hive-ci = {
autoStart = true;
ephemeral = false;
# Private network namespace, attached to the hive bridge so the
# runner reaches the forge via the gateway — and cannot reach
# host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.).
# Requires `forge.behindGateway = true` (asserted in the options
# block above). See docs/network.md.
privateNetwork = true;
hostBridge = networkCfg.bridgeName;
bindMounts = {
# Pre-filled by hive-ci-prefetch.service (host-side) before the
# container starts. Read-only: the container reads TOKEN= from
# here; the core admin token never enters the container.
"/run/hive-ci/runner-token" = {
hostPath = "/run/hive-ci/runner-token";
isReadOnly = true;
};
}
# Self-signed mode: bind ONLY the public hive CA cert (never the
# `hive-tls` state dir — it holds the CA + leaf private keys) so the
# runner's Node actions can trust the gateway/forge self-signed leaf
# (see NODE_EXTRA_CA_CERTS in the container config). Source generated
# by the host `hive-tls-ca` service; the container@hive-ci ordering
# below guarantees it exists before this mount is set up.
// lib.optionalAttrs useSelfSigned {
${caContainerPath} = {
hostPath = caHostPath;
isReadOnly = true;
};
};
config =
{ pkgs, lib, ... }:
{
system.stateVersion = "26.05";
# Point the forge domain at the bridge IP so the runner can
# reach the forge through the gateway — both for registration /
# polling (runner URL below) and for artifact uploads (the
# Forgejo Actions artifact API uses ROOT_URL, i.e. the public
# forge domain, not a localhost URL). The gateway vhost for
# `forgeCfg.domain` proxies all `/` → forge; `addSSL = true`
# means HTTP:80 is served without redirect alongside HTTPS:443.
# Ports 80 and 443 are always open on the bridge firewall (see
# hive-network.nix). No DNS lookup needed — /etc/hosts wins.
networking.extraHosts = "${networkCfg.bridgeIp} ${forgeCfg.domain}";
# DNS: use the hive resolver on the bridge IP (dnsmasq in
# hive-gateway) for external lookups (git checkout, crate
# registries, etc.). The bridge→loopback DROP rule does not
# affect traffic destined for the bridge IP itself.
networking.nameservers = [ networkCfg.bridgeIp ];
# Bridge-attached via privateNetwork=true + hostBridge. The
# gateway's dnsmasq serves a DHCP pool covering all usable bridge
# addresses (see dhcp-range in hive-gateway.nix) — agents and
# service containers alike receive IPs dynamically.
networking.interfaces.eth0.useDHCP = true;
# nspawn containers can't create user-namespaces, so nix
# sandboxing always fails. Fall back to unsandboxed builds.
# Moot once every nix invocation in the container routes
# through the host daemon (the daemon governs sandboxing).
# See docs/gotchas.md and harness-base.nix.
nix.settings.sandbox-fallback = lib.mkForce true;
nix.settings.experimental-features = [
"nix-command"
"flakes"
];
# package is top-level on gitea-actions-runner, not per-instance.
services.gitea-actions-runner.package = cfg.package;
services.gitea-actions-runner.instances.hive = {
enable = true;
name = cfg.name;
# Route through the gateway (bridge IP, port 80) so the
# runner never touches host-loopback. The forge domain
# resolves to the bridge IP via networking.extraHosts above;
# the gateway vhost `forgeCfg.domain` proxies to the forge
# on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect).
url = "http://${forgeCfg.domain}";
# EnvironmentFile providing TOKEN= — pre-filled by the
# host-side hive-ci-prefetch.service before the container
# starts; bind-mounted read-only from /run/hive-ci/runner-token
# on the host.
tokenFile = "/run/hive-ci/runner-token";
labels = cfg.labels;
settings = {
runner.capacity = cfg.concurrency;
# Per-job wall-clock cap — see the `jobTimeout` option.
runner.timeout = cfg.jobTimeout;
};
};
# No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted
# read-only from the host (pre-filled before container start).
# nspawn creates the /run/hive-ci/ mount-point directory
# automatically before launching the container's init.
# No hive-ci-register.service inside the container: all forge
# API calls (runner validation, token fetch) moved to the
# host-side hive-ci-prefetch.service. The core admin token
# never enters this container.
# git is already in the gitea-actions-runner service PATH (the
# nixpkgs module builds it from the package's runtime deps).
# nix is required for `nix flake check` / `nix build` workflow
# steps — it's not included by the upstream module.
# Use the `path` service attribute (generates ExecSearchPath=)
# to prepend nix's bin dir to PATH without touching the
# environment.PATH the nixpkgs module sets — overriding that
# would lose git, curl, nodejs, and other runner deps.
environment.systemPackages = [
pkgs.git
pkgs.nix
];
systemd.services."gitea-runner-hive" = {
path = [ pkgs.nix ];
# Trust the hive CA in Node-based actions. With self-signed TLS,
# forgejo's ROOT_URL is `https://forge.<domain>` (CA-signed leaf),
# so actions like `upload-artifact` — whose Node HTTP client uses
# Node's *bundled* CA bundle, not the system store — reject the
# chain with "unable to verify the first certificate". Pointing
# NODE_EXTRA_CA_CERTS at the bind-mounted CA adds it to Node's
# roots for every action, hive-wide. Inherited by the job
# processes the runner spawns (host execution mode). Only set in
# self-signed mode; with an operator cert / ACME the public CA
# already validates and the bind-mount is absent.
environment = lib.mkIf useSelfSigned {
NODE_EXTRA_CA_CERTS = caContainerPath;
};
# Gate runner start (and therefore job registration/claiming) on
# the in-container nix daemon being reachable. After a hive-ci
# restart the runner re-registers and immediately claims any
# queued jobs — which can beat the nix daemon coming up: its
# `nix-daemon.socket` carries
# `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket` and is
# skipped until /nix/var is read-write, so the first
# nix-dependent build dispatches into a cold daemon and
# hangs/retries (observed: a 55m48s `nix flake check` vs the
# normal ~30s, a build-offload stall, not a code failure).
#
# Ordering `after`/`wants` the socket unit does NOT fix this — a
# condition-skipped unit satisfies systemd ordering immediately,
# so the runner would still start before the daemon is live.
# Instead block in ExecStartPre by polling the daemon until it
# actually answers; this is topology-agnostic (works whether the
# daemon is in-container or a shared host socket). `mkBefore` so
# this runs ahead of any pre-steps the upstream module adds.
serviceConfig.ExecStartPre = lib.mkBefore [
(pkgs.writeShellScript "wait-nix-daemon" ''
# Up to ~180s; the daemon is normally up within seconds, this
# only bites in the post-restart cold window. Non-fatal shape:
# if it never comes up the unit fails cleanly (recoverable)
# rather than the runner claiming jobs into a dead daemon.
#
# `--store daemon` is load-bearing. A bare `nix store ping`
# uses the `auto` store, which — when run as root with the
# daemon socket absent — silently resolves to a LOCAL store
# (root can write /nix/store directly) and pings successfully.
# systemd service units don't source the profile that sets
# `NIX_REMOTE=daemon`, so this is the real environment here.
# At cold boot the daemon socket IS absent: nix-daemon.socket
# carries `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket`
# and is condition-skipped until /nix/var goes read-write. So
# a bare ping would pass against the local store while the
# daemon is still down — defeating the gate's whole purpose
# (the runner would start and claim jobs the daemon can't yet
# service). Pinning `--store daemon` makes the poll verify the
# actual daemon socket, so the gate honours its contract and
# waits until the daemon — not a local fallback — answers.
for _ in $(seq 1 90); do
if ${pkgs.nix}/bin/nix store ping --store daemon >/dev/null 2>&1; then
exit 0
fi
sleep 2
done
echo "nix daemon not reachable after 180s" >&2
exit 1
'')
];
};
};
};
};
}