401 lines
20 KiB
Nix
401 lines
20 KiB
Nix
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.forge.ci;
|
|
forgeCfg = config.services.hyperhive.forge;
|
|
gatewayCfg = config.services.hyperhive.gateway;
|
|
networkCfg = config.services.hyperhive.network;
|
|
tlsCfg = config.services.hyperhive.tls;
|
|
|
|
# Self-signed TLS is the gateway default (no operator cert / ACME). When
|
|
# active, forgejo's ROOT_URL is `https://forge.<domain>` and the leaf is
|
|
# signed by the host hive CA — so the runner's Node-based actions (e.g.
|
|
# `upload-artifact`, which POSTs to the ROOT_URL-derived artifact endpoint)
|
|
# reject the chain, since Node trusts only its bundled CA bundle, not the
|
|
# system store. Trust the hive CA explicitly via NODE_EXTRA_CA_CERTS below.
|
|
# The bind-mount + `container@` ordering that makes the CA reachable are
|
|
# shared with hive-forge via the `hive-ca-trust` helper; only the Node
|
|
# `NODE_EXTRA_CA_CERTS` consumption is hive-ci-specific.
|
|
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
useSelfSigned = caTrust.useSelfSigned;
|
|
caContainerPath = caTrust.caContainerPath;
|
|
|
|
in
|
|
{
|
|
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
# Uses a private network namespace (bridge-connected, not host netns)
|
|
# so CI build scripts cannot reach host-loopback services (dashboard,
|
|
# forge internal port, etc.) — a key defence against prompt-injection
|
|
# via PR nix builds. The runner reaches the forge via the
|
|
# gateway at `http://${forgeCfg.domain}` (resolved to the bridge IP
|
|
# via `networking.extraHosts`; gateway port 80 is always open on the
|
|
# bridge; `addSSL = true` means HTTP is served alongside HTTPS without
|
|
# a redirect). See docs/network.md.
|
|
# Container is non-ephemeral: the runner's registered credentials
|
|
# survive restarts (gitea-actions-runner writes them to its stateDir
|
|
# on first registration and reuses them on every subsequent start).
|
|
#
|
|
# Credential isolation: the forge admin token (`forge-core-token`)
|
|
# never enters the hive-ci container. hive-c0re holds it and performs
|
|
# all forge API calls (runner validation + registration-token mint,
|
|
# in `forge/ci_runner.rs`); via hive-priv it writes only the runner
|
|
# registration token to the host env-file `/run/hive-ci/runner-token`,
|
|
# which the container bind-mounts read-only. The container never has
|
|
# access to the wider admin token.
|
|
#
|
|
# Nix builds inside the container use the shared /nix/store (standard
|
|
# nixos-container behaviour) with sandbox-fallback = true, because
|
|
# nspawn containers can't create the user-namespaces that nix sandboxing
|
|
# requires. See docs/gotchas.md.
|
|
|
|
options.services.hyperhive.forge.ci = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
example = true;
|
|
description = ''
|
|
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
Grouped under `services.hyperhive.forge` because the runner is
|
|
tightly coupled to the forge instance it registers against.
|
|
Disabled by default; the internal forge it registers against is
|
|
always present (mandatory), so enabling this is all that's needed.
|
|
|
|
On first start the container auto-registers against hive-forge using
|
|
hive-c0re's admin token — no manual token provisioning needed.
|
|
Runner credentials are persisted in the container's state dir and
|
|
reused on every subsequent boot.
|
|
'';
|
|
};
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-ci";
|
|
example = "prod-hive";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel. Defaults to
|
|
"hive-ci"; override when multiple hives share a Forgejo instance.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; multiple parallel
|
|
jobs share the container's nix store and cargo registry cache.
|
|
Higher values trade memory + CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). Workflow
|
|
files target this runner with `runs-on: [hive-ci]`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
|
|
jobTimeout = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "1h";
|
|
example = "3h";
|
|
description = ''
|
|
Per-job wall-clock timeout the runner enforces (act_runner's
|
|
`runner.timeout`). A job that exceeds it is killed, so a hung or
|
|
runaway build is bounded instead of holding the runner's single
|
|
slot indefinitely. Default `1h` comfortably covers a cold-cache
|
|
nix build while still bounding a stuck job; raise it (e.g.
|
|
`"3h"`) if you legitimately run jobs longer than that. Accepts a
|
|
Go duration string (`30m`, `1h`, `2h30m`). Note: this is
|
|
enforced by the runner process, so it only fires while that
|
|
process is itself healthy.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
# `forge.behindGateway = true` (the default) is required because the
|
|
# CI container uses private networking and reaches the forge through
|
|
# the gateway vhost. Without the gateway vhost there is no HTTP
|
|
# listener for `forgeCfg.domain` on the bridge that the runner can
|
|
# connect to.
|
|
assertions = [
|
|
{
|
|
assertion = forgeCfg.behindGateway;
|
|
message = ''
|
|
services.hyperhive.forge.ci.enable requires
|
|
services.hyperhive.forge.behindGateway = true.
|
|
The CI container runs with a private network namespace and
|
|
reaches the forge through the gateway vhost on the bridge IP.
|
|
Set behindGateway = true (it defaults to true alongside
|
|
services.hyperhive.enable).
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Create /run/hive-ci/ on the host and seed runner-token with a
|
|
# placeholder. The container bind-mounts this file read-only; hive-c0re
|
|
# (via hive-priv's RegisterCiRunner) overwrites it with the real
|
|
# registration token when it registers the runner out of band. The
|
|
# placeholder keeps the runner's EnvironmentFile present from first boot,
|
|
# before c0re has registered — the runner's ExecStartPre precond (below)
|
|
# distinguishes the placeholder from a real token.
|
|
systemd.tmpfiles.rules = [
|
|
"d /run/hive-ci 0700 root root -"
|
|
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
|
];
|
|
|
|
# Tell hive-c0re that CI is enabled so its startup sweep registers the
|
|
# runner. Registration moved OFF the container's boot-critical path into
|
|
# hive-c0re (`forge/ci_runner.rs`): it validates the persisted `.runner`
|
|
# against the forge and, when absent/stale, mints a registration token and
|
|
# hands it to hive-priv to write `/run/hive-ci/runner-token` + restart the
|
|
# runner. The forge admin token stays in hive-c0re; only the registration
|
|
# token reaches the host env-file the container mounts read-only.
|
|
systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_CI_ENABLED = "1";
|
|
|
|
# `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
|
|
# in self-signed mode (see the hive-ca-trust helper). Runner registration
|
|
# is no longer on the boot-critical path — hive-c0re owns it out of band —
|
|
# so the container boots immediately and needs no start-timeout bump. The
|
|
# former `TimeoutStartSec = mkForce "180s"` band-aid (which papered over a
|
|
# boot-path forge + core-token round-trip that could trip the nspawn start
|
|
# timeout into a ~60s restart loop) is gone with that move.
|
|
systemd.services."container@hive-ci" = caTrust.containerOrdering;
|
|
|
|
containers.hive-ci = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Private network namespace, attached to the hive bridge so the
|
|
# runner reaches the forge via the gateway — and cannot reach
|
|
# host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.).
|
|
# Requires `forge.behindGateway = true` (asserted in the options
|
|
# block above). See docs/network.md.
|
|
privateNetwork = true;
|
|
hostBridge = networkCfg.bridgeName;
|
|
|
|
bindMounts = {
|
|
# Pre-filled by hive-c0re (via hive-priv) with the runner
|
|
# registration token; tmpfiles seeds a `TOKEN=placeholder` before
|
|
# that. Read-only: the container reads TOKEN= from here; the core
|
|
# admin token never enters the container.
|
|
"/run/hive-ci/runner-token" = {
|
|
hostPath = "/run/hive-ci/runner-token";
|
|
isReadOnly = true;
|
|
};
|
|
}
|
|
# Self-signed mode: bind the public hive CA cert read-only so the
|
|
# runner's Node actions trust the gateway/forge leaf (consumed via
|
|
# NODE_EXTRA_CA_CERTS in the container config). Shared bind-mount +
|
|
# ordering come from the hive-ca-trust helper.
|
|
// caTrust.bindMount;
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# Point the forge domain at the bridge IP so the runner can
|
|
# reach the forge through the gateway — both for registration /
|
|
# polling (runner URL below) and for artifact uploads (the
|
|
# Forgejo Actions artifact API uses ROOT_URL, i.e. the public
|
|
# forge domain, not a localhost URL). The gateway vhost for
|
|
# `forgeCfg.domain` proxies all `/` → forge; `addSSL = true`
|
|
# means HTTP:80 is served without redirect alongside HTTPS:443.
|
|
# Ports 80 and 443 are always open on the bridge firewall (see
|
|
# hive-network.nix). No DNS lookup needed — /etc/hosts wins.
|
|
networking.extraHosts = "${networkCfg.bridgeIp} ${forgeCfg.domain}";
|
|
# DNS: use the hive resolver on the bridge IP (dnsmasq in
|
|
# hive-gateway) for external lookups (git checkout, crate
|
|
# registries, etc.). The bridge→loopback DROP rule does not
|
|
# affect traffic destined for the bridge IP itself.
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
# Bridge-attached via privateNetwork=true + hostBridge. The
|
|
# gateway's dnsmasq serves a DHCP pool covering all usable bridge
|
|
# addresses (see dhcp-range in hive-gateway.nix) — agents and
|
|
# service containers alike receive IPs dynamically.
|
|
networking.interfaces.eth0.useDHCP = true;
|
|
|
|
# nspawn containers can't create user-namespaces, so nix
|
|
# sandboxing always fails. Fall back to unsandboxed builds.
|
|
# Moot once every nix invocation in the container routes
|
|
# through the host daemon (the daemon governs sandboxing).
|
|
# See docs/gotchas.md and harness-base.nix.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# package is top-level on gitea-actions-runner, not per-instance.
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.hive = {
|
|
enable = true;
|
|
name = cfg.name;
|
|
# Route through the gateway (bridge IP, port 80) so the
|
|
# runner never touches host-loopback. The forge domain
|
|
# resolves to the bridge IP via networking.extraHosts above;
|
|
# the gateway vhost `forgeCfg.domain` proxies to the forge
|
|
# on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect).
|
|
url = "http://${forgeCfg.domain}";
|
|
# EnvironmentFile providing TOKEN= — pre-filled by hive-c0re
|
|
# (via hive-priv) with the runner registration token;
|
|
# bind-mounted read-only from /run/hive-ci/runner-token on the
|
|
# host. The runner's ExecStartPre precond gates on this being a
|
|
# real (non-placeholder) token or an already-registered .runner.
|
|
tokenFile = "/run/hive-ci/runner-token";
|
|
labels = cfg.labels;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
# Per-job wall-clock cap — see the `jobTimeout` option.
|
|
runner.timeout = cfg.jobTimeout;
|
|
};
|
|
};
|
|
|
|
# No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted
|
|
# read-only from the host (pre-filled before container start).
|
|
# nspawn creates the /run/hive-ci/ mount-point directory
|
|
# automatically before launching the container's init.
|
|
|
|
# No hive-ci-register.service inside the container: all forge
|
|
# API calls (runner validation, token fetch) live in hive-c0re
|
|
# (forge/ci_runner.rs), out of band. The core admin token never
|
|
# enters this container.
|
|
|
|
# git is already in the gitea-actions-runner service PATH (the
|
|
# nixpkgs module builds it from the package's runtime deps).
|
|
# nix is required for `nix flake check` / `nix build` workflow
|
|
# steps — it's not included by the upstream module.
|
|
# Use the `path` service attribute (generates ExecSearchPath=)
|
|
# to prepend nix's bin dir to PATH without touching the
|
|
# environment.PATH the nixpkgs module sets — overriding that
|
|
# would lose git, curl, nodejs, and other runner deps.
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
];
|
|
|
|
systemd.services."gitea-runner-hive" = {
|
|
path = [ pkgs.nix ];
|
|
# Trust the hive CA in Node-based actions. With self-signed TLS,
|
|
# forgejo's ROOT_URL is `https://forge.<domain>` (CA-signed leaf),
|
|
# so actions like `upload-artifact` — whose Node HTTP client uses
|
|
# Node's *bundled* CA bundle, not the system store — reject the
|
|
# chain with "unable to verify the first certificate". Pointing
|
|
# NODE_EXTRA_CA_CERTS at the bind-mounted CA adds it to Node's
|
|
# roots for every action, hive-wide. Inherited by the job
|
|
# processes the runner spawns (host execution mode). Only set in
|
|
# self-signed mode; with an operator cert / ACME the public CA
|
|
# already validates and the bind-mount is absent.
|
|
environment = lib.mkIf useSelfSigned {
|
|
NODE_EXTRA_CA_CERTS = caContainerPath;
|
|
};
|
|
# Gate runner start (and therefore job registration/claiming) on
|
|
# the in-container nix daemon being reachable. After a hive-ci
|
|
# restart the runner re-registers and immediately claims any
|
|
# queued jobs — which can beat the nix daemon coming up: its
|
|
# `nix-daemon.socket` carries
|
|
# `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket` and is
|
|
# skipped until /nix/var is read-write, so the first
|
|
# nix-dependent build dispatches into a cold daemon and
|
|
# hangs/retries (observed: a 55m48s `nix flake check` vs the
|
|
# normal ~30s, a build-offload stall, not a code failure).
|
|
#
|
|
# Ordering `after`/`wants` the socket unit does NOT fix this — a
|
|
# condition-skipped unit satisfies systemd ordering immediately,
|
|
# so the runner would still start before the daemon is live.
|
|
# Instead block in ExecStartPre by polling the daemon until it
|
|
# actually answers; this is topology-agnostic (works whether the
|
|
# daemon is in-container or a shared host socket). `mkBefore` so
|
|
# this runs ahead of any pre-steps the upstream module adds.
|
|
serviceConfig.ExecStartPre = lib.mkBefore [
|
|
# Fail fast unless the runner can actually come up. Registration
|
|
# is done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints
|
|
# a token and, via hive-priv, writes /run/hive-ci/runner-token +
|
|
# restarts this unit — so the runner never does the forge
|
|
# round-trip on the boot path (that was the old prefetch's
|
|
# boot-critical wait that could trip the nspawn start timeout).
|
|
# Pass iff already registered (.runner present) OR a real
|
|
# (non-placeholder) token is in place; otherwise exit 1 so the
|
|
# runner just stays down (Restart=on-failure retries) until c0re
|
|
# registers it, instead of blocking the container's start. Runs
|
|
# ahead of the nix-daemon wait below — no point waiting for the
|
|
# daemon if there are no credentials to come up with.
|
|
(pkgs.writeShellScript "hive-ci-runner-precond" ''
|
|
RUNNER=/var/lib/gitea-runner/hive/.runner
|
|
TOKEN_FILE=/run/hive-ci/runner-token
|
|
if [ -f "$RUNNER" ]; then exit 0; fi
|
|
if [ -s "$TOKEN_FILE" ] && ! ${pkgs.gnugrep}/bin/grep -q '^TOKEN=placeholder$' "$TOKEN_FILE"; then
|
|
exit 0
|
|
fi
|
|
echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2
|
|
exit 1
|
|
'')
|
|
(pkgs.writeShellScript "wait-nix-daemon" ''
|
|
# Up to ~180s; the daemon is normally up within seconds, this
|
|
# only bites in the post-restart cold window. Non-fatal shape:
|
|
# if it never comes up the unit fails cleanly (recoverable)
|
|
# rather than the runner claiming jobs into a dead daemon.
|
|
#
|
|
# `--store daemon` is load-bearing. A bare `nix store ping`
|
|
# uses the `auto` store, which — when run as root with the
|
|
# daemon socket absent — silently resolves to a LOCAL store
|
|
# (root can write /nix/store directly) and pings successfully.
|
|
# systemd service units don't source the profile that sets
|
|
# `NIX_REMOTE=daemon`, so this is the real environment here.
|
|
# At cold boot the daemon socket IS absent: nix-daemon.socket
|
|
# carries `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket`
|
|
# and is condition-skipped until /nix/var goes read-write. So
|
|
# a bare ping would pass against the local store while the
|
|
# daemon is still down — defeating the gate's whole purpose
|
|
# (the runner would start and claim jobs the daemon can't yet
|
|
# service). Pinning `--store daemon` makes the poll verify the
|
|
# actual daemon socket, so the gate honours its contract and
|
|
# waits until the daemon — not a local fallback — answers.
|
|
for _ in $(seq 1 90); do
|
|
if ${pkgs.nix}/bin/nix store ping --store daemon >/dev/null 2>&1; then
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "nix daemon not reachable after 180s" >&2
|
|
exit 1
|
|
'')
|
|
];
|
|
# Registration is out of band: hive-c0re explicitly restarts this
|
|
# unit once it writes the real token, which is the primary path. As
|
|
# a safety net, if the precond above fires first (no creds yet at
|
|
# boot) retry until c0re registers, rather than staying down — with
|
|
# no start-limit rate cap so it keeps retrying however long forge /
|
|
# the core token take to settle.
|
|
serviceConfig.Restart = lib.mkForce "on-failure";
|
|
# `mkForce`: the upstream gitea-actions-runner module also sets
|
|
# RestartSec (2), so ours needs the higher priority or the two
|
|
# collide into an eval error — same as the sibling `Restart`.
|
|
serviceConfig.RestartSec = lib.mkForce 15;
|
|
startLimitIntervalSec = 0;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|