hyperhive/nix/host-modules/hive-ci.nix
atlas 4423da4899 deploy: move the forge CI runner toggle out of swarm
Last of the swarm.* -> deploy.* moves for the enable-shaped options.
Whether this host also runs the CI runner is a per-machine decision, so
it was sitting in the namespace that has to be identical swarm-wide.

Renames all five options, not just enable: name, concurrency, labels and
package all describe the runner THIS host would run, and leaving them
behind would keep the original defect for four more options. One rename
entry each, because ci is a plain attrset rather than a submodule type,
so there is no parent path to rename in one go.

deploy.forgejo is the name deploy.nix's own header already specifies,
including this exact case -- "ci (does this host run the runner too) is
exactly that shape". It is the only entry with no enable: the forge is
not optional, it deploys with hyperhive itself, so running the runner is
the only deployment decision it has.

Also fixes hive-c0re/src/forge/ci_runner.rs, whose doc comment named
services.hyperhive.forge.ci.enable -- missing the swarm. segment, so it
had never resolved to a real option.
2026-08-30 16:07:21 +02:00

459 lines
23 KiB
Nix

{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.deploy.forgejo.ci;
forgeCfg = config.services.hyperhive.swarm.forge;
gatewayCfg = config.services.hyperhive.gateway;
networkCfg = config.services.hyperhive.network;
tlsCfg = config.services.hyperhive.tls;
# Self-signed TLS is the gateway default (no operator cert / ACME). When
# active, forgejo's ROOT_URL is `https://forge.<domain>` and the leaf is
# signed by the host hive CA — so the runner's Node-based actions (e.g.
# `upload-artifact`, which POSTs to the ROOT_URL-derived artifact endpoint)
# reject the chain, since Node trusts only its bundled CA bundle, not the
# system store. Trust the hive CA explicitly via NODE_EXTRA_CA_CERTS below.
# The bind-mount + `container@` ordering that makes the CA reachable are
# shared with hive-forge via the `hive-ca-trust` helper; only the Node
# `NODE_EXTRA_CA_CERTS` consumption is hive-ci-specific.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
useSelfSigned = caTrust.useSelfSigned;
caContainerPath = caTrust.caContainerPath;
# The runner daemon needs the hive CA too, and `NODE_EXTRA_CA_CERTS` above
# does not give it to them: that variable is Node's, and it is ADDITIVE.
# `gitea-runner` is Go, whose trust store is REPLACING -- it reads
# `SSL_CERT_FILE` and then trusts only what that file contains. So the two
# consumers need different treatment from the same CA, which is exactly the
# split the helper documents at its head.
#
# Without this the daemon fails every startup call with
# `x509: certificate signed by unknown authority` the moment it reaches a
# TLS endpoint -- and it reaches one whenever the address it is registered
# at redirects to the gateway's https vhost. The unit then crash-loops on
# `Restart=on-failure`, no job is ever picked up, and nothing reports it as
# broken: every pull request simply sits at "Waiting to run".
#
# ⚠️ This is deliberately NOT claimed as the whole story of that outage --
# what address the daemon holds is a separate question. It is here because
# the trust gap is real under every explanation of it, and adding a CA to
# one unit cannot break a path that already works.
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = "hive-ci";
consumers = [ "gitea-runner-hive" ];
};
in
{
# Forgejo Actions runner in a `hive-ci` nixos-container.
# Uses a private network namespace (bridge-connected, not host netns)
# so CI build scripts cannot reach host-loopback services (dashboard,
# forge internal port, etc.) — a key defence against prompt-injection
# via PR nix builds. The runner reaches the forge via the
# gateway at `http://${forgeCfg.domain}` (resolved to the bridge IP
# via `networking.extraHosts`; gateway port 80 is always open on the
# bridge; `addSSL = true` means HTTP is served alongside HTTPS without
# a redirect). See docs/network.md.
# Container is non-ephemeral: the runner's registered credentials
# survive restarts (gitea-actions-runner writes them to its stateDir
# on first registration and reuses them on every subsequent start).
#
# Credential isolation: the forge admin token (`forge-core-token`)
# never enters the hive-ci container. hive-c0re holds it and performs
# all forge API calls (runner validation + registration-token mint,
# in `forge/ci_runner.rs`); via hive-priv it writes only the runner
# registration token to the host env-file `/run/hive-ci/runner-token`,
# which the container bind-mounts read-only. The container never has
# access to the wider admin token.
#
# Nix builds inside the container use the shared /nix/store (standard
# nixos-container behaviour) with sandbox-fallback = true, because
# nspawn containers can't create the user-namespaces that nix sandboxing
# requires. See docs/gotchas.md.
options.services.hyperhive.deploy.forgejo.ci = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = ''
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
Grouped under `services.hyperhive.swarm.forge` because the runner is
tightly coupled to the forge instance it registers against.
Disabled by default; the internal forge it registers against is
always present (mandatory), so enabling this is all that's needed.
On first start the container auto-registers against hive-forge using
hive-c0re's admin token no manual token provisioning needed.
Runner credentials are persisted in the container's state dir and
reused on every subsequent boot.
'';
};
name = lib.mkOption {
type = lib.types.str;
default = "hive-ci";
example = "prod-hive";
description = ''
Runner name as shown in the Forgejo admin panel. Defaults to
"hive-ci"; override when multiple hives share a Forgejo instance.
'';
};
concurrency = lib.mkOption {
type = lib.types.ints.positive;
default = 1;
example = 4;
description = ''
Maximum number of workflow jobs the runner executes in parallel.
Each job gets its own temporary working directory; multiple parallel
jobs share the container's nix store and cargo registry cache.
Higher values trade memory + CPU headroom for throughput.
'';
};
labels = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ "hive-ci:host" ];
example = [
"hive-ci:host"
"nix:host"
];
description = ''
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
commands directly in the container (no docker/podman). Workflow
files target this runner with `runs-on: [hive-ci]`.
'';
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.gitea-actions-runner;
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
description = "gitea-actions-runner package.";
};
jobTimeout = lib.mkOption {
type = lib.types.str;
default = "1h";
example = "3h";
description = ''
Per-job wall-clock timeout the runner enforces (act_runner's
`runner.timeout`). A job that exceeds it is killed, so a hung or
runaway build is bounded instead of holding the runner's single
slot indefinitely. Default `1h` comfortably covers a cold-cache
nix build while still bounding a stuck job; raise it (e.g.
`"3h"`) if you legitimately run jobs longer than that. Accepts a
Go duration string (`30m`, `1h`, `2h30m`). Note: this is
enforced by the runner process, so it only fires while that
process is itself healthy.
'';
};
};
config = lib.mkIf cfg.enable {
# `forge.behindGateway = true` (the default) is required because the
# CI container uses private networking and reaches the forge through
# the gateway vhost. Without the gateway vhost there is no HTTP
# listener for `forgeCfg.domain` on the bridge that the runner can
# connect to.
assertions = [
{
assertion = forgeCfg.behindGateway;
message = ''
services.hyperhive.deploy.forgejo.ci.enable requires
services.hyperhive.swarm.forge.behindGateway = true.
The CI container runs with a private network namespace and
reaches the forge through the gateway vhost on the bridge IP.
Set behindGateway = true (it defaults to true alongside
services.hyperhive.enable).
'';
}
];
# Create /run/hive-ci/ on the host and seed runner-token with a
# placeholder. The container bind-mounts this file read-only; hive-c0re
# (via hive-priv's RegisterCiRunner) overwrites it with the real
# registration token when it registers the runner out of band. The
# placeholder keeps the runner's EnvironmentFile present from first boot,
# before c0re has registered — the runner's ExecStartPre precond (below)
# distinguishes the placeholder from a real token.
systemd.tmpfiles.rules = [
"d /run/hive-ci 0700 root root -"
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
];
# Tell hive-c0re that CI is enabled so its startup sweep registers the
# runner. Registration moved OFF the container's boot-critical path into
# hive-c0re (`forge/ci_runner.rs`): it validates the persisted `.runner`
# against the forge and, when absent/stale, mints a registration token and
# hands it to hive-priv to write `/run/hive-ci/runner-token` + restart the
# runner. The forge admin token stays in hive-c0re; only the registration
# token reaches the host env-file the container mounts read-only.
systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_CI_ENABLED = "1";
# `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
# in self-signed mode (see the hive-ca-trust helper). Runner registration
# is no longer on the boot-critical path — hive-c0re owns it out of band —
# so the container boots immediately and needs no start-timeout bump. The
# former `TimeoutStartSec = mkForce "180s"` band-aid (which papered over a
# boot-path forge + core-token round-trip that could trip the nspawn start
# timeout into a ~60s restart loop) is gone with that move.
systemd.services."container@hive-ci" = caTrust.containerOrdering;
containers.hive-ci = {
autoStart = true;
ephemeral = false;
# Private network namespace, attached to the hive bridge so the
# runner reaches the forge via the gateway — and cannot reach
# host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.).
# Requires `forge.behindGateway = true` (asserted in the options
# block above). See docs/network.md.
privateNetwork = true;
hostBridge = networkCfg.bridgeName;
bindMounts = {
# Pre-filled by hive-c0re (via hive-priv) with the runner
# registration token; tmpfiles seeds a `TOKEN=placeholder` before
# that. Read-only: the container reads TOKEN= from here; the core
# admin token never enters the container.
"/run/hive-ci/runner-token" = {
hostPath = "/run/hive-ci/runner-token";
isReadOnly = true;
};
# Route the container's nix through the HOST nix-daemon (as the agent
# containers do) instead of an in-container daemon: with
# NIX_REMOTE=daemon below, builds run on the host daemon and inherit
# its buildMachines + max-jobs. Note that `fallback` is NOT inherited
# that way — it's a client-side option transmitted per connection, so
# the container sets it itself (see nix.settings.fallback below).
# Bind the *directory* (not the socket file) so the mount stays live
# across a host nix-daemon restart, which recreates the socket inode.
"/nix/var/nix/daemon-socket" = {
hostPath = "/nix/var/nix/daemon-socket";
isReadOnly = false;
};
}
# Self-signed mode: bind the public hive CA cert read-only so the
# runner's Node actions trust the gateway/forge leaf (consumed via
# NODE_EXTRA_CA_CERTS in the container config). Shared bind-mount +
# ordering come from the hive-ca-trust helper.
// caTrust.bindMount;
config =
{ pkgs, lib, ... }:
{
# Assembles system CAs + the hive CA into one bundle and sets
# `SSL_CERT_FILE` on the runner unit. See `caBundleModule` above for
# why the Node variable beside it is not enough.
imports = [ caBundleModule ];
system.stateVersion = "26.05";
# Point the forge domain at the bridge IP so the runner can
# reach the forge through the gateway — both for registration /
# polling (runner URL below) and for artifact uploads (the
# Forgejo Actions artifact API uses ROOT_URL, i.e. the public
# forge domain, not a localhost URL). The gateway vhost for
# `forgeCfg.domain` proxies all `/` → forge; `addSSL = true`
# means HTTP:80 is served without redirect alongside HTTPS:443.
# Ports 80 and 443 are always open on the bridge firewall (see
# hive-network.nix). No DNS lookup needed — /etc/hosts wins.
networking.extraHosts = "${networkCfg.bridgeIp} ${forgeCfg.domain}";
# DNS: use the hive resolver on the bridge IP (dnsmasq in
# hive-gateway) for external lookups (git checkout, crate
# registries, etc.). The bridge→loopback DROP rule does not
# affect traffic destined for the bridge IP itself.
networking.nameservers = [ networkCfg.bridgeIp ];
# Bridge-attached via privateNetwork=true + hostBridge. The
# gateway's dnsmasq serves a DHCP pool covering all usable bridge
# addresses (see dhcp-range in hive-gateway.nix) — agents and
# service containers alike receive IPs dynamically.
networking.interfaces.eth0.useDHCP = true;
# nspawn containers can't create user-namespaces, so nix
# sandboxing always fails. Fall back to unsandboxed builds.
# Moot once every nix invocation in the container routes
# through the host daemon (the daemon governs sandboxing).
# See docs/gotchas.md and nix/agent-modules/default.nix.
nix.settings.sandbox-fallback = lib.mkForce true;
# Degrade to a local build when a remote builder is unreachable
# rather than failing the check. `fallback` is a client-side
# option (transmitted per daemon connection), so routing through
# the host daemon does NOT inherit the host's value — the
# container has to set it. Same reasoning as the agent
# containers; see nix/agent-modules/default.nix.
nix.settings.fallback = true;
nix.settings.experimental-features = [
"nix-command"
"flakes"
];
# Build through the HOST nix-daemon (bind-mounted socket above), not
# an in-container daemon. NIX_REMOTE=daemon makes root services (the
# runner) use the daemon socket rather than store=auto → local store.
# Disable BOTH the service and its .socket unit: the .socket owns the
# listen path /nix/var/nix/daemon-socket/socket and would otherwise
# race the bind mount over the same path.
systemd.globalEnvironment.NIX_REMOTE = "daemon";
systemd.services.nix-daemon.enable = false;
systemd.sockets.nix-daemon.enable = false;
# package is top-level on gitea-actions-runner, not per-instance.
services.gitea-actions-runner.package = cfg.package;
services.gitea-actions-runner.instances.hive = {
enable = true;
name = cfg.name;
# Route through the gateway (bridge IP, port 80) so the
# runner never touches host-loopback. The forge domain
# resolves to the bridge IP via networking.extraHosts above;
# the gateway vhost `forgeCfg.domain` proxies to the forge
# on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect).
url = "http://${forgeCfg.domain}";
# EnvironmentFile providing TOKEN= — pre-filled by hive-c0re
# (via hive-priv) with the runner registration token;
# bind-mounted read-only from /run/hive-ci/runner-token on the
# host. The runner's ExecStartPre precond gates on this being a
# real (non-placeholder) token or an already-registered .runner.
tokenFile = "/run/hive-ci/runner-token";
labels = cfg.labels;
settings = {
runner.capacity = cfg.concurrency;
# Per-job wall-clock cap — see the `jobTimeout` option.
runner.timeout = cfg.jobTimeout;
};
};
# No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted
# read-only from the host (pre-filled before container start).
# nspawn creates the /run/hive-ci/ mount-point directory
# automatically before launching the container's init.
# No hive-ci-register.service inside the container: all forge
# API calls (runner validation, token fetch) live in hive-c0re
# (forge/ci_runner.rs), out of band. The core admin token never
# enters this container.
# git is already in the gitea-actions-runner service PATH (the
# nixpkgs module builds it from the package's runtime deps).
# nix is required for `nix flake check` / `nix build` workflow
# steps — it's not included by the upstream module.
# Use the `path` service attribute (generates ExecSearchPath=)
# to prepend nix's bin dir to PATH without touching the
# environment.PATH the nixpkgs module sets — overriding that
# would lose git, curl, nodejs, and other runner deps.
environment.systemPackages = [
pkgs.git
pkgs.nix
];
systemd.services."gitea-runner-hive" = {
path = [ pkgs.nix ];
# A CI runner must be able to EXECUTE what it builds, and by
# default it cannot: the upstream unit uses `DynamicUser`, and
# systemd mounts a dynamic unit's state directory `noexec`.
# Measured on the live runner rather than inferred — from a job's
# own /proc/self/mountinfo:
#
# /var/lib/private ro,nosuid,nodev,noexec (tmpfs)
# /var/lib/private/gitea-runner rw,nosuid,nodev,noexec,idmapped
#
# with /tmp as the control at rw,nosuid,nodev — no noexec, and a
# script there executes fine. Nothing noticed for as long as every
# workflow built inside the nix sandbox and executed nothing in the
# workspace; the first job that compiled a build script into the
# runner's own target dir died with EACCES on execve, reported as
# "could not execute process ... (never executed)".
#
# Both spellings are listed because `ExecPaths=` resolves against
# the HOST root, where the dynamic-user layout makes
# /var/lib/gitea-runner a symlink into private/. The `-` prefix
# means "ignore if absent", so whichever layout the unit ends up
# with, the other is a no-op rather than a startup failure.
serviceConfig.ExecPaths = [
"-/var/lib/gitea-runner"
"-/var/lib/private/gitea-runner"
];
# Trust the hive CA in Node-based actions. With self-signed TLS,
# forgejo's ROOT_URL is `https://forge.<domain>` (CA-signed leaf),
# so actions like `upload-artifact` — whose Node HTTP client uses
# Node's *bundled* CA bundle, not the system store — reject the
# chain with "unable to verify the first certificate". Pointing
# NODE_EXTRA_CA_CERTS at the bind-mounted CA adds it to Node's
# roots for every action, hive-wide. Inherited by the job
# processes the runner spawns (host execution mode). Only set in
# self-signed mode; with an operator cert / ACME the public CA
# already validates and the bind-mount is absent.
environment = lib.mkIf useSelfSigned {
NODE_EXTRA_CA_CERTS = caContainerPath;
};
# Gate runner start on credentials being present. Registration is
# done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints a
# token and, via hive-priv, writes /run/hive-ci/runner-token +
# restarts this unit. Pass iff already registered (.runner present)
# OR a real (non-placeholder) token is in place; else exit 1 so the
# runner stays down (Restart=on-failure retries) until c0re
# registers it, rather than blocking the container's start.
#
# No nix-daemon wait anymore: builds route through the HOST daemon
# (bound socket above), which is up before the container — nix's
# own connect retries cover any brief window. `mkBefore` runs this
# ahead of any upstream pre-steps.
#
# The `+` prefix runs this step with FULL PRIVILEGES instead of as
# the unit's (dynamic) `gitea-runner` user. The token file is 0600
# root-owned, so an unprivileged check cannot read it — and it did
# not fail closed: `[ -s ]` succeeds on a stat alone, `grep` then
# died with EACCES, and `! grep` turned that error into `true`, so
# the gate exited 0 on an unreadable file. It reported "a real token
# is present" without ever having looked, for its entire existence.
#
# This grants the runner nothing new: `tokenFile` above becomes
# `EnvironmentFile=`, which systemd already reads AS ROOT before
# dropping privileges, so the payload never passes through an
# unprivileged reader either way. The `+` only lets the gate observe
# what systemd observes.
serviceConfig.ExecStartPre = lib.mkBefore [
"+${pkgs.writeShellScript "hive-ci-runner-precond" ''
RUNNER=/var/lib/gitea-runner/hive/.runner
TOKEN_FILE=/run/hive-ci/runner-token
if [ -f "$RUNNER" ]; then exit 0; fi
# Readability is asserted SEPARATELY and fails loudly. Folding
# it into the test below is what hid the bug: an unreadable file
# and a file holding a real token produced the same verdict, so
# "I could not look" was indistinguishable from "I looked and it
# is fine".
if [ -e "$TOKEN_FILE" ] && [ ! -r "$TOKEN_FILE" ]; then
echo "hive-ci runner: $TOKEN_FILE exists but is unreadable by this step; the credential gate cannot run" >&2
exit 1
fi
if [ -s "$TOKEN_FILE" ] && ! ${pkgs.gnugrep}/bin/grep -q '^TOKEN=placeholder$' "$TOKEN_FILE"; then
exit 0
fi
echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2
exit 1
''}"
];
# Registration is out of band: hive-c0re explicitly restarts this
# unit once it writes the real token, which is the primary path. As
# a safety net, if the precond above fires first (no creds yet at
# boot) retry until c0re registers, rather than staying down — with
# no start-limit rate cap so it keeps retrying however long forge /
# the core token take to settle.
serviceConfig.Restart = lib.mkForce "on-failure";
# `mkForce`: the upstream gitea-actions-runner module also sets
# RestartSec (2), so ours needs the higher priority or the two
# collide into an eval error — same as the sibling `Restart`.
serviceConfig.RestartSec = lib.mkForce 15;
startLimitIntervalSec = 0;
};
};
};
};
}