38c222fadd
Forgejo is a swarm-global service, so its operator-facing host options move to services.hyperhive.swarm.forge (and .swarm.forge.ci) as the first of the namespace consolidation. Existing hive configs keep evaluating: swarm-renames.nix maps every moved leaf with mkRenamedOptionModule, which also emits a deprecation warning naming both the old and new path, so an operator is told what to rename rather than discovering it from a failed eval. The per-agent hyperhive.forge.url does NOT move. It is a client pointer at whatever forge an agent talks to - it shares a word with the service and nothing else, and the two are already documented as separate option surfaces. Verified by evaluating the host module, since no Rust gate evaluates nix: setting the old paths and reading the new ones yields the values (httpPort 3999, ci.concurrency 7), and config.warnings carries the rename notice.
382 lines
19 KiB
Nix
382 lines
19 KiB
Nix
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.swarm.forge.ci;
|
|
forgeCfg = config.services.hyperhive.swarm.forge;
|
|
gatewayCfg = config.services.hyperhive.gateway;
|
|
networkCfg = config.services.hyperhive.network;
|
|
tlsCfg = config.services.hyperhive.tls;
|
|
|
|
# Self-signed TLS is the gateway default (no operator cert / ACME). When
|
|
# active, forgejo's ROOT_URL is `https://forge.<domain>` and the leaf is
|
|
# signed by the host hive CA — so the runner's Node-based actions (e.g.
|
|
# `upload-artifact`, which POSTs to the ROOT_URL-derived artifact endpoint)
|
|
# reject the chain, since Node trusts only its bundled CA bundle, not the
|
|
# system store. Trust the hive CA explicitly via NODE_EXTRA_CA_CERTS below.
|
|
# The bind-mount + `container@` ordering that makes the CA reachable are
|
|
# shared with hive-forge via the `hive-ca-trust` helper; only the Node
|
|
# `NODE_EXTRA_CA_CERTS` consumption is hive-ci-specific.
|
|
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
useSelfSigned = caTrust.useSelfSigned;
|
|
caContainerPath = caTrust.caContainerPath;
|
|
|
|
in
|
|
{
|
|
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
# Uses a private network namespace (bridge-connected, not host netns)
|
|
# so CI build scripts cannot reach host-loopback services (dashboard,
|
|
# forge internal port, etc.) — a key defence against prompt-injection
|
|
# via PR nix builds. The runner reaches the forge via the
|
|
# gateway at `http://${forgeCfg.domain}` (resolved to the bridge IP
|
|
# via `networking.extraHosts`; gateway port 80 is always open on the
|
|
# bridge; `addSSL = true` means HTTP is served alongside HTTPS without
|
|
# a redirect). See docs/network.md.
|
|
# Container is non-ephemeral: the runner's registered credentials
|
|
# survive restarts (gitea-actions-runner writes them to its stateDir
|
|
# on first registration and reuses them on every subsequent start).
|
|
#
|
|
# Credential isolation: the forge admin token (`forge-core-token`)
|
|
# never enters the hive-ci container. hive-c0re holds it and performs
|
|
# all forge API calls (runner validation + registration-token mint,
|
|
# in `forge/ci_runner.rs`); via hive-priv it writes only the runner
|
|
# registration token to the host env-file `/run/hive-ci/runner-token`,
|
|
# which the container bind-mounts read-only. The container never has
|
|
# access to the wider admin token.
|
|
#
|
|
# Nix builds inside the container use the shared /nix/store (standard
|
|
# nixos-container behaviour) with sandbox-fallback = true, because
|
|
# nspawn containers can't create the user-namespaces that nix sandboxing
|
|
# requires. See docs/gotchas.md.
|
|
|
|
options.services.hyperhive.swarm.forge.ci = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
example = true;
|
|
description = ''
|
|
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
Grouped under `services.hyperhive.swarm.forge` because the runner is
|
|
tightly coupled to the forge instance it registers against.
|
|
Disabled by default; the internal forge it registers against is
|
|
always present (mandatory), so enabling this is all that's needed.
|
|
|
|
On first start the container auto-registers against hive-forge using
|
|
hive-c0re's admin token — no manual token provisioning needed.
|
|
Runner credentials are persisted in the container's state dir and
|
|
reused on every subsequent boot.
|
|
'';
|
|
};
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-ci";
|
|
example = "prod-hive";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel. Defaults to
|
|
"hive-ci"; override when multiple hives share a Forgejo instance.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; multiple parallel
|
|
jobs share the container's nix store and cargo registry cache.
|
|
Higher values trade memory + CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). Workflow
|
|
files target this runner with `runs-on: [hive-ci]`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
|
|
jobTimeout = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "1h";
|
|
example = "3h";
|
|
description = ''
|
|
Per-job wall-clock timeout the runner enforces (act_runner's
|
|
`runner.timeout`). A job that exceeds it is killed, so a hung or
|
|
runaway build is bounded instead of holding the runner's single
|
|
slot indefinitely. Default `1h` comfortably covers a cold-cache
|
|
nix build while still bounding a stuck job; raise it (e.g.
|
|
`"3h"`) if you legitimately run jobs longer than that. Accepts a
|
|
Go duration string (`30m`, `1h`, `2h30m`). Note: this is
|
|
enforced by the runner process, so it only fires while that
|
|
process is itself healthy.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
# `forge.behindGateway = true` (the default) is required because the
|
|
# CI container uses private networking and reaches the forge through
|
|
# the gateway vhost. Without the gateway vhost there is no HTTP
|
|
# listener for `forgeCfg.domain` on the bridge that the runner can
|
|
# connect to.
|
|
assertions = [
|
|
{
|
|
assertion = forgeCfg.behindGateway;
|
|
message = ''
|
|
services.hyperhive.swarm.forge.ci.enable requires
|
|
services.hyperhive.swarm.forge.behindGateway = true.
|
|
The CI container runs with a private network namespace and
|
|
reaches the forge through the gateway vhost on the bridge IP.
|
|
Set behindGateway = true (it defaults to true alongside
|
|
services.hyperhive.enable).
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Create /run/hive-ci/ on the host and seed runner-token with a
|
|
# placeholder. The container bind-mounts this file read-only; hive-c0re
|
|
# (via hive-priv's RegisterCiRunner) overwrites it with the real
|
|
# registration token when it registers the runner out of band. The
|
|
# placeholder keeps the runner's EnvironmentFile present from first boot,
|
|
# before c0re has registered — the runner's ExecStartPre precond (below)
|
|
# distinguishes the placeholder from a real token.
|
|
systemd.tmpfiles.rules = [
|
|
"d /run/hive-ci 0700 root root -"
|
|
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
|
];
|
|
|
|
# Tell hive-c0re that CI is enabled so its startup sweep registers the
|
|
# runner. Registration moved OFF the container's boot-critical path into
|
|
# hive-c0re (`forge/ci_runner.rs`): it validates the persisted `.runner`
|
|
# against the forge and, when absent/stale, mints a registration token and
|
|
# hands it to hive-priv to write `/run/hive-ci/runner-token` + restart the
|
|
# runner. The forge admin token stays in hive-c0re; only the registration
|
|
# token reaches the host env-file the container mounts read-only.
|
|
systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_CI_ENABLED = "1";
|
|
|
|
# `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
|
|
# in self-signed mode (see the hive-ca-trust helper). Runner registration
|
|
# is no longer on the boot-critical path — hive-c0re owns it out of band —
|
|
# so the container boots immediately and needs no start-timeout bump. The
|
|
# former `TimeoutStartSec = mkForce "180s"` band-aid (which papered over a
|
|
# boot-path forge + core-token round-trip that could trip the nspawn start
|
|
# timeout into a ~60s restart loop) is gone with that move.
|
|
systemd.services."container@hive-ci" = caTrust.containerOrdering;
|
|
|
|
containers.hive-ci = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Private network namespace, attached to the hive bridge so the
|
|
# runner reaches the forge via the gateway — and cannot reach
|
|
# host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.).
|
|
# Requires `forge.behindGateway = true` (asserted in the options
|
|
# block above). See docs/network.md.
|
|
privateNetwork = true;
|
|
hostBridge = networkCfg.bridgeName;
|
|
|
|
bindMounts = {
|
|
# Pre-filled by hive-c0re (via hive-priv) with the runner
|
|
# registration token; tmpfiles seeds a `TOKEN=placeholder` before
|
|
# that. Read-only: the container reads TOKEN= from here; the core
|
|
# admin token never enters the container.
|
|
"/run/hive-ci/runner-token" = {
|
|
hostPath = "/run/hive-ci/runner-token";
|
|
isReadOnly = true;
|
|
};
|
|
# Route the container's nix through the HOST nix-daemon (as the agent
|
|
# containers do) instead of an in-container daemon: with
|
|
# NIX_REMOTE=daemon below, builds run on the host daemon and inherit
|
|
# its buildMachines + max-jobs. Note that `fallback` is NOT inherited
|
|
# that way — it's a client-side option transmitted per connection, so
|
|
# the container sets it itself (see nix.settings.fallback below).
|
|
# Bind the *directory* (not the socket file) so the mount stays live
|
|
# across a host nix-daemon restart, which recreates the socket inode.
|
|
"/nix/var/nix/daemon-socket" = {
|
|
hostPath = "/nix/var/nix/daemon-socket";
|
|
isReadOnly = false;
|
|
};
|
|
}
|
|
# Self-signed mode: bind the public hive CA cert read-only so the
|
|
# runner's Node actions trust the gateway/forge leaf (consumed via
|
|
# NODE_EXTRA_CA_CERTS in the container config). Shared bind-mount +
|
|
# ordering come from the hive-ca-trust helper.
|
|
// caTrust.bindMount;
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# Point the forge domain at the bridge IP so the runner can
|
|
# reach the forge through the gateway — both for registration /
|
|
# polling (runner URL below) and for artifact uploads (the
|
|
# Forgejo Actions artifact API uses ROOT_URL, i.e. the public
|
|
# forge domain, not a localhost URL). The gateway vhost for
|
|
# `forgeCfg.domain` proxies all `/` → forge; `addSSL = true`
|
|
# means HTTP:80 is served without redirect alongside HTTPS:443.
|
|
# Ports 80 and 443 are always open on the bridge firewall (see
|
|
# hive-network.nix). No DNS lookup needed — /etc/hosts wins.
|
|
networking.extraHosts = "${networkCfg.bridgeIp} ${forgeCfg.domain}";
|
|
# DNS: use the hive resolver on the bridge IP (dnsmasq in
|
|
# hive-gateway) for external lookups (git checkout, crate
|
|
# registries, etc.). The bridge→loopback DROP rule does not
|
|
# affect traffic destined for the bridge IP itself.
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
# Bridge-attached via privateNetwork=true + hostBridge. The
|
|
# gateway's dnsmasq serves a DHCP pool covering all usable bridge
|
|
# addresses (see dhcp-range in hive-gateway.nix) — agents and
|
|
# service containers alike receive IPs dynamically.
|
|
networking.interfaces.eth0.useDHCP = true;
|
|
|
|
# nspawn containers can't create user-namespaces, so nix
|
|
# sandboxing always fails. Fall back to unsandboxed builds.
|
|
# Moot once every nix invocation in the container routes
|
|
# through the host daemon (the daemon governs sandboxing).
|
|
# See docs/gotchas.md and harness-base.nix.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
# Degrade to a local build when a remote builder is unreachable
|
|
# rather than failing the check. `fallback` is a client-side
|
|
# option (transmitted per daemon connection), so routing through
|
|
# the host daemon does NOT inherit the host's value — the
|
|
# container has to set it. Same reasoning as the agent
|
|
# containers; see nix/agent-modules/default.nix.
|
|
nix.settings.fallback = true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# Build through the HOST nix-daemon (bind-mounted socket above), not
|
|
# an in-container daemon. NIX_REMOTE=daemon makes root services (the
|
|
# runner) use the daemon socket rather than store=auto → local store.
|
|
# Disable BOTH the service and its .socket unit: the .socket owns the
|
|
# listen path /nix/var/nix/daemon-socket/socket and would otherwise
|
|
# race the bind mount over the same path.
|
|
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
|
systemd.services.nix-daemon.enable = false;
|
|
systemd.sockets.nix-daemon.enable = false;
|
|
|
|
# package is top-level on gitea-actions-runner, not per-instance.
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.hive = {
|
|
enable = true;
|
|
name = cfg.name;
|
|
# Route through the gateway (bridge IP, port 80) so the
|
|
# runner never touches host-loopback. The forge domain
|
|
# resolves to the bridge IP via networking.extraHosts above;
|
|
# the gateway vhost `forgeCfg.domain` proxies to the forge
|
|
# on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect).
|
|
url = "http://${forgeCfg.domain}";
|
|
# EnvironmentFile providing TOKEN= — pre-filled by hive-c0re
|
|
# (via hive-priv) with the runner registration token;
|
|
# bind-mounted read-only from /run/hive-ci/runner-token on the
|
|
# host. The runner's ExecStartPre precond gates on this being a
|
|
# real (non-placeholder) token or an already-registered .runner.
|
|
tokenFile = "/run/hive-ci/runner-token";
|
|
labels = cfg.labels;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
# Per-job wall-clock cap — see the `jobTimeout` option.
|
|
runner.timeout = cfg.jobTimeout;
|
|
};
|
|
};
|
|
|
|
# No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted
|
|
# read-only from the host (pre-filled before container start).
|
|
# nspawn creates the /run/hive-ci/ mount-point directory
|
|
# automatically before launching the container's init.
|
|
|
|
# No hive-ci-register.service inside the container: all forge
|
|
# API calls (runner validation, token fetch) live in hive-c0re
|
|
# (forge/ci_runner.rs), out of band. The core admin token never
|
|
# enters this container.
|
|
|
|
# git is already in the gitea-actions-runner service PATH (the
|
|
# nixpkgs module builds it from the package's runtime deps).
|
|
# nix is required for `nix flake check` / `nix build` workflow
|
|
# steps — it's not included by the upstream module.
|
|
# Use the `path` service attribute (generates ExecSearchPath=)
|
|
# to prepend nix's bin dir to PATH without touching the
|
|
# environment.PATH the nixpkgs module sets — overriding that
|
|
# would lose git, curl, nodejs, and other runner deps.
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
];
|
|
|
|
systemd.services."gitea-runner-hive" = {
|
|
path = [ pkgs.nix ];
|
|
# Trust the hive CA in Node-based actions. With self-signed TLS,
|
|
# forgejo's ROOT_URL is `https://forge.<domain>` (CA-signed leaf),
|
|
# so actions like `upload-artifact` — whose Node HTTP client uses
|
|
# Node's *bundled* CA bundle, not the system store — reject the
|
|
# chain with "unable to verify the first certificate". Pointing
|
|
# NODE_EXTRA_CA_CERTS at the bind-mounted CA adds it to Node's
|
|
# roots for every action, hive-wide. Inherited by the job
|
|
# processes the runner spawns (host execution mode). Only set in
|
|
# self-signed mode; with an operator cert / ACME the public CA
|
|
# already validates and the bind-mount is absent.
|
|
environment = lib.mkIf useSelfSigned {
|
|
NODE_EXTRA_CA_CERTS = caContainerPath;
|
|
};
|
|
# Gate runner start on credentials being present. Registration is
|
|
# done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints a
|
|
# token and, via hive-priv, writes /run/hive-ci/runner-token +
|
|
# restarts this unit. Pass iff already registered (.runner present)
|
|
# OR a real (non-placeholder) token is in place; else exit 1 so the
|
|
# runner stays down (Restart=on-failure retries) until c0re
|
|
# registers it, rather than blocking the container's start.
|
|
#
|
|
# No nix-daemon wait anymore: builds route through the HOST daemon
|
|
# (bound socket above), which is up before the container — nix's
|
|
# own connect retries cover any brief window. `mkBefore` runs this
|
|
# ahead of any upstream pre-steps.
|
|
serviceConfig.ExecStartPre = lib.mkBefore [
|
|
(pkgs.writeShellScript "hive-ci-runner-precond" ''
|
|
RUNNER=/var/lib/gitea-runner/hive/.runner
|
|
TOKEN_FILE=/run/hive-ci/runner-token
|
|
if [ -f "$RUNNER" ]; then exit 0; fi
|
|
if [ -s "$TOKEN_FILE" ] && ! ${pkgs.gnugrep}/bin/grep -q '^TOKEN=placeholder$' "$TOKEN_FILE"; then
|
|
exit 0
|
|
fi
|
|
echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2
|
|
exit 1
|
|
'')
|
|
];
|
|
# Registration is out of band: hive-c0re explicitly restarts this
|
|
# unit once it writes the real token, which is the primary path. As
|
|
# a safety net, if the precond above fires first (no creds yet at
|
|
# boot) retry until c0re registers, rather than staying down — with
|
|
# no start-limit rate cap so it keeps retrying however long forge /
|
|
# the core token take to settle.
|
|
serviceConfig.Restart = lib.mkForce "on-failure";
|
|
# `mkForce`: the upstream gitea-actions-runner module also sets
|
|
# RestartSec (2), so ours needs the higher priority or the two
|
|
# collide into an eval error — same as the sibling `Restart`.
|
|
serviceConfig.RestartSec = lib.mkForce 15;
|
|
startLimitIntervalSec = 0;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|