{ pkgs, lib, config, ... }: let cfg = config.services.hyperhive.forge.ci; forgeCfg = config.services.hyperhive.forge; gatewayCfg = config.services.hyperhive.gateway; networkCfg = config.services.hyperhive.network; tlsCfg = config.services.hyperhive.tls; # Self-signed TLS is the gateway default (no operator cert / ACME). When # active, forgejo's ROOT_URL is `https://forge.` and the leaf is # signed by the host hive CA — so the runner's Node-based actions (e.g. # `upload-artifact`, which POSTs to the ROOT_URL-derived artifact endpoint) # reject the chain, since Node trusts only its bundled CA bundle, not the # system store. Trust the hive CA explicitly via NODE_EXTRA_CA_CERTS below. # The bind-mount + `container@` ordering that makes the CA reachable are # shared with hive-forge via the `hive-ca-trust` helper; only the Node # `NODE_EXTRA_CA_CERTS` consumption is hive-ci-specific. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; useSelfSigned = caTrust.useSelfSigned; caContainerPath = caTrust.caContainerPath; in { # Forgejo Actions runner in a `hive-ci` nixos-container. # Uses a private network namespace (bridge-connected, not host netns) # so CI build scripts cannot reach host-loopback services (dashboard, # forge internal port, etc.) — a key defence against prompt-injection # via PR nix builds. The runner reaches the forge via the # gateway at `http://${forgeCfg.domain}` (resolved to the bridge IP # via `networking.extraHosts`; gateway port 80 is always open on the # bridge; `addSSL = true` means HTTP is served alongside HTTPS without # a redirect). See docs/network.md. # Container is non-ephemeral: the runner's registered credentials # survive restarts (gitea-actions-runner writes them to its stateDir # on first registration and reuses them on every subsequent start). # # Credential isolation: the forge admin token (`forge-core-token`) # never enters the hive-ci container. hive-c0re holds it and performs # all forge API calls (runner validation + registration-token mint, # in `forge/ci_runner.rs`); via hive-priv it writes only the runner # registration token to the host env-file `/run/hive-ci/runner-token`, # which the container bind-mounts read-only. The container never has # access to the wider admin token. # # Nix builds inside the container use the shared /nix/store (standard # nixos-container behaviour) with sandbox-fallback = true, because # nspawn containers can't create the user-namespaces that nix sandboxing # requires. See docs/gotchas.md. options.services.hyperhive.forge.ci = { enable = lib.mkOption { type = lib.types.bool; default = false; example = true; description = '' Run a Forgejo Actions runner in a `hive-ci` nixos-container. Grouped under `services.hyperhive.forge` because the runner is tightly coupled to the forge instance it registers against. Disabled by default; the internal forge it registers against is always present (mandatory), so enabling this is all that's needed. On first start the container auto-registers against hive-forge using hive-c0re's admin token — no manual token provisioning needed. Runner credentials are persisted in the container's state dir and reused on every subsequent boot. ''; }; name = lib.mkOption { type = lib.types.str; default = "hive-ci"; example = "prod-hive"; description = '' Runner name as shown in the Forgejo admin panel. Defaults to "hive-ci"; override when multiple hives share a Forgejo instance. ''; }; concurrency = lib.mkOption { type = lib.types.ints.positive; default = 1; example = 4; description = '' Maximum number of workflow jobs the runner executes in parallel. Each job gets its own temporary working directory; multiple parallel jobs share the container's nix store and cargo registry cache. Higher values trade memory + CPU headroom for throughput. ''; }; labels = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ "hive-ci:host" ]; example = [ "hive-ci:host" "nix:host" ]; description = '' Runner labels in `:` format. The `host` scheme runs commands directly in the container (no docker/podman). Workflow files target this runner with `runs-on: [hive-ci]`. ''; }; package = lib.mkOption { type = lib.types.package; default = pkgs.gitea-actions-runner; defaultText = lib.literalExpression "pkgs.gitea-actions-runner"; description = "gitea-actions-runner package."; }; jobTimeout = lib.mkOption { type = lib.types.str; default = "1h"; example = "3h"; description = '' Per-job wall-clock timeout the runner enforces (act_runner's `runner.timeout`). A job that exceeds it is killed, so a hung or runaway build is bounded instead of holding the runner's single slot indefinitely. Default `1h` comfortably covers a cold-cache nix build while still bounding a stuck job; raise it (e.g. `"3h"`) if you legitimately run jobs longer than that. Accepts a Go duration string (`30m`, `1h`, `2h30m`). Note: this is enforced by the runner process, so it only fires while that process is itself healthy. ''; }; }; config = lib.mkIf cfg.enable { # `forge.behindGateway = true` (the default) is required because the # CI container uses private networking and reaches the forge through # the gateway vhost. Without the gateway vhost there is no HTTP # listener for `forgeCfg.domain` on the bridge that the runner can # connect to. assertions = [ { assertion = forgeCfg.behindGateway; message = '' services.hyperhive.forge.ci.enable requires services.hyperhive.forge.behindGateway = true. The CI container runs with a private network namespace and reaches the forge through the gateway vhost on the bridge IP. Set behindGateway = true (it defaults to true alongside services.hyperhive.enable). ''; } ]; # Create /run/hive-ci/ on the host and seed runner-token with a # placeholder. The container bind-mounts this file read-only; hive-c0re # (via hive-priv's RegisterCiRunner) overwrites it with the real # registration token when it registers the runner out of band. The # placeholder keeps the runner's EnvironmentFile present from first boot, # before c0re has registered — the runner's ExecStartPre precond (below) # distinguishes the placeholder from a real token. systemd.tmpfiles.rules = [ "d /run/hive-ci 0700 root root -" "f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder" ]; # Tell hive-c0re that CI is enabled so its startup sweep registers the # runner. Registration moved OFF the container's boot-critical path into # hive-c0re (`forge/ci_runner.rs`): it validates the persisted `.runner` # against the forge and, when absent/stale, mints a registration token and # hands it to hive-priv to write `/run/hive-ci/runner-token` + restart the # runner. The forge admin token stays in hive-c0re; only the registration # token reaches the host env-file the container mounts read-only. systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_CI_ENABLED = "1"; # `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service` # in self-signed mode (see the hive-ca-trust helper). Runner registration # is no longer on the boot-critical path — hive-c0re owns it out of band — # so the container boots immediately and needs no start-timeout bump. The # former `TimeoutStartSec = mkForce "180s"` band-aid (which papered over a # boot-path forge + core-token round-trip that could trip the nspawn start # timeout into a ~60s restart loop) is gone with that move. systemd.services."container@hive-ci" = caTrust.containerOrdering; containers.hive-ci = { autoStart = true; ephemeral = false; # Private network namespace, attached to the hive bridge so the # runner reaches the forge via the gateway — and cannot reach # host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.). # Requires `forge.behindGateway = true` (asserted in the options # block above). See docs/network.md. privateNetwork = true; hostBridge = networkCfg.bridgeName; bindMounts = { # Pre-filled by hive-c0re (via hive-priv) with the runner # registration token; tmpfiles seeds a `TOKEN=placeholder` before # that. Read-only: the container reads TOKEN= from here; the core # admin token never enters the container. "/run/hive-ci/runner-token" = { hostPath = "/run/hive-ci/runner-token"; isReadOnly = true; }; } # Self-signed mode: bind the public hive CA cert read-only so the # runner's Node actions trust the gateway/forge leaf (consumed via # NODE_EXTRA_CA_CERTS in the container config). Shared bind-mount + # ordering come from the hive-ca-trust helper. // caTrust.bindMount; config = { pkgs, lib, ... }: { system.stateVersion = "26.05"; # Point the forge domain at the bridge IP so the runner can # reach the forge through the gateway — both for registration / # polling (runner URL below) and for artifact uploads (the # Forgejo Actions artifact API uses ROOT_URL, i.e. the public # forge domain, not a localhost URL). The gateway vhost for # `forgeCfg.domain` proxies all `/` → forge; `addSSL = true` # means HTTP:80 is served without redirect alongside HTTPS:443. # Ports 80 and 443 are always open on the bridge firewall (see # hive-network.nix). No DNS lookup needed — /etc/hosts wins. networking.extraHosts = "${networkCfg.bridgeIp} ${forgeCfg.domain}"; # DNS: use the hive resolver on the bridge IP (dnsmasq in # hive-gateway) for external lookups (git checkout, crate # registries, etc.). The bridge→loopback DROP rule does not # affect traffic destined for the bridge IP itself. networking.nameservers = [ networkCfg.bridgeIp ]; # Bridge-attached via privateNetwork=true + hostBridge. The # gateway's dnsmasq serves a DHCP pool covering all usable bridge # addresses (see dhcp-range in hive-gateway.nix) — agents and # service containers alike receive IPs dynamically. networking.interfaces.eth0.useDHCP = true; # nspawn containers can't create user-namespaces, so nix # sandboxing always fails. Fall back to unsandboxed builds. # Moot once every nix invocation in the container routes # through the host daemon (the daemon governs sandboxing). # See docs/gotchas.md and harness-base.nix. nix.settings.sandbox-fallback = lib.mkForce true; nix.settings.experimental-features = [ "nix-command" "flakes" ]; # package is top-level on gitea-actions-runner, not per-instance. services.gitea-actions-runner.package = cfg.package; services.gitea-actions-runner.instances.hive = { enable = true; name = cfg.name; # Route through the gateway (bridge IP, port 80) so the # runner never touches host-loopback. The forge domain # resolves to the bridge IP via networking.extraHosts above; # the gateway vhost `forgeCfg.domain` proxies to the forge # on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect). url = "http://${forgeCfg.domain}"; # EnvironmentFile providing TOKEN= — pre-filled by hive-c0re # (via hive-priv) with the runner registration token; # bind-mounted read-only from /run/hive-ci/runner-token on the # host. The runner's ExecStartPre precond gates on this being a # real (non-placeholder) token or an already-registered .runner. tokenFile = "/run/hive-ci/runner-token"; labels = cfg.labels; settings = { runner.capacity = cfg.concurrency; # Per-job wall-clock cap — see the `jobTimeout` option. runner.timeout = cfg.jobTimeout; }; }; # No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted # read-only from the host (pre-filled before container start). # nspawn creates the /run/hive-ci/ mount-point directory # automatically before launching the container's init. # No hive-ci-register.service inside the container: all forge # API calls (runner validation, token fetch) live in hive-c0re # (forge/ci_runner.rs), out of band. The core admin token never # enters this container. # git is already in the gitea-actions-runner service PATH (the # nixpkgs module builds it from the package's runtime deps). # nix is required for `nix flake check` / `nix build` workflow # steps — it's not included by the upstream module. # Use the `path` service attribute (generates ExecSearchPath=) # to prepend nix's bin dir to PATH without touching the # environment.PATH the nixpkgs module sets — overriding that # would lose git, curl, nodejs, and other runner deps. environment.systemPackages = [ pkgs.git pkgs.nix ]; systemd.services."gitea-runner-hive" = { path = [ pkgs.nix ]; # Trust the hive CA in Node-based actions. With self-signed TLS, # forgejo's ROOT_URL is `https://forge.` (CA-signed leaf), # so actions like `upload-artifact` — whose Node HTTP client uses # Node's *bundled* CA bundle, not the system store — reject the # chain with "unable to verify the first certificate". Pointing # NODE_EXTRA_CA_CERTS at the bind-mounted CA adds it to Node's # roots for every action, hive-wide. Inherited by the job # processes the runner spawns (host execution mode). Only set in # self-signed mode; with an operator cert / ACME the public CA # already validates and the bind-mount is absent. environment = lib.mkIf useSelfSigned { NODE_EXTRA_CA_CERTS = caContainerPath; }; # Gate runner start (and therefore job registration/claiming) on # the in-container nix daemon being reachable. After a hive-ci # restart the runner re-registers and immediately claims any # queued jobs — which can beat the nix daemon coming up: its # `nix-daemon.socket` carries # `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket` and is # skipped until /nix/var is read-write, so the first # nix-dependent build dispatches into a cold daemon and # hangs/retries (observed: a 55m48s `nix flake check` vs the # normal ~30s, a build-offload stall, not a code failure). # # Ordering `after`/`wants` the socket unit does NOT fix this — a # condition-skipped unit satisfies systemd ordering immediately, # so the runner would still start before the daemon is live. # Instead block in ExecStartPre by polling the daemon until it # actually answers; this is topology-agnostic (works whether the # daemon is in-container or a shared host socket). `mkBefore` so # this runs ahead of any pre-steps the upstream module adds. serviceConfig.ExecStartPre = lib.mkBefore [ # Fail fast unless the runner can actually come up. Registration # is done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints # a token and, via hive-priv, writes /run/hive-ci/runner-token + # restarts this unit — so the runner never does the forge # round-trip on the boot path (that was the old prefetch's # boot-critical wait that could trip the nspawn start timeout). # Pass iff already registered (.runner present) OR a real # (non-placeholder) token is in place; otherwise exit 1 so the # runner just stays down (Restart=on-failure retries) until c0re # registers it, instead of blocking the container's start. Runs # ahead of the nix-daemon wait below — no point waiting for the # daemon if there are no credentials to come up with. (pkgs.writeShellScript "hive-ci-runner-precond" '' RUNNER=/var/lib/gitea-runner/hive/.runner TOKEN_FILE=/run/hive-ci/runner-token if [ -f "$RUNNER" ]; then exit 0; fi if [ -s "$TOKEN_FILE" ] && ! ${pkgs.gnugrep}/bin/grep -q '^TOKEN=placeholder$' "$TOKEN_FILE"; then exit 0 fi echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2 exit 1 '') (pkgs.writeShellScript "wait-nix-daemon" '' # Up to ~180s; the daemon is normally up within seconds, this # only bites in the post-restart cold window. Non-fatal shape: # if it never comes up the unit fails cleanly (recoverable) # rather than the runner claiming jobs into a dead daemon. # # `--store daemon` is load-bearing. A bare `nix store ping` # uses the `auto` store, which — when run as root with the # daemon socket absent — silently resolves to a LOCAL store # (root can write /nix/store directly) and pings successfully. # systemd service units don't source the profile that sets # `NIX_REMOTE=daemon`, so this is the real environment here. # At cold boot the daemon socket IS absent: nix-daemon.socket # carries `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket` # and is condition-skipped until /nix/var goes read-write. So # a bare ping would pass against the local store while the # daemon is still down — defeating the gate's whole purpose # (the runner would start and claim jobs the daemon can't yet # service). Pinning `--store daemon` makes the poll verify the # actual daemon socket, so the gate honours its contract and # waits until the daemon — not a local fallback — answers. for _ in $(seq 1 90); do if ${pkgs.nix}/bin/nix store ping --store daemon >/dev/null 2>&1; then exit 0 fi sleep 2 done echo "nix daemon not reachable after 180s" >&2 exit 1 '') ]; # Registration is out of band: hive-c0re explicitly restarts this # unit once it writes the real token, which is the primary path. As # a safety net, if the precond above fires first (no creds yet at # boot) retry until c0re registers, rather than staying down — with # no start-limit rate cap so it keeps retrying however long forge / # the core token take to settle. serviceConfig.Restart = lib.mkForce "on-failure"; # `mkForce`: the upstream gitea-actions-runner module also sets # RestartSec (2), so ours needs the higher priority or the two # collide into an eval error — same as the sibling `Restart`. serviceConfig.RestartSec = lib.mkForce 15; startLimitIntervalSec = 0; }; }; }; }; }