{ pkgs, lib, config, ... }: let cfg = config.services.hyperhive.deploy.forgejo.ci; forgeCfg = config.services.hyperhive.swarm.forge; gatewayCfg = config.services.hyperhive.gateway; networkCfg = config.services.hyperhive.network; tlsCfg = config.services.hyperhive.tls; # Self-signed TLS is the gateway default (no operator cert / ACME). When # active, forgejo's ROOT_URL is `https://forge.` and the leaf is # signed by the host hive CA — so the runner's Node-based actions (e.g. # `upload-artifact`, which POSTs to the ROOT_URL-derived artifact endpoint) # reject the chain, since Node trusts only its bundled CA bundle, not the # system store. Trust the hive CA explicitly via NODE_EXTRA_CA_CERTS below. # The bind-mount + `container@` ordering that makes the CA reachable are # shared with hive-forge via the `hive-ca-trust` helper; only the Node # `NODE_EXTRA_CA_CERTS` consumption is hive-ci-specific. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; useSelfSigned = caTrust.useSelfSigned; caContainerPath = caTrust.caContainerPath; # The runner daemon needs the hive CA too, and `NODE_EXTRA_CA_CERTS` above # does not give it to them: that variable is Node's, and it is ADDITIVE. # `gitea-runner` is Go, whose trust store is REPLACING -- it reads # `SSL_CERT_FILE` and then trusts only what that file contains. So the two # consumers need different treatment from the same CA, which is exactly the # split the helper documents at its head. # # Without this the daemon fails every startup call with # `x509: certificate signed by unknown authority` the moment it reaches a # TLS endpoint -- and it reaches one whenever the address it is registered # at redirects to the gateway's https vhost. The unit then crash-loops on # `Restart=on-failure`, no job is ever picked up, and nothing reports it as # broken: every pull request simply sits at "Waiting to run". # # ⚠️ This is deliberately NOT claimed as the whole story of that outage -- # what address the daemon holds is a separate question. It is here because # the trust gap is real under every explanation of it, and adding a CA to # one unit cannot break a path that already works. caBundleModule = caTrust.trustBundle { inherit pkgs; name = "hive-ci"; consumers = [ "gitea-runner-hive" ]; }; in { # Forgejo Actions runner in a `hive-ci` nixos-container. # Uses a private network namespace (bridge-connected, not host netns) # so CI build scripts cannot reach host-loopback services (dashboard, # forge internal port, etc.) — a key defence against prompt-injection # via PR nix builds. The runner reaches the forge via the # gateway at `http://${forgeCfg.domain}` (resolved to the bridge IP # via `networking.extraHosts`; gateway port 80 is always open on the # bridge; `addSSL = true` means HTTP is served alongside HTTPS without # a redirect). See docs/network.md. # Container is non-ephemeral: the runner's registered credentials # survive restarts (gitea-actions-runner writes them to its stateDir # on first registration and reuses them on every subsequent start). # # Credential isolation: the forge admin token (`forge-core-token`) # never enters the hive-ci container. hive-c0re holds it and performs # all forge API calls (runner validation + registration-token mint, # in `forge/ci_runner.rs`); via hive-priv it writes only the runner # registration token to the host env-file `/run/hive-ci/runner-token`, # which the container bind-mounts read-only. The container never has # access to the wider admin token. # # Nix builds inside the container use the shared /nix/store (standard # nixos-container behaviour) with sandbox-fallback = true, because # nspawn containers can't create the user-namespaces that nix sandboxing # requires. See docs/gotchas.md. options.services.hyperhive.deploy.forgejo.ci = { enable = lib.mkOption { type = lib.types.bool; default = false; example = true; description = '' Run a Forgejo Actions runner in a `hive-ci` nixos-container. Grouped under `services.hyperhive.swarm.forge` because the runner is tightly coupled to the forge instance it registers against. Disabled by default; the internal forge it registers against is always present (mandatory), so enabling this is all that's needed. On first start the container auto-registers against hive-forge using hive-c0re's admin token — no manual token provisioning needed. Runner credentials are persisted in the container's state dir and reused on every subsequent boot. ''; }; name = lib.mkOption { type = lib.types.str; default = "hive-ci"; example = "prod-hive"; description = '' Runner name as shown in the Forgejo admin panel. Defaults to "hive-ci"; override when multiple hives share a Forgejo instance. ''; }; concurrency = lib.mkOption { type = lib.types.ints.positive; default = 1; example = 4; description = '' Maximum number of workflow jobs the runner executes in parallel. Each job gets its own temporary working directory; multiple parallel jobs share the container's nix store and cargo registry cache. Higher values trade memory + CPU headroom for throughput. ''; }; labels = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ "hive-ci:host" ]; example = [ "hive-ci:host" "nix:host" ]; description = '' Runner labels in `:` format. The `host` scheme runs commands directly in the container (no docker/podman). Workflow files target this runner with `runs-on: [hive-ci]`. ''; }; package = lib.mkOption { type = lib.types.package; default = pkgs.gitea-actions-runner; defaultText = lib.literalExpression "pkgs.gitea-actions-runner"; description = "gitea-actions-runner package."; }; jobTimeout = lib.mkOption { type = lib.types.str; default = "1h"; example = "3h"; description = '' Per-job wall-clock timeout the runner enforces (act_runner's `runner.timeout`). A job that exceeds it is killed, so a hung or runaway build is bounded instead of holding the runner's single slot indefinitely. Default `1h` comfortably covers a cold-cache nix build while still bounding a stuck job; raise it (e.g. `"3h"`) if you legitimately run jobs longer than that. Accepts a Go duration string (`30m`, `1h`, `2h30m`). Note: this is enforced by the runner process, so it only fires while that process is itself healthy. ''; }; }; config = lib.mkIf cfg.enable { # `forge.behindGateway = true` (the default) is required because the # CI container uses private networking and reaches the forge through # the gateway vhost. Without the gateway vhost there is no HTTP # listener for `forgeCfg.domain` on the bridge that the runner can # connect to. assertions = [ { assertion = forgeCfg.behindGateway; message = '' services.hyperhive.deploy.forgejo.ci.enable requires services.hyperhive.swarm.forge.behindGateway = true. The CI container runs with a private network namespace and reaches the forge through the gateway vhost on the bridge IP. Set behindGateway = true (it defaults to true alongside services.hyperhive.enable). ''; } ]; # Create /run/hive-ci/ on the host and seed runner-token with a # placeholder. The container bind-mounts this file read-only; hive-c0re # (via hive-priv's RegisterCiRunner) overwrites it with the real # registration token when it registers the runner out of band. The # placeholder keeps the runner's EnvironmentFile present from first boot, # before c0re has registered — the runner's ExecStartPre precond (below) # distinguishes the placeholder from a real token. systemd.tmpfiles.rules = [ "d /run/hive-ci 0700 root root -" "f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder" ]; # Tell hive-c0re that CI is enabled so its startup sweep registers the # runner. Registration moved OFF the container's boot-critical path into # hive-c0re (`forge/ci_runner.rs`): it validates the persisted `.runner` # against the forge and, when absent/stale, mints a registration token and # hands it to hive-priv to write `/run/hive-ci/runner-token` + restart the # runner. The forge admin token stays in hive-c0re; only the registration # token reaches the host env-file the container mounts read-only. systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_CI_ENABLED = "1"; # `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service` # in self-signed mode (see the hive-ca-trust helper). Runner registration # is no longer on the boot-critical path — hive-c0re owns it out of band — # so the container boots immediately and needs no start-timeout bump. The # former `TimeoutStartSec = mkForce "180s"` band-aid (which papered over a # boot-path forge + core-token round-trip that could trip the nspawn start # timeout into a ~60s restart loop) is gone with that move. systemd.services."container@hive-ci" = caTrust.containerOrdering; containers.hive-ci = { autoStart = true; ephemeral = false; # Private network namespace, attached to the hive bridge so the # runner reaches the forge via the gateway — and cannot reach # host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.). # Requires `forge.behindGateway = true` (asserted in the options # block above). See docs/network.md. privateNetwork = true; hostBridge = networkCfg.bridgeName; bindMounts = { # Pre-filled by hive-c0re (via hive-priv) with the runner # registration token; tmpfiles seeds a `TOKEN=placeholder` before # that. Read-only: the container reads TOKEN= from here; the core # admin token never enters the container. "/run/hive-ci/runner-token" = { hostPath = "/run/hive-ci/runner-token"; isReadOnly = true; }; # Route the container's nix through the HOST nix-daemon (as the agent # containers do) instead of an in-container daemon: with # NIX_REMOTE=daemon below, builds run on the host daemon and inherit # its buildMachines + max-jobs. Note that `fallback` is NOT inherited # that way — it's a client-side option transmitted per connection, so # the container sets it itself (see nix.settings.fallback below). # Bind the *directory* (not the socket file) so the mount stays live # across a host nix-daemon restart, which recreates the socket inode. "/nix/var/nix/daemon-socket" = { hostPath = "/nix/var/nix/daemon-socket"; isReadOnly = false; }; } # Self-signed mode: bind the public hive CA cert read-only so the # runner's Node actions trust the gateway/forge leaf (consumed via # NODE_EXTRA_CA_CERTS in the container config). Shared bind-mount + # ordering come from the hive-ca-trust helper. // caTrust.bindMount; config = { pkgs, lib, ... }: { # Assembles system CAs + the hive CA into one bundle and sets # `SSL_CERT_FILE` on the runner unit. See `caBundleModule` above for # why the Node variable beside it is not enough. imports = [ caBundleModule ]; system.stateVersion = "26.05"; # Point the forge domain at the bridge IP so the runner can # reach the forge through the gateway — both for registration / # polling (runner URL below) and for artifact uploads (the # Forgejo Actions artifact API uses ROOT_URL, i.e. the public # forge domain, not a localhost URL). The gateway vhost for # `forgeCfg.domain` proxies all `/` → forge; `addSSL = true` # means HTTP:80 is served without redirect alongside HTTPS:443. # Ports 80 and 443 are always open on the bridge firewall (see # hive-network.nix). No DNS lookup needed — /etc/hosts wins. networking.extraHosts = "${networkCfg.bridgeIp} ${forgeCfg.domain}"; # DNS: use the hive resolver on the bridge IP (dnsmasq in # hive-gateway) for external lookups (git checkout, crate # registries, etc.). The bridge→loopback DROP rule does not # affect traffic destined for the bridge IP itself. networking.nameservers = [ networkCfg.bridgeIp ]; # Bridge-attached via privateNetwork=true + hostBridge. The # gateway's dnsmasq serves a DHCP pool covering all usable bridge # addresses (see dhcp-range in hive-gateway.nix) — agents and # service containers alike receive IPs dynamically. networking.interfaces.eth0.useDHCP = true; # nspawn containers can't create user-namespaces, so nix # sandboxing always fails. Fall back to unsandboxed builds. # Moot once every nix invocation in the container routes # through the host daemon (the daemon governs sandboxing). # See docs/gotchas.md and nix/agent-modules/default.nix. nix.settings.sandbox-fallback = lib.mkForce true; # Degrade to a local build when a remote builder is unreachable # rather than failing the check. `fallback` is a client-side # option (transmitted per daemon connection), so routing through # the host daemon does NOT inherit the host's value — the # container has to set it. Same reasoning as the agent # containers; see nix/agent-modules/default.nix. nix.settings.fallback = true; nix.settings.experimental-features = [ "nix-command" "flakes" ]; # Build through the HOST nix-daemon (bind-mounted socket above), not # an in-container daemon. NIX_REMOTE=daemon makes root services (the # runner) use the daemon socket rather than store=auto → local store. # Disable BOTH the service and its .socket unit: the .socket owns the # listen path /nix/var/nix/daemon-socket/socket and would otherwise # race the bind mount over the same path. systemd.globalEnvironment.NIX_REMOTE = "daemon"; systemd.services.nix-daemon.enable = false; systemd.sockets.nix-daemon.enable = false; # package is top-level on gitea-actions-runner, not per-instance. services.gitea-actions-runner.package = cfg.package; services.gitea-actions-runner.instances.hive = { enable = true; name = cfg.name; # Route through the gateway (bridge IP, port 80) so the # runner never touches host-loopback. The forge domain # resolves to the bridge IP via networking.extraHosts above; # the gateway vhost `forgeCfg.domain` proxies to the forge # on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect). url = "http://${forgeCfg.domain}"; # EnvironmentFile providing TOKEN= — pre-filled by hive-c0re # (via hive-priv) with the runner registration token; # bind-mounted read-only from /run/hive-ci/runner-token on the # host. The runner's ExecStartPre precond gates on this being a # real (non-placeholder) token or an already-registered .runner. tokenFile = "/run/hive-ci/runner-token"; labels = cfg.labels; settings = { runner.capacity = cfg.concurrency; # Per-job wall-clock cap — see the `jobTimeout` option. runner.timeout = cfg.jobTimeout; }; }; # No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted # read-only from the host (pre-filled before container start). # nspawn creates the /run/hive-ci/ mount-point directory # automatically before launching the container's init. # No hive-ci-register.service inside the container: all forge # API calls (runner validation, token fetch) live in hive-c0re # (forge/ci_runner.rs), out of band. The core admin token never # enters this container. # git is already in the gitea-actions-runner service PATH (the # nixpkgs module builds it from the package's runtime deps). # nix is required for `nix flake check` / `nix build` workflow # steps — it's not included by the upstream module. # Use the `path` service attribute (generates ExecSearchPath=) # to prepend nix's bin dir to PATH without touching the # environment.PATH the nixpkgs module sets — overriding that # would lose git, curl, nodejs, and other runner deps. environment.systemPackages = [ pkgs.git pkgs.nix ]; systemd.services."gitea-runner-hive" = { path = [ pkgs.nix ]; # A CI runner must be able to EXECUTE what it builds, and by # default it cannot: the upstream unit uses `DynamicUser`, and # systemd mounts a dynamic unit's state directory `noexec`. # Measured on the live runner rather than inferred — from a job's # own /proc/self/mountinfo: # # /var/lib/private ro,nosuid,nodev,noexec (tmpfs) # /var/lib/private/gitea-runner rw,nosuid,nodev,noexec,idmapped # # with /tmp as the control at rw,nosuid,nodev — no noexec, and a # script there executes fine. Nothing noticed for as long as every # workflow built inside the nix sandbox and executed nothing in the # workspace; the first job that compiled a build script into the # runner's own target dir died with EACCES on execve, reported as # "could not execute process ... (never executed)". # # Both spellings are listed because `ExecPaths=` resolves against # the HOST root, where the dynamic-user layout makes # /var/lib/gitea-runner a symlink into private/. The `-` prefix # means "ignore if absent", so whichever layout the unit ends up # with, the other is a no-op rather than a startup failure. serviceConfig.ExecPaths = [ "-/var/lib/gitea-runner" "-/var/lib/private/gitea-runner" ]; # Trust the hive CA in Node-based actions. With self-signed TLS, # forgejo's ROOT_URL is `https://forge.` (CA-signed leaf), # so actions like `upload-artifact` — whose Node HTTP client uses # Node's *bundled* CA bundle, not the system store — reject the # chain with "unable to verify the first certificate". Pointing # NODE_EXTRA_CA_CERTS at the bind-mounted CA adds it to Node's # roots for every action, hive-wide. Inherited by the job # processes the runner spawns (host execution mode). Only set in # self-signed mode; with an operator cert / ACME the public CA # already validates and the bind-mount is absent. environment = lib.mkIf useSelfSigned { NODE_EXTRA_CA_CERTS = caContainerPath; }; # Gate runner start on credentials being present. Registration is # done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints a # token and, via hive-priv, writes /run/hive-ci/runner-token + # restarts this unit. Pass iff already registered (.runner present) # OR a real (non-placeholder) token is in place; else exit 1 so the # runner stays down (Restart=on-failure retries) until c0re # registers it, rather than blocking the container's start. # # No nix-daemon wait anymore: builds route through the HOST daemon # (bound socket above), which is up before the container — nix's # own connect retries cover any brief window. `mkBefore` runs this # ahead of any upstream pre-steps. # # The `+` prefix runs this step with FULL PRIVILEGES instead of as # the unit's (dynamic) `gitea-runner` user. The token file is 0600 # root-owned, so an unprivileged check cannot read it — and it did # not fail closed: `[ -s ]` succeeds on a stat alone, `grep` then # died with EACCES, and `! grep` turned that error into `true`, so # the gate exited 0 on an unreadable file. It reported "a real token # is present" without ever having looked, for its entire existence. # # This grants the runner nothing new: `tokenFile` above becomes # `EnvironmentFile=`, which systemd already reads AS ROOT before # dropping privileges, so the payload never passes through an # unprivileged reader either way. The `+` only lets the gate observe # what systemd observes. serviceConfig.ExecStartPre = lib.mkBefore [ "+${pkgs.writeShellScript "hive-ci-runner-precond" '' RUNNER=/var/lib/gitea-runner/hive/.runner TOKEN_FILE=/run/hive-ci/runner-token if [ -f "$RUNNER" ]; then exit 0; fi # Readability is asserted SEPARATELY and fails loudly. Folding # it into the test below is what hid the bug: an unreadable file # and a file holding a real token produced the same verdict, so # "I could not look" was indistinguishable from "I looked and it # is fine". if [ -e "$TOKEN_FILE" ] && [ ! -r "$TOKEN_FILE" ]; then echo "hive-ci runner: $TOKEN_FILE exists but is unreadable by this step; the credential gate cannot run" >&2 exit 1 fi if [ -s "$TOKEN_FILE" ] && ! ${pkgs.gnugrep}/bin/grep -q '^TOKEN=placeholder$' "$TOKEN_FILE"; then exit 0 fi echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2 exit 1 ''}" ]; # Registration is out of band: hive-c0re explicitly restarts this # unit once it writes the real token, which is the primary path. As # a safety net, if the precond above fires first (no creds yet at # boot) retry until c0re registers, rather than staying down — with # no start-limit rate cap so it keeps retrying however long forge / # the core token take to settle. serviceConfig.Restart = lib.mkForce "on-failure"; # `mkForce`: the upstream gitea-actions-runner module also sets # RestartSec (2), so ours needs the higher priority or the two # collide into an eval error — same as the sibling `Restart`. serviceConfig.RestartSec = lib.mkForce 15; startLimitIntervalSec = 0; }; }; }; }; }