078ea74ba9
hive-ci is the one container on the hive bridge that used DHCP (networking.interfaces.eth0.useDHCP = true, from #2310 / #2336). But the bridge has no DHCP server: dnsmasq on the bridge is DNS-only and agent containers get deterministic static IPs (lifecycle::agent_network_ip), so the DHCP client never gets a lease and the container hangs at boot: A start job is running for DHCP Client (5s / 1min 31s) Assign a static address on eth0 (top host address of the subnet) plus a default route via the bridge gateway, and drop useDHCP. Preserves the #2310 netns isolation (no host-loopback reach) while letting the container boot and reach the forge through the gateway. Refs #2310, #2336.
570 lines
27 KiB
Nix
570 lines
27 KiB
Nix
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.forge.ci;
|
|
forgeCfg = config.services.hyperhive.forge;
|
|
gatewayCfg = config.services.hyperhive.gateway;
|
|
networkCfg = config.services.hyperhive.network;
|
|
tlsCfg = config.services.hyperhive.tls;
|
|
|
|
# Static bridge address for the hive-ci container. The hive bridge has
|
|
# NO DHCP server: agent containers get deterministic static IPs
|
|
# (`lifecycle::agent_network_ip` hashes each name across .2..broadcast-1)
|
|
# and dnsmasq on the bridge is DNS-only. hive-ci is the one service
|
|
# container on the bridge, so it needs a static address too — `useDHCP`
|
|
# here only hangs the boot waiting for a lease nothing serves. Reserve the
|
|
# top host address of the (default /24) subnet; a clash with an agent that
|
|
# happens to hash here is the same rename-to-resolve case as any
|
|
# agent/agent IP collision. Operators on a non-/24 bridge (or with
|
|
# `bridgeIp` set to the top address) should pick a free host address.
|
|
ciBridgeOctets = lib.splitString "." networkCfg.bridgeIp;
|
|
ciBridgeIp = "${lib.elemAt ciBridgeOctets 0}.${lib.elemAt ciBridgeOctets 1}.${lib.elemAt ciBridgeOctets 2}.254";
|
|
|
|
# Self-signed TLS is the gateway default (no operator cert / ACME). When
|
|
# active, forgejo's ROOT_URL is `https://forge.<domain>` and the leaf is
|
|
# signed by the host hive CA — so the runner's Node-based actions (e.g.
|
|
# `upload-artifact`, which POSTs to the ROOT_URL-derived artifact endpoint)
|
|
# reject the chain, since Node trusts only its bundled CA bundle, not the
|
|
# system store. Trust the hive CA explicitly via NODE_EXTRA_CA_CERTS below.
|
|
# `gateway.useSelfSigned` is the gateway module's single source of truth
|
|
# for the self-signed condition (no duplicated derivation here).
|
|
useSelfSigned = gatewayCfg.useSelfSigned;
|
|
caHostPath = "${tlsCfg.stateDir}/ca.pem";
|
|
caContainerPath = "/run/hive-ca/ca.pem";
|
|
|
|
# hive-c0re writes its own admin token here on first forge startup.
|
|
# The token has read:admin + write:admin scopes — sufficient to call
|
|
# the runner registration-token API endpoint.
|
|
# This path is HOST-ONLY. It is never bind-mounted into hive-ci.
|
|
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
|
|
|
|
# Container state root on the host. Non-ephemeral containers keep
|
|
# their filesystem here across reboots. The prefetch service accesses
|
|
# the runner's .runner file via this path to validate credentials
|
|
# without entering the container.
|
|
containerRoot = "/var/lib/nixos-containers/hive-ci";
|
|
|
|
# Host-side oneshot. Runs before `container@hive-ci.service`.
|
|
#
|
|
# The core token stays on the host. The container only ever sees the
|
|
# TOKEN= env-file populated here, never the core token itself.
|
|
#
|
|
# Flow:
|
|
# 1. If .runner exists: validate the runner ID against forge.
|
|
# Waits up to 60s for forge-core-token (hive-c0re writes it after
|
|
# forge container starts and admin is provisioned — this lags
|
|
# hive-c0re.service becoming active on first boot).
|
|
# 404 → purge .runner (re-registration needed).
|
|
# 000 (forge unreachable) → keep credentials, write placeholder.
|
|
# other non-200 → purge .runner.
|
|
# Token absent after 60s → keep credentials (safe — the runner
|
|
# holds valid creds; next boot will validate properly).
|
|
# 2. If .runner absent or just purged: wait up to 60s for both
|
|
# forge-core-token to appear AND forge API to respond, then
|
|
# fetch a fresh registration token and write TOKEN=<real>.
|
|
# Exits non-zero if both timeout — systemd logs the failure;
|
|
# the placeholder from tmpfiles means the container still starts
|
|
# but the runner will error. Operator restarts the service once
|
|
# forge is healthy.
|
|
prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
|
|
set -euo pipefail
|
|
TOKEN_FILE=/run/hive-ci/runner-token
|
|
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
|
RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
|
|
|
|
# Validate existing .runner credentials against forge. Purge if
|
|
# the runner was deleted (404) or the file is malformed. Keep if
|
|
# forge is unreachable (000) — transient outage shouldn't wipe creds.
|
|
#
|
|
# We wait up to 60s for forge-core-token to appear (hive-c0re writes
|
|
# it after provisioning the forge admin, which requires the forge
|
|
# container to start and become ready — this can lag behind
|
|
# hive-c0re.service becoming "active" on first boot). The same loop
|
|
# doubles as a wait for forge's API to become ready.
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
|
|
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
|
|
echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
else
|
|
# Wait for the core token before validating (same timeout as below).
|
|
CORE_TOKEN=""
|
|
for i in $(seq 1 60); do
|
|
if [ -f "${coreTokenPath}" ]; then
|
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
if [ -z "''${CORE_TOKEN:-}" ]; then
|
|
echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
|
|
else
|
|
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
|
|
-H "Authorization: token $CORE_TOKEN" \
|
|
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
|
|
if [ "$HTTP" = "404" ]; then
|
|
echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
elif [ "$HTTP" = "000" ]; then
|
|
echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
|
|
elif [ "$HTTP" != "200" ]; then
|
|
echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# .runner valid → write placeholder; gitea-actions-runner skips
|
|
# re-registration when .runner exists (TOKEN value is irrelevant).
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
echo "TOKEN=placeholder" > "$TOKEN_FILE"
|
|
chmod 600 "$TOKEN_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
# First boot or stale creds purged — wait for forge-core-token, then
|
|
# fetch a fresh registration token. Single retry loop covers both:
|
|
# waiting for hive-c0re to write the token file AND for forge's API
|
|
# to become responsive (they race on first boot).
|
|
REG_TOKEN=""
|
|
for i in $(seq 1 60); do
|
|
if [ ! -f "${coreTokenPath}" ]; then
|
|
echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
|
|
sleep 1
|
|
continue
|
|
fi
|
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
|
# Capture the HTTP status so a stale/invalid core token (401/403) is
|
|
# distinguished from a transient forge hiccup. With the old bare
|
|
# `curl -sf | jq`, a forge-core-token that's stale for the current
|
|
# forge (e.g. after a forge rebuild) 401s and fails silently every
|
|
# attempt for the full 60s loop, then exits with a misleading
|
|
# "core token absent or forge unreachable" — masking the real cause.
|
|
# Fail fast + loudly on 401/403 so the failure mode is legible and
|
|
# the operator/hive-c0re knows to re-mint forge-core-token.
|
|
RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \
|
|
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
|
-H "Authorization: token $CORE_TOKEN" || printf '\n000')
|
|
HTTP=$(printf '%s' "$RESP" | tail -n1)
|
|
BODY=$(printf '%s' "$RESP" | sed '$d')
|
|
case "$HTTP" in
|
|
2*)
|
|
REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token)
|
|
if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi
|
|
echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2
|
|
;;
|
|
401 | 403)
|
|
echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2
|
|
;;
|
|
esac
|
|
sleep 1
|
|
done
|
|
|
|
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
|
|
echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
|
|
chmod 600 "$TOKEN_FILE"
|
|
'';
|
|
in
|
|
{
|
|
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
# Uses a private network namespace (bridge-connected, not host netns)
|
|
# so CI build scripts cannot reach host-loopback services (dashboard,
|
|
# forge internal port, etc.) — a key defence against prompt-injection
|
|
# via PR nix builds. The runner reaches the forge via the
|
|
# gateway at `http://${forgeCfg.domain}` (resolved to the bridge IP
|
|
# via `networking.extraHosts`; gateway port 80 is always open on the
|
|
# bridge; `addSSL = true` means HTTP is served alongside HTTPS without
|
|
# a redirect). See docs/network.md.
|
|
# Container is non-ephemeral: the runner's registered credentials
|
|
# survive restarts (gitea-actions-runner writes them to its stateDir
|
|
# on first registration and reuses them on every subsequent start).
|
|
#
|
|
# Credential isolation: the forge admin token (`forge-core-token`)
|
|
# never enters the hive-ci container. A host-side oneshot service
|
|
# (`hive-ci-prefetch.service`) performs all forge API calls before
|
|
# the container starts and writes only the runner registration token
|
|
# to `/run/hive-ci/runner-token`. The container bind-mounts this
|
|
# file read-only and never has access to the wider admin token.
|
|
#
|
|
# Nix builds inside the container use the shared /nix/store (standard
|
|
# nixos-container behaviour) with sandbox-fallback = true, because
|
|
# nspawn containers can't create the user-namespaces that nix sandboxing
|
|
# requires. See docs/gotchas.md.
|
|
|
|
options.services.hyperhive.forge.ci = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
example = true;
|
|
description = ''
|
|
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
Grouped under `services.hyperhive.forge` because the runner is
|
|
tightly coupled to the forge instance it registers against.
|
|
Disabled by default; the internal forge it registers against is
|
|
always present (mandatory), so enabling this is all that's needed.
|
|
|
|
On first start the container auto-registers against hive-forge using
|
|
hive-c0re's admin token — no manual token provisioning needed.
|
|
Runner credentials are persisted in the container's state dir and
|
|
reused on every subsequent boot.
|
|
'';
|
|
};
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-ci";
|
|
example = "prod-hive";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel. Defaults to
|
|
"hive-ci"; override when multiple hives share a Forgejo instance.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; multiple parallel
|
|
jobs share the container's nix store and cargo registry cache.
|
|
Higher values trade memory + CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). Workflow
|
|
files target this runner with `runs-on: [hive-ci]`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
|
|
jobTimeout = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "1h";
|
|
example = "3h";
|
|
description = ''
|
|
Per-job wall-clock timeout the runner enforces (act_runner's
|
|
`runner.timeout`). A job that exceeds it is killed, so a hung or
|
|
runaway build is bounded instead of holding the runner's single
|
|
slot indefinitely. Default `1h` comfortably covers a cold-cache
|
|
nix build while still bounding a stuck job; raise it (e.g.
|
|
`"3h"`) if you legitimately run jobs longer than that. Accepts a
|
|
Go duration string (`30m`, `1h`, `2h30m`). Note: this is
|
|
enforced by the runner process, so it only fires while that
|
|
process is itself healthy.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
# `forge.behindGateway = true` (the default) is required because the
|
|
# CI container uses private networking and reaches the forge through
|
|
# the gateway vhost. Without the gateway vhost there is no HTTP
|
|
# listener for `forgeCfg.domain` on the bridge that the runner can
|
|
# connect to.
|
|
assertions = [
|
|
{
|
|
assertion = forgeCfg.behindGateway;
|
|
message = ''
|
|
services.hyperhive.forge.ci.enable requires
|
|
services.hyperhive.forge.behindGateway = true.
|
|
The CI container runs with a private network namespace and
|
|
reaches the forge through the gateway vhost on the bridge IP.
|
|
Set behindGateway = true (it defaults to true alongside
|
|
services.hyperhive.enable).
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Create /run/hive-ci/ on the host and seed runner-token with a
|
|
# placeholder. hive-ci-prefetch.service overwrites it with the real
|
|
# token (or a fresh placeholder) before the container starts. The
|
|
# placeholder ensures the EnvironmentFile is always present even if
|
|
# the prefetch service hasn't run yet (e.g. tmpfiles-setup timing).
|
|
systemd.tmpfiles.rules = [
|
|
"d /run/hive-ci 0700 root root -"
|
|
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
|
];
|
|
|
|
# Host-side oneshot: validates/refreshes runner credentials before
|
|
# the container starts. The core admin token stays on the host and
|
|
# is never bind-mounted into the container. Runs on every boot so
|
|
# stale .runner credentials (runner deleted from forge) are detected
|
|
# and the container re-registers on the next start.
|
|
systemd.services.hive-ci-prefetch = {
|
|
description = "Pre-fetch hive-ci runner registration token (host-side)";
|
|
# Run before the container starts but after tmpfiles so the token
|
|
# file directory exists. After hive-c0re so the forge token is
|
|
# likely written (best-effort — the script handles the absent case).
|
|
after = [
|
|
"systemd-tmpfiles-setup.service"
|
|
"hive-c0re.service"
|
|
];
|
|
before = [ "container@hive-ci.service" ];
|
|
wantedBy = [ "container@hive-ci.service" ];
|
|
# partOf binds this oneshot's lifecycle to the container: when the
|
|
# container is stopped or restarted, systemd propagates that to this
|
|
# unit so it re-runs on the NEXT container start. Without this, the
|
|
# RemainAfterExit=true oneshot stays "active (exited)" forever after
|
|
# its first run — so a container restart skips it and the stale
|
|
# runner-token file (a placeholder from a boot where the forge token
|
|
# wasn't ready yet, or a registration token consumed/rotated since)
|
|
# is never refreshed. The in-container register service then fails
|
|
# with "runner registration token not found". partOf guarantees a
|
|
# fresh token is fetched before every container (re)start.
|
|
#
|
|
# Unit name: a declarative `containers.<name>` is the host unit
|
|
# `container@<name>.service` (the nspawn template), NOT
|
|
# `nixos-container@…`. The earlier `nixos-container@hive-ci.service`
|
|
# matched no real unit, so before/wantedBy/partOf were silent
|
|
# no-ops — the partOf never bound, the oneshot stayed
|
|
# `active (exited)`, and the token was never refreshed on restart.
|
|
# Confirmed against the live `container@hive-matrix.service` unit
|
|
# during the matrix-outage incident.
|
|
partOf = [ "container@hive-ci.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = prefetchScript;
|
|
# Pin the journal identity; ExecStart is a writeShellScript whose
|
|
# store-path basename would otherwise be the journal identifier.
|
|
SyslogIdentifier = "hive-ci-prefetch";
|
|
};
|
|
};
|
|
|
|
# Self-signed mode: the CA cert the bind-mount above sources is
|
|
# generated by the host `hive-tls-ca` service. Order the container after
|
|
# it so the bind source exists before nspawn sets the mount up (a
|
|
# condition-skipped/late CA would otherwise fail the container start).
|
|
systemd.services."container@hive-ci" = lib.mkIf useSelfSigned {
|
|
after = [ "hive-tls-ca.service" ];
|
|
requires = [ "hive-tls-ca.service" ];
|
|
};
|
|
|
|
containers.hive-ci = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Private network namespace, attached to the hive bridge so the
|
|
# runner reaches the forge via the gateway — and cannot reach
|
|
# host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.).
|
|
# Requires `forge.behindGateway = true` (asserted in the options
|
|
# block above). See docs/network.md.
|
|
privateNetwork = true;
|
|
hostBridge = networkCfg.bridgeName;
|
|
|
|
bindMounts = {
|
|
# Pre-filled by hive-ci-prefetch.service (host-side) before the
|
|
# container starts. Read-only: the container reads TOKEN= from
|
|
# here; the core admin token never enters the container.
|
|
"/run/hive-ci/runner-token" = {
|
|
hostPath = "/run/hive-ci/runner-token";
|
|
isReadOnly = true;
|
|
};
|
|
}
|
|
# Self-signed mode: bind ONLY the public hive CA cert (never the
|
|
# `hive-tls` state dir — it holds the CA + leaf private keys) so the
|
|
# runner's Node actions can trust the gateway/forge self-signed leaf
|
|
# (see NODE_EXTRA_CA_CERTS in the container config). Source generated
|
|
# by the host `hive-tls-ca` service; the container@hive-ci ordering
|
|
# below guarantees it exists before this mount is set up.
|
|
// lib.optionalAttrs useSelfSigned {
|
|
${caContainerPath} = {
|
|
hostPath = caHostPath;
|
|
isReadOnly = true;
|
|
};
|
|
};
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# Point the forge domain at the bridge IP so the runner can
|
|
# reach the forge through the gateway — both for registration /
|
|
# polling (runner URL below) and for artifact uploads (the
|
|
# Forgejo Actions artifact API uses ROOT_URL, i.e. the public
|
|
# forge domain, not a localhost URL). The gateway vhost for
|
|
# `forgeCfg.domain` proxies all `/` → forge; `addSSL = true`
|
|
# means HTTP:80 is served without redirect alongside HTTPS:443.
|
|
# Ports 80 and 443 are always open on the bridge firewall (see
|
|
# hive-network.nix). No DNS lookup needed — /etc/hosts wins.
|
|
networking.extraHosts = "${networkCfg.bridgeIp} ${forgeCfg.domain}";
|
|
# DNS: use the hive resolver on the bridge IP (dnsmasq in
|
|
# hive-gateway) for external lookups (git checkout, crate
|
|
# registries, etc.). The bridge→loopback DROP rule does not
|
|
# affect traffic destined for the bridge IP itself.
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
# With privateNetwork=true + hostBridge the container's veth
|
|
# (eth0) is bridge-attached. There is no DHCP server on the hive
|
|
# bridge (dnsmasq is DNS-only; agents use static IPs), so assign a
|
|
# static address + default route via the bridge gateway rather than
|
|
# DHCP — `useDHCP` here just hangs boot on a lease that never
|
|
# arrives. See `ciBridgeIp` above.
|
|
networking.interfaces.eth0.ipv4.addresses = [
|
|
{
|
|
address = ciBridgeIp;
|
|
prefixLength = networkCfg.bridgePrefixLength;
|
|
}
|
|
];
|
|
networking.defaultGateway = {
|
|
address = networkCfg.bridgeIp;
|
|
interface = "eth0";
|
|
};
|
|
|
|
# nspawn containers can't create user-namespaces, so nix
|
|
# sandboxing always fails. Fall back to unsandboxed builds.
|
|
# Moot once every nix invocation in the container routes
|
|
# through the host daemon (the daemon governs sandboxing).
|
|
# See docs/gotchas.md and harness-base.nix.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# package is top-level on gitea-actions-runner, not per-instance.
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.hive = {
|
|
enable = true;
|
|
name = cfg.name;
|
|
# Route through the gateway (bridge IP, port 80) so the
|
|
# runner never touches host-loopback. The forge domain
|
|
# resolves to the bridge IP via networking.extraHosts above;
|
|
# the gateway vhost `forgeCfg.domain` proxies to the forge
|
|
# on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect).
|
|
url = "http://${forgeCfg.domain}";
|
|
# EnvironmentFile providing TOKEN= — pre-filled by the
|
|
# host-side hive-ci-prefetch.service before the container
|
|
# starts; bind-mounted read-only from /run/hive-ci/runner-token
|
|
# on the host.
|
|
tokenFile = "/run/hive-ci/runner-token";
|
|
labels = cfg.labels;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
# Per-job wall-clock cap — see the `jobTimeout` option.
|
|
runner.timeout = cfg.jobTimeout;
|
|
};
|
|
};
|
|
|
|
# No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted
|
|
# read-only from the host (pre-filled before container start).
|
|
# nspawn creates the /run/hive-ci/ mount-point directory
|
|
# automatically before launching the container's init.
|
|
|
|
# No hive-ci-register.service inside the container: all forge
|
|
# API calls (runner validation, token fetch) moved to the
|
|
# host-side hive-ci-prefetch.service. The core admin token
|
|
# never enters this container.
|
|
|
|
# git is already in the gitea-actions-runner service PATH (the
|
|
# nixpkgs module builds it from the package's runtime deps).
|
|
# nix is required for `nix flake check` / `nix build` workflow
|
|
# steps — it's not included by the upstream module.
|
|
# Use the `path` service attribute (generates ExecSearchPath=)
|
|
# to prepend nix's bin dir to PATH without touching the
|
|
# environment.PATH the nixpkgs module sets — overriding that
|
|
# would lose git, curl, nodejs, and other runner deps.
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
];
|
|
|
|
systemd.services."gitea-runner-hive" = {
|
|
path = [ pkgs.nix ];
|
|
# Trust the hive CA in Node-based actions. With self-signed TLS,
|
|
# forgejo's ROOT_URL is `https://forge.<domain>` (CA-signed leaf),
|
|
# so actions like `upload-artifact` — whose Node HTTP client uses
|
|
# Node's *bundled* CA bundle, not the system store — reject the
|
|
# chain with "unable to verify the first certificate". Pointing
|
|
# NODE_EXTRA_CA_CERTS at the bind-mounted CA adds it to Node's
|
|
# roots for every action, hive-wide. Inherited by the job
|
|
# processes the runner spawns (host execution mode). Only set in
|
|
# self-signed mode; with an operator cert / ACME the public CA
|
|
# already validates and the bind-mount is absent.
|
|
environment = lib.mkIf useSelfSigned {
|
|
NODE_EXTRA_CA_CERTS = caContainerPath;
|
|
};
|
|
# Gate runner start (and therefore job registration/claiming) on
|
|
# the in-container nix daemon being reachable. After a hive-ci
|
|
# restart the runner re-registers and immediately claims any
|
|
# queued jobs — which can beat the nix daemon coming up: its
|
|
# `nix-daemon.socket` carries
|
|
# `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket` and is
|
|
# skipped until /nix/var is read-write, so the first
|
|
# nix-dependent build dispatches into a cold daemon and
|
|
# hangs/retries (observed: a 55m48s `nix flake check` vs the
|
|
# normal ~30s, a build-offload stall, not a code failure).
|
|
#
|
|
# Ordering `after`/`wants` the socket unit does NOT fix this — a
|
|
# condition-skipped unit satisfies systemd ordering immediately,
|
|
# so the runner would still start before the daemon is live.
|
|
# Instead block in ExecStartPre by polling the daemon until it
|
|
# actually answers; this is topology-agnostic (works whether the
|
|
# daemon is in-container or a shared host socket). `mkBefore` so
|
|
# this runs ahead of any pre-steps the upstream module adds.
|
|
serviceConfig.ExecStartPre = lib.mkBefore [
|
|
(pkgs.writeShellScript "wait-nix-daemon" ''
|
|
# Up to ~180s; the daemon is normally up within seconds, this
|
|
# only bites in the post-restart cold window. Non-fatal shape:
|
|
# if it never comes up the unit fails cleanly (recoverable)
|
|
# rather than the runner claiming jobs into a dead daemon.
|
|
#
|
|
# `--store daemon` is load-bearing. A bare `nix store ping`
|
|
# uses the `auto` store, which — when run as root with the
|
|
# daemon socket absent — silently resolves to a LOCAL store
|
|
# (root can write /nix/store directly) and pings successfully.
|
|
# systemd service units don't source the profile that sets
|
|
# `NIX_REMOTE=daemon`, so this is the real environment here.
|
|
# At cold boot the daemon socket IS absent: nix-daemon.socket
|
|
# carries `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket`
|
|
# and is condition-skipped until /nix/var goes read-write. So
|
|
# a bare ping would pass against the local store while the
|
|
# daemon is still down — defeating the gate's whole purpose
|
|
# (the runner would start and claim jobs the daemon can't yet
|
|
# service). Pinning `--store daemon` makes the poll verify the
|
|
# actual daemon socket, so the gate honours its contract and
|
|
# waits until the daemon — not a local fallback — answers.
|
|
for _ in $(seq 1 90); do
|
|
if ${pkgs.nix}/bin/nix store ping --store daemon >/dev/null 2>&1; then
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "nix daemon not reachable after 180s" >&2
|
|
exit 1
|
|
'')
|
|
];
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|