bd77b94518
The container-level nix.settings.fallback = lib.mkForce true (added as an interim safety net so a down remote builder degrades gracefully rather than hard-failing CI) is rendered redundant by the host-daemon routing fix: with NIX_REMOTE=daemon set for all systemd units in the container, every nix build invocation routes through the host daemon, which already has its own fallback policy. The container setting would only apply to nix invocations that bypass the daemon entirely — a case that should not exist post-deploy. NOTE: draft — gated on the daemon-routing change (PR that adds NIX_REMOTE=daemon to systemd.globalEnvironment + hive-priv service env) being deployed and confirmed working in production before merging.
547 lines
26 KiB
Nix
547 lines
26 KiB
Nix
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.forge.ci;
|
|
forgeCfg = config.services.hyperhive.forge;
|
|
gatewayCfg = config.services.hyperhive.gateway;
|
|
networkCfg = config.services.hyperhive.network;
|
|
tlsCfg = config.services.hyperhive.tls;
|
|
|
|
# Self-signed TLS is the gateway default (no operator cert / ACME). When
|
|
# active, forgejo's ROOT_URL is `https://forge.<domain>` and the leaf is
|
|
# signed by the host hive CA — so the runner's Node-based actions (e.g.
|
|
# `upload-artifact`, which POSTs to the ROOT_URL-derived artifact endpoint)
|
|
# reject the chain, since Node trusts only its bundled CA bundle, not the
|
|
# system store. Trust the hive CA explicitly via NODE_EXTRA_CA_CERTS below.
|
|
# `gateway.useSelfSigned` is the gateway module's single source of truth
|
|
# for the self-signed condition (no duplicated derivation here).
|
|
useSelfSigned = gatewayCfg.useSelfSigned;
|
|
caHostPath = "${tlsCfg.stateDir}/ca.pem";
|
|
caContainerPath = "/run/hive-ca/ca.pem";
|
|
|
|
# hive-c0re writes its own admin token here on first forge startup.
|
|
# The token has read:admin + write:admin scopes — sufficient to call
|
|
# the runner registration-token API endpoint.
|
|
# This path is HOST-ONLY. It is never bind-mounted into hive-ci.
|
|
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
|
|
|
|
# Container state root on the host. Non-ephemeral containers keep
|
|
# their filesystem here across reboots. The prefetch service accesses
|
|
# the runner's .runner file via this path to validate credentials
|
|
# without entering the container.
|
|
containerRoot = "/var/lib/nixos-containers/hive-ci";
|
|
|
|
# Host-side oneshot. Runs before `container@hive-ci.service`.
|
|
#
|
|
# The core token stays on the host. The container only ever sees the
|
|
# TOKEN= env-file populated here, never the core token itself.
|
|
#
|
|
# Flow:
|
|
# 1. If .runner exists: validate the runner ID against forge.
|
|
# Waits up to 60s for forge-core-token (hive-c0re writes it after
|
|
# forge container starts and admin is provisioned — this lags
|
|
# hive-c0re.service becoming active on first boot).
|
|
# 404 → purge .runner (re-registration needed).
|
|
# 000 (forge unreachable) → keep credentials, write placeholder.
|
|
# other non-200 → purge .runner.
|
|
# Token absent after 60s → keep credentials (safe — the runner
|
|
# holds valid creds; next boot will validate properly).
|
|
# 2. If .runner absent or just purged: wait up to 60s for both
|
|
# forge-core-token to appear AND forge API to respond, then
|
|
# fetch a fresh registration token and write TOKEN=<real>.
|
|
# Exits non-zero if both timeout — systemd logs the failure;
|
|
# the placeholder from tmpfiles means the container still starts
|
|
# but the runner will error. Operator restarts the service once
|
|
# forge is healthy.
|
|
prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
|
|
set -euo pipefail
|
|
TOKEN_FILE=/run/hive-ci/runner-token
|
|
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
|
RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
|
|
|
|
# Validate existing .runner credentials against forge. Purge if
|
|
# the runner was deleted (404) or the file is malformed. Keep if
|
|
# forge is unreachable (000) — transient outage shouldn't wipe creds.
|
|
#
|
|
# We wait up to 60s for forge-core-token to appear (hive-c0re writes
|
|
# it after provisioning the forge admin, which requires the forge
|
|
# container to start and become ready — this can lag behind
|
|
# hive-c0re.service becoming "active" on first boot). The same loop
|
|
# doubles as a wait for forge's API to become ready.
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
|
|
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
|
|
echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
else
|
|
# Wait for the core token before validating (same timeout as below).
|
|
CORE_TOKEN=""
|
|
for i in $(seq 1 60); do
|
|
if [ -f "${coreTokenPath}" ]; then
|
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
if [ -z "''${CORE_TOKEN:-}" ]; then
|
|
echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
|
|
else
|
|
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
|
|
-H "Authorization: token $CORE_TOKEN" \
|
|
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
|
|
if [ "$HTTP" = "404" ]; then
|
|
echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
elif [ "$HTTP" = "000" ]; then
|
|
echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
|
|
elif [ "$HTTP" != "200" ]; then
|
|
echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# .runner valid → write placeholder; gitea-actions-runner skips
|
|
# re-registration when .runner exists (TOKEN value is irrelevant).
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
echo "TOKEN=placeholder" > "$TOKEN_FILE"
|
|
chmod 600 "$TOKEN_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
# First boot or stale creds purged — wait for forge-core-token, then
|
|
# fetch a fresh registration token. Single retry loop covers both:
|
|
# waiting for hive-c0re to write the token file AND for forge's API
|
|
# to become responsive (they race on first boot).
|
|
REG_TOKEN=""
|
|
for i in $(seq 1 60); do
|
|
if [ ! -f "${coreTokenPath}" ]; then
|
|
echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
|
|
sleep 1
|
|
continue
|
|
fi
|
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
|
# Capture the HTTP status so a stale/invalid core token (401/403) is
|
|
# distinguished from a transient forge hiccup. With the old bare
|
|
# `curl -sf | jq`, a forge-core-token that's stale for the current
|
|
# forge (e.g. after a forge rebuild) 401s and fails silently every
|
|
# attempt for the full 60s loop, then exits with a misleading
|
|
# "core token absent or forge unreachable" — masking the real cause.
|
|
# Fail fast + loudly on 401/403 so the failure mode is legible and
|
|
# the operator/hive-c0re knows to re-mint forge-core-token.
|
|
RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \
|
|
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
|
-H "Authorization: token $CORE_TOKEN" || printf '\n000')
|
|
HTTP=$(printf '%s' "$RESP" | tail -n1)
|
|
BODY=$(printf '%s' "$RESP" | sed '$d')
|
|
case "$HTTP" in
|
|
2*)
|
|
REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token)
|
|
if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi
|
|
echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2
|
|
;;
|
|
401 | 403)
|
|
echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2
|
|
;;
|
|
esac
|
|
sleep 1
|
|
done
|
|
|
|
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
|
|
echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
|
|
chmod 600 "$TOKEN_FILE"
|
|
'';
|
|
in
|
|
{
|
|
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
# Uses a private network namespace (bridge-connected, not host netns)
|
|
# so CI build scripts cannot reach host-loopback services (dashboard,
|
|
# forge internal port, etc.) — a key defence against prompt-injection
|
|
# via PR nix builds. The runner reaches the forge via the
|
|
# gateway at `http://${forgeCfg.domain}` (resolved to the bridge IP
|
|
# via `networking.extraHosts`; gateway port 80 is always open on the
|
|
# bridge; `addSSL = true` means HTTP is served alongside HTTPS without
|
|
# a redirect). See docs/network.md.
|
|
# Container is non-ephemeral: the runner's registered credentials
|
|
# survive restarts (gitea-actions-runner writes them to its stateDir
|
|
# on first registration and reuses them on every subsequent start).
|
|
#
|
|
# Credential isolation: the forge admin token (`forge-core-token`)
|
|
# never enters the hive-ci container. A host-side oneshot service
|
|
# (`hive-ci-prefetch.service`) performs all forge API calls before
|
|
# the container starts and writes only the runner registration token
|
|
# to `/run/hive-ci/runner-token`. The container bind-mounts this
|
|
# file read-only and never has access to the wider admin token.
|
|
#
|
|
# Nix builds inside the container use the shared /nix/store (standard
|
|
# nixos-container behaviour) with sandbox-fallback = true, because
|
|
# nspawn containers can't create the user-namespaces that nix sandboxing
|
|
# requires. See docs/gotchas.md.
|
|
|
|
options.services.hyperhive.forge.ci = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
example = true;
|
|
description = ''
|
|
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
Grouped under `services.hyperhive.forge` because the runner is
|
|
tightly coupled to the forge instance it registers against.
|
|
Disabled by default; the internal forge it registers against is
|
|
always present (mandatory), so enabling this is all that's needed.
|
|
|
|
On first start the container auto-registers against hive-forge using
|
|
hive-c0re's admin token — no manual token provisioning needed.
|
|
Runner credentials are persisted in the container's state dir and
|
|
reused on every subsequent boot.
|
|
'';
|
|
};
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-ci";
|
|
example = "prod-hive";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel. Defaults to
|
|
"hive-ci"; override when multiple hives share a Forgejo instance.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; multiple parallel
|
|
jobs share the container's nix store and cargo registry cache.
|
|
Higher values trade memory + CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). Workflow
|
|
files target this runner with `runs-on: [hive-ci]`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
|
|
jobTimeout = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "1h";
|
|
example = "3h";
|
|
description = ''
|
|
Per-job wall-clock timeout the runner enforces (act_runner's
|
|
`runner.timeout`). A job that exceeds it is killed, so a hung or
|
|
runaway build is bounded instead of holding the runner's single
|
|
slot indefinitely. Default `1h` comfortably covers a cold-cache
|
|
nix build while still bounding a stuck job; raise it (e.g.
|
|
`"3h"`) if you legitimately run jobs longer than that. Accepts a
|
|
Go duration string (`30m`, `1h`, `2h30m`). Note: this is
|
|
enforced by the runner process, so it only fires while that
|
|
process is itself healthy.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
# `forge.behindGateway = true` (the default) is required because the
|
|
# CI container uses private networking and reaches the forge through
|
|
# the gateway vhost. Without the gateway vhost there is no HTTP
|
|
# listener for `forgeCfg.domain` on the bridge that the runner can
|
|
# connect to.
|
|
assertions = [
|
|
{
|
|
assertion = forgeCfg.behindGateway;
|
|
message = ''
|
|
services.hyperhive.forge.ci.enable requires
|
|
services.hyperhive.forge.behindGateway = true.
|
|
The CI container runs with a private network namespace and
|
|
reaches the forge through the gateway vhost on the bridge IP.
|
|
Set behindGateway = true (it defaults to true alongside
|
|
services.hyperhive.enable).
|
|
'';
|
|
}
|
|
];
|
|
|
|
# Create /run/hive-ci/ on the host and seed runner-token with a
|
|
# placeholder. hive-ci-prefetch.service overwrites it with the real
|
|
# token (or a fresh placeholder) before the container starts. The
|
|
# placeholder ensures the EnvironmentFile is always present even if
|
|
# the prefetch service hasn't run yet (e.g. tmpfiles-setup timing).
|
|
systemd.tmpfiles.rules = [
|
|
"d /run/hive-ci 0700 root root -"
|
|
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
|
];
|
|
|
|
# Host-side oneshot: validates/refreshes runner credentials before
|
|
# the container starts. The core admin token stays on the host and
|
|
# is never bind-mounted into the container. Runs on every boot so
|
|
# stale .runner credentials (runner deleted from forge) are detected
|
|
# and the container re-registers on the next start.
|
|
systemd.services.hive-ci-prefetch = {
|
|
description = "Pre-fetch hive-ci runner registration token (host-side)";
|
|
# Run before the container starts but after tmpfiles so the token
|
|
# file directory exists. After hive-c0re so the forge token is
|
|
# likely written (best-effort — the script handles the absent case).
|
|
after = [
|
|
"systemd-tmpfiles-setup.service"
|
|
"hive-c0re.service"
|
|
];
|
|
before = [ "container@hive-ci.service" ];
|
|
wantedBy = [ "container@hive-ci.service" ];
|
|
# partOf binds this oneshot's lifecycle to the container: when the
|
|
# container is stopped or restarted, systemd propagates that to this
|
|
# unit so it re-runs on the NEXT container start. Without this, the
|
|
# RemainAfterExit=true oneshot stays "active (exited)" forever after
|
|
# its first run — so a container restart skips it and the stale
|
|
# runner-token file (a placeholder from a boot where the forge token
|
|
# wasn't ready yet, or a registration token consumed/rotated since)
|
|
# is never refreshed. The in-container register service then fails
|
|
# with "runner registration token not found". partOf guarantees a
|
|
# fresh token is fetched before every container (re)start.
|
|
#
|
|
# Unit name: a declarative `containers.<name>` is the host unit
|
|
# `container@<name>.service` (the nspawn template), NOT
|
|
# `nixos-container@…`. The earlier `nixos-container@hive-ci.service`
|
|
# matched no real unit, so before/wantedBy/partOf were silent
|
|
# no-ops — the partOf never bound, the oneshot stayed
|
|
# `active (exited)`, and the token was never refreshed on restart.
|
|
# Confirmed against the live `container@hive-matrix.service` unit
|
|
# during the matrix-outage incident.
|
|
partOf = [ "container@hive-ci.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = prefetchScript;
|
|
# Pin the journal identity; ExecStart is a writeShellScript whose
|
|
# store-path basename would otherwise be the journal identifier.
|
|
SyslogIdentifier = "hive-ci-prefetch";
|
|
};
|
|
};
|
|
|
|
# Self-signed mode: the CA cert the bind-mount above sources is
|
|
# generated by the host `hive-tls-ca` service. Order the container after
|
|
# it so the bind source exists before nspawn sets the mount up (a
|
|
# condition-skipped/late CA would otherwise fail the container start).
|
|
systemd.services."container@hive-ci" = lib.mkIf useSelfSigned {
|
|
after = [ "hive-tls-ca.service" ];
|
|
requires = [ "hive-tls-ca.service" ];
|
|
};
|
|
|
|
containers.hive-ci = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Private network namespace, attached to the hive bridge so the
|
|
# runner reaches the forge via the gateway — and cannot reach
|
|
# host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.).
|
|
# Requires `forge.behindGateway = true` (asserted in the options
|
|
# block above). See docs/network.md.
|
|
privateNetwork = true;
|
|
hostBridge = networkCfg.bridgeName;
|
|
|
|
bindMounts = {
|
|
# Pre-filled by hive-ci-prefetch.service (host-side) before the
|
|
# container starts. Read-only: the container reads TOKEN= from
|
|
# here; the core admin token never enters the container.
|
|
"/run/hive-ci/runner-token" = {
|
|
hostPath = "/run/hive-ci/runner-token";
|
|
isReadOnly = true;
|
|
};
|
|
}
|
|
# Self-signed mode: bind ONLY the public hive CA cert (never the
|
|
# `hive-tls` state dir — it holds the CA + leaf private keys) so the
|
|
# runner's Node actions can trust the gateway/forge self-signed leaf
|
|
# (see NODE_EXTRA_CA_CERTS in the container config). Source generated
|
|
# by the host `hive-tls-ca` service; the container@hive-ci ordering
|
|
# below guarantees it exists before this mount is set up.
|
|
// lib.optionalAttrs useSelfSigned {
|
|
${caContainerPath} = {
|
|
hostPath = caHostPath;
|
|
isReadOnly = true;
|
|
};
|
|
};
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# Point the forge domain at the bridge IP so the runner can
|
|
# reach the forge through the gateway — both for registration /
|
|
# polling (runner URL below) and for artifact uploads (the
|
|
# Forgejo Actions artifact API uses ROOT_URL, i.e. the public
|
|
# forge domain, not a localhost URL). The gateway vhost for
|
|
# `forgeCfg.domain` proxies all `/` → forge; `addSSL = true`
|
|
# means HTTP:80 is served without redirect alongside HTTPS:443.
|
|
# Ports 80 and 443 are always open on the bridge firewall (see
|
|
# hive-network.nix). No DNS lookup needed — /etc/hosts wins.
|
|
networking.extraHosts = "${networkCfg.bridgeIp} ${forgeCfg.domain}";
|
|
# DNS: use the hive resolver on the bridge IP (dnsmasq in
|
|
# hive-gateway) for external lookups (git checkout, crate
|
|
# registries, etc.). The bridge→loopback DROP rule does not
|
|
# affect traffic destined for the bridge IP itself.
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
# With privateNetwork=true + hostBridge the container's veth
|
|
# is bridge-attached. Enable DHCP so the container gets an IP
|
|
# from dnsmasq on the bridge (hive-gateway serves the bridge
|
|
# subnet). Matches how normal hive agent containers acquire
|
|
# their addresses.
|
|
networking.useDHCP = true;
|
|
|
|
# nspawn containers can't create user-namespaces, so nix
|
|
# sandboxing always fails. Fall back to unsandboxed builds.
|
|
# Moot once every nix invocation in the container routes
|
|
# through the host daemon (the daemon governs sandboxing).
|
|
# See docs/gotchas.md and harness-base.nix.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# package is top-level on gitea-actions-runner, not per-instance.
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.hive = {
|
|
enable = true;
|
|
name = cfg.name;
|
|
# Route through the gateway (bridge IP, port 80) so the
|
|
# runner never touches host-loopback. The forge domain
|
|
# resolves to the bridge IP via networking.extraHosts above;
|
|
# the gateway vhost `forgeCfg.domain` proxies to the forge
|
|
# on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect).
|
|
url = "http://${forgeCfg.domain}";
|
|
# EnvironmentFile providing TOKEN= — pre-filled by the
|
|
# host-side hive-ci-prefetch.service before the container
|
|
# starts; bind-mounted read-only from /run/hive-ci/runner-token
|
|
# on the host.
|
|
tokenFile = "/run/hive-ci/runner-token";
|
|
labels = cfg.labels;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
# Per-job wall-clock cap — see the `jobTimeout` option.
|
|
runner.timeout = cfg.jobTimeout;
|
|
};
|
|
};
|
|
|
|
# No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted
|
|
# read-only from the host (pre-filled before container start).
|
|
# nspawn creates the /run/hive-ci/ mount-point directory
|
|
# automatically before launching the container's init.
|
|
|
|
# No hive-ci-register.service inside the container: all forge
|
|
# API calls (runner validation, token fetch) moved to the
|
|
# host-side hive-ci-prefetch.service. The core admin token
|
|
# never enters this container.
|
|
|
|
# git is already in the gitea-actions-runner service PATH (the
|
|
# nixpkgs module builds it from the package's runtime deps).
|
|
# nix is required for `nix flake check` / `nix build` workflow
|
|
# steps — it's not included by the upstream module.
|
|
# Use the `path` service attribute (generates ExecSearchPath=)
|
|
# to prepend nix's bin dir to PATH without touching the
|
|
# environment.PATH the nixpkgs module sets — overriding that
|
|
# would lose git, curl, nodejs, and other runner deps.
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
];
|
|
|
|
systemd.services."gitea-runner-hive" = {
|
|
path = [ pkgs.nix ];
|
|
# Trust the hive CA in Node-based actions. With self-signed TLS,
|
|
# forgejo's ROOT_URL is `https://forge.<domain>` (CA-signed leaf),
|
|
# so actions like `upload-artifact` — whose Node HTTP client uses
|
|
# Node's *bundled* CA bundle, not the system store — reject the
|
|
# chain with "unable to verify the first certificate". Pointing
|
|
# NODE_EXTRA_CA_CERTS at the bind-mounted CA adds it to Node's
|
|
# roots for every action, hive-wide. Inherited by the job
|
|
# processes the runner spawns (host execution mode). Only set in
|
|
# self-signed mode; with an operator cert / ACME the public CA
|
|
# already validates and the bind-mount is absent.
|
|
environment = lib.mkIf useSelfSigned {
|
|
NODE_EXTRA_CA_CERTS = caContainerPath;
|
|
};
|
|
# Gate runner start (and therefore job registration/claiming) on
|
|
# the in-container nix daemon being reachable. After a hive-ci
|
|
# restart the runner re-registers and immediately claims any
|
|
# queued jobs — which can beat the nix daemon coming up: its
|
|
# `nix-daemon.socket` carries
|
|
# `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket` and is
|
|
# skipped until /nix/var is read-write, so the first
|
|
# nix-dependent build dispatches into a cold daemon and
|
|
# hangs/retries (observed: a 55m48s `nix flake check` vs the
|
|
# normal ~30s, a build-offload stall, not a code failure).
|
|
#
|
|
# Ordering `after`/`wants` the socket unit does NOT fix this — a
|
|
# condition-skipped unit satisfies systemd ordering immediately,
|
|
# so the runner would still start before the daemon is live.
|
|
# Instead block in ExecStartPre by polling the daemon until it
|
|
# actually answers; this is topology-agnostic (works whether the
|
|
# daemon is in-container or a shared host socket). `mkBefore` so
|
|
# this runs ahead of any pre-steps the upstream module adds.
|
|
serviceConfig.ExecStartPre = lib.mkBefore [
|
|
(pkgs.writeShellScript "wait-nix-daemon" ''
|
|
# Up to ~180s; the daemon is normally up within seconds, this
|
|
# only bites in the post-restart cold window. Non-fatal shape:
|
|
# if it never comes up the unit fails cleanly (recoverable)
|
|
# rather than the runner claiming jobs into a dead daemon.
|
|
#
|
|
# `--store daemon` is load-bearing. A bare `nix store ping`
|
|
# uses the `auto` store, which — when run as root with the
|
|
# daemon socket absent — silently resolves to a LOCAL store
|
|
# (root can write /nix/store directly) and pings successfully.
|
|
# systemd service units don't source the profile that sets
|
|
# `NIX_REMOTE=daemon`, so this is the real environment here.
|
|
# At cold boot the daemon socket IS absent: nix-daemon.socket
|
|
# carries `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket`
|
|
# and is condition-skipped until /nix/var goes read-write. So
|
|
# a bare ping would pass against the local store while the
|
|
# daemon is still down — defeating the gate's whole purpose
|
|
# (the runner would start and claim jobs the daemon can't yet
|
|
# service). Pinning `--store daemon` makes the poll verify the
|
|
# actual daemon socket, so the gate honours its contract and
|
|
# waits until the daemon — not a local fallback — answers.
|
|
for _ in $(seq 1 90); do
|
|
if ${pkgs.nix}/bin/nix store ping --store daemon >/dev/null 2>&1; then
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "nix daemon not reachable after 180s" >&2
|
|
exit 1
|
|
'')
|
|
];
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|