9d816431dc
hive-ci-register.service now runs unconditionally on every boot (not just when .runner is absent). Before fetching a registration token it validates existing .runner credentials via the forge admin API: - 200: runner still registered, write dummy token and exit - 404: runner deleted from forge, purge .runner and re-register - 000: forge unreachable, keep credentials (runner surfaces the error) - other non-200 or malformed .runner: purge and re-register Removes ConditionPathExists so stale credentials from a wiped forge no longer block the runner indefinitely. Updates docs/ci.md to match.
278 lines
11 KiB
Nix
278 lines
11 KiB
Nix
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.forge.ci;
|
|
forgeCfg = config.services.hyperhive.forge;
|
|
|
|
# hive-c0re writes its own admin token here on first forge startup.
|
|
# The token has read:admin + write:admin scopes — sufficient to call
|
|
# the runner registration-token API endpoint.
|
|
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
|
|
|
|
# Oneshot run before gitea-runner-hive.service on every boot.
|
|
#
|
|
# The nixpkgs gitea-actions-runner module uses tokenFile as a systemd
|
|
# EnvironmentFile (sets TOKEN= in the environment). EnvironmentFile is
|
|
# loaded before any ExecStartPre, so the file MUST exist at service start —
|
|
# an ExecStartPre override is too late. We solve this with:
|
|
# 1. tmpfiles: pre-create /run/hive-ci/runner-token with TOKEN=placeholder
|
|
# 2. hive-ci-register.service (this script): runs on every boot:
|
|
# a. If .runner exists: validate the runner ID against the forge admin
|
|
# API. If the runner was deleted from forge (404), delete .runner so
|
|
# the next step re-registers. If forge is unreachable (000), keep
|
|
# the existing credentials (the runner itself will surface the error).
|
|
# If valid, write a dummy token and exit — runner reuses .runner creds.
|
|
# b. If .runner absent (first boot or stale creds were purged): fetch a
|
|
# fresh registration token and write TOKEN=<real> to the EnvironmentFile.
|
|
# 3. gitea-runner-hive.service: After=hive-ci-register.service
|
|
registerScript = pkgs.writeShellScript "hive-ci-register" ''
|
|
set -euo pipefail
|
|
TOKEN_FILE=/run/hive-ci/runner-token
|
|
CORE_TOKEN=$(cat /run/hive-ci/core-token)
|
|
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
|
RUNNER_FILE=/var/lib/gitea-runner/hive/.runner
|
|
|
|
# If .runner exists, validate runner credentials against forge.
|
|
# Delete .runner if the runner entry is gone (404) — this triggers
|
|
# fresh re-registration below. Skip validation if forge is unreachable
|
|
# (000) so a transient forge outage doesn't wipe valid credentials.
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
|
|
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
|
|
echo "hive-ci: .runner malformed (no id), purging for re-registration" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
else
|
|
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
|
|
-H "Authorization: token $CORE_TOKEN" \
|
|
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
|
|
if [ "$HTTP" = "404" ]; then
|
|
echo "hive-ci: runner $RUNNER_ID not found in forge, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
elif [ "$HTTP" = "000" ]; then
|
|
echo "hive-ci: forge unreachable, keeping existing .runner credentials" >&2
|
|
elif [ "$HTTP" != "200" ]; then
|
|
echo "hive-ci: runner validation returned HTTP $HTTP, purging .runner" >&2
|
|
rm -f "$RUNNER_FILE"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# If .runner still valid, write dummy token and exit — nixpkgs register
|
|
# step exits early when .runner exists; TOKEN value is irrelevant.
|
|
if [ -f "$RUNNER_FILE" ]; then
|
|
echo "TOKEN=placeholder" > "$TOKEN_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
# First boot or stale creds purged — fetch a fresh registration token.
|
|
# Retry up to 30s for forge to come up (containers autoStart in parallel).
|
|
REG_TOKEN=""
|
|
for i in $(seq 1 30); do
|
|
REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \
|
|
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
|
-H "Authorization: token $CORE_TOKEN" \
|
|
| ${pkgs.jq}/bin/jq -r .token) && break
|
|
sleep 1
|
|
done
|
|
|
|
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
|
|
echo "hive-ci: failed to fetch runner registration token from forge" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Write in EnvironmentFile format: TOKEN=<value>.
|
|
# File is already 0600 (set by tmpfiles on boot).
|
|
echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
|
|
'';
|
|
in
|
|
{
|
|
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
# Shares host netns (same as hive-forge), so the runner reaches
|
|
# the forge at `http://127.0.0.1:<httpPort>` without extra plumbing.
|
|
# Container is non-ephemeral: the runner's registered credentials
|
|
# survive restarts (gitea-actions-runner writes them to its stateDir
|
|
# on first registration and reuses them on every subsequent start).
|
|
#
|
|
# Auto-registration: on every boot a oneshot service validates existing
|
|
# runner credentials against forge (purging stale/.runner if the runner
|
|
# was deleted from forge). On first boot (or after credential purge) it
|
|
# fetches a fresh registration token via the forge admin API, using the
|
|
# core token hive-c0re writes to /var/lib/hyperhive/forge-core-token.
|
|
# No manual token handling needed — `forge.ci.enable = true` is the
|
|
# full operator bootstrap.
|
|
#
|
|
# Nix builds inside the container use the shared /nix/store (standard
|
|
# nixos-container behaviour) with sandbox-fallback = true, because
|
|
# nspawn containers can't create the user-namespaces that nix sandboxing
|
|
# requires. See docs/gotchas.md.
|
|
|
|
options.services.hyperhive.forge.ci = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
example = true;
|
|
description = ''
|
|
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
Grouped under `services.hyperhive.forge` because the runner is
|
|
tightly coupled to the forge instance it registers against.
|
|
Disabled by default; `services.hyperhive.forge.enable = true` is
|
|
a prerequisite (enforced by assertion).
|
|
|
|
On first start the container auto-registers against hive-forge using
|
|
hive-c0re's admin token — no manual token provisioning needed.
|
|
Runner credentials are persisted in the container's state dir and
|
|
reused on every subsequent boot.
|
|
'';
|
|
};
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-ci";
|
|
example = "prod-hive";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel. Defaults to
|
|
"hive-ci"; override when multiple hives share a Forgejo instance.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; multiple parallel
|
|
jobs share the container's nix store and cargo registry cache.
|
|
Higher values trade memory + CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). Workflow
|
|
files target this runner with `runs-on: [hive-ci]`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = forgeCfg.enable;
|
|
message = ''
|
|
services.hyperhive.forge.ci.enable = true requires
|
|
services.hyperhive.forge.enable = true — the runner registers
|
|
against the hive-forge Forgejo instance.
|
|
'';
|
|
}
|
|
];
|
|
|
|
containers.hive-ci = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Shared host netns: runner reaches hive-forge at localhost.
|
|
privateNetwork = false;
|
|
|
|
bindMounts = {
|
|
# Core token — used by hive-ci-register.service on first boot.
|
|
# Read-only: the container only reads it, never modifies it.
|
|
"/run/hive-ci/core-token" = {
|
|
hostPath = coreTokenPath;
|
|
isReadOnly = true;
|
|
};
|
|
};
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# nspawn containers can't create user-namespaces, so nix
|
|
# sandboxing always fails. Fall back to unsandboxed builds.
|
|
# See docs/gotchas.md.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# package is top-level on gitea-actions-runner, not per-instance.
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.hive = {
|
|
enable = true;
|
|
name = cfg.name;
|
|
url = "http://127.0.0.1:${toString forgeCfg.httpPort}";
|
|
# EnvironmentFile providing TOKEN= for the nixpkgs register step.
|
|
# Pre-created by tmpfiles (placeholder); overwritten with the real
|
|
# token by hive-ci-register.service on first boot only.
|
|
tokenFile = "/run/hive-ci/runner-token";
|
|
labels = cfg.labels;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
# Generous timeout for cold-cache nix builds.
|
|
runner.timeout = "3h";
|
|
};
|
|
};
|
|
|
|
# Pre-create the EnvironmentFile so gitea-runner-hive.service can
|
|
# always load it. The nixpkgs module sets EnvironmentFile=tokenFile
|
|
# which systemd loads before any ExecStartPre — the file must exist
|
|
# at service-start time. hive-ci-register.service overwrites this
|
|
# placeholder before the runner starts (real token on first/re-reg,
|
|
# dummy placeholder when .runner credentials are still valid).
|
|
#
|
|
# Note: `f` doesn't create parent directories, but /run/hive-ci/
|
|
# is guaranteed to exist by the time container systemd starts:
|
|
# nspawn creates mount-point directories for all bindMounts before
|
|
# launching the container's init. So the dir is there when
|
|
# systemd-tmpfiles-setup.service runs.
|
|
systemd.tmpfiles.rules = [
|
|
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
|
];
|
|
|
|
# Oneshot that validates/refreshes runner credentials on every boot:
|
|
# validates existing .runner against forge (purges if stale/404),
|
|
# then fetches a fresh registration token if .runner is absent.
|
|
# Runs before gitea-runner-hive.service via the After= dependency.
|
|
systemd.services.hive-ci-register = {
|
|
description = "Validate/register hive-ci runner against Forgejo";
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = registerScript;
|
|
};
|
|
};
|
|
|
|
# Runner must start after the register oneshot so the EnvironmentFile
|
|
# contains the real token on first boot.
|
|
systemd.services."gitea-runner-hive".after = [ "hive-ci-register.service" ];
|
|
systemd.services."gitea-runner-hive".wants = [ "hive-ci-register.service" ];
|
|
|
|
# git is required by the runner's checkout step.
|
|
# Everything else (nix, rust tools) is either part of NixOS
|
|
# by default or pulled in hermetically by nix flake check.
|
|
# curl/jq in the register script use absolute store paths.
|
|
environment.systemPackages = [ pkgs.git ];
|
|
};
|
|
};
|
|
};
|
|
}
|