242 lines
9.5 KiB
Nix
242 lines
9.5 KiB
Nix
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.forge.ci;
|
|
forgeCfg = config.services.hyperhive.forge;
|
|
|
|
# hive-c0re writes its own admin token here on first forge startup.
|
|
# The token has read:admin + write:admin scopes — sufficient to call
|
|
# the runner registration-token API endpoint.
|
|
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
|
|
|
|
# Oneshot run before gitea-runner-hive.service starts.
|
|
#
|
|
# The nixpkgs gitea-actions-runner module uses tokenFile as a systemd
|
|
# EnvironmentFile (sets TOKEN= in the environment). EnvironmentFile is
|
|
# loaded before any ExecStartPre, so the file MUST exist at service start —
|
|
# an ExecStartPre override is too late. We solve this with:
|
|
# 1. tmpfiles: pre-create /run/hive-ci/runner-token with TOKEN=placeholder
|
|
# 2. hive-ci-register.service (this script): on first boot only,
|
|
# fetch the real token from forge and overwrite with TOKEN=<real>.
|
|
# 3. gitea-runner-hive.service: After=hive-ci-register.service
|
|
#
|
|
# On subsequent boots: .runner credentials file exists so the nixpkgs
|
|
# register step exits early — TOKEN value in the EnvironmentFile is
|
|
# irrelevant (placeholder is fine).
|
|
registerScript = pkgs.writeShellScript "hive-ci-register" ''
|
|
set -euo pipefail
|
|
TOKEN_FILE=/run/hive-ci/runner-token
|
|
CORE_TOKEN=$(cat /run/hive-ci/core-token)
|
|
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
|
|
|
# Retry up to 30s for forge to come up (containers autoStart in parallel).
|
|
for i in $(seq 1 30); do
|
|
REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \
|
|
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
|
-H "Authorization: token $CORE_TOKEN" \
|
|
| ${pkgs.jq}/bin/jq -r .token) && break
|
|
sleep 1
|
|
done
|
|
|
|
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
|
|
echo "hive-ci: failed to fetch runner registration token from forge" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Write in EnvironmentFile format: TOKEN=<value>.
|
|
# File is already 0600 (set by tmpfiles on boot).
|
|
echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
|
|
'';
|
|
in
|
|
{
|
|
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
# Shares host netns (same as hive-forge), so the runner reaches
|
|
# the forge at `http://127.0.0.1:<httpPort>` without extra plumbing.
|
|
# Container is non-ephemeral: the runner's registered credentials
|
|
# survive restarts (gitea-actions-runner writes them to its stateDir
|
|
# on first registration and reuses them on every subsequent start).
|
|
#
|
|
# Auto-registration: on first boot a oneshot service fetches a runner
|
|
# registration token from the forge's admin API using the core token
|
|
# hive-c0re writes to /var/lib/hyperhive/forge-core-token. No manual
|
|
# token handling needed — `forge.ci.enable = true` is the full operator
|
|
# bootstrap.
|
|
#
|
|
# Nix builds inside the container use the shared /nix/store (standard
|
|
# nixos-container behaviour) with sandbox-fallback = true, because
|
|
# nspawn containers can't create the user-namespaces that nix sandboxing
|
|
# requires. See docs/gotchas.md.
|
|
|
|
options.services.hyperhive.forge.ci = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
example = true;
|
|
description = ''
|
|
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
Grouped under `services.hyperhive.forge` because the runner is
|
|
tightly coupled to the forge instance it registers against.
|
|
Disabled by default; `services.hyperhive.forge.enable = true` is
|
|
a prerequisite (enforced by assertion).
|
|
|
|
On first start the container auto-registers against hive-forge using
|
|
hive-c0re's admin token — no manual token provisioning needed.
|
|
Runner credentials are persisted in the container's state dir and
|
|
reused on every subsequent boot.
|
|
'';
|
|
};
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hive-ci";
|
|
example = "prod-hive";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel. Defaults to
|
|
"hive-ci"; override when multiple hives share a Forgejo instance.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; multiple parallel
|
|
jobs share the container's nix store and cargo registry cache.
|
|
Higher values trade memory + CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). Workflow
|
|
files target this runner with `runs-on: [hive-ci]`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = forgeCfg.enable;
|
|
message = ''
|
|
services.hyperhive.forge.ci.enable = true requires
|
|
services.hyperhive.forge.enable = true — the runner registers
|
|
against the hive-forge Forgejo instance.
|
|
'';
|
|
}
|
|
];
|
|
|
|
containers.hive-ci = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Shared host netns: runner reaches hive-forge at localhost.
|
|
privateNetwork = false;
|
|
|
|
bindMounts = {
|
|
# Core token — used by hive-ci-register.service on first boot.
|
|
# Read-only: the container only reads it, never modifies it.
|
|
"/run/hive-ci/core-token" = {
|
|
hostPath = coreTokenPath;
|
|
isReadOnly = true;
|
|
};
|
|
};
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
system.stateVersion = "26.05";
|
|
|
|
# nspawn containers can't create user-namespaces, so nix
|
|
# sandboxing always fails. Fall back to unsandboxed builds.
|
|
# See docs/gotchas.md.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# package is top-level on gitea-actions-runner, not per-instance.
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.hive = {
|
|
enable = true;
|
|
name = cfg.name;
|
|
url = "http://127.0.0.1:${toString forgeCfg.httpPort}";
|
|
# EnvironmentFile providing TOKEN= for the nixpkgs register step.
|
|
# Pre-created by tmpfiles (placeholder); overwritten with the real
|
|
# token by hive-ci-register.service on first boot only.
|
|
tokenFile = "/run/hive-ci/runner-token";
|
|
labels = cfg.labels;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
# Generous timeout for cold-cache nix builds.
|
|
runner.timeout = "3h";
|
|
};
|
|
};
|
|
|
|
# Pre-create the EnvironmentFile so gitea-runner-hive.service can
|
|
# always load it. The nixpkgs module sets EnvironmentFile=tokenFile
|
|
# which systemd loads before any ExecStartPre — the file must exist
|
|
# at service-start time. On first boot hive-ci-register.service
|
|
# overwrites this placeholder with the real token before the runner
|
|
# starts. On subsequent boots the placeholder is harmless because
|
|
# the nixpkgs register step exits early when .runner already exists.
|
|
#
|
|
# Note: `f` doesn't create parent directories, but /run/hive-ci/
|
|
# is guaranteed to exist by the time container systemd starts:
|
|
# nspawn creates mount-point directories for all bindMounts before
|
|
# launching the container's init. So the dir is there when
|
|
# systemd-tmpfiles-setup.service runs.
|
|
systemd.tmpfiles.rules = [
|
|
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
|
];
|
|
|
|
# Oneshot that fetches a runner registration token from the forge
|
|
# admin API and writes it to the EnvironmentFile. Runs only on
|
|
# first boot (ConditionPathExists gates on .runner absence) and
|
|
# before gitea-runner-hive.service via the After= dependency below.
|
|
systemd.services.hive-ci-register = {
|
|
description = "Register hive-ci runner against Forgejo (first boot)";
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = registerScript;
|
|
};
|
|
# Only needed before .runner credentials exist.
|
|
unitConfig.ConditionPathExists = "!/var/lib/gitea-runner/hive/.runner";
|
|
};
|
|
|
|
# Runner must start after the register oneshot so the EnvironmentFile
|
|
# contains the real token on first boot.
|
|
systemd.services."gitea-runner-hive".after = [ "hive-ci-register.service" ];
|
|
systemd.services."gitea-runner-hive".wants = [ "hive-ci-register.service" ];
|
|
|
|
# git is required by the runner's checkout step.
|
|
# Everything else (nix, rust tools) is either part of NixOS
|
|
# by default or pulled in hermetically by nix flake check.
|
|
# curl/jq in the register script use absolute store paths.
|
|
environment.systemPackages = [ pkgs.git ];
|
|
};
|
|
};
|
|
};
|
|
}
|