Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/host-modules/swarm-ci-runner.nix
atlas 2f05e2db28 nix: re-run the host's store fetches on a ten-minute timer
Seven host oneshots fetch a secret from the swarm secret store into a file
their consumer loads at start: swarm-bao-grafana-oidc, swarm-bao-otel-oidc,
swarm-bao-matrix-oidc, swarm-bao-ci-runner, swarm-bao-forge-oidc,
swarm-bao-hive-otel-oidc and swarm-bao-forwarder-oidc. Each was
RemainAfterExit with no timer, so it ran at boot or deploy and never again:
a secret re-minted in the store reached its consumer only through an
operator restarting the fetch.

lib/store-refresh-timer.nix gives each one a timer (OnUnitInactiveSec 10min,
RandomizedDelaySec 1min, the agent's own store re-read interval), and each
drops RemainAfterExit so a timer start is not a no-op. Each script already
writes the file only when secret_differs, and refresh_consumer/refresh_unit
restart the consumer only when the file is newer than its last start, so a
run reading an unchanged value restarts nothing.

Two docs said these units do not run on a timer; credentials.md's
ci-runner row said the runner token is not re-pulled.

Part of #5017 (slice T).
2026-10-13 00:03:20 +02:00

520 lines
22 KiB
Nix

# The swarm's Forgejo Actions runner, in a `swarm-ci-runner` container on the
# host that sets `deploy.ci-runner.enable`. It reaches the forge by name and
# reads its registration token from the secret store under a certificate of
# its own, so nothing here asks whether this host runs the forge or a hive.
#
# ⚠️ Two properties here are load-bearing and neither is visible from the
# config that implements it:
#
# - The container gets its OWN network namespace, not the host's, so a PR's
# nix build cannot reach host-loopback services. That is a
# prompt-injection boundary; moving this to host netns for convenience
# removes it. `docs/networking/network.md::Container isolation`.
# - The forge ADMIN token never enters this container. swarm-controller holds
# it and publishes only the registration token, which the runner's own store
# identity reads. `docs/scheduler/ci.md::Registration`.
#
# Non-ephemeral on purpose: the runner writes its registered credentials
# (`.runner`) to its state dir and reuses them, so an ephemeral container
# would register a new runner on every restart.
{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.deploy.ci-runner;
forgeCfg = config.services.hyperhive.swarm.forge;
baoCfg = config.services.hyperhive.swarm.bao;
baoDeploy = config.services.hyperhive.deploy.bao;
gatewayCfg = config.services.hyperhive.gateway;
networkCfg = config.services.hyperhive.network;
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
machine = "swarm-ci-runner";
# The upstream module names its unit `gitea-runner-<instance>`.
instance = "swarm";
runnerUnit = "gitea-runner-${instance}";
# Holds the registration token and registers; see its definition.
registerUnit = "swarm-ci-runner-register";
# What the runner registers against and records in `.runner`. The precond
# below compares the two, so both come from here.
forgeUrl = "https://${forgeCfg.domain}";
# The forge's gateway serves a leaf from the store's `pki` mount in
# self-signed mode.
# The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store,
# so it gets the assembled bundle; the Node actions it runs get
# `NODE_EXTRA_CA_CERTS`, which is additive, further down.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = machine;
consumers = [
runnerUnit
registerUnit
];
};
# The upstream unit is `DynamicUser`, so its state dir is
# `/var/lib/private/gitea-runner`; `/var/lib/gitea-runner` is a symlink to
# it, which is the spelling a step run as root (`+`) can follow.
runnerFile = "/var/lib/gitea-runner/${instance}/.runner";
# Inside the container, root-only. The token file holds the bare token and
# is a `LoadCredential=` of the register unit alone; the host unit below
# writes it through the container's root, at `hostTokenFile`.
stateDir = "/var/lib/swarm-ci-runner";
tokenFile = "${stateDir}/registration-token";
tokenCredential = "registration-token";
hostTokenFile = "/var/lib/nixos-containers/${machine}${tokenFile}";
# The epoch second the daemon last started. The precond reads the daemon's
# journal from there.
daemonStartedFile = "${stateDir}/daemon-started";
# The nix half of `swarm_secret_client::ci::registration_token_path`, in the
# form `bao kv get` takes (no KV v2 `data/`).
storePath = "secret/swarm/services/ci-runner/registration-token";
# What Forgejo answers a runner it has no record of — deleted in the admin
# panel, or a forge whose database was replaced — on every call but
# `Register` (`routers/api/actions/runner/interceptor.go`). The daemon logs
# it and exits 1 when its startup `Declare` gets it. ⚠️ Nothing this module
# prints may contain it: the precond would read its own line as a rejection.
rejectedMessage = "unregistered runner";
haveClientIdentity =
baoDeploy.ciRunnerClientCertFile != null && baoDeploy.ciRunnerClientKeyFile != null;
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
refreshConsumer = import ./lib/refresh-consumer.nix { };
storeRetry = import ./lib/store-retry.nix { };
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
privateNetwork = true;
in
{
options.services.hyperhive.deploy.ci-runner = {
# `enable` is in ./deploy.nix.
name = lib.mkOption {
type = lib.types.str;
default = "swarm-ci-runner";
description = ''
Runner name as shown in the Forgejo admin panel.
'';
};
concurrency = lib.mkOption {
type = lib.types.ints.positive;
default = 1;
example = 4;
description = ''
Maximum number of workflow jobs the runner executes in parallel.
Each job gets its own temporary working directory; parallel jobs
share the host's nix store and the container's cargo registry
cache. Higher values trade memory and CPU headroom for throughput.
'';
};
labels = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ "hive-ci:host" ];
example = [
"hive-ci:host"
"nix:host"
];
description = ''
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
commands directly in the container (no docker/podman). This repo's
workflows target the default with `runs-on: hive-ci`.
'';
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.gitea-actions-runner;
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
description = "gitea-actions-runner package.";
};
jobTimeout = lib.mkOption {
type = lib.types.str;
default = "1h";
example = "3h";
description = ''
Per-job wall-clock timeout the runner enforces (act_runner's
`runner.timeout`). A job that exceeds it is killed, so a hung build
does not hold a runner slot indefinitely. Accepts a Go duration
string (`30m`, `1h`, `2h30m`). Enforced by the runner process, so
it only fires while that process is healthy.
'';
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = haveClientIdentity;
message = ''
services.hyperhive.deploy.ci-runner.enable requires the runner's
swarm-secret-store client identity: set both
services.hyperhive.deploy.bao.ciRunnerClientCertFile
services.hyperhive.deploy.bao.ciRunnerClientKeyFile
swarm-bao-ci-runner.service reads the forge's runner registration
token with it, and the runner cannot register without that token.
Both default to the pair glue-bao-tls.nix mints when
services.hyperhive.deploy.bao.enable is true (the host that runs
the store). Otherwise copy ci-runner.pem and ci-runner-key.pem from
the store host's /var/lib/swarm-bao-pki/ and name them in those two
options.
'';
}
];
# The container hangs off the bridge, takes its address from the
# gateway's DHCP and resolves through its dnsmasq. The gateway also runs
# `hive-tls-ca`, which writes the trust bundle bound in below.
services.hyperhive.network.enable = lib.mkDefault true;
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# Shaped after ./swarm-grafana.nix's `swarm-bao-grafana-oidc`, except that
# a missing token fails too: swarm-controller publishes it at its start and
# every five minutes, so a retry is what turns "no value there" into one.
systemd.services.swarm-bao-ci-runner = {
description = "fetch the CI runner's registration token from the swarm secret store";
after = lib.optionals baoDeploy.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
before = [ "container@${machine}.service" ];
wantedBy = [
"multi-user.target"
"container@${machine}.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.systemd
];
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
# No `RemainAfterExit`: the timer below starts this again, and
# starting an active unit is a no-op.
Type = "oneshot";
SyslogIdentifier = "swarm-bao-ci-runner";
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.ciRunnerClientCertFile;
BAO_CLIENT_KEY = baoDeploy.ciRunnerClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# `-token-only` answers on stdout and needs no token helper.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with the CI runner's certificate; leaving its registration token as it is." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
if ! token="$(bao kv get -field=value ${lib.escapeShellArg storePath} 2>"$err")"; then
echo "swarm-bao did not return ${storePath}; retrying until swarm-controller publishes it." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
if [ -z "$token" ]; then
echo "swarm-bao returned an empty ${storePath}; retrying." >&2
exit 1
fi
# Root-owned: systemd reads a `LoadCredential=` source as root, before
# the runner drops to its dynamic user.
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostTokenFile)}
if secret_differs ${lib.escapeShellArg hostTokenFile} "$token"; then
atomic_write_secret 0600 "" ${lib.escapeShellArg hostTokenFile} "$token"
fi
# `LoadCredential=` copies the file at start, so a runner still waiting
# to register sees a new token only once it restarts.
refresh_consumer ${lib.escapeShellArg machine} ${runnerUnit}.service ${lib.escapeShellArg hostTokenFile}
'';
};
systemd.timers.swarm-bao-ci-runner = storeRefreshTimer "re-read the CI runner's registration token from the swarm secret store";
systemd.services."container@${machine}" = caTrust.containerOrdering;
containers.${machine} = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
bindMounts = {
# Builds go through the HOST nix-daemon, so they inherit its
# buildMachines and max-jobs. The directory rather than the socket
# file, so the mount survives a daemon restart replacing the inode.
"/nix/var/nix/daemon-socket" = {
hostPath = "/nix/var/nix/daemon-socket";
isReadOnly = false;
};
}
// caTrust.bindMount;
config =
{
pkgs,
lib,
config,
...
}:
let
runnerCfg = config.services.gitea-actions-runner;
instanceCfg = runnerCfg.instances.${instance};
# The same file upstream generates and hands the daemon.
runnerConfigFile = (pkgs.formats.yaml { }).generate "config.yaml" instanceCfg.settings;
# In the form the daemon rewrites `.runner`'s labels to on start
# (act_runner's `labels.Parse` then `ToStrings`): `<name>:<scheme>`
# with `host` for a missing scheme, then `:<arg>` if there is one.
normaliseLabel =
label:
let
parts = lib.splitString ":" label;
scheme = if lib.length parts >= 2 then lib.elemAt parts 1 else "host";
arg = lib.concatStringsSep ":" (lib.drop 2 parts);
in
"${lib.head parts}:${scheme}" + lib.optionalString (arg != "") ":${arg}";
registeredLabels = lib.concatStringsSep "," (
lib.sort lib.lessThan (map normaliseLabel instanceCfg.labels)
);
in
{
imports = [
../container-modules/swarm-container.nix
caBundleModule
];
# Its own netns, so it keeps its own firewall; resolvconf stays on
# and serves `networking.nameservers` below.
services.hyperhive.swarmContainer = {
inherit privateNetwork;
writesOwnResolvConf = false;
};
# The gateway's dnsmasq on the bridge answers the forge's name with
# the bridge IP where this host runs the forge, and forwards it
# upstream everywhere else.
networking.nameservers = [ networkCfg.bridgeIp ];
networking.interfaces.eth0.useDHCP = true;
# nspawn containers can't create user namespaces, so nix sandboxing
# fails; see docs/process/gotchas.md. `fallback` is a client-side
# option sent per daemon connection, so the host's value does not
# reach builds started here.
nix.settings.sandbox-fallback = lib.mkForce true;
nix.settings.fallback = true;
nix.settings.experimental-features = [
"nix-command"
"flakes"
];
# Both the service and its socket are off: the socket unit owns
# `/nix/var/nix/daemon-socket/socket` and would race the bind mount.
systemd.globalEnvironment.NIX_REMOTE = "daemon";
systemd.services.nix-daemon.enable = false;
systemd.sockets.nix-daemon.enable = false;
systemd.tmpfiles.rules = [
"d ${stateDir} 0700 root root -"
# `LoadCredential=` fails the unit when the file is missing; an
# empty one lets the register unit report the missing token.
"f ${tokenFile} 0600 root root -"
# Unused: nothing in this module writes or reads them. The first
# holds a token; the others are upstream's re-registration markers.
"r ${stateDir}/registration-token.env"
"r /var/lib/private/gitea-runner/${instance}/.token-hash"
"r /var/lib/private/gitea-runner/${instance}/.labels"
];
services.gitea-actions-runner.package = cfg.package;
services.gitea-actions-runner.instances.${instance} = {
enable = true;
inherit (cfg) name labels;
url = forgeUrl;
# Upstream refuses an instance without a token source, and turns
# this one into the `EnvironmentFile=` the daemon unit drops.
inherit tokenFile;
settings = {
runner.capacity = cfg.concurrency;
runner.timeout = cfg.jobTimeout;
};
};
environment.systemPackages = [
pkgs.git
pkgs.nix
];
# The only unit that can read the registration token: systemd hands
# a credential to every process of the unit that loads it, and the
# daemon's processes include every host job. Leaves a usable
# `.runner` behind or fails.
#
# The daemon's dynamic user and state dir: units naming the same
# `DynamicUser` share its uid, so `.runner` is the daemon's own.
systemd.services.${registerUnit} = {
description = "register the CI runner with the forge when it has no .runner";
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
environment = {
HOME = "/var/lib/gitea-runner/${instance}";
# In the form the daemon rewrites `.runner`'s labels to.
SWARM_CI_RUNNER_LABELS = registeredLabels;
};
serviceConfig = {
Type = "oneshot";
DynamicUser = true;
User = "gitea-runner";
StateDirectory = "gitea-runner";
LoadCredential = [ "${tokenCredential}:${tokenFile}" ];
# `register` retries an unreachable forge without end; under the
# daemon's own start timeout, so the failure is this unit's.
TimeoutStartSec = 60;
# Removes `.runner` in the cases where reusing it would fail, so
# the register step registers again.
#
# `+`: root, because the system journal is out of the dynamic
# user's reach.
ExecStartPre = "+${pkgs.writeShellScript "swarm-ci-runner-precond" ''
set -eu
runner=${runnerFile}
started=${daemonStartedFile}
# The forge refused the registration the daemon last started
# with. Read on the next start rather than when the daemon
# exits, so journald has its last lines by then.
if [ -f "$started" ]; then
since="$(< "$started")"
${pkgs.coreutils}/bin/rm -f "$started"
if ${pkgs.systemd}/bin/journalctl -u ${runnerUnit}.service --since "@$since" -o cat --no-pager \
| ${pkgs.gnugrep}/bin/grep -qF ${lib.escapeShellArg rejectedMessage}; then
echo "the forge has no record of this runner's registration; registering again" >&2
${pkgs.coreutils}/bin/rm -f "$runner"
fi
fi
# `.runner` records the address it registered at, and the
# daemon dials that one, not the configured one.
if [ -f "$runner" ] \
&& [ "$(${pkgs.jq}/bin/jq -r .address "$runner")" != ${lib.escapeShellArg forgeUrl} ]; then
echo "$runner names another forge address; registering again at ${forgeUrl}" >&2
${pkgs.coreutils}/bin/rm -f "$runner"
fi
# The daemon declares the labels `.runner` records, not the
# configured ones.
if [ -f "$runner" ] \
&& [ "$(${pkgs.jq}/bin/jq -r '.labels | sort | join(",")' "$runner")" != "$SWARM_CI_RUNNER_LABELS" ]; then
echo "$runner names other labels; registering again with $SWARM_CI_RUNNER_LABELS" >&2
${pkgs.coreutils}/bin/rm -f "$runner"
fi
''}";
};
script = ''
set -u
dir="$STATE_DIRECTORY/${instance}"
mkdir -p "$dir"
cd "$dir"
if [ -e .runner ]; then
exit 0
fi
token="$(< "$CREDENTIALS_DIRECTORY/${tokenCredential}")"
if [ -z "$token" ]; then
echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2
exit 1
fi
exec ${lib.getExe runnerCfg.package} register --no-interactive \
--instance ${lib.escapeShellArg instanceCfg.url} \
--token "$token" \
--name ${lib.escapeShellArg instanceCfg.name} \
--labels ${lib.escapeShellArg (lib.concatStringsSep "," instanceCfg.labels)} \
--config ${runnerConfigFile}
'';
};
systemd.services.${runnerUnit} = {
# `path` adds to the PATH the upstream module builds instead of
# replacing it.
path = [ pkgs.nix ];
# The upstream unit is `DynamicUser`, whose state dir systemd mounts
# `noexec`, so a job could not execute what it builds in its
# workspace. `ExecPaths=` resolves against the host root, where
# `/var/lib/gitea-runner` is a symlink into `private/`; `-` makes
# whichever spelling is absent a no-op.
serviceConfig.ExecPaths = [
"-/var/lib/gitea-runner"
"-/var/lib/private/gitea-runner"
];
environment = lib.mkIf caTrust.useSelfSigned {
NODE_EXTRA_CA_CERTS = caTrust.caContainerPath;
};
# A host job inherits the daemon's whole environment and can read
# its credentials, so the daemon carries the token in neither.
serviceConfig.EnvironmentFile = lib.mkForce [ ];
# Replaces upstream's pre-start step, which registers from
# `$TOKEN`, so the register unit is the one registration path.
#
# Started from here rather than through `Requires=`: a failed
# requirement fails the start job, which `Restart=` does not retry,
# while a failed pre-start step does.
serviceConfig.ExecStartPre = lib.mkForce [
"+${pkgs.systemd}/bin/systemctl start ${registerUnit}.service"
"+${pkgs.writeShellScript "swarm-ci-runner-mark-start" ''
${pkgs.coreutils}/bin/date +%s > ${daemonStartedFile}
''}"
];
# Restarted until the token arrives, however long the store and
# the controller take. `mkForce`: upstream sets both.
serviceConfig.Restart = lib.mkForce "on-failure";
serviceConfig.RestartSec = lib.mkForce 15;
startLimitIntervalSec = 0;
};
};
};
};
}