e2da5509ae
The CI runner was a hive deploy module: hive-ci.nix ran it beside the forge, and hive-c0re registered it through the forge's admin API with the `core` token, writing the registration token through hive-priv. It could only run on the forge's host, and it kept an admin-scoped forge token on a hive. Now the runner is a swarm service: - swarm-ci-runner.nix runs it in a `swarm-ci-runner` container on the host that sets deploy.ci-runner.enable, reaching the forge by name. It needs neither the forge nor a hive there. - swarm-controller reads the forge's runner registration token with its own admin token and writes it to swarm/services/ci-runner/ registration-token whenever it differs from what it last wrote. Its grant on that path is create/update only. - The runner's own leaf (CN swarm-ci-runner, role and policy of the same name) reads that one path. swarm-bao-ci-runner.service fetches it into the runner's EnvironmentFile and restarts the runner on a change. - A pre-start step removes .runner when the daemon's journal since its last start holds Forgejo's "unregistered runner" refusal (the daemon exits 1 when its startup Declare gets it), or when .runner names another forge address; upstream's pre-start step then registers again. Removed from the hive side: hive-ci.nix and deploy.forgejo.ci.*, forge/ci_runner.rs, PrivRequest::RegisterCiRunner and its handler and ReadWritePaths entry, InfraContainer::Ci with hivectl's --ci and the dashboard's hive-ci journal entry, HYPERHIVE_FORGE_CI_ENABLED, and read:admin,write:admin from CORE_TOKEN_SCOPES: nothing else in hive-c0re calls /api/v1/admin/*. The old ci options, under both swarm.forge.ci and deploy.forgejo.ci, now fail evaluation naming deploy.ci-runner. The forge's half of the old ci.enable, the actions/checkout mirror and DEFAULT_ACTIONS_URL = self, is deploy.forgejo.actionsFromSelf. Closes #4954
421 lines
17 KiB
Nix
421 lines
17 KiB
Nix
# The swarm's Forgejo Actions runner, in a `swarm-ci-runner` container on the
|
|
# host that sets `deploy.ci-runner.enable`. It reaches the forge by name and
|
|
# reads its registration token from the secret store under a certificate of
|
|
# its own, so nothing here asks whether this host runs the forge or a hive.
|
|
#
|
|
# ⚠️ Two properties here are load-bearing and neither is visible from the
|
|
# config that implements it:
|
|
#
|
|
# - The container gets its OWN network namespace, not the host's, so a PR's
|
|
# nix build cannot reach host-loopback services. That is a
|
|
# prompt-injection boundary; moving this to host netns for convenience
|
|
# removes it. `docs/networking/network.md::Container isolation`.
|
|
# - The forge ADMIN token never enters this container. swarm-controller holds
|
|
# it and publishes only the registration token, which the runner's own store
|
|
# identity reads. `docs/scheduler/ci.md::Registration`.
|
|
#
|
|
# Non-ephemeral on purpose: the runner writes its registered credentials
|
|
# (`.runner`) to its state dir and reuses them, so an ephemeral container
|
|
# would register a new runner on every restart.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.deploy.ci-runner;
|
|
forgeCfg = config.services.hyperhive.swarm.forge;
|
|
baoCfg = config.services.hyperhive.swarm.bao;
|
|
baoDeploy = config.services.hyperhive.deploy.bao;
|
|
gatewayCfg = config.services.hyperhive.gateway;
|
|
networkCfg = config.services.hyperhive.network;
|
|
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
|
|
|
|
machine = "swarm-ci-runner";
|
|
# The upstream module names its unit `gitea-runner-<instance>`.
|
|
instance = "swarm";
|
|
runnerUnit = "gitea-runner-${instance}";
|
|
|
|
# What the runner registers against and records in `.runner`. The precond
|
|
# below compares the two, so both come from here.
|
|
forgeUrl = "https://${forgeCfg.domain}";
|
|
|
|
# The forge's gateway serves a leaf from the swarm CA in self-signed mode.
|
|
# The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store,
|
|
# so it gets the assembled bundle; the Node actions it runs get
|
|
# `NODE_EXTRA_CA_CERTS`, which is additive, further down.
|
|
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
caBundleModule = caTrust.trustBundle {
|
|
inherit pkgs;
|
|
name = machine;
|
|
consumers = [ runnerUnit ];
|
|
};
|
|
|
|
# The upstream unit is `DynamicUser`, so its state dir is
|
|
# `/var/lib/private/gitea-runner`; `/var/lib/gitea-runner` is a symlink to
|
|
# it, which is the spelling a step run as root (`+`) can follow.
|
|
runnerFile = "/var/lib/gitea-runner/${instance}/.runner";
|
|
|
|
# Inside the container, root-only. The token file is the runner's
|
|
# `EnvironmentFile` (`TOKEN=<value>`); the host unit below writes it through
|
|
# the container's root, at `hostTokenFile`.
|
|
stateDir = "/var/lib/swarm-ci-runner";
|
|
tokenFile = "${stateDir}/registration-token.env";
|
|
hostTokenFile = "/var/lib/nixos-containers/${machine}${tokenFile}";
|
|
# The epoch second the daemon last started. The precond reads the daemon's
|
|
# journal from there.
|
|
daemonStartedFile = "${stateDir}/daemon-started";
|
|
|
|
# The nix half of `swarm_secret_client::ci::registration_token_path`, in the
|
|
# form `bao kv get` takes (no KV v2 `data/`).
|
|
storePath = "secret/swarm/services/ci-runner/registration-token";
|
|
|
|
# What Forgejo answers a runner it has no record of — deleted in the admin
|
|
# panel, or a forge whose database was replaced — on every call but
|
|
# `Register` (`routers/api/actions/runner/interceptor.go`). The daemon logs
|
|
# it and exits 1 when its startup `Declare` gets it. ⚠️ Nothing this module
|
|
# prints may contain it: the precond would read its own line as a rejection.
|
|
rejectedMessage = "unregistered runner";
|
|
|
|
haveClientIdentity =
|
|
baoDeploy.ciRunnerClientCertFile != null && baoDeploy.ciRunnerClientKeyFile != null;
|
|
|
|
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
|
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
|
storeRetry = import ./lib/store-retry.nix { };
|
|
|
|
privateNetwork = true;
|
|
in
|
|
{
|
|
options.services.hyperhive.deploy.ci-runner = {
|
|
# `enable` is in ./deploy.nix.
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "swarm-ci-runner";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; parallel jobs
|
|
share the host's nix store and the container's cargo registry
|
|
cache. Higher values trade memory and CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). This repo's
|
|
workflows target the default with `runs-on: hive-ci`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
|
|
jobTimeout = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "1h";
|
|
example = "3h";
|
|
description = ''
|
|
Per-job wall-clock timeout the runner enforces (act_runner's
|
|
`runner.timeout`). A job that exceeds it is killed, so a hung build
|
|
does not hold a runner slot indefinitely. Accepts a Go duration
|
|
string (`30m`, `1h`, `2h30m`). Enforced by the runner process, so
|
|
it only fires while that process is healthy.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = haveClientIdentity;
|
|
message = ''
|
|
services.hyperhive.deploy.ci-runner.enable requires the runner's
|
|
swarm-secret-store client identity: set both
|
|
|
|
services.hyperhive.deploy.bao.ciRunnerClientCertFile
|
|
services.hyperhive.deploy.bao.ciRunnerClientKeyFile
|
|
|
|
swarm-bao-ci-runner.service reads the forge's runner registration
|
|
token with it, and the runner cannot register without that token.
|
|
|
|
On the host that runs the store, glue-bao-tls.nix supplies both as
|
|
defaults. Elsewhere, copy ci-runner.pem and ci-runner-key.pem from
|
|
the store host's /var/lib/swarm-bao-pki/ and name them here.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The container hangs off the bridge, takes its address from the
|
|
# gateway's DHCP and resolves through its dnsmasq. The gateway also runs
|
|
# `hive-tls-ca`, which writes the trust bundle bound in below.
|
|
services.hyperhive.network.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# Shaped after ./swarm-grafana.nix's `swarm-bao-grafana-oidc`, except that
|
|
# a missing token fails too: swarm-controller publishes it at its start and
|
|
# every five minutes, so a retry is what turns "no value there" into one.
|
|
systemd.services.swarm-bao-ci-runner = {
|
|
description = "fetch the CI runner's registration token from the swarm secret store";
|
|
after = lib.optionals baoDeploy.enable [
|
|
"swarm-bao-pki.service"
|
|
"container@${baoCfg.machine}.service"
|
|
];
|
|
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
|
|
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
|
|
before = [ "container@${machine}.service" ];
|
|
wantedBy = [
|
|
"multi-user.target"
|
|
"container@${machine}.service"
|
|
];
|
|
path = [
|
|
baoDeploy.package
|
|
pkgs.coreutils
|
|
pkgs.systemd
|
|
];
|
|
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
|
serviceConfig = storeRetry.serviceConfig // {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
SyslogIdentifier = "swarm-bao-ci-runner";
|
|
TimeoutStartSec = 30;
|
|
};
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = baoDeploy.ciRunnerClientCertFile;
|
|
BAO_CLIENT_KEY = baoDeploy.ciRunnerClientKeyFile;
|
|
}
|
|
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
|
BAO_CACERT = baoDeploy.serverCaFile;
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
|
|
${atomicWriteSecret}
|
|
${refreshConsumer}
|
|
|
|
err="$(mktemp)"
|
|
trap 'rm -f "$err"' EXIT
|
|
|
|
# `-token-only` answers on stdout and needs no token helper.
|
|
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
|
echo "could not log in to swarm-bao with the CI runner's certificate; leaving its registration token as it is." >&2
|
|
if [ -s "$err" ]; then
|
|
cat "$err" >&2
|
|
else
|
|
echo "bao failed without writing a diagnostic." >&2
|
|
fi
|
|
exit 1
|
|
fi
|
|
export BAO_TOKEN
|
|
|
|
if ! token="$(bao kv get -field=value ${lib.escapeShellArg storePath} 2>"$err")"; then
|
|
echo "swarm-bao did not return ${storePath}; retrying until swarm-controller publishes it." >&2
|
|
if [ -s "$err" ]; then
|
|
cat "$err" >&2
|
|
else
|
|
echo "bao failed without writing a diagnostic." >&2
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$token" ]; then
|
|
echo "swarm-bao returned an empty ${storePath}; retrying." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Root-owned: systemd reads an `EnvironmentFile` as root, before the
|
|
# runner drops to its dynamic user.
|
|
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostTokenFile)}
|
|
if secret_differs ${lib.escapeShellArg hostTokenFile} "TOKEN=$token"; then
|
|
atomic_write_secret 0600 "" ${lib.escapeShellArg hostTokenFile} "TOKEN=$token"
|
|
fi
|
|
|
|
# `EnvironmentFile=` is read once, at start. A changed token makes the
|
|
# upstream pre-start step register again.
|
|
refresh_consumer ${lib.escapeShellArg machine} ${runnerUnit}.service ${lib.escapeShellArg hostTokenFile}
|
|
'';
|
|
};
|
|
|
|
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
|
|
|
containers.${machine} = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
|
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
|
extraFlags = [ "--link-journal=host" ];
|
|
inherit privateNetwork;
|
|
hostBridge = networkCfg.bridgeName;
|
|
|
|
bindMounts = {
|
|
# Builds go through the HOST nix-daemon, so they inherit its
|
|
# buildMachines and max-jobs. The directory rather than the socket
|
|
# file, so the mount survives a daemon restart replacing the inode.
|
|
"/nix/var/nix/daemon-socket" = {
|
|
hostPath = "/nix/var/nix/daemon-socket";
|
|
isReadOnly = false;
|
|
};
|
|
}
|
|
// caTrust.bindMount;
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
imports = [
|
|
../container-modules/swarm-container.nix
|
|
caBundleModule
|
|
];
|
|
|
|
# Its own netns, so it keeps its own firewall; resolvconf stays on
|
|
# and serves `networking.nameservers` below.
|
|
services.hyperhive.swarmContainer = {
|
|
inherit privateNetwork;
|
|
writesOwnResolvConf = false;
|
|
};
|
|
|
|
# The gateway's dnsmasq on the bridge answers the forge's name with
|
|
# the bridge IP where this host runs the forge, and forwards it
|
|
# upstream everywhere else.
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
networking.interfaces.eth0.useDHCP = true;
|
|
|
|
# nspawn containers can't create user namespaces, so nix sandboxing
|
|
# fails; see docs/process/gotchas.md. `fallback` is a client-side
|
|
# option sent per daemon connection, so the host's value does not
|
|
# reach builds started here.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.fallback = true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# Both the service and its socket are off: the socket unit owns
|
|
# `/nix/var/nix/daemon-socket/socket` and would race the bind mount.
|
|
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
|
systemd.services.nix-daemon.enable = false;
|
|
systemd.sockets.nix-daemon.enable = false;
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d ${stateDir} 0700 root root -"
|
|
# `EnvironmentFile=` fails the unit when the file is missing; an
|
|
# empty one leaves `TOKEN` unset for the precond to report.
|
|
"f ${tokenFile} 0600 root root -"
|
|
];
|
|
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.${instance} = {
|
|
enable = true;
|
|
inherit (cfg) name labels;
|
|
url = forgeUrl;
|
|
inherit tokenFile;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
runner.timeout = cfg.jobTimeout;
|
|
};
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
];
|
|
|
|
systemd.services.${runnerUnit} = {
|
|
# `path` adds to the PATH the upstream module builds instead of
|
|
# replacing it.
|
|
path = [ pkgs.nix ];
|
|
# The upstream unit is `DynamicUser`, whose state dir systemd mounts
|
|
# `noexec`, so a job could not execute what it builds in its
|
|
# workspace. `ExecPaths=` resolves against the host root, where
|
|
# `/var/lib/gitea-runner` is a symlink into `private/`; `-` makes
|
|
# whichever spelling is absent a no-op.
|
|
serviceConfig.ExecPaths = [
|
|
"-/var/lib/gitea-runner"
|
|
"-/var/lib/private/gitea-runner"
|
|
];
|
|
environment = lib.mkIf caTrust.useSelfSigned {
|
|
NODE_EXTRA_CA_CERTS = caTrust.caContainerPath;
|
|
};
|
|
# Runs before upstream's pre-start step, which registers whenever
|
|
# `.runner` is absent or the token or labels changed. This one
|
|
# removes `.runner` in the two cases upstream cannot see, and holds
|
|
# the unit down until there is a token.
|
|
#
|
|
# `+`: root, because `.runner`, the token and the system journal
|
|
# are all out of the dynamic user's reach.
|
|
serviceConfig.ExecStartPre = lib.mkMerge [
|
|
(lib.mkBefore [
|
|
"+${pkgs.writeShellScript "swarm-ci-runner-precond" ''
|
|
set -eu
|
|
runner=${runnerFile}
|
|
started=${daemonStartedFile}
|
|
|
|
# The forge refused the registration the daemon last started
|
|
# with. Read on the next start rather than when the daemon
|
|
# exits, so journald has its last lines by then.
|
|
if [ -f "$started" ]; then
|
|
since="$(< "$started")"
|
|
${pkgs.coreutils}/bin/rm -f "$started"
|
|
if ${pkgs.systemd}/bin/journalctl -u ${runnerUnit}.service --since "@$since" -o cat --no-pager \
|
|
| ${pkgs.gnugrep}/bin/grep -qF ${lib.escapeShellArg rejectedMessage}; then
|
|
echo "the forge has no record of this runner's registration; registering again" >&2
|
|
${pkgs.coreutils}/bin/rm -f "$runner"
|
|
fi
|
|
fi
|
|
|
|
# `.runner` records the address it registered at, and the
|
|
# daemon dials that one, not the configured one.
|
|
if [ -f "$runner" ] \
|
|
&& [ "$(${pkgs.jq}/bin/jq -r .address "$runner")" != ${lib.escapeShellArg forgeUrl} ]; then
|
|
echo "$runner names another forge address; registering again at ${forgeUrl}" >&2
|
|
${pkgs.coreutils}/bin/rm -f "$runner"
|
|
fi
|
|
|
|
# Upstream registers on a changed token hash, an empty token
|
|
# included, and a registration with no token fails.
|
|
if [ -z "''${TOKEN:-}" ]; then
|
|
echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2
|
|
exit 1
|
|
fi
|
|
''}"
|
|
])
|
|
(lib.mkAfter [
|
|
"+${pkgs.writeShellScript "swarm-ci-runner-mark-start" ''
|
|
${pkgs.coreutils}/bin/date +%s > ${daemonStartedFile}
|
|
''}"
|
|
])
|
|
];
|
|
# Restarted until the token arrives, however long the store and
|
|
# the controller take. `mkForce`: upstream sets both.
|
|
serviceConfig.Restart = lib.mkForce "on-failure";
|
|
serviceConfig.RestartSec = lib.mkForce 15;
|
|
startLimitIntervalSec = 0;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|