c6ca9ab845
Each container now runs the journal forwarder the queue's and the builder's containers run, pushing to a receiver of its own on the swarm collector. Each authenticates as a new machine OIDC client of its own (swarm.<service>.otel.clientId), registered by a glue file wherever authelia runs, minted by authelia and published to the store by the secret publisher. Grafana's browser login client is unchanged. The fetch unit inside each container reads the client secret with the store identity the host already holds for that service (the runner's, Grafana's, and the log store responder's), now also handed into the container as an nspawn system credential. Each of those three policies gains one read stanza on its forwarder's client path. The host units that use those identities are unchanged. The three containers drop --link-journal=host, and the host collector stops reading them.
622 lines
26 KiB
Nix
622 lines
26 KiB
Nix
# The swarm's Forgejo Actions runner, in a `swarm-ci-runner` container on the
|
|
# host that sets `deploy.ci-runner.enable`. It reaches the forge by name and
|
|
# reads its registration token from the secret store under a certificate of
|
|
# its own, so nothing here asks whether this host runs the forge or a hive.
|
|
#
|
|
# ⚠️ Two properties here are load-bearing and neither is visible from the
|
|
# config that implements it:
|
|
#
|
|
# - The container gets its OWN network namespace, not the host's, so a PR's
|
|
# nix build cannot reach host-loopback services. That is a
|
|
# prompt-injection boundary; moving this to host netns for convenience
|
|
# removes it. `docs/networking/network.md::Container isolation`.
|
|
# - The forge ADMIN token never enters this container. swarm-controller holds
|
|
# it and publishes only the registration token, which the runner's own store
|
|
# identity reads. `docs/scheduler/ci.md::Registration`.
|
|
#
|
|
# Non-ephemeral on purpose: the runner writes its registered credentials
|
|
# (`.runner`) to its state dir and reuses them, so an ephemeral container
|
|
# would register a new runner on every restart.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.deploy.ci-runner;
|
|
forgeCfg = config.services.hyperhive.swarm.forge;
|
|
baoCfg = config.services.hyperhive.swarm.bao;
|
|
baoDeploy = config.services.hyperhive.deploy.bao;
|
|
gatewayCfg = config.services.hyperhive.gateway;
|
|
networkCfg = config.services.hyperhive.network;
|
|
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
|
|
|
|
machine = "swarm-ci-runner";
|
|
# The upstream module names its unit `gitea-runner-<instance>`.
|
|
instance = "swarm";
|
|
runnerUnit = "gitea-runner-${instance}";
|
|
# Holds the registration token and registers; see its definition.
|
|
registerUnit = "swarm-ci-runner-register";
|
|
|
|
# What the runner registers against and records in `.runner`. The precond
|
|
# below compares the two, so both come from here.
|
|
forgeUrl = "https://${forgeCfg.domain}";
|
|
|
|
# The forge's gateway serves a leaf from the store's `pki` mount in
|
|
# self-signed mode.
|
|
# The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store,
|
|
# so it gets the assembled bundle; the Node actions it runs get
|
|
# `NODE_EXTRA_CA_CERTS`, which is additive, further down.
|
|
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
caBundleModule = caTrust.trustBundle {
|
|
inherit pkgs;
|
|
name = machine;
|
|
consumers = [
|
|
runnerUnit
|
|
registerUnit
|
|
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
|
|
# collector's and authelia's names.
|
|
"opentelemetry-collector"
|
|
];
|
|
};
|
|
|
|
# The container's journal forwarder pushes to its own receiver on the swarm
|
|
# collector, the route `serviceProducers.swarm-ci-runner` in
|
|
# ./swarm-otel-service.nix declares. The port suffix is elided on 443, as in
|
|
# ./swarm-bao.nix.
|
|
swarmOtelCfg = config.services.hyperhive.swarm.otel;
|
|
forwarderClientId = config.services.hyperhive.swarm.ci-runner.otel.clientId;
|
|
gatewayPortSuffix = lib.optionalString (
|
|
gatewayCfg.httpsPort != 443
|
|
) ":${toString gatewayCfg.httpsPort}";
|
|
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
|
|
forwarderTokenUrl = "${toString config.services.hyperhive.swarm.authelia.url}/api/oidc/token";
|
|
forwarderUnit = "swarm-journal-forwarder-oidc";
|
|
# Loopback in the container's own netns, and distinct from the other service
|
|
# containers' forwarders all the same.
|
|
forwarderTelemetryPort = 8895;
|
|
|
|
# The runner's store identity, handed into the container as system
|
|
# credentials (credential id → host source) for the forwarder's fetch unit.
|
|
storeFiles = lib.filterAttrs (_: source: source != null) {
|
|
ci-runner-bao-cert = baoDeploy.ciRunnerClientCertFile;
|
|
ci-runner-bao-key = baoDeploy.ciRunnerClientKeyFile;
|
|
ci-runner-bao-server-ca = baoDeploy.serverCaFile;
|
|
};
|
|
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
|
# one word.
|
|
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
|
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
|
|
|
# The upstream unit is `DynamicUser`, so its state dir is
|
|
# `/var/lib/private/gitea-runner`; `/var/lib/gitea-runner` is a symlink to
|
|
# it, which is the spelling a step run as root (`+`) can follow.
|
|
runnerFile = "/var/lib/gitea-runner/${instance}/.runner";
|
|
|
|
stateDir = "/var/lib/swarm-ci-runner";
|
|
# Root-only, on tmpfs: the token never reaches persistent disk, and after a
|
|
# reboot the host unit below fetches it again. The host unit writes it at
|
|
# `hostTokenFile`; the container sees the directory read-only at `tokenDir`,
|
|
# where the token file is a `LoadCredential=` of the register unit alone.
|
|
hostTokenDir = "/run/${machine}";
|
|
hostTokenFile = "${hostTokenDir}/registration-token";
|
|
tokenDir = "/run/${machine}";
|
|
tokenFile = "${tokenDir}/registration-token";
|
|
tokenCredential = "registration-token";
|
|
# The epoch second the daemon last started. The precond reads the daemon's
|
|
# journal from there.
|
|
daemonStartedFile = "${stateDir}/daemon-started";
|
|
|
|
# The nix half of `swarm_secret_client::ci::registration_token_path`, in the
|
|
# form `bao kv get` takes (no KV v2 `data/`).
|
|
storePath = "secret/swarm/services/ci-runner/registration-token";
|
|
|
|
# What Forgejo answers a runner it has no record of — deleted in the admin
|
|
# panel, or a forge whose database was replaced — on every call but
|
|
# `Register` (`routers/api/actions/runner/interceptor.go`). The daemon logs
|
|
# it and exits 1 when its startup `Declare` gets it. ⚠️ Nothing this module
|
|
# prints may contain it: the precond would read its own line as a rejection.
|
|
rejectedMessage = "unregistered runner";
|
|
|
|
haveClientIdentity =
|
|
baoDeploy.ciRunnerClientCertFile != null && baoDeploy.ciRunnerClientKeyFile != null;
|
|
|
|
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
|
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
|
storeRetry = import ./lib/store-retry.nix { };
|
|
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
|
|
|
|
privateNetwork = true;
|
|
in
|
|
{
|
|
options.services.hyperhive.swarm.ci-runner.otel.clientId = lib.mkOption {
|
|
type = lib.types.str;
|
|
readOnly = true;
|
|
default = "swarm-ci-runner-collector";
|
|
description = ''
|
|
OAuth2 client id the journal forwarder inside the CI runner's container
|
|
authenticates as, and the audience it asks its token for.
|
|
|
|
Read-only because three hosts have to agree on it: authelia registers
|
|
the client (`glue-ci-runner-otel-oidc-client.nix`), the swarm collector
|
|
admits it on the container's receiver (`swarm.otel.serviceProducers`),
|
|
and the secret publisher carries its secret to the store.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.ci-runner = {
|
|
# `enable` is in ./deploy.nix.
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "swarm-ci-runner";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; parallel jobs
|
|
share the host's nix store and the container's cargo registry
|
|
cache. Higher values trade memory and CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). This repo's
|
|
workflows target the default with `runs-on: hive-ci`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
|
|
jobTimeout = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "1h";
|
|
example = "3h";
|
|
description = ''
|
|
Per-job wall-clock timeout the runner enforces (act_runner's
|
|
`runner.timeout`). A job that exceeds it is killed, so a hung build
|
|
does not hold a runner slot indefinitely. Accepts a Go duration
|
|
string (`30m`, `1h`, `2h30m`). Enforced by the runner process, so
|
|
it only fires while that process is healthy.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = haveClientIdentity;
|
|
message = ''
|
|
services.hyperhive.deploy.ci-runner.enable requires the runner's
|
|
swarm-secret-store client identity: set both
|
|
|
|
services.hyperhive.deploy.bao.ciRunnerClientCertFile
|
|
services.hyperhive.deploy.bao.ciRunnerClientKeyFile
|
|
|
|
swarm-bao-ci-runner.service reads the forge's runner registration
|
|
token with it, and the runner cannot register without that token.
|
|
|
|
Both default to the pair glue-bao-tls.nix mints when
|
|
services.hyperhive.deploy.bao.enable is true (the host that runs
|
|
the store). Otherwise copy ci-runner.pem and ci-runner-key.pem from
|
|
the store host's /var/lib/swarm-bao-pki/ and name them in those two
|
|
options.
|
|
'';
|
|
}
|
|
{
|
|
assertion = lib.all oneWord (lib.attrValues storeFiles);
|
|
message = ''
|
|
The CI runner's store identity paths must not contain whitespace:
|
|
nspawn's --load-credential receives them through nixos-container's
|
|
unquoted extra flags.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The container hangs off the bridge, takes its address from the
|
|
# gateway's DHCP and resolves through its dnsmasq. The gateway also runs
|
|
# `hive-tls-ca`, which writes the trust bundle bound in below.
|
|
services.hyperhive.network.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# Shaped after ./swarm-grafana.nix's `swarm-bao-grafana-oidc`, except that
|
|
# a missing token fails too: swarm-controller publishes it at its start and
|
|
# every five minutes, so a retry is what turns "no value there" into one.
|
|
systemd.services.swarm-bao-ci-runner = {
|
|
description = "fetch the CI runner's registration token from the swarm secret store";
|
|
# Ordered after the store where it runs here:
|
|
# ./glue-bao-readers-store-order.nix.
|
|
before = [ "container@${machine}.service" ];
|
|
wantedBy = [
|
|
"multi-user.target"
|
|
"container@${machine}.service"
|
|
];
|
|
path = [
|
|
baoDeploy.package
|
|
pkgs.coreutils
|
|
pkgs.systemd
|
|
];
|
|
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
|
serviceConfig = storeRetry.serviceConfig // {
|
|
# No `RemainAfterExit`: the timer below starts this again, and
|
|
# starting an active unit is a no-op.
|
|
Type = "oneshot";
|
|
SyslogIdentifier = "swarm-bao-ci-runner";
|
|
TimeoutStartSec = 30;
|
|
};
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = baoDeploy.ciRunnerClientCertFile;
|
|
BAO_CLIENT_KEY = baoDeploy.ciRunnerClientKeyFile;
|
|
}
|
|
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
|
BAO_CACERT = baoDeploy.serverCaFile;
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
|
|
${atomicWriteSecret}
|
|
${refreshConsumer}
|
|
|
|
# The container's bind source, so it starts even when the store does not
|
|
# answer; tmpfiles creates it too, but unordered with the container on
|
|
# a switch. Root-owned: systemd reads a `LoadCredential=` source as
|
|
# root, before the runner drops to its dynamic user.
|
|
install -d -m 0700 ${lib.escapeShellArg hostTokenDir}
|
|
|
|
err="$(mktemp)"
|
|
trap 'rm -f "$err"' EXIT
|
|
|
|
# `-token-only` answers on stdout and needs no token helper.
|
|
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
|
echo "could not log in to swarm-bao with the CI runner's certificate; leaving its registration token as it is." >&2
|
|
if [ -s "$err" ]; then
|
|
cat "$err" >&2
|
|
else
|
|
echo "bao failed without writing a diagnostic." >&2
|
|
fi
|
|
exit 1
|
|
fi
|
|
export BAO_TOKEN
|
|
|
|
if ! token="$(bao kv get -field=value ${lib.escapeShellArg storePath} 2>"$err")"; then
|
|
echo "swarm-bao did not return ${storePath}; retrying until swarm-controller publishes it." >&2
|
|
if [ -s "$err" ]; then
|
|
cat "$err" >&2
|
|
else
|
|
echo "bao failed without writing a diagnostic." >&2
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$token" ]; then
|
|
echo "swarm-bao returned an empty ${storePath}; retrying." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if secret_differs ${lib.escapeShellArg hostTokenFile} "$token"; then
|
|
atomic_write_secret 0600 "" ${lib.escapeShellArg hostTokenFile} "$token"
|
|
fi
|
|
|
|
# `LoadCredential=` copies the file at start, so a runner still waiting
|
|
# to register sees a new token only once it restarts.
|
|
refresh_consumer ${lib.escapeShellArg machine} ${runnerUnit}.service ${lib.escapeShellArg hostTokenFile}
|
|
'';
|
|
};
|
|
|
|
systemd.timers.swarm-bao-ci-runner = storeRefreshTimer "re-read the CI runner's registration token from the swarm secret store";
|
|
|
|
# nspawn fails to start the container when a credential source is
|
|
# missing, so where this host mints the leaf the container pulls the
|
|
# minting unit in and waits for it, as ./swarm-builder.nix's does.
|
|
systemd.services."container@${machine}" = lib.mkMerge [
|
|
caTrust.containerOrdering
|
|
{
|
|
after = [ "swarm-bao-pki.service" ];
|
|
wants = [ "swarm-bao-pki.service" ];
|
|
}
|
|
];
|
|
|
|
systemd.tmpfiles.rules = [
|
|
# The bind source must exist when the container starts.
|
|
"d ${hostTokenDir} 0700 root root -"
|
|
# `LoadCredential=` fails the unit when the file is missing; an empty
|
|
# one lets the register unit report the missing token.
|
|
"f ${hostTokenFile} 0600 root root -"
|
|
];
|
|
|
|
containers.${machine} = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# No `--link-journal=host`: the journal stays in the guest, where the
|
|
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
|
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
|
|
inherit privateNetwork;
|
|
hostBridge = networkCfg.bridgeName;
|
|
|
|
bindMounts = {
|
|
# Builds go through the HOST nix-daemon, so they inherit its
|
|
# buildMachines and max-jobs. The directory rather than the socket
|
|
# file, so the mount survives a daemon restart replacing the inode.
|
|
"/nix/var/nix/daemon-socket" = {
|
|
hostPath = "/nix/var/nix/daemon-socket";
|
|
isReadOnly = false;
|
|
};
|
|
${tokenDir} = {
|
|
hostPath = hostTokenDir;
|
|
isReadOnly = true;
|
|
};
|
|
}
|
|
// caTrust.bindMount;
|
|
|
|
config =
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
runnerCfg = config.services.gitea-actions-runner;
|
|
instanceCfg = runnerCfg.instances.${instance};
|
|
# The same file upstream generates and hands the daemon.
|
|
runnerConfigFile = (pkgs.formats.yaml { }).generate "config.yaml" instanceCfg.settings;
|
|
# In the form the daemon rewrites `.runner`'s labels to on start
|
|
# (act_runner's `labels.Parse` then `ToStrings`): `<name>:<scheme>`
|
|
# with `host` for a missing scheme, then `:<arg>` if there is one.
|
|
normaliseLabel =
|
|
label:
|
|
let
|
|
parts = lib.splitString ":" label;
|
|
scheme = if lib.length parts >= 2 then lib.elemAt parts 1 else "host";
|
|
arg = lib.concatStringsSep ":" (lib.drop 2 parts);
|
|
in
|
|
"${lib.head parts}:${scheme}" + lib.optionalString (arg != "") ":${arg}";
|
|
registeredLabels = lib.concatStringsSep "," (
|
|
lib.sort lib.lessThan (map normaliseLabel instanceCfg.labels)
|
|
);
|
|
in
|
|
{
|
|
imports = [
|
|
../container-modules/swarm-container.nix
|
|
../container-modules/swarm-journal-forwarder.nix
|
|
caBundleModule
|
|
];
|
|
|
|
# Authenticated as the forwarder's own OIDC client, with the runner's
|
|
# store identity: one identity for the whole container. It is a
|
|
# system credential, so the fetch unit loads it.
|
|
services.hyperhive.swarmJournalForwarder = {
|
|
enable = true;
|
|
serviceName = "ci-runner";
|
|
endpoint = forwarderEndpoint;
|
|
clientId = forwarderClientId;
|
|
tokenUrl = forwarderTokenUrl;
|
|
telemetryPort = forwarderTelemetryPort;
|
|
store = {
|
|
package = baoDeploy.package;
|
|
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
|
|
clientCertFile = forwarderCredential "ci-runner-bao-cert";
|
|
clientKeyFile = forwarderCredential "ci-runner-bao-key";
|
|
caFile =
|
|
if storeFiles ? ci-runner-bao-server-ca then
|
|
forwarderCredential "ci-runner-bao-server-ca"
|
|
else
|
|
null;
|
|
};
|
|
};
|
|
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
|
|
|
|
# Its own netns, so it keeps its own firewall; resolvconf stays on
|
|
# and serves `networking.nameservers` below.
|
|
services.hyperhive.swarmContainer = {
|
|
inherit privateNetwork;
|
|
writesOwnResolvConf = false;
|
|
};
|
|
|
|
# The gateway's dnsmasq on the bridge answers the forge's name with
|
|
# the bridge IP where this host runs the forge, and forwards it
|
|
# upstream everywhere else.
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
networking.interfaces.eth0.useDHCP = true;
|
|
|
|
# nspawn containers can't create user namespaces, so nix sandboxing
|
|
# fails; see docs/process/gotchas.md. `fallback` is a client-side
|
|
# option sent per daemon connection, so the host's value does not
|
|
# reach builds started here.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.fallback = true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# Both the service and its socket are off: the socket unit owns
|
|
# `/nix/var/nix/daemon-socket/socket` and would race the bind mount.
|
|
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
|
systemd.services.nix-daemon.enable = false;
|
|
systemd.sockets.nix-daemon.enable = false;
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d ${stateDir} 0700 root root -"
|
|
# Unused: nothing in this module writes or reads them. The first
|
|
# two hold a token; the others are upstream's re-registration
|
|
# markers.
|
|
"r ${stateDir}/registration-token"
|
|
"r ${stateDir}/registration-token.env"
|
|
"r /var/lib/private/gitea-runner/${instance}/.token-hash"
|
|
"r /var/lib/private/gitea-runner/${instance}/.labels"
|
|
];
|
|
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.${instance} = {
|
|
enable = true;
|
|
inherit (cfg) name labels;
|
|
url = forgeUrl;
|
|
# Upstream refuses an instance without a token source, and turns
|
|
# this one into the `EnvironmentFile=` the daemon unit drops.
|
|
inherit tokenFile;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
runner.timeout = cfg.jobTimeout;
|
|
};
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
];
|
|
|
|
# The only unit that can read the registration token: systemd hands
|
|
# a credential to every process of the unit that loads it, and the
|
|
# daemon's processes include every host job. Leaves a usable
|
|
# `.runner` behind or fails.
|
|
#
|
|
# The daemon's dynamic user and state dir: units naming the same
|
|
# `DynamicUser` share its uid, so `.runner` is the daemon's own.
|
|
systemd.services.${registerUnit} = {
|
|
description = "register the CI runner with the forge when it has no .runner";
|
|
wants = [ "network-online.target" ];
|
|
after = [ "network-online.target" ];
|
|
environment = {
|
|
HOME = "/var/lib/gitea-runner/${instance}";
|
|
# In the form the daemon rewrites `.runner`'s labels to.
|
|
SWARM_CI_RUNNER_LABELS = registeredLabels;
|
|
};
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
DynamicUser = true;
|
|
User = "gitea-runner";
|
|
StateDirectory = "gitea-runner";
|
|
LoadCredential = [ "${tokenCredential}:${tokenFile}" ];
|
|
# `register` retries an unreachable forge without end; under the
|
|
# daemon's own start timeout, so the failure is this unit's.
|
|
TimeoutStartSec = 60;
|
|
# Removes `.runner` in the cases where reusing it would fail, so
|
|
# the register step registers again.
|
|
#
|
|
# `+`: root, because the system journal is out of the dynamic
|
|
# user's reach.
|
|
ExecStartPre = "+${pkgs.writeShellScript "swarm-ci-runner-precond" ''
|
|
set -eu
|
|
runner=${runnerFile}
|
|
started=${daemonStartedFile}
|
|
|
|
# The forge refused the registration the daemon last started
|
|
# with. Read on the next start rather than when the daemon
|
|
# exits, so journald has its last lines by then.
|
|
if [ -f "$started" ]; then
|
|
since="$(< "$started")"
|
|
${pkgs.coreutils}/bin/rm -f "$started"
|
|
if ${pkgs.systemd}/bin/journalctl -u ${runnerUnit}.service --since "@$since" -o cat --no-pager \
|
|
| ${pkgs.gnugrep}/bin/grep -qF ${lib.escapeShellArg rejectedMessage}; then
|
|
echo "the forge has no record of this runner's registration; registering again" >&2
|
|
${pkgs.coreutils}/bin/rm -f "$runner"
|
|
fi
|
|
fi
|
|
|
|
# `.runner` records the address it registered at, and the
|
|
# daemon dials that one, not the configured one.
|
|
if [ -f "$runner" ] \
|
|
&& [ "$(${pkgs.jq}/bin/jq -r .address "$runner")" != ${lib.escapeShellArg forgeUrl} ]; then
|
|
echo "$runner names another forge address; registering again at ${forgeUrl}" >&2
|
|
${pkgs.coreutils}/bin/rm -f "$runner"
|
|
fi
|
|
|
|
# The daemon declares the labels `.runner` records, not the
|
|
# configured ones.
|
|
if [ -f "$runner" ] \
|
|
&& [ "$(${pkgs.jq}/bin/jq -r '.labels | sort | join(",")' "$runner")" != "$SWARM_CI_RUNNER_LABELS" ]; then
|
|
echo "$runner names other labels; registering again with $SWARM_CI_RUNNER_LABELS" >&2
|
|
${pkgs.coreutils}/bin/rm -f "$runner"
|
|
fi
|
|
''}";
|
|
};
|
|
script = ''
|
|
set -u
|
|
dir="$STATE_DIRECTORY/${instance}"
|
|
mkdir -p "$dir"
|
|
cd "$dir"
|
|
if [ -e .runner ]; then
|
|
exit 0
|
|
fi
|
|
|
|
token="$(< "$CREDENTIALS_DIRECTORY/${tokenCredential}")"
|
|
if [ -z "$token" ]; then
|
|
echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2
|
|
exit 1
|
|
fi
|
|
exec ${lib.getExe runnerCfg.package} register --no-interactive \
|
|
--instance ${lib.escapeShellArg instanceCfg.url} \
|
|
--token "$token" \
|
|
--name ${lib.escapeShellArg instanceCfg.name} \
|
|
--labels ${lib.escapeShellArg (lib.concatStringsSep "," instanceCfg.labels)} \
|
|
--config ${runnerConfigFile}
|
|
'';
|
|
};
|
|
|
|
systemd.services.${runnerUnit} = {
|
|
# `path` adds to the PATH the upstream module builds instead of
|
|
# replacing it.
|
|
path = [ pkgs.nix ];
|
|
# The upstream unit is `DynamicUser`, whose state dir systemd mounts
|
|
# `noexec`, so a job could not execute what it builds in its
|
|
# workspace. `ExecPaths=` resolves against the host root, where
|
|
# `/var/lib/gitea-runner` is a symlink into `private/`; `-` makes
|
|
# whichever spelling is absent a no-op.
|
|
serviceConfig.ExecPaths = [
|
|
"-/var/lib/gitea-runner"
|
|
"-/var/lib/private/gitea-runner"
|
|
];
|
|
environment = lib.mkIf caTrust.useSelfSigned {
|
|
NODE_EXTRA_CA_CERTS = caTrust.caContainerPath;
|
|
};
|
|
# A host job inherits the daemon's whole environment and can read
|
|
# its credentials, so the daemon carries the token in neither.
|
|
serviceConfig.EnvironmentFile = lib.mkForce [ ];
|
|
# Replaces upstream's pre-start step, which registers from
|
|
# `$TOKEN`, so the register unit is the one registration path.
|
|
#
|
|
# Started from here rather than through `Requires=`: a failed
|
|
# requirement fails the start job, which `Restart=` does not retry,
|
|
# while a failed pre-start step does.
|
|
serviceConfig.ExecStartPre = lib.mkForce [
|
|
"+${pkgs.systemd}/bin/systemctl start ${registerUnit}.service"
|
|
"+${pkgs.writeShellScript "swarm-ci-runner-mark-start" ''
|
|
${pkgs.coreutils}/bin/date +%s > ${daemonStartedFile}
|
|
''}"
|
|
];
|
|
# Restarted until the token arrives, however long the store and
|
|
# the controller take. `mkForce`: upstream sets both.
|
|
serviceConfig.Restart = lib.mkForce "on-failure";
|
|
serviceConfig.RestartSec = lib.mkForce 15;
|
|
startLimitIntervalSec = 0;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|