vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-ci-runner: load the registration token in a register unit of its own

Browse source 
systemd hands a unit's credentials to every process of that unit, and a
host job is a process of the runner daemon, so the `LoadCredential=` on
gitea-runner-swarm still let any workflow read the registration token
from `$CREDENTIALS_DIRECTORY`.

swarm-ci-runner-register.service, a oneshot under the daemon's dynamic
user and state dir, now holds the only `LoadCredential=` and runs the
precond and `act_runner register` (only when `.runner` is missing). The
daemon unit has no credential, no `EnvironmentFile` and no `TOKEN`; its
pre-start step starts the register unit, so a missing token fails the
daemon's start and `Restart=on-failure` retries it.

The precond compares `.runner`'s labels in the form the daemon rewrites
them to (`nix` -> `nix:host`); compared verbatim, a scheme-less label
re-registered the runner on every start.

tmpfiles removes the unused `registration-token.env` and upstream's
`.token-hash`/`.labels` markers.
This commit is contained in:
atlas 2026-10-05 16:33:24 +02:00
commit 8ffceea8e3
4 changed files with 184 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

168
nix/host-modules/swarm-ci-runner.nix
View file

@ -36,6 +36,8 @@ let
# The upstream module names its unit `gitea-runner-<instance>`.
instance = "swarm";
runnerUnit = "gitea-runner-${instance}";
# Holds the registration token and registers; see its definition.
registerUnit = "swarm-ci-runner-register";
# What the runner registers against and records in `.runner`. The precond
# below compares the two, so both come from here.
@ -49,7 +51,10 @@ let
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = machine;
consumers = [ runnerUnit ];
consumers = [
runnerUnit
registerUnit
];
};
# The upstream unit is `DynamicUser`, so its state dir is
@ -58,9 +63,8 @@ let
runnerFile = "/var/lib/gitea-runner/${instance}/.runner";
# Inside the container, root-only. The token file holds the bare token and
# reaches the runner unit only as a `LoadCredential=`, which only the
# register step reads; the host unit below writes it through the
# container's root, at `hostTokenFile`.
# is a `LoadCredential=` of the register unit alone; the host unit below
# writes it through the container's root, at `hostTokenFile`.
stateDir = "/var/lib/swarm-ci-runner";
tokenFile = "${stateDir}/registration-token";
tokenCredential = "registration-token";
@ -295,8 +299,20 @@ in
instanceCfg = runnerCfg.instances.${instance};
# The same file upstream generates and hands the daemon.
runnerConfigFile = (pkgs.formats.yaml { }).generate "config.yaml" instanceCfg.settings;
# In the form the precond reads back from `.runner`.
registeredLabels = lib.concatStringsSep "," (lib.sort lib.lessThan instanceCfg.labels);
# In the form the daemon rewrites `.runner`'s labels to on start
# (act_runner's `labels.Parse` then `ToStrings`): `<name>:<scheme>`
# with `host` for a missing scheme, then `:<arg>` if there is one.
normaliseLabel =
label:
let
parts = lib.splitString ":" label;
scheme = if lib.length parts >= 2 then lib.elemAt parts 1 else "host";
arg = lib.concatStringsSep ":" (lib.drop 2 parts);
in
"${lib.head parts}:${scheme}" + lib.optionalString (arg != "") ":${arg}";
registeredLabels = lib.concatStringsSep "," (
lib.sort lib.lessThan (map normaliseLabel instanceCfg.labels)
);
in
{
imports = [
@ -337,9 +353,13 @@ in
systemd.tmpfiles.rules = [
"d ${stateDir} 0700 root root -"
# `LoadCredential=` fails the unit when the file is missing; an
# empty one lets a registered runner start before any token has
# arrived, and the register step report the missing token.
# empty one lets the register unit report the missing token.
"f ${tokenFile} 0600 root root -"
# Unused: nothing in this module writes or reads them. The first
# holds a token; the others are upstream's re-registration markers.
"r ${stateDir}/registration-token.env"
"r /var/lib/private/gitea-runner/${instance}/.token-hash"
"r /var/lib/private/gitea-runner/${instance}/.labels"
];
services.gitea-actions-runner.package = cfg.package;
@ -349,7 +369,7 @@ in
inherit (cfg) name labels;
url = forgeUrl;
# Upstream refuses an instance without a token source, and turns
# this one into the `EnvironmentFile=` the unit below drops.
# this one into the `EnvironmentFile=` the daemon unit drops.
inherit tokenFile;
settings = {
runner.capacity = cfg.concurrency;
@ -362,36 +382,37 @@ in
pkgs.nix
];
systemd.services.${runnerUnit} = {
# `path` adds to the PATH the upstream module builds instead of
# replacing it.
path = [ pkgs.nix ];
# The upstream unit is `DynamicUser`, whose state dir systemd mounts
# `noexec`, so a job could not execute what it builds in its
# workspace. `ExecPaths=` resolves against the host root, where
# `/var/lib/gitea-runner` is a symlink into `private/`; `-` makes
# whichever spelling is absent a no-op.
serviceConfig.ExecPaths = [
"-/var/lib/gitea-runner"
"-/var/lib/private/gitea-runner"
];
environment = lib.mkIf caTrust.useSelfSigned {
NODE_EXTRA_CA_CERTS = caTrust.caContainerPath;
# The only unit that can read the registration token: systemd hands
# a credential to every process of the unit that loads it, and the
# daemon's processes include every host job. Leaves a usable
# `.runner` behind or fails.
#
# The daemon's dynamic user and state dir: units naming the same
# `DynamicUser` share its uid, so `.runner` is the daemon's own.
systemd.services.${registerUnit} = {
description = "register the CI runner with the forge when it has no .runner";
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
environment = {
HOME = "/var/lib/gitea-runner/${instance}";
# In the form the daemon rewrites `.runner`'s labels to.
SWARM_CI_RUNNER_LABELS = registeredLabels;
};
# The registration token reaches the unit only as a credential,
# which the register step reads. A host job inherits the daemon's
# whole environment, so neither carries it.
serviceConfig.EnvironmentFile = lib.mkForce [ ];
serviceConfig.LoadCredential = [ "${tokenCredential}:${tokenFile}" ];
# One registration path: upstream's pre-start step registers from
# `$TOKEN`, so this list replaces it.
serviceConfig.ExecStartPre = lib.mkForce [
serviceConfig = {
Type = "oneshot";
DynamicUser = true;
User = "gitea-runner";
StateDirectory = "gitea-runner";
LoadCredential = [ "${tokenCredential}:${tokenFile}" ];
# `register` retries an unreachable forge without end; under the
# daemon's own start timeout, so the failure is this unit's.
TimeoutStartSec = 60;
# Removes `.runner` in the cases where reusing it would fail, so
# the register step after it registers again.
# the register step registers again.
#
# `+`: root, because the system journal is out of the dynamic
# user's reach.
"+${pkgs.writeShellScript "swarm-ci-runner-precond" ''
ExecStartPre = "+${pkgs.writeShellScript "swarm-ci-runner-precond" ''
set -eu
runner=${runnerFile}
started=${daemonStartedFile}
@ -420,35 +441,62 @@ in
# The daemon declares the labels `.runner` records, not the
# configured ones.
if [ -f "$runner" ] \
&& [ "$(${pkgs.jq}/bin/jq -r '.labels | sort | join(",")' "$runner")" != ${lib.escapeShellArg registeredLabels} ]; then
echo "$runner names other labels; registering again with ${registeredLabels}" >&2
&& [ "$(${pkgs.jq}/bin/jq -r '.labels | sort | join(",")' "$runner")" != "$SWARM_CI_RUNNER_LABELS" ]; then
echo "$runner names other labels; registering again with $SWARM_CI_RUNNER_LABELS" >&2
${pkgs.coreutils}/bin/rm -f "$runner"
fi
''}"
# Registers only when `.runner` is absent. As the dynamic user,
# so `.runner` is the daemon's own. The derivation itself rather
# than its path, so ../module-eval reads its text without building.
(pkgs.writeShellScript "swarm-ci-runner-register" ''
set -eu
dir="$STATE_DIRECTORY/${instance}"
${pkgs.coreutils}/bin/mkdir -p "$dir"
cd "$dir"
if [ -e .runner ]; then
exit 0
fi
''}";
};
script = ''
set -u
dir="$STATE_DIRECTORY/${instance}"
mkdir -p "$dir"
cd "$dir"
if [ -e .runner ]; then
exit 0
fi
token="$(< "$CREDENTIALS_DIRECTORY/${tokenCredential}")"
if [ -z "$token" ]; then
echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2
exit 1
fi
exec ${lib.getExe runnerCfg.package} register --no-interactive \
--instance ${lib.escapeShellArg instanceCfg.url} \
--token "$token" \
--name ${lib.escapeShellArg instanceCfg.name} \
--labels ${lib.escapeShellArg (lib.concatStringsSep "," instanceCfg.labels)} \
--config ${runnerConfigFile}
'')
token="$(< "$CREDENTIALS_DIRECTORY/${tokenCredential}")"
if [ -z "$token" ]; then
echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2
exit 1
fi
exec ${lib.getExe runnerCfg.package} register --no-interactive \
--instance ${lib.escapeShellArg instanceCfg.url} \
--token "$token" \
--name ${lib.escapeShellArg instanceCfg.name} \
--labels ${lib.escapeShellArg (lib.concatStringsSep "," instanceCfg.labels)} \
--config ${runnerConfigFile}
'';
};
systemd.services.${runnerUnit} = {
# `path` adds to the PATH the upstream module builds instead of
# replacing it.
path = [ pkgs.nix ];
# The upstream unit is `DynamicUser`, whose state dir systemd mounts
# `noexec`, so a job could not execute what it builds in its
# workspace. `ExecPaths=` resolves against the host root, where
# `/var/lib/gitea-runner` is a symlink into `private/`; `-` makes
# whichever spelling is absent a no-op.
serviceConfig.ExecPaths = [
"-/var/lib/gitea-runner"
"-/var/lib/private/gitea-runner"
];
environment = lib.mkIf caTrust.useSelfSigned {
NODE_EXTRA_CA_CERTS = caTrust.caContainerPath;
};
# A host job inherits the daemon's whole environment and can read
# its credentials, so the daemon carries the token in neither.
serviceConfig.EnvironmentFile = lib.mkForce [ ];
# Replaces upstream's pre-start step, which registers from
# `$TOKEN`, so the register unit is the one registration path.
#
# Started from here rather than through `Requires=`: a failed
# requirement fails the start job, which `Restart=` does not retry,
# while a failed pre-start step does.
serviceConfig.ExecStartPre = lib.mkForce [
"+${pkgs.systemd}/bin/systemctl start ${registerUnit}.service"
"+${pkgs.writeShellScript "swarm-ci-runner-mark-start" ''
${pkgs.coreutils}/bin/date +%s > ${daemonStartedFile}
''}"