swarm-ci-runner: give the registration token only to the register step
The runner got its registration token through an `EnvironmentFile` (`TOKEN=…`), so the daemon carried it for its whole life, and for a `host` label act_runner copies its environment into every job step: any workflow could read a token that registers runners on the forge. The host unit now writes the bare token, and the runner unit loads it with `LoadCredential=`. The upstream pre-start step, which registered from `$TOKEN`, is replaced by one register step that reads the token from `$CREDENTIALS_DIRECTORY` and runs `act_runner register` only when `.runner` is missing, with the instance URL, name, labels and config file upstream used. The precond now also removes `.runner` when its labels differ from the configured ones, which upstream's step covered. A changed token alone no longer re-registers. Host jobs can still read `.runner` itself (#4979). Closes #4974
This commit is contained in:
atlas
parent
84347f122c
commit
0302b29ca4
4 changed files with 158 additions and 71 deletions
|
|
@ -57,11 +57,13 @@ let
|
|||
# it, which is the spelling a step run as root (`+`) can follow.
|
||||
runnerFile = "/var/lib/gitea-runner/${instance}/.runner";
|
||||
|
||||
# Inside the container, root-only. The token file is the runner's
|
||||
# `EnvironmentFile` (`TOKEN=<value>`); the host unit below writes it through
|
||||
# the container's root, at `hostTokenFile`.
|
||||
# Inside the container, root-only. The token file holds the bare token and
|
||||
# reaches the runner unit only as a `LoadCredential=`, which only the
|
||||
# register step reads; the host unit below writes it through the
|
||||
# container's root, at `hostTokenFile`.
|
||||
stateDir = "/var/lib/swarm-ci-runner";
|
||||
tokenFile = "${stateDir}/registration-token.env";
|
||||
tokenFile = "${stateDir}/registration-token";
|
||||
tokenCredential = "registration-token";
|
||||
hostTokenFile = "/var/lib/nixos-containers/${machine}${tokenFile}";
|
||||
# The epoch second the daemon last started. The precond reads the daemon's
|
||||
# journal from there.
|
||||
|
|
@ -246,15 +248,15 @@ in
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# Root-owned: systemd reads an `EnvironmentFile` as root, before the
|
||||
# runner drops to its dynamic user.
|
||||
# Root-owned: systemd reads a `LoadCredential=` source as root, before
|
||||
# the runner drops to its dynamic user.
|
||||
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostTokenFile)}
|
||||
if secret_differs ${lib.escapeShellArg hostTokenFile} "TOKEN=$token"; then
|
||||
atomic_write_secret 0600 "" ${lib.escapeShellArg hostTokenFile} "TOKEN=$token"
|
||||
if secret_differs ${lib.escapeShellArg hostTokenFile} "$token"; then
|
||||
atomic_write_secret 0600 "" ${lib.escapeShellArg hostTokenFile} "$token"
|
||||
fi
|
||||
|
||||
# `EnvironmentFile=` is read once, at start. A changed token makes the
|
||||
# upstream pre-start step register again.
|
||||
# `LoadCredential=` copies the file at start, so a runner still waiting
|
||||
# to register sees a new token only once it restarts.
|
||||
refresh_consumer ${lib.escapeShellArg machine} ${runnerUnit}.service ${lib.escapeShellArg hostTokenFile}
|
||||
'';
|
||||
};
|
||||
|
|
@ -282,7 +284,20 @@ in
|
|||
// caTrust.bindMount;
|
||||
|
||||
config =
|
||||
{ pkgs, lib, ... }:
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
runnerCfg = config.services.gitea-actions-runner;
|
||||
instanceCfg = runnerCfg.instances.${instance};
|
||||
# The same file upstream generates and hands the daemon.
|
||||
runnerConfigFile = (pkgs.formats.yaml { }).generate "config.yaml" instanceCfg.settings;
|
||||
# In the form the precond reads back from `.runner`.
|
||||
registeredLabels = lib.concatStringsSep "," (lib.sort lib.lessThan instanceCfg.labels);
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
../container-modules/swarm-container.nix
|
||||
|
|
@ -321,8 +336,9 @@ in
|
|||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${stateDir} 0700 root root -"
|
||||
# `EnvironmentFile=` fails the unit when the file is missing; an
|
||||
# empty one leaves `TOKEN` unset for the precond to report.
|
||||
# `LoadCredential=` fails the unit when the file is missing; an
|
||||
# empty one lets a registered runner start before any token has
|
||||
# arrived, and the register step report the missing token.
|
||||
"f ${tokenFile} 0600 root root -"
|
||||
];
|
||||
|
||||
|
|
@ -332,6 +348,8 @@ in
|
|||
enable = true;
|
||||
inherit (cfg) name labels;
|
||||
url = forgeUrl;
|
||||
# Upstream refuses an instance without a token source, and turns
|
||||
# this one into the `EnvironmentFile=` the unit below drops.
|
||||
inherit tokenFile;
|
||||
settings = {
|
||||
runner.capacity = cfg.concurrency;
|
||||
|
|
@ -360,54 +378,80 @@ in
|
|||
environment = lib.mkIf caTrust.useSelfSigned {
|
||||
NODE_EXTRA_CA_CERTS = caTrust.caContainerPath;
|
||||
};
|
||||
# Runs before upstream's pre-start step, which registers whenever
|
||||
# `.runner` is absent or the token or labels changed. This one
|
||||
# removes `.runner` in the two cases upstream cannot see, and holds
|
||||
# the unit down until there is a token.
|
||||
#
|
||||
# `+`: root, because `.runner`, the token and the system journal
|
||||
# are all out of the dynamic user's reach.
|
||||
serviceConfig.ExecStartPre = lib.mkMerge [
|
||||
(lib.mkBefore [
|
||||
"+${pkgs.writeShellScript "swarm-ci-runner-precond" ''
|
||||
set -eu
|
||||
runner=${runnerFile}
|
||||
started=${daemonStartedFile}
|
||||
# The registration token reaches the unit only as a credential,
|
||||
# which the register step reads. A host job inherits the daemon's
|
||||
# whole environment, so neither carries it.
|
||||
serviceConfig.EnvironmentFile = lib.mkForce [ ];
|
||||
serviceConfig.LoadCredential = [ "${tokenCredential}:${tokenFile}" ];
|
||||
# One registration path: upstream's pre-start step registers from
|
||||
# `$TOKEN`, so this list replaces it.
|
||||
serviceConfig.ExecStartPre = lib.mkForce [
|
||||
# Removes `.runner` in the cases where reusing it would fail, so
|
||||
# the register step after it registers again.
|
||||
#
|
||||
# `+`: root, because the system journal is out of the dynamic
|
||||
# user's reach.
|
||||
"+${pkgs.writeShellScript "swarm-ci-runner-precond" ''
|
||||
set -eu
|
||||
runner=${runnerFile}
|
||||
started=${daemonStartedFile}
|
||||
|
||||
# The forge refused the registration the daemon last started
|
||||
# with. Read on the next start rather than when the daemon
|
||||
# exits, so journald has its last lines by then.
|
||||
if [ -f "$started" ]; then
|
||||
since="$(< "$started")"
|
||||
${pkgs.coreutils}/bin/rm -f "$started"
|
||||
if ${pkgs.systemd}/bin/journalctl -u ${runnerUnit}.service --since "@$since" -o cat --no-pager \
|
||||
| ${pkgs.gnugrep}/bin/grep -qF ${lib.escapeShellArg rejectedMessage}; then
|
||||
echo "the forge has no record of this runner's registration; registering again" >&2
|
||||
${pkgs.coreutils}/bin/rm -f "$runner"
|
||||
fi
|
||||
fi
|
||||
|
||||
# `.runner` records the address it registered at, and the
|
||||
# daemon dials that one, not the configured one.
|
||||
if [ -f "$runner" ] \
|
||||
&& [ "$(${pkgs.jq}/bin/jq -r .address "$runner")" != ${lib.escapeShellArg forgeUrl} ]; then
|
||||
echo "$runner names another forge address; registering again at ${forgeUrl}" >&2
|
||||
# The forge refused the registration the daemon last started
|
||||
# with. Read on the next start rather than when the daemon
|
||||
# exits, so journald has its last lines by then.
|
||||
if [ -f "$started" ]; then
|
||||
since="$(< "$started")"
|
||||
${pkgs.coreutils}/bin/rm -f "$started"
|
||||
if ${pkgs.systemd}/bin/journalctl -u ${runnerUnit}.service --since "@$since" -o cat --no-pager \
|
||||
| ${pkgs.gnugrep}/bin/grep -qF ${lib.escapeShellArg rejectedMessage}; then
|
||||
echo "the forge has no record of this runner's registration; registering again" >&2
|
||||
${pkgs.coreutils}/bin/rm -f "$runner"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Upstream registers on a changed token hash, an empty token
|
||||
# included, and a registration with no token fails.
|
||||
if [ -z "''${TOKEN:-}" ]; then
|
||||
echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2
|
||||
exit 1
|
||||
fi
|
||||
''}"
|
||||
])
|
||||
(lib.mkAfter [
|
||||
"+${pkgs.writeShellScript "swarm-ci-runner-mark-start" ''
|
||||
${pkgs.coreutils}/bin/date +%s > ${daemonStartedFile}
|
||||
''}"
|
||||
])
|
||||
# `.runner` records the address it registered at, and the
|
||||
# daemon dials that one, not the configured one.
|
||||
if [ -f "$runner" ] \
|
||||
&& [ "$(${pkgs.jq}/bin/jq -r .address "$runner")" != ${lib.escapeShellArg forgeUrl} ]; then
|
||||
echo "$runner names another forge address; registering again at ${forgeUrl}" >&2
|
||||
${pkgs.coreutils}/bin/rm -f "$runner"
|
||||
fi
|
||||
|
||||
# The daemon declares the labels `.runner` records, not the
|
||||
# configured ones.
|
||||
if [ -f "$runner" ] \
|
||||
&& [ "$(${pkgs.jq}/bin/jq -r '.labels | sort | join(",")' "$runner")" != ${lib.escapeShellArg registeredLabels} ]; then
|
||||
echo "$runner names other labels; registering again with ${registeredLabels}" >&2
|
||||
${pkgs.coreutils}/bin/rm -f "$runner"
|
||||
fi
|
||||
''}"
|
||||
# Registers only when `.runner` is absent. As the dynamic user,
|
||||
# so `.runner` is the daemon's own. The derivation itself rather
|
||||
# than its path, so ../module-eval reads its text without building.
|
||||
(pkgs.writeShellScript "swarm-ci-runner-register" ''
|
||||
set -eu
|
||||
dir="$STATE_DIRECTORY/${instance}"
|
||||
${pkgs.coreutils}/bin/mkdir -p "$dir"
|
||||
cd "$dir"
|
||||
if [ -e .runner ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
token="$(< "$CREDENTIALS_DIRECTORY/${tokenCredential}")"
|
||||
if [ -z "$token" ]; then
|
||||
echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2
|
||||
exit 1
|
||||
fi
|
||||
exec ${lib.getExe runnerCfg.package} register --no-interactive \
|
||||
--instance ${lib.escapeShellArg instanceCfg.url} \
|
||||
--token "$token" \
|
||||
--name ${lib.escapeShellArg instanceCfg.name} \
|
||||
--labels ${lib.escapeShellArg (lib.concatStringsSep "," instanceCfg.labels)} \
|
||||
--config ${runnerConfigFile}
|
||||
'')
|
||||
"+${pkgs.writeShellScript "swarm-ci-runner-mark-start" ''
|
||||
${pkgs.coreutils}/bin/date +%s > ${daemonStartedFile}
|
||||
''}"
|
||||
];
|
||||
# Restarted until the token arrives, however long the store and
|
||||
# the controller take. `mkForce`: upstream sets both.
|
||||
|
|
|
|||
Loading…
Reference in a new issue