Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/module-eval/forge-placement.nix
atlas 23807c5d0c swarm-ci-runner: keep the registration token on the host's /run, not in the container's root
swarm-bao-ci-runner still reads the token from bao under the runner's own
ci-runner leaf, but now writes it to /run/swarm-ci-runner on the host
(tmpfs), which the container bind-mounts read-only for the register
unit's LoadCredential. Before, it wrote the token to
/var/lib/nixos-containers/swarm-ci-runner/var/lib/swarm-ci-runner/registration-token
on persistent disk, where a superseded token stayed after rotation and
went into backups of the container root. The container's tmpfiles now
remove that old file.

Rotation is unchanged: the ten-minute timer rewrites the file when the
value differs and refresh_consumer restarts the runner. After a reboot
the token is fetched again; until the store answers, a runner that is
already registered keeps running and only a new registration waits.

Part of #4998 (slice 1).
2026-10-15 21:44:46 +02:00

497 lines
22 KiB
Nix

# `checks.module-eval-forge-placement` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
#
# Where the forge runs. A swarm has one, on the host with
# `deploy.forgejo.enable`; every other hive is a client of it, and the parts
# of a split deployment that must not assume every host runs a forge
# (the OIDC client, the controller's token, the CI runner) have to hold.
{
pkgs,
lib,
self,
nixosSystem,
shared,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
hive
runGroup
;
inherit (shared) bare;
allLocal = hive { deploy.singleHostSwarm = true; };
inherit (shared) servicesHere forgeHere;
# The shared services here, the forge somewhere else. Every derivation in
# ../host-modules/swarm-required-services.nix is `mkDefault`, so this stays
# expressible — and it is also the authelia-without-forge host the OIDC
# client case needs.
servicesForgeElsewhere = hive {
deploy.allSwarmServices = true;
deploy.forgejo.enable = false;
};
# The forge with neither authelia nor the store, its store identity placed by
# hand: the other half of the split, and the deployment the store fetch
# exists for.
forgeAlone = hive {
deploy.forgejo.enable = true;
deploy.bao.forgePublisherClientCertFile = "/etc/pki/forge-publisher.pem";
deploy.bao.forgePublisherClientKeyFile = "/etc/pki/forge-publisher-key.pem";
};
# The runner alone: no hive, no forge, no store, its store identity placed
# by hand. `nix` is a label without a scheme.
ciAlone = hive {
deploy.hive-controller.enable = false;
deploy.ci-runner.enable = true;
deploy.ci-runner.labels = [
"hive-ci:host"
"nix"
];
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
};
ciNoIdentity = hive {
deploy.hive-controller.enable = false;
deploy.ci-runner.enable = true;
};
# A host config that still names the runner under the forge, every option.
ciUnderForge = hive {
deploy.forgejo.ci.enable = true;
deploy.forgejo.ci.name = "old-runner";
deploy.forgejo.ci.concurrency = 3;
deploy.forgejo.ci.labels = [ "nix:host" ];
deploy.forgejo.ci.package = pkgs.hello;
deploy.forgejo.ci.jobTimeout = "2h";
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
};
# The old name alone, with no store identity placed and none defaulted.
ciUnderForgeNoIdentity = hive {
deploy.hive-controller.enable = false;
deploy.forgejo.ci.enable = true;
};
# The older spelling, under the swarm-level forge options.
ciUnderSwarmForge = hive {
swarm.forge.ci.enable = true;
swarm.forge.ci.name = "older-runner";
swarm.forge.ci.concurrency = 2;
swarm.forge.ci.labels = [ "nix:host" ];
swarm.forge.ci.package = pkgs.hello;
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
};
runsForge = m: m.services.hyperhive.deploy.forgejo.enable && m.containers ? hive-forge;
forgeClientIds =
m:
map (c: c.id) (
lib.filter (
c: c.id == m.services.hyperhive.swarm.forge.sso.clientId
) m.services.hyperhive.swarm.authelia.oidc.clients
);
# Matched on the option the message names, same reasoning as
# ./grafana.nix's `grafanaRefusedFor`.
refusedOver = m: needle: lib.any (a: !a.assertion && lib.hasInfix needle a.message) m.assertions;
publisher = m: m.systemd.services.swarm-bao-forge-publisher;
oidcFetch = m: m.systemd.services.swarm-bao-forge-oidc;
forgeSettings = forgeHere.containers.hive-forge.config.services.forgejo.settings;
runnerService = ciAlone.containers.swarm-ci-runner.config.systemd.services.gitea-runner-swarm;
registerService =
ciAlone.containers.swarm-ci-runner.config.systemd.services.swarm-ci-runner-register;
cases = [
{
# The absence the whole option exists for: a second forge in a swarm
# is a split brain nobody notices, so a hive that has not been told it
# is the forge's host runs none of its surface.
name = "a hive that is not the forge's host runs no forge";
ok =
!bare.services.hyperhive.deploy.forgejo.enable
&& !(bare.containers ? hive-forge)
&& !(bare.systemd.services ? swarm-bao-forge-publisher)
&& !(lib.elem "forge.t.local" bare.services.hyperhive.gateway.localNames);
}
{
# All three are published targets; ./swarm-otel-identity.nix covers them.
name = "a host running the collector, authelia, grafana and nats scrapes none of them on loopback";
ok =
let
t = servicesHere.services.hyperhive.swarm.otel.scrapeTargets;
in
servicesHere.services.hyperhive.deploy.nats.enable
&& !(t ? authelia)
&& !(t ? grafana)
&& !(t ? nats);
}
{
name = "hosting the swarm's shared services runs the forge";
ok = runsForge servicesHere;
}
{
name = "the all-local mode runs the forge";
ok = runsForge allLocal && lib.elem "forge.t.local" allLocal.services.hyperhive.gateway.localNames;
}
{
name = "an explicit deploy.forgejo.enable runs the forge on its own";
ok = runsForge forgeHere && !forgeHere.services.hyperhive.deploy.allSwarmServices;
}
{
# `mkDefault`, not a plain assignment: the forge stays placeable on a
# host of its own. `nats` is the control, so the case cannot pass on a
# fixture where nothing came on.
name = "placing the forge elsewhere survives the switch that would enable it";
ok =
!(servicesForgeElsewhere.containers ? hive-forge)
&& servicesForgeElsewhere.services.hyperhive.deploy.nats.enable;
}
{
# A client is a row in authelia's config, so it is declared where
# authelia runs. With the forge's module gated, registering it from
# there would leave a split swarm's forge unknown to its IdP.
name = "the forge's OIDC client is registered wherever authelia runs, and only there";
ok =
forgeClientIds servicesForgeElsewhere == [ "forgejo" ]
&& forgeClientIds allLocal == [ "forgejo" ]
&& forgeClientIds forgeAlone == [ ];
}
{
name = "the forge's OIDC callback is the one forgejo sends";
ok =
servicesForgeElsewhere.services.hyperhive.swarm.forge.sso.redirectUri
== "https://forge.t.local/user/oauth2/authelia/callback";
}
{
# The runner reaches the forge by name and reads its token with its own
# certificate, so nothing about the host it runs on is asked.
name = "the CI runner runs on a host with neither the forge nor a hive";
ok =
ciAlone.containers ? swarm-ci-runner
&& !(ciAlone.containers ? hive-forge)
&& !ciAlone.services.hyperhive.deploy.hive-controller.enable
&& !(refusedOver ciAlone "ci-runner")
&&
ciAlone.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT
== "/etc/pki/bao-ci-runner.pem";
}
{
# The store's units exist only on its host; `Requires=` on an absent one
# fails the fetch's start. The arm where the store is here is in
# ./swarm-services-switch.nix.
name = "the CI runner's token fetch names none of the store's units on a host without the store";
ok =
let
u = ciAlone.systemd.services.swarm-bao-ci-runner;
storeUnits = [
"swarm-bao-pki.service"
"container@${ciAlone.services.hyperhive.swarm.bao.machine}.service"
];
in
!(lib.any (n: lib.elem n (u.after ++ u.wants ++ u.requires)) storeUnits);
}
{
# Without the identity the token never arrives; the control is the
# placed pair above.
name = "the CI runner is refused without its store identity";
ok = refusedOver ciNoIdentity "deploy.bao.ciRunnerClientCertFile";
}
{
# systemd hands a credential to every process of the unit that loads
# it, and a host job is one of the daemon's processes and inherits its
# environment; a token in either lets any workflow register runners.
name = "the CI runner daemon can read no registration token";
ok =
let
sc = runnerService.serviceConfig;
in
!(runnerService.environment ? TOKEN)
&& lib.toList (sc.EnvironmentFile or [ ]) == [ ]
&& !(lib.any (lib.hasPrefix "TOKEN=") (lib.toList (sc.Environment or [ ])))
&& !(sc ? LoadCredential)
&& !(sc ? LoadCredentialEncrypted)
&& !(sc ? SetCredential);
}
{
name = "the CI register unit alone loads the registration token, and runs before the daemon";
ok =
let
sc = registerService.serviceConfig;
in
sc.LoadCredential == [ "registration-token:/run/swarm-ci-runner/registration-token" ]
&& sc.Type == "oneshot"
&& sc.User == runnerService.serviceConfig.User
&& sc.DynamicUser
&& sc.StateDirectory == runnerService.serviceConfig.StateDirectory
&& lib.any (
s: lib.hasSuffix "/bin/systemctl start swarm-ci-runner-register.service" s
) runnerService.serviceConfig.ExecStartPre
&& !(lib.any (
s: lib.hasInfix "gitea-register-runner" (toString s)
) runnerService.serviceConfig.ExecStartPre);
}
{
# A copy on persistent disk outlives the token's rotation and lands in
# backups of the container's root.
name = "the CI runner's registration token is written to the host's /run only, and the old copy removed";
ok =
let
script = ciAlone.systemd.services.swarm-bao-ci-runner.script;
mount = ciAlone.containers.swarm-ci-runner.bindMounts."/run/swarm-ci-runner";
guestRules = ciAlone.containers.swarm-ci-runner.config.systemd.tmpfiles.rules;
in
lib.hasInfix "atomic_write_secret 0600 \"\" /run/swarm-ci-runner/registration-token" script
&& !(lib.hasInfix "/var/lib/nixos-containers" script)
&& mount.hostPath == "/run/swarm-ci-runner"
&& mount.isReadOnly
&& lib.elem "d /run/swarm-ci-runner 0700 root root -" ciAlone.systemd.tmpfiles.rules
&& lib.elem "r /var/lib/swarm-ci-runner/registration-token" guestRules
&& !(lib.any (lib.hasPrefix "f /var/lib/swarm-ci-runner/registration-token") guestRules);
}
{
name = "the CI register unit registers from its credential, and only without a .runner";
ok =
let
text = registerService.script;
beforeRegister = lib.head (lib.splitString "register --no-interactive" text);
in
lib.hasInfix "if [ -e .runner ]; then\n exit 0\nfi" beforeRegister
&& lib.hasInfix ''token="$(< "$CREDENTIALS_DIRECTORY/registration-token")"'' beforeRegister
&& lib.hasInfix ''--token "$token"'' text
&& !(lib.hasInfix "export" text)
&& !(lib.hasInfix "TOKEN" text);
}
{
# The daemon rewrites `.runner`'s labels to this form on start; compared
# against the configured strings verbatim, a scheme-less label would
# discard `.runner` and register a new runner on every start.
name = "the CI runner compares .runner's labels in the form the daemon writes them";
ok = registerService.environment.SWARM_CI_RUNNER_LABELS == "hive-ci:host,nix:host";
}
{
# The control is the container: it only exists if `enable` arrived.
name = "a CI option under deploy.forgejo sets the same option under deploy.ci-runner";
ok =
let
r = ciUnderForge.services.hyperhive.deploy.ci-runner;
in
r.enable
&& r.name == "old-runner"
&& r.concurrency == 3
&& r.labels == [ "nix:host" ]
&& r.package == pkgs.hello
&& r.jobTimeout == "2h"
&& ciUnderForge.containers ? swarm-ci-runner
&& lib.any (lib.hasInfix "services.hyperhive.deploy.forgejo.ci.enable") ciUnderForge.warnings
&& !(lib.any (a: !a.assertion) ciUnderForge.assertions);
}
{
# The control is the container: it only exists if `enable` arrived.
name = "a CI option under swarm.forge sets the same option under deploy.ci-runner";
ok =
let
r = ciUnderSwarmForge.services.hyperhive.deploy.ci-runner;
in
r.enable
&& r.name == "older-runner"
&& r.concurrency == 2
&& r.labels == [ "nix:host" ]
&& r.package == pkgs.hello
&& ciUnderSwarmForge.containers ? swarm-ci-runner
&& lib.any (lib.hasInfix "services.hyperhive.swarm.forge.ci.enable") ciUnderSwarmForge.warnings
&& !(lib.any (a: !a.assertion) ciUnderSwarmForge.assertions);
}
{
# The rename carries `enable` but not the identity: an old config with
# only the old name is refused, and the message says what to set. The
# control is the placed pair in `ciUnderForge` above.
name = "an old CI enable without a store identity is refused naming the options to set";
ok =
refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.ciRunnerClientCertFile"
&& refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.ciRunnerClientKeyFile"
&& refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.enable";
}
{
# The forge's host publishes the controller's token under its own leaf,
# which the store mints beside it here. `value=@` is what keeps the token
# out of the process's arguments.
name = "the forge's host publishes the controller's forge token to the store under its own leaf";
ok =
let
u = publisher allLocal;
in
u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/forge-publisher.pem"
&& u.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/forge-publisher-key.pem"
&& lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/forge/token value=@\"$src\"" u.script
&& lib.hasInfix "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token" u.script
&& lib.elem "container@hive-forge.service" u.wantedBy
# The deploy that adds the unit starts it only through a target.
&& lib.elem "multi-user.target" u.wantedBy
&& !(allLocal.systemd.services ? hive-forge-swarm-controller-token);
}
{
# What makes it re-run. A oneshot left `active (exited)` makes every later
# start a no-op, so `RemainAfterExit` would silence every trigger: the
# targets and the container's start (`wantedBy`, above) and a write of
# the token file. A missing file exits 0, so a switch with the container
# down does not fail.
name = "the forge publisher re-runs on each container start and each write of the token file";
ok =
let
p = allLocal.systemd.paths.swarm-bao-forge-publisher;
in
!((publisher allLocal).serviceConfig.RemainAfterExit or false)
&&
p.pathConfig.PathChanged
== "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token"
&& p.pathConfig.Unit == "swarm-bao-forge-publisher.service"
&& lib.hasInfix "the path unit publishes it when it does" (publisher allLocal).script
&& lib.hasInfix "exit 0" (publisher allLocal).script
&& lib.elem "multi-user.target" p.wantedBy
&& !(bare.systemd.paths ? swarm-bao-forge-publisher);
}
{
# The absence arm: a forge host without the store has no leaf unless one
# is placed, and the unit says which options to set.
name = "a forge host with no publisher leaf renders a publisher that fails naming the options";
ok =
let
s = (publisher forgeHere).script;
in
lib.hasInfix "forgePublisherClientCertFile" s
&& lib.hasInfix "exit 1" s
&& !(lib.hasInfix "bao kv put" s);
}
{
# The forge's OIDC secret has one route, the store, under the forge
# publisher's leaf. No unit copies it out of authelia's tree, even on a
# host that runs authelia, and nothing defines the removed path option.
name = "the forge reads its OIDC secret from the store under the forge publisher's leaf";
ok =
let
u = oidcFetch forgeAlone;
d = forgeAlone.services.hyperhive.deploy;
in
!d.authelia.enable
&& !d.bao.enable
&& !(refusedOver forgeAlone "deploy.forgejo.sso.clientSecretFile")
&& !(refusedOver allLocal "deploy.forgejo.sso.clientSecretFile")
&& u.environment.BAO_CLIENT_CERT == "/etc/pki/forge-publisher.pem"
&& u.environment.BAO_CLIENT_KEY == "/etc/pki/forge-publisher-key.pem"
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/forgejo/oidc/client" u.script
&& lib.hasInfix "refresh_consumer hive-forge forgejo-sso-source.service" u.script
# An unpublished or empty secret fails the unit, so `storeRetry` retries it.
&& !(lib.hasInfix "exit 0" u.script)
&& u.serviceConfig.Restart == "on-failure"
&& lib.elem "container@hive-forge.service" u.before
&& lib.elem "multi-user.target" u.wantedBy
&& (oidcFetch allLocal).environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/forge-publisher.pem"
&& !(forgeAlone.systemd.services ? hive-forge-oidc-secret)
&& !(allLocal.systemd.services ? hive-forge-oidc-secret)
&& !(bare.systemd.services ? swarm-bao-forge-oidc);
}
{
# The fetched secret lives in a host tmpfs directory bound read-only into
# the container, and reaches the login source as a credential. Nothing
# writes it under the container's persistent root, and the copy an
# earlier fetch left there is removed.
name = "the forge's OIDC secret reaches its login source from /run, not from persistent disk";
ok =
let
s = (oidcFetch forgeAlone).script;
c = forgeAlone.containers.hive-forge;
src = c.config.systemd.services.forgejo-sso-source;
rules = forgeAlone.systemd.tmpfiles.rules;
in
lib.hasInfix "atomic_write_secret 0400" s
&& lib.hasInfix "/run/hive-forge-oidc/forgejo.secret" s
&& !(lib.hasInfix "/var/lib/nixos-containers/hive-forge" s)
&& c.bindMounts."/run/hive-forge-oidc".hostPath == "/run/hive-forge-oidc"
&& c.bindMounts."/run/hive-forge-oidc".isReadOnly
&& lib.elem "oidc-client-secret:/run/hive-forge-oidc/forgejo.secret" src.serviceConfig.LoadCredential
&& lib.hasInfix "$CREDENTIALS_DIRECTORY/oidc-client-secret" src.script
&& lib.elem "d /run/hive-forge-oidc 0700 root root -" rules
&& lib.elem "r /var/lib/nixos-containers/hive-forge/var/lib/forgejo-oidc/forgejo.secret" rules;
}
{
# Beside the store, the fetch waits for the unit writing the forge
# publisher's role, the leaf it logs in with.
name = "the forge's OIDC secret fetch is ordered after the forge publisher's policy unit";
ok =
let
u = oidcFetch allLocal;
in
lib.elem "swarm-bao-forge-publisher-policy.service" u.after
&& lib.elem "swarm-bao-forge-publisher-policy.service" u.wants;
}
{
# The absence arm: no publisher leaf, no secret, and the unit says which
# options to set.
name = "a forge host with no publisher leaf renders an OIDC secret fetch that fails naming the options";
ok =
let
u = oidcFetch forgeHere;
in
!(u.environment ? BAO_CLIENT_CERT)
&& lib.hasInfix "forgePublisherClientCertFile" u.script
&& lib.hasInfix "exit 1" u.script
&& !(lib.hasInfix "bao login" u.script);
}
{
# The login source fetches the issuer's discovery document by name
# wherever the forge runs, so it waits for the resolver on a forge host
# without authelia too.
name = "the login source waits for the container's resolver on a forge host without authelia";
ok = lib.elem "forgejo-sso-source.service" (
forgeAlone.containers.hive-forge.config.systemd.services.swarm-bridge-dns.before
);
}
{
# The host copy is removed where the forge runs: nothing reads it.
name = "the forge's host removes its old copy of the controller's forge token";
ok =
lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" forgeHere.systemd.tmpfiles.rules
&& !(lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" bare.systemd.tmpfiles.rules);
}
{
# The only thing that makes a human's forge account: nothing else
# creates one.
name = "a first authelia login creates the forge account, named by preferred_username";
ok =
let
o = forgeSettings.oauth2_client;
in
o.ENABLE_AUTO_REGISTRATION == true && o.USERNAME == "preferred_username";
}
{
# `auto` would give an SSO user whatever local account carries their
# name — an agent's, or `core`'s.
name = "an SSO login adopts an existing forge account only with that account's password";
ok = forgeSettings.oauth2_client.ACCOUNT_LINKING == "login";
}
{
# Both halves: local sign-up stays off, and nothing turns on the one
# setting forgejo's auto-registration does check.
name = "local sign-up stays off without blocking the SSO registration";
ok =
forgeSettings.service.DISABLE_REGISTRATION == true
&& !(forgeSettings.service.ALLOW_ONLY_INTERNAL_REGISTRATION or false);
}
];
in
runGroup "forge-placement" cases