2d2d7529fc
The publisher was a oneshot with RemainAfterExit=true, started only through `wantedBy = container@hive-forge.service`. Once it had run it stayed `active (exited)`, so a later container start made its start a no-op: a token re-minted without a host reboot or a unit change never reached the store. - RemainAfterExit is dropped, so each start runs the script again. - systemd.paths.swarm-bao-forge-publisher watches the token file in the container (PathChanged) and starts the publisher when the mint writes it, the same shape as swarm-secret-publish's path unit. - The code comment, secrets.md and credentials.md now name both triggers; forge-placement pins the missing RemainAfterExit and the path unit. Refs #4998
359 lines
15 KiB
Nix
359 lines
15 KiB
Nix
# `checks.module-eval-forge-placement` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
#
|
|
# Where the forge runs. A swarm has one, on the host with
|
|
# `deploy.forgejo.enable`; every other hive is a client of it, and the parts
|
|
# of a split deployment that must not assume every host runs a forge
|
|
# (the OIDC client, the controller's token, the CI runner) have to hold.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
bare = hive { };
|
|
allLocal = hive { deploy.singleHostSwarm = true; };
|
|
servicesHere = hive { deploy.allSwarmServices = true; };
|
|
forgeHere = hive { deploy.forgejo.enable = true; };
|
|
|
|
# The shared services here, the forge somewhere else. Every derivation in
|
|
# ../host-modules/swarm-required-services.nix is `mkDefault`, so this stays
|
|
# expressible — and it is also the authelia-without-forge host the OIDC
|
|
# client case needs.
|
|
servicesForgeElsewhere = hive {
|
|
deploy.allSwarmServices = true;
|
|
deploy.forgejo.enable = false;
|
|
};
|
|
|
|
# The forge without authelia: the other half of the split.
|
|
forgeNoAuthelia = hive {
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
|
|
};
|
|
|
|
# The runner alone: no hive, no forge, no store, its store identity placed
|
|
# by hand. `nix` is a label without a scheme.
|
|
ciAlone = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.ci-runner.enable = true;
|
|
deploy.ci-runner.labels = [
|
|
"hive-ci:host"
|
|
"nix"
|
|
];
|
|
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
|
|
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
|
|
};
|
|
ciNoIdentity = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.ci-runner.enable = true;
|
|
};
|
|
|
|
# A host config that still names the runner under the forge, every option.
|
|
ciUnderForge = hive {
|
|
deploy.forgejo.ci.enable = true;
|
|
deploy.forgejo.ci.name = "old-runner";
|
|
deploy.forgejo.ci.concurrency = 3;
|
|
deploy.forgejo.ci.labels = [ "nix:host" ];
|
|
deploy.forgejo.ci.package = pkgs.hello;
|
|
deploy.forgejo.ci.jobTimeout = "2h";
|
|
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
|
|
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
|
|
};
|
|
# The old name alone, with no store identity placed and none defaulted.
|
|
ciUnderForgeNoIdentity = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.forgejo.ci.enable = true;
|
|
};
|
|
# The older spelling, under the swarm-level forge options.
|
|
ciUnderSwarmForge = hive {
|
|
swarm.forge.ci.enable = true;
|
|
swarm.forge.ci.name = "older-runner";
|
|
swarm.forge.ci.concurrency = 2;
|
|
swarm.forge.ci.labels = [ "nix:host" ];
|
|
swarm.forge.ci.package = pkgs.hello;
|
|
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
|
|
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
|
|
};
|
|
|
|
runsForge = m: m.services.hyperhive.deploy.forgejo.enable && m.containers ? hive-forge;
|
|
|
|
forgeClientIds =
|
|
m:
|
|
map (c: c.id) (
|
|
lib.filter (
|
|
c: c.id == m.services.hyperhive.swarm.forge.sso.clientId
|
|
) m.services.hyperhive.swarm.authelia.oidc.clients
|
|
);
|
|
|
|
# Matched on the option the message names, same reasoning as
|
|
# ./grafana.nix's `grafanaRefusedFor`.
|
|
refusedOver = m: needle: lib.any (a: !a.assertion && lib.hasInfix needle a.message) m.assertions;
|
|
|
|
publisher = m: m.systemd.services.swarm-bao-forge-publisher;
|
|
|
|
forgeSettings = forgeHere.containers.hive-forge.config.services.forgejo.settings;
|
|
|
|
runnerService = ciAlone.containers.swarm-ci-runner.config.systemd.services.gitea-runner-swarm;
|
|
registerService =
|
|
ciAlone.containers.swarm-ci-runner.config.systemd.services.swarm-ci-runner-register;
|
|
|
|
cases = [
|
|
{
|
|
# The absence the whole option exists for: a second forge in a swarm
|
|
# is a split brain nobody notices, so a hive that has not been told it
|
|
# is the forge's host runs none of its surface.
|
|
name = "a hive that is not the forge's host runs no forge";
|
|
ok =
|
|
!bare.services.hyperhive.deploy.forgejo.enable
|
|
&& !(bare.containers ? hive-forge)
|
|
&& !(bare.systemd.services ? swarm-bao-forge-publisher)
|
|
&& !(lib.elem "forge.t.local" bare.services.hyperhive.gateway.localNames)
|
|
&& !(refusedOver bare "deploy.forgejo.sso.clientSecretFile");
|
|
}
|
|
{
|
|
name = "hosting the swarm's shared services runs the forge";
|
|
ok = runsForge servicesHere;
|
|
}
|
|
{
|
|
name = "the all-local mode runs the forge";
|
|
ok = runsForge allLocal && lib.elem "forge.t.local" allLocal.services.hyperhive.gateway.localNames;
|
|
}
|
|
{
|
|
name = "an explicit deploy.forgejo.enable runs the forge on its own";
|
|
ok = runsForge forgeHere && !forgeHere.services.hyperhive.deploy.allSwarmServices;
|
|
}
|
|
{
|
|
# `mkDefault`, not a plain assignment: the forge stays placeable on a
|
|
# host of its own. `nats` is the control, so the case cannot pass on a
|
|
# fixture where nothing came on.
|
|
name = "placing the forge elsewhere survives the switch that would enable it";
|
|
ok =
|
|
!(servicesForgeElsewhere.containers ? hive-forge)
|
|
&& servicesForgeElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# A client is a row in authelia's config, so it is declared where
|
|
# authelia runs. With the forge's module gated, registering it from
|
|
# there would leave a split swarm's forge unknown to its IdP.
|
|
name = "the forge's OIDC client is registered wherever authelia runs, and only there";
|
|
ok =
|
|
forgeClientIds servicesForgeElsewhere == [ "forgejo" ]
|
|
&& forgeClientIds allLocal == [ "forgejo" ]
|
|
&& forgeClientIds forgeNoAuthelia == [ ];
|
|
}
|
|
{
|
|
name = "the forge's OIDC callback is the one forgejo sends";
|
|
ok =
|
|
servicesForgeElsewhere.services.hyperhive.swarm.forge.sso.redirectUri
|
|
== "https://forge.t.local/user/oauth2/authelia/callback";
|
|
}
|
|
{
|
|
# The runner reaches the forge by name and reads its token with its own
|
|
# certificate, so nothing about the host it runs on is asked.
|
|
name = "the CI runner runs on a host with neither the forge nor a hive";
|
|
ok =
|
|
ciAlone.containers ? swarm-ci-runner
|
|
&& !(ciAlone.containers ? hive-forge)
|
|
&& !ciAlone.services.hyperhive.deploy.hive-controller.enable
|
|
&& !(refusedOver ciAlone "ci-runner")
|
|
&&
|
|
ciAlone.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT
|
|
== "/etc/pki/bao-ci-runner.pem";
|
|
}
|
|
{
|
|
# Without the identity the token never arrives; the control is the
|
|
# placed pair above.
|
|
name = "the CI runner is refused without its store identity";
|
|
ok = refusedOver ciNoIdentity "deploy.bao.ciRunnerClientCertFile";
|
|
}
|
|
{
|
|
# systemd hands a credential to every process of the unit that loads
|
|
# it, and a host job is one of the daemon's processes and inherits its
|
|
# environment; a token in either lets any workflow register runners.
|
|
name = "the CI runner daemon can read no registration token";
|
|
ok =
|
|
let
|
|
sc = runnerService.serviceConfig;
|
|
in
|
|
!(runnerService.environment ? TOKEN)
|
|
&& lib.toList (sc.EnvironmentFile or [ ]) == [ ]
|
|
&& !(lib.any (lib.hasPrefix "TOKEN=") (lib.toList (sc.Environment or [ ])))
|
|
&& !(sc ? LoadCredential)
|
|
&& !(sc ? LoadCredentialEncrypted)
|
|
&& !(sc ? SetCredential);
|
|
}
|
|
{
|
|
name = "the CI register unit alone loads the registration token, and runs before the daemon";
|
|
ok =
|
|
let
|
|
sc = registerService.serviceConfig;
|
|
in
|
|
sc.LoadCredential == [ "registration-token:/var/lib/swarm-ci-runner/registration-token" ]
|
|
&& sc.Type == "oneshot"
|
|
&& sc.User == runnerService.serviceConfig.User
|
|
&& sc.DynamicUser
|
|
&& sc.StateDirectory == runnerService.serviceConfig.StateDirectory
|
|
&& lib.any (
|
|
s: lib.hasSuffix "/bin/systemctl start swarm-ci-runner-register.service" s
|
|
) runnerService.serviceConfig.ExecStartPre
|
|
&& !(lib.any (
|
|
s: lib.hasInfix "gitea-register-runner" (toString s)
|
|
) runnerService.serviceConfig.ExecStartPre);
|
|
}
|
|
{
|
|
name = "the CI register unit registers from its credential, and only without a .runner";
|
|
ok =
|
|
let
|
|
text = registerService.script;
|
|
beforeRegister = lib.head (lib.splitString "register --no-interactive" text);
|
|
in
|
|
lib.hasInfix "if [ -e .runner ]; then\n exit 0\nfi" beforeRegister
|
|
&& lib.hasInfix ''token="$(< "$CREDENTIALS_DIRECTORY/registration-token")"'' beforeRegister
|
|
&& lib.hasInfix ''--token "$token"'' text
|
|
&& !(lib.hasInfix "export" text)
|
|
&& !(lib.hasInfix "TOKEN" text);
|
|
}
|
|
{
|
|
# The daemon rewrites `.runner`'s labels to this form on start; compared
|
|
# against the configured strings verbatim, a scheme-less label would
|
|
# discard `.runner` and register a new runner on every start.
|
|
name = "the CI runner compares .runner's labels in the form the daemon writes them";
|
|
ok = registerService.environment.SWARM_CI_RUNNER_LABELS == "hive-ci:host,nix:host";
|
|
}
|
|
{
|
|
# The control is the container: it only exists if `enable` arrived.
|
|
name = "a CI option under deploy.forgejo sets the same option under deploy.ci-runner";
|
|
ok =
|
|
let
|
|
r = ciUnderForge.services.hyperhive.deploy.ci-runner;
|
|
in
|
|
r.enable
|
|
&& r.name == "old-runner"
|
|
&& r.concurrency == 3
|
|
&& r.labels == [ "nix:host" ]
|
|
&& r.package == pkgs.hello
|
|
&& r.jobTimeout == "2h"
|
|
&& ciUnderForge.containers ? swarm-ci-runner
|
|
&& lib.any (lib.hasInfix "services.hyperhive.deploy.forgejo.ci.enable") ciUnderForge.warnings
|
|
&& !(lib.any (a: !a.assertion) ciUnderForge.assertions);
|
|
}
|
|
{
|
|
# The control is the container: it only exists if `enable` arrived.
|
|
name = "a CI option under swarm.forge sets the same option under deploy.ci-runner";
|
|
ok =
|
|
let
|
|
r = ciUnderSwarmForge.services.hyperhive.deploy.ci-runner;
|
|
in
|
|
r.enable
|
|
&& r.name == "older-runner"
|
|
&& r.concurrency == 2
|
|
&& r.labels == [ "nix:host" ]
|
|
&& r.package == pkgs.hello
|
|
&& ciUnderSwarmForge.containers ? swarm-ci-runner
|
|
&& lib.any (lib.hasInfix "services.hyperhive.swarm.forge.ci.enable") ciUnderSwarmForge.warnings
|
|
&& !(lib.any (a: !a.assertion) ciUnderSwarmForge.assertions);
|
|
}
|
|
{
|
|
# The rename carries `enable` but not the identity: an old config with
|
|
# only the old name is refused, and the message says what to set. The
|
|
# control is the placed pair in `ciUnderForge` above.
|
|
name = "an old CI enable without a store identity is refused naming the options to set";
|
|
ok =
|
|
refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.ciRunnerClientCertFile"
|
|
&& refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.ciRunnerClientKeyFile"
|
|
&& refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.enable";
|
|
}
|
|
{
|
|
# The forge's host publishes the controller's token under its own leaf,
|
|
# which the store mints beside it here. `value=@` is what keeps the token
|
|
# out of the process's arguments.
|
|
name = "the forge's host publishes the controller's forge token to the store under its own leaf";
|
|
ok =
|
|
let
|
|
u = publisher allLocal;
|
|
in
|
|
u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/forge-publisher.pem"
|
|
&& u.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/forge-publisher-key.pem"
|
|
&& lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/forge/token value=@\"$src\"" u.script
|
|
&& lib.hasInfix "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token" u.script
|
|
&& lib.elem "container@hive-forge.service" u.wantedBy
|
|
&& !(allLocal.systemd.services ? hive-forge-swarm-controller-token);
|
|
}
|
|
{
|
|
# What makes it re-run. A oneshot left `active (exited)` makes every later
|
|
# start a no-op, so `RemainAfterExit` would silence both triggers: the
|
|
# container's start (`wantedBy`, above) and a write of the token file.
|
|
name = "the forge publisher re-runs on each container start and each write of the token file";
|
|
ok =
|
|
let
|
|
p = allLocal.systemd.paths.swarm-bao-forge-publisher;
|
|
in
|
|
!((publisher allLocal).serviceConfig.RemainAfterExit or false)
|
|
&&
|
|
p.pathConfig.PathChanged
|
|
== "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token"
|
|
&& p.pathConfig.Unit == "swarm-bao-forge-publisher.service"
|
|
&& lib.elem "multi-user.target" p.wantedBy
|
|
&& !(bare.systemd.paths ? swarm-bao-forge-publisher);
|
|
}
|
|
{
|
|
# The absence arm: a forge host without the store has no leaf unless one
|
|
# is placed, and the unit says which options to set.
|
|
name = "a forge host with no publisher leaf renders a publisher that fails naming the options";
|
|
ok =
|
|
let
|
|
s = (publisher forgeHere).script;
|
|
in
|
|
lib.hasInfix "forgePublisherClientCertFile" s
|
|
&& lib.hasInfix "exit 1" s
|
|
&& !(lib.hasInfix "bao kv put" s);
|
|
}
|
|
{
|
|
# The host copy is removed where the forge runs: nothing reads it.
|
|
name = "the forge's host removes its old copy of the controller's forge token";
|
|
ok =
|
|
lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" forgeHere.systemd.tmpfiles.rules
|
|
&& !(lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" bare.systemd.tmpfiles.rules);
|
|
}
|
|
{
|
|
# The only thing that makes a human's forge account: nothing else
|
|
# creates one.
|
|
name = "a first authelia login creates the forge account, named by preferred_username";
|
|
ok =
|
|
let
|
|
o = forgeSettings.oauth2_client;
|
|
in
|
|
o.ENABLE_AUTO_REGISTRATION == true && o.USERNAME == "preferred_username";
|
|
}
|
|
{
|
|
# `auto` would give an SSO user whatever local account carries their
|
|
# name — an agent's, or `core`'s.
|
|
name = "an SSO login adopts an existing forge account only with that account's password";
|
|
ok = forgeSettings.oauth2_client.ACCOUNT_LINKING == "login";
|
|
}
|
|
{
|
|
# Both halves: local sign-up stays off, and nothing turns on the one
|
|
# setting forgejo's auto-registration does check.
|
|
name = "local sign-up stays off without blocking the SSO registration";
|
|
ok =
|
|
forgeSettings.service.DISABLE_REGISTRATION == true
|
|
&& !(forgeSettings.service.ALLOW_ONLY_INTERNAL_REGISTRATION or false);
|
|
}
|
|
];
|
|
in
|
|
runGroup "forge-placement" cases
|