597fdebaf3
The publisher was wanted only by container@hive-forge.service. A switch starts a new unit only through the targets that want it, and the container is already running, so the deploy that introduced the publisher left it dead until the next container restart or boot; the path unit cannot help, because the mint's only-once guard never rewrites the file. - The publisher is also wanted by multi-user.target, like swarm-secret-publish. - A token file that is not there yet exits 0 instead of waiting 120s and failing: the path unit runs the publisher when the mint writes it, so a switch with the forge container down does not fail. TimeoutStartSec is 60. - swarm-controller loses its After= on the publisher: a switch issues the two jobs separately, so it ordered nothing there, and the background re-read covers the gap. - The controller's background re-read logs at info once the backoff is capped, one line per five minutes, so a forge host that never got its publisher leaf stays visible; earlier retries stay at debug. Refs #4998
365 lines
15 KiB
Nix
365 lines
15 KiB
Nix
# `checks.module-eval-forge-placement` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
#
|
|
# Where the forge runs. A swarm has one, on the host with
|
|
# `deploy.forgejo.enable`; every other hive is a client of it, and the parts
|
|
# of a split deployment that must not assume every host runs a forge
|
|
# (the OIDC client, the controller's token, the CI runner) have to hold.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
bare = hive { };
|
|
allLocal = hive { deploy.singleHostSwarm = true; };
|
|
servicesHere = hive { deploy.allSwarmServices = true; };
|
|
forgeHere = hive { deploy.forgejo.enable = true; };
|
|
|
|
# The shared services here, the forge somewhere else. Every derivation in
|
|
# ../host-modules/swarm-required-services.nix is `mkDefault`, so this stays
|
|
# expressible — and it is also the authelia-without-forge host the OIDC
|
|
# client case needs.
|
|
servicesForgeElsewhere = hive {
|
|
deploy.allSwarmServices = true;
|
|
deploy.forgejo.enable = false;
|
|
};
|
|
|
|
# The forge without authelia: the other half of the split.
|
|
forgeNoAuthelia = hive {
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
|
|
};
|
|
|
|
# The runner alone: no hive, no forge, no store, its store identity placed
|
|
# by hand. `nix` is a label without a scheme.
|
|
ciAlone = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.ci-runner.enable = true;
|
|
deploy.ci-runner.labels = [
|
|
"hive-ci:host"
|
|
"nix"
|
|
];
|
|
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
|
|
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
|
|
};
|
|
ciNoIdentity = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.ci-runner.enable = true;
|
|
};
|
|
|
|
# A host config that still names the runner under the forge, every option.
|
|
ciUnderForge = hive {
|
|
deploy.forgejo.ci.enable = true;
|
|
deploy.forgejo.ci.name = "old-runner";
|
|
deploy.forgejo.ci.concurrency = 3;
|
|
deploy.forgejo.ci.labels = [ "nix:host" ];
|
|
deploy.forgejo.ci.package = pkgs.hello;
|
|
deploy.forgejo.ci.jobTimeout = "2h";
|
|
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
|
|
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
|
|
};
|
|
# The old name alone, with no store identity placed and none defaulted.
|
|
ciUnderForgeNoIdentity = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.forgejo.ci.enable = true;
|
|
};
|
|
# The older spelling, under the swarm-level forge options.
|
|
ciUnderSwarmForge = hive {
|
|
swarm.forge.ci.enable = true;
|
|
swarm.forge.ci.name = "older-runner";
|
|
swarm.forge.ci.concurrency = 2;
|
|
swarm.forge.ci.labels = [ "nix:host" ];
|
|
swarm.forge.ci.package = pkgs.hello;
|
|
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
|
|
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
|
|
};
|
|
|
|
runsForge = m: m.services.hyperhive.deploy.forgejo.enable && m.containers ? hive-forge;
|
|
|
|
forgeClientIds =
|
|
m:
|
|
map (c: c.id) (
|
|
lib.filter (
|
|
c: c.id == m.services.hyperhive.swarm.forge.sso.clientId
|
|
) m.services.hyperhive.swarm.authelia.oidc.clients
|
|
);
|
|
|
|
# Matched on the option the message names, same reasoning as
|
|
# ./grafana.nix's `grafanaRefusedFor`.
|
|
refusedOver = m: needle: lib.any (a: !a.assertion && lib.hasInfix needle a.message) m.assertions;
|
|
|
|
publisher = m: m.systemd.services.swarm-bao-forge-publisher;
|
|
|
|
forgeSettings = forgeHere.containers.hive-forge.config.services.forgejo.settings;
|
|
|
|
runnerService = ciAlone.containers.swarm-ci-runner.config.systemd.services.gitea-runner-swarm;
|
|
registerService =
|
|
ciAlone.containers.swarm-ci-runner.config.systemd.services.swarm-ci-runner-register;
|
|
|
|
cases = [
|
|
{
|
|
# The absence the whole option exists for: a second forge in a swarm
|
|
# is a split brain nobody notices, so a hive that has not been told it
|
|
# is the forge's host runs none of its surface.
|
|
name = "a hive that is not the forge's host runs no forge";
|
|
ok =
|
|
!bare.services.hyperhive.deploy.forgejo.enable
|
|
&& !(bare.containers ? hive-forge)
|
|
&& !(bare.systemd.services ? swarm-bao-forge-publisher)
|
|
&& !(lib.elem "forge.t.local" bare.services.hyperhive.gateway.localNames)
|
|
&& !(refusedOver bare "deploy.forgejo.sso.clientSecretFile");
|
|
}
|
|
{
|
|
name = "hosting the swarm's shared services runs the forge";
|
|
ok = runsForge servicesHere;
|
|
}
|
|
{
|
|
name = "the all-local mode runs the forge";
|
|
ok = runsForge allLocal && lib.elem "forge.t.local" allLocal.services.hyperhive.gateway.localNames;
|
|
}
|
|
{
|
|
name = "an explicit deploy.forgejo.enable runs the forge on its own";
|
|
ok = runsForge forgeHere && !forgeHere.services.hyperhive.deploy.allSwarmServices;
|
|
}
|
|
{
|
|
# `mkDefault`, not a plain assignment: the forge stays placeable on a
|
|
# host of its own. `nats` is the control, so the case cannot pass on a
|
|
# fixture where nothing came on.
|
|
name = "placing the forge elsewhere survives the switch that would enable it";
|
|
ok =
|
|
!(servicesForgeElsewhere.containers ? hive-forge)
|
|
&& servicesForgeElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# A client is a row in authelia's config, so it is declared where
|
|
# authelia runs. With the forge's module gated, registering it from
|
|
# there would leave a split swarm's forge unknown to its IdP.
|
|
name = "the forge's OIDC client is registered wherever authelia runs, and only there";
|
|
ok =
|
|
forgeClientIds servicesForgeElsewhere == [ "forgejo" ]
|
|
&& forgeClientIds allLocal == [ "forgejo" ]
|
|
&& forgeClientIds forgeNoAuthelia == [ ];
|
|
}
|
|
{
|
|
name = "the forge's OIDC callback is the one forgejo sends";
|
|
ok =
|
|
servicesForgeElsewhere.services.hyperhive.swarm.forge.sso.redirectUri
|
|
== "https://forge.t.local/user/oauth2/authelia/callback";
|
|
}
|
|
{
|
|
# The runner reaches the forge by name and reads its token with its own
|
|
# certificate, so nothing about the host it runs on is asked.
|
|
name = "the CI runner runs on a host with neither the forge nor a hive";
|
|
ok =
|
|
ciAlone.containers ? swarm-ci-runner
|
|
&& !(ciAlone.containers ? hive-forge)
|
|
&& !ciAlone.services.hyperhive.deploy.hive-controller.enable
|
|
&& !(refusedOver ciAlone "ci-runner")
|
|
&&
|
|
ciAlone.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT
|
|
== "/etc/pki/bao-ci-runner.pem";
|
|
}
|
|
{
|
|
# Without the identity the token never arrives; the control is the
|
|
# placed pair above.
|
|
name = "the CI runner is refused without its store identity";
|
|
ok = refusedOver ciNoIdentity "deploy.bao.ciRunnerClientCertFile";
|
|
}
|
|
{
|
|
# systemd hands a credential to every process of the unit that loads
|
|
# it, and a host job is one of the daemon's processes and inherits its
|
|
# environment; a token in either lets any workflow register runners.
|
|
name = "the CI runner daemon can read no registration token";
|
|
ok =
|
|
let
|
|
sc = runnerService.serviceConfig;
|
|
in
|
|
!(runnerService.environment ? TOKEN)
|
|
&& lib.toList (sc.EnvironmentFile or [ ]) == [ ]
|
|
&& !(lib.any (lib.hasPrefix "TOKEN=") (lib.toList (sc.Environment or [ ])))
|
|
&& !(sc ? LoadCredential)
|
|
&& !(sc ? LoadCredentialEncrypted)
|
|
&& !(sc ? SetCredential);
|
|
}
|
|
{
|
|
name = "the CI register unit alone loads the registration token, and runs before the daemon";
|
|
ok =
|
|
let
|
|
sc = registerService.serviceConfig;
|
|
in
|
|
sc.LoadCredential == [ "registration-token:/var/lib/swarm-ci-runner/registration-token" ]
|
|
&& sc.Type == "oneshot"
|
|
&& sc.User == runnerService.serviceConfig.User
|
|
&& sc.DynamicUser
|
|
&& sc.StateDirectory == runnerService.serviceConfig.StateDirectory
|
|
&& lib.any (
|
|
s: lib.hasSuffix "/bin/systemctl start swarm-ci-runner-register.service" s
|
|
) runnerService.serviceConfig.ExecStartPre
|
|
&& !(lib.any (
|
|
s: lib.hasInfix "gitea-register-runner" (toString s)
|
|
) runnerService.serviceConfig.ExecStartPre);
|
|
}
|
|
{
|
|
name = "the CI register unit registers from its credential, and only without a .runner";
|
|
ok =
|
|
let
|
|
text = registerService.script;
|
|
beforeRegister = lib.head (lib.splitString "register --no-interactive" text);
|
|
in
|
|
lib.hasInfix "if [ -e .runner ]; then\n exit 0\nfi" beforeRegister
|
|
&& lib.hasInfix ''token="$(< "$CREDENTIALS_DIRECTORY/registration-token")"'' beforeRegister
|
|
&& lib.hasInfix ''--token "$token"'' text
|
|
&& !(lib.hasInfix "export" text)
|
|
&& !(lib.hasInfix "TOKEN" text);
|
|
}
|
|
{
|
|
# The daemon rewrites `.runner`'s labels to this form on start; compared
|
|
# against the configured strings verbatim, a scheme-less label would
|
|
# discard `.runner` and register a new runner on every start.
|
|
name = "the CI runner compares .runner's labels in the form the daemon writes them";
|
|
ok = registerService.environment.SWARM_CI_RUNNER_LABELS == "hive-ci:host,nix:host";
|
|
}
|
|
{
|
|
# The control is the container: it only exists if `enable` arrived.
|
|
name = "a CI option under deploy.forgejo sets the same option under deploy.ci-runner";
|
|
ok =
|
|
let
|
|
r = ciUnderForge.services.hyperhive.deploy.ci-runner;
|
|
in
|
|
r.enable
|
|
&& r.name == "old-runner"
|
|
&& r.concurrency == 3
|
|
&& r.labels == [ "nix:host" ]
|
|
&& r.package == pkgs.hello
|
|
&& r.jobTimeout == "2h"
|
|
&& ciUnderForge.containers ? swarm-ci-runner
|
|
&& lib.any (lib.hasInfix "services.hyperhive.deploy.forgejo.ci.enable") ciUnderForge.warnings
|
|
&& !(lib.any (a: !a.assertion) ciUnderForge.assertions);
|
|
}
|
|
{
|
|
# The control is the container: it only exists if `enable` arrived.
|
|
name = "a CI option under swarm.forge sets the same option under deploy.ci-runner";
|
|
ok =
|
|
let
|
|
r = ciUnderSwarmForge.services.hyperhive.deploy.ci-runner;
|
|
in
|
|
r.enable
|
|
&& r.name == "older-runner"
|
|
&& r.concurrency == 2
|
|
&& r.labels == [ "nix:host" ]
|
|
&& r.package == pkgs.hello
|
|
&& ciUnderSwarmForge.containers ? swarm-ci-runner
|
|
&& lib.any (lib.hasInfix "services.hyperhive.swarm.forge.ci.enable") ciUnderSwarmForge.warnings
|
|
&& !(lib.any (a: !a.assertion) ciUnderSwarmForge.assertions);
|
|
}
|
|
{
|
|
# The rename carries `enable` but not the identity: an old config with
|
|
# only the old name is refused, and the message says what to set. The
|
|
# control is the placed pair in `ciUnderForge` above.
|
|
name = "an old CI enable without a store identity is refused naming the options to set";
|
|
ok =
|
|
refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.ciRunnerClientCertFile"
|
|
&& refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.ciRunnerClientKeyFile"
|
|
&& refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.enable";
|
|
}
|
|
{
|
|
# The forge's host publishes the controller's token under its own leaf,
|
|
# which the store mints beside it here. `value=@` is what keeps the token
|
|
# out of the process's arguments.
|
|
name = "the forge's host publishes the controller's forge token to the store under its own leaf";
|
|
ok =
|
|
let
|
|
u = publisher allLocal;
|
|
in
|
|
u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/forge-publisher.pem"
|
|
&& u.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/forge-publisher-key.pem"
|
|
&& lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/forge/token value=@\"$src\"" u.script
|
|
&& lib.hasInfix "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token" u.script
|
|
&& lib.elem "container@hive-forge.service" u.wantedBy
|
|
# The deploy that adds the unit starts it only through a target.
|
|
&& lib.elem "multi-user.target" u.wantedBy
|
|
&& !(allLocal.systemd.services ? hive-forge-swarm-controller-token);
|
|
}
|
|
{
|
|
# What makes it re-run. A oneshot left `active (exited)` makes every later
|
|
# start a no-op, so `RemainAfterExit` would silence every trigger: the
|
|
# targets and the container's start (`wantedBy`, above) and a write of
|
|
# the token file. A missing file exits 0, so a switch with the container
|
|
# down does not fail.
|
|
name = "the forge publisher re-runs on each container start and each write of the token file";
|
|
ok =
|
|
let
|
|
p = allLocal.systemd.paths.swarm-bao-forge-publisher;
|
|
in
|
|
!((publisher allLocal).serviceConfig.RemainAfterExit or false)
|
|
&&
|
|
p.pathConfig.PathChanged
|
|
== "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token"
|
|
&& p.pathConfig.Unit == "swarm-bao-forge-publisher.service"
|
|
&& lib.hasInfix "the path unit publishes it when it does" (publisher allLocal).script
|
|
&& lib.hasInfix "exit 0" (publisher allLocal).script
|
|
&& lib.elem "multi-user.target" p.wantedBy
|
|
&& !(bare.systemd.paths ? swarm-bao-forge-publisher);
|
|
}
|
|
{
|
|
# The absence arm: a forge host without the store has no leaf unless one
|
|
# is placed, and the unit says which options to set.
|
|
name = "a forge host with no publisher leaf renders a publisher that fails naming the options";
|
|
ok =
|
|
let
|
|
s = (publisher forgeHere).script;
|
|
in
|
|
lib.hasInfix "forgePublisherClientCertFile" s
|
|
&& lib.hasInfix "exit 1" s
|
|
&& !(lib.hasInfix "bao kv put" s);
|
|
}
|
|
{
|
|
# The host copy is removed where the forge runs: nothing reads it.
|
|
name = "the forge's host removes its old copy of the controller's forge token";
|
|
ok =
|
|
lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" forgeHere.systemd.tmpfiles.rules
|
|
&& !(lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" bare.systemd.tmpfiles.rules);
|
|
}
|
|
{
|
|
# The only thing that makes a human's forge account: nothing else
|
|
# creates one.
|
|
name = "a first authelia login creates the forge account, named by preferred_username";
|
|
ok =
|
|
let
|
|
o = forgeSettings.oauth2_client;
|
|
in
|
|
o.ENABLE_AUTO_REGISTRATION == true && o.USERNAME == "preferred_username";
|
|
}
|
|
{
|
|
# `auto` would give an SSO user whatever local account carries their
|
|
# name — an agent's, or `core`'s.
|
|
name = "an SSO login adopts an existing forge account only with that account's password";
|
|
ok = forgeSettings.oauth2_client.ACCOUNT_LINKING == "login";
|
|
}
|
|
{
|
|
# Both halves: local sign-up stays off, and nothing turns on the one
|
|
# setting forgejo's auto-registration does check.
|
|
name = "local sign-up stays off without blocking the SSO registration";
|
|
ok =
|
|
forgeSettings.service.DISABLE_REGISTRATION == true
|
|
&& !(forgeSettings.service.ALLOW_ONLY_INTERNAL_REGISTRATION or false);
|
|
}
|
|
];
|
|
in
|
|
runGroup "forge-placement" cases
|