9407db69b5
Each store reader (forge's OIDC fetch and token publisher, matrix's OIDC fetch, the hive collector's fetch, Grafana's OIDC fetch and admin password, the queue's callout-xkey unit, the CI runner's token fetch, the swarm collector's OIDC fetch) used to read `deploy.bao.enable` in its own module to add `after`/`wants` on `container@<store machine>` and `after`/`requires` on `swarm-bao-pki.service`. Per the #3742 ruling, that cross-service read moves into glue-bao-readers-store-order.nix, which sets the same edges when the store and the reader are both on this host and restates each reader's own gate so no stub unit is rendered. Rendered `after`/`wants`/`requires` are identical as sets to main on 16 fixtures, store here and elsewhere. New module-eval cases in existing fixtures pin both arms. Part of #3742.
465 lines
20 KiB
Nix
465 lines
20 KiB
Nix
# `checks.module-eval-forge-placement` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
#
|
|
# Where the forge runs. A swarm has one, on the host with
|
|
# `deploy.forgejo.enable`; every other hive is a client of it, and the parts
|
|
# of a split deployment that must not assume every host runs a forge
|
|
# (the OIDC client, the controller's token, the CI runner) have to hold.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
inherit (shared) bare;
|
|
allLocal = hive { deploy.singleHostSwarm = true; };
|
|
inherit (shared) servicesHere forgeHere;
|
|
|
|
# The shared services here, the forge somewhere else. Every derivation in
|
|
# ../host-modules/swarm-required-services.nix is `mkDefault`, so this stays
|
|
# expressible — and it is also the authelia-without-forge host the OIDC
|
|
# client case needs.
|
|
servicesForgeElsewhere = hive {
|
|
deploy.allSwarmServices = true;
|
|
deploy.forgejo.enable = false;
|
|
};
|
|
|
|
# The forge without authelia: the other half of the split, with its secret's
|
|
# path and its store identity chosen by hand.
|
|
forgeNoAuthelia = hive {
|
|
deploy.forgejo.enable = true;
|
|
deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
|
|
deploy.bao.forgePublisherClientCertFile = "/etc/pki/forge-publisher.pem";
|
|
deploy.bao.forgePublisherClientKeyFile = "/etc/pki/forge-publisher-key.pem";
|
|
};
|
|
|
|
# The forge with neither authelia nor the store, its store identity placed by
|
|
# hand: the deployment the store fetch exists for.
|
|
forgeAlone = hive {
|
|
deploy.forgejo.enable = true;
|
|
deploy.bao.forgePublisherClientCertFile = "/etc/pki/forge-publisher.pem";
|
|
deploy.bao.forgePublisherClientKeyFile = "/etc/pki/forge-publisher-key.pem";
|
|
};
|
|
|
|
# The runner alone: no hive, no forge, no store, its store identity placed
|
|
# by hand. `nix` is a label without a scheme.
|
|
ciAlone = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.ci-runner.enable = true;
|
|
deploy.ci-runner.labels = [
|
|
"hive-ci:host"
|
|
"nix"
|
|
];
|
|
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
|
|
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
|
|
};
|
|
ciNoIdentity = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.ci-runner.enable = true;
|
|
};
|
|
|
|
# A host config that still names the runner under the forge, every option.
|
|
ciUnderForge = hive {
|
|
deploy.forgejo.ci.enable = true;
|
|
deploy.forgejo.ci.name = "old-runner";
|
|
deploy.forgejo.ci.concurrency = 3;
|
|
deploy.forgejo.ci.labels = [ "nix:host" ];
|
|
deploy.forgejo.ci.package = pkgs.hello;
|
|
deploy.forgejo.ci.jobTimeout = "2h";
|
|
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
|
|
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
|
|
};
|
|
# The old name alone, with no store identity placed and none defaulted.
|
|
ciUnderForgeNoIdentity = hive {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.forgejo.ci.enable = true;
|
|
};
|
|
# The older spelling, under the swarm-level forge options.
|
|
ciUnderSwarmForge = hive {
|
|
swarm.forge.ci.enable = true;
|
|
swarm.forge.ci.name = "older-runner";
|
|
swarm.forge.ci.concurrency = 2;
|
|
swarm.forge.ci.labels = [ "nix:host" ];
|
|
swarm.forge.ci.package = pkgs.hello;
|
|
deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem";
|
|
deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem";
|
|
};
|
|
|
|
runsForge = m: m.services.hyperhive.deploy.forgejo.enable && m.containers ? hive-forge;
|
|
|
|
forgeClientIds =
|
|
m:
|
|
map (c: c.id) (
|
|
lib.filter (
|
|
c: c.id == m.services.hyperhive.swarm.forge.sso.clientId
|
|
) m.services.hyperhive.swarm.authelia.oidc.clients
|
|
);
|
|
|
|
# Matched on the option the message names, same reasoning as
|
|
# ./grafana.nix's `grafanaRefusedFor`.
|
|
refusedOver = m: needle: lib.any (a: !a.assertion && lib.hasInfix needle a.message) m.assertions;
|
|
|
|
publisher = m: m.systemd.services.swarm-bao-forge-publisher;
|
|
oidcFetch = m: m.systemd.services.swarm-bao-forge-oidc;
|
|
|
|
forgeSettings = forgeHere.containers.hive-forge.config.services.forgejo.settings;
|
|
|
|
runnerService = ciAlone.containers.swarm-ci-runner.config.systemd.services.gitea-runner-swarm;
|
|
registerService =
|
|
ciAlone.containers.swarm-ci-runner.config.systemd.services.swarm-ci-runner-register;
|
|
|
|
cases = [
|
|
{
|
|
# The absence the whole option exists for: a second forge in a swarm
|
|
# is a split brain nobody notices, so a hive that has not been told it
|
|
# is the forge's host runs none of its surface.
|
|
name = "a hive that is not the forge's host runs no forge";
|
|
ok =
|
|
!bare.services.hyperhive.deploy.forgejo.enable
|
|
&& !(bare.containers ? hive-forge)
|
|
&& !(bare.systemd.services ? swarm-bao-forge-publisher)
|
|
&& !(lib.elem "forge.t.local" bare.services.hyperhive.gateway.localNames)
|
|
&& !(refusedOver bare "deploy.forgejo.sso.clientSecretFile");
|
|
}
|
|
{
|
|
name = "hosting the swarm's shared services runs the forge";
|
|
ok = runsForge servicesHere;
|
|
}
|
|
{
|
|
name = "the all-local mode runs the forge";
|
|
ok = runsForge allLocal && lib.elem "forge.t.local" allLocal.services.hyperhive.gateway.localNames;
|
|
}
|
|
{
|
|
name = "an explicit deploy.forgejo.enable runs the forge on its own";
|
|
ok = runsForge forgeHere && !forgeHere.services.hyperhive.deploy.allSwarmServices;
|
|
}
|
|
{
|
|
# `mkDefault`, not a plain assignment: the forge stays placeable on a
|
|
# host of its own. `nats` is the control, so the case cannot pass on a
|
|
# fixture where nothing came on.
|
|
name = "placing the forge elsewhere survives the switch that would enable it";
|
|
ok =
|
|
!(servicesForgeElsewhere.containers ? hive-forge)
|
|
&& servicesForgeElsewhere.services.hyperhive.deploy.nats.enable;
|
|
}
|
|
{
|
|
# A client is a row in authelia's config, so it is declared where
|
|
# authelia runs. With the forge's module gated, registering it from
|
|
# there would leave a split swarm's forge unknown to its IdP.
|
|
name = "the forge's OIDC client is registered wherever authelia runs, and only there";
|
|
ok =
|
|
forgeClientIds servicesForgeElsewhere == [ "forgejo" ]
|
|
&& forgeClientIds allLocal == [ "forgejo" ]
|
|
&& forgeClientIds forgeNoAuthelia == [ ];
|
|
}
|
|
{
|
|
name = "the forge's OIDC callback is the one forgejo sends";
|
|
ok =
|
|
servicesForgeElsewhere.services.hyperhive.swarm.forge.sso.redirectUri
|
|
== "https://forge.t.local/user/oauth2/authelia/callback";
|
|
}
|
|
{
|
|
# The runner reaches the forge by name and reads its token with its own
|
|
# certificate, so nothing about the host it runs on is asked.
|
|
name = "the CI runner runs on a host with neither the forge nor a hive";
|
|
ok =
|
|
ciAlone.containers ? swarm-ci-runner
|
|
&& !(ciAlone.containers ? hive-forge)
|
|
&& !ciAlone.services.hyperhive.deploy.hive-controller.enable
|
|
&& !(refusedOver ciAlone "ci-runner")
|
|
&&
|
|
ciAlone.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT
|
|
== "/etc/pki/bao-ci-runner.pem";
|
|
}
|
|
{
|
|
# The store's units exist only on its host; `Requires=` on an absent one
|
|
# fails the fetch's start. The arm where the store is here is in
|
|
# ./swarm-services-switch.nix.
|
|
name = "the CI runner's token fetch names none of the store's units on a host without the store";
|
|
ok =
|
|
let
|
|
u = ciAlone.systemd.services.swarm-bao-ci-runner;
|
|
storeUnits = [
|
|
"swarm-bao-pki.service"
|
|
"container@${ciAlone.services.hyperhive.swarm.bao.machine}.service"
|
|
];
|
|
in
|
|
!(lib.any (n: lib.elem n (u.after ++ u.wants ++ u.requires)) storeUnits);
|
|
}
|
|
{
|
|
# Without the identity the token never arrives; the control is the
|
|
# placed pair above.
|
|
name = "the CI runner is refused without its store identity";
|
|
ok = refusedOver ciNoIdentity "deploy.bao.ciRunnerClientCertFile";
|
|
}
|
|
{
|
|
# systemd hands a credential to every process of the unit that loads
|
|
# it, and a host job is one of the daemon's processes and inherits its
|
|
# environment; a token in either lets any workflow register runners.
|
|
name = "the CI runner daemon can read no registration token";
|
|
ok =
|
|
let
|
|
sc = runnerService.serviceConfig;
|
|
in
|
|
!(runnerService.environment ? TOKEN)
|
|
&& lib.toList (sc.EnvironmentFile or [ ]) == [ ]
|
|
&& !(lib.any (lib.hasPrefix "TOKEN=") (lib.toList (sc.Environment or [ ])))
|
|
&& !(sc ? LoadCredential)
|
|
&& !(sc ? LoadCredentialEncrypted)
|
|
&& !(sc ? SetCredential);
|
|
}
|
|
{
|
|
name = "the CI register unit alone loads the registration token, and runs before the daemon";
|
|
ok =
|
|
let
|
|
sc = registerService.serviceConfig;
|
|
in
|
|
sc.LoadCredential == [ "registration-token:/var/lib/swarm-ci-runner/registration-token" ]
|
|
&& sc.Type == "oneshot"
|
|
&& sc.User == runnerService.serviceConfig.User
|
|
&& sc.DynamicUser
|
|
&& sc.StateDirectory == runnerService.serviceConfig.StateDirectory
|
|
&& lib.any (
|
|
s: lib.hasSuffix "/bin/systemctl start swarm-ci-runner-register.service" s
|
|
) runnerService.serviceConfig.ExecStartPre
|
|
&& !(lib.any (
|
|
s: lib.hasInfix "gitea-register-runner" (toString s)
|
|
) runnerService.serviceConfig.ExecStartPre);
|
|
}
|
|
{
|
|
name = "the CI register unit registers from its credential, and only without a .runner";
|
|
ok =
|
|
let
|
|
text = registerService.script;
|
|
beforeRegister = lib.head (lib.splitString "register --no-interactive" text);
|
|
in
|
|
lib.hasInfix "if [ -e .runner ]; then\n exit 0\nfi" beforeRegister
|
|
&& lib.hasInfix ''token="$(< "$CREDENTIALS_DIRECTORY/registration-token")"'' beforeRegister
|
|
&& lib.hasInfix ''--token "$token"'' text
|
|
&& !(lib.hasInfix "export" text)
|
|
&& !(lib.hasInfix "TOKEN" text);
|
|
}
|
|
{
|
|
# The daemon rewrites `.runner`'s labels to this form on start; compared
|
|
# against the configured strings verbatim, a scheme-less label would
|
|
# discard `.runner` and register a new runner on every start.
|
|
name = "the CI runner compares .runner's labels in the form the daemon writes them";
|
|
ok = registerService.environment.SWARM_CI_RUNNER_LABELS == "hive-ci:host,nix:host";
|
|
}
|
|
{
|
|
# The control is the container: it only exists if `enable` arrived.
|
|
name = "a CI option under deploy.forgejo sets the same option under deploy.ci-runner";
|
|
ok =
|
|
let
|
|
r = ciUnderForge.services.hyperhive.deploy.ci-runner;
|
|
in
|
|
r.enable
|
|
&& r.name == "old-runner"
|
|
&& r.concurrency == 3
|
|
&& r.labels == [ "nix:host" ]
|
|
&& r.package == pkgs.hello
|
|
&& r.jobTimeout == "2h"
|
|
&& ciUnderForge.containers ? swarm-ci-runner
|
|
&& lib.any (lib.hasInfix "services.hyperhive.deploy.forgejo.ci.enable") ciUnderForge.warnings
|
|
&& !(lib.any (a: !a.assertion) ciUnderForge.assertions);
|
|
}
|
|
{
|
|
# The control is the container: it only exists if `enable` arrived.
|
|
name = "a CI option under swarm.forge sets the same option under deploy.ci-runner";
|
|
ok =
|
|
let
|
|
r = ciUnderSwarmForge.services.hyperhive.deploy.ci-runner;
|
|
in
|
|
r.enable
|
|
&& r.name == "older-runner"
|
|
&& r.concurrency == 2
|
|
&& r.labels == [ "nix:host" ]
|
|
&& r.package == pkgs.hello
|
|
&& ciUnderSwarmForge.containers ? swarm-ci-runner
|
|
&& lib.any (lib.hasInfix "services.hyperhive.swarm.forge.ci.enable") ciUnderSwarmForge.warnings
|
|
&& !(lib.any (a: !a.assertion) ciUnderSwarmForge.assertions);
|
|
}
|
|
{
|
|
# The rename carries `enable` but not the identity: an old config with
|
|
# only the old name is refused, and the message says what to set. The
|
|
# control is the placed pair in `ciUnderForge` above.
|
|
name = "an old CI enable without a store identity is refused naming the options to set";
|
|
ok =
|
|
refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.ciRunnerClientCertFile"
|
|
&& refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.ciRunnerClientKeyFile"
|
|
&& refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.enable";
|
|
}
|
|
{
|
|
# The forge's host publishes the controller's token under its own leaf,
|
|
# which the store mints beside it here. `value=@` is what keeps the token
|
|
# out of the process's arguments.
|
|
name = "the forge's host publishes the controller's forge token to the store under its own leaf";
|
|
ok =
|
|
let
|
|
u = publisher allLocal;
|
|
in
|
|
u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/forge-publisher.pem"
|
|
&& u.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/forge-publisher-key.pem"
|
|
&& lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/forge/token value=@\"$src\"" u.script
|
|
&& lib.hasInfix "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token" u.script
|
|
&& lib.elem "container@hive-forge.service" u.wantedBy
|
|
# The deploy that adds the unit starts it only through a target.
|
|
&& lib.elem "multi-user.target" u.wantedBy
|
|
&& !(allLocal.systemd.services ? hive-forge-swarm-controller-token);
|
|
}
|
|
{
|
|
# What makes it re-run. A oneshot left `active (exited)` makes every later
|
|
# start a no-op, so `RemainAfterExit` would silence every trigger: the
|
|
# targets and the container's start (`wantedBy`, above) and a write of
|
|
# the token file. A missing file exits 0, so a switch with the container
|
|
# down does not fail.
|
|
name = "the forge publisher re-runs on each container start and each write of the token file";
|
|
ok =
|
|
let
|
|
p = allLocal.systemd.paths.swarm-bao-forge-publisher;
|
|
in
|
|
!((publisher allLocal).serviceConfig.RemainAfterExit or false)
|
|
&&
|
|
p.pathConfig.PathChanged
|
|
== "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token"
|
|
&& p.pathConfig.Unit == "swarm-bao-forge-publisher.service"
|
|
&& lib.hasInfix "the path unit publishes it when it does" (publisher allLocal).script
|
|
&& lib.hasInfix "exit 0" (publisher allLocal).script
|
|
&& lib.elem "multi-user.target" p.wantedBy
|
|
&& !(bare.systemd.paths ? swarm-bao-forge-publisher);
|
|
}
|
|
{
|
|
# The absence arm: a forge host without the store has no leaf unless one
|
|
# is placed, and the unit says which options to set.
|
|
name = "a forge host with no publisher leaf renders a publisher that fails naming the options";
|
|
ok =
|
|
let
|
|
s = (publisher forgeHere).script;
|
|
in
|
|
lib.hasInfix "forgePublisherClientCertFile" s
|
|
&& lib.hasInfix "exit 1" s
|
|
&& !(lib.hasInfix "bao kv put" s);
|
|
}
|
|
{
|
|
# The forge's OIDC secret has one route, the store, under the forge
|
|
# publisher's leaf, landing where `clientSecretFile` points inside the
|
|
# container. No unit copies it out of authelia's tree, even on a host
|
|
# that runs authelia, and a forge host without authelia needs no path
|
|
# set by hand.
|
|
name = "the forge reads its OIDC secret from the store under the forge publisher's leaf";
|
|
ok =
|
|
let
|
|
u = oidcFetch forgeAlone;
|
|
d = forgeAlone.services.hyperhive.deploy;
|
|
in
|
|
!d.authelia.enable
|
|
&& !d.bao.enable
|
|
&& d.forgejo.sso.clientSecretFile == "/var/lib/forgejo-oidc/forgejo.secret"
|
|
&& !(refusedOver forgeAlone "deploy.forgejo.sso.clientSecretFile")
|
|
&& u.environment.BAO_CLIENT_CERT == "/etc/pki/forge-publisher.pem"
|
|
&& u.environment.BAO_CLIENT_KEY == "/etc/pki/forge-publisher-key.pem"
|
|
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/forgejo/oidc/client" u.script
|
|
&& lib.hasInfix "/var/lib/nixos-containers/hive-forge/var/lib/forgejo-oidc/forgejo.secret" u.script
|
|
&& lib.hasInfix "refresh_consumer hive-forge forgejo-sso-source.service" u.script
|
|
# An unpublished or empty secret fails the unit, so `storeRetry` retries it.
|
|
&& !(lib.hasInfix "exit 0" u.script)
|
|
&& u.serviceConfig.Restart == "on-failure"
|
|
&& lib.elem "container@hive-forge.service" u.before
|
|
&& lib.elem "multi-user.target" u.wantedBy
|
|
&& (oidcFetch allLocal).environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/forge-publisher.pem"
|
|
&& !(forgeAlone.systemd.services ? hive-forge-oidc-secret)
|
|
&& !(allLocal.systemd.services ? hive-forge-oidc-secret)
|
|
&& !(bare.systemd.services ? swarm-bao-forge-oidc);
|
|
}
|
|
{
|
|
# An operator-chosen path is where the fetched secret lands.
|
|
name = "the forge's OIDC secret lands where clientSecretFile points";
|
|
ok =
|
|
let
|
|
s = (oidcFetch forgeNoAuthelia).script;
|
|
in
|
|
lib.hasInfix "/var/lib/nixos-containers/hive-forge/var/lib/forgejo-oidc/by-hand.secret" s;
|
|
}
|
|
{
|
|
# Beside the store, the fetch waits for the unit writing the forge
|
|
# publisher's role, the leaf it logs in with.
|
|
name = "the forge's OIDC secret fetch is ordered after the forge publisher's policy unit";
|
|
ok =
|
|
let
|
|
u = oidcFetch allLocal;
|
|
in
|
|
lib.elem "swarm-bao-forge-publisher-policy.service" u.after
|
|
&& lib.elem "swarm-bao-forge-publisher-policy.service" u.wants;
|
|
}
|
|
{
|
|
# The absence arm: no publisher leaf, no secret, and the unit says which
|
|
# options to set.
|
|
name = "a forge host with no publisher leaf renders an OIDC secret fetch that fails naming the options";
|
|
ok =
|
|
let
|
|
u = oidcFetch forgeHere;
|
|
in
|
|
!(u.environment ? BAO_CLIENT_CERT)
|
|
&& lib.hasInfix "forgePublisherClientCertFile" u.script
|
|
&& lib.hasInfix "exit 1" u.script
|
|
&& !(lib.hasInfix "bao login" u.script);
|
|
}
|
|
{
|
|
# The login source fetches the issuer's discovery document by name
|
|
# wherever the forge runs, so it waits for the resolver on a forge host
|
|
# without authelia too.
|
|
name = "the login source waits for the container's resolver on a forge host without authelia";
|
|
ok = lib.elem "forgejo-sso-source.service" (
|
|
forgeAlone.containers.hive-forge.config.systemd.services.swarm-bridge-dns.before
|
|
);
|
|
}
|
|
{
|
|
# The host copy is removed where the forge runs: nothing reads it.
|
|
name = "the forge's host removes its old copy of the controller's forge token";
|
|
ok =
|
|
lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" forgeHere.systemd.tmpfiles.rules
|
|
&& !(lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" bare.systemd.tmpfiles.rules);
|
|
}
|
|
{
|
|
# The only thing that makes a human's forge account: nothing else
|
|
# creates one.
|
|
name = "a first authelia login creates the forge account, named by preferred_username";
|
|
ok =
|
|
let
|
|
o = forgeSettings.oauth2_client;
|
|
in
|
|
o.ENABLE_AUTO_REGISTRATION == true && o.USERNAME == "preferred_username";
|
|
}
|
|
{
|
|
# `auto` would give an SSO user whatever local account carries their
|
|
# name — an agent's, or `core`'s.
|
|
name = "an SSO login adopts an existing forge account only with that account's password";
|
|
ok = forgeSettings.oauth2_client.ACCOUNT_LINKING == "login";
|
|
}
|
|
{
|
|
# Both halves: local sign-up stays off, and nothing turns on the one
|
|
# setting forgejo's auto-registration does check.
|
|
name = "local sign-up stays off without blocking the SSO registration";
|
|
ok =
|
|
forgeSettings.service.DISABLE_REGISTRATION == true
|
|
&& !(forgeSettings.service.ALLOW_ONLY_INTERNAL_REGISTRATION or false);
|
|
}
|
|
];
|
|
in
|
|
runGroup "forge-placement" cases
|