# `checks.module-eval-forge-placement` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). # # Where the forge runs. A swarm has one, on the host with # `deploy.forgejo.enable`; every other hive is a client of it, and the parts # of a split deployment that must not assume every host runs a forge # (the OIDC client, the controller's token, the CI runner) have to hold. { pkgs, lib, self, nixosSystem, shared, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive runGroup ; inherit (shared) bare; allLocal = hive { deploy.singleHostSwarm = true; }; inherit (shared) servicesHere forgeHere; # The shared services here, the forge somewhere else. Every derivation in # ../host-modules/swarm-required-services.nix is `mkDefault`, so this stays # expressible — and it is also the authelia-without-forge host the OIDC # client case needs. servicesForgeElsewhere = hive { deploy.allSwarmServices = true; deploy.forgejo.enable = false; }; # The forge with neither authelia nor the store, its store identity placed by # hand: the other half of the split, and the deployment the store fetch # exists for. forgeAlone = hive { deploy.forgejo.enable = true; deploy.bao.forgePublisherClientCertFile = "/etc/pki/forge-publisher.pem"; deploy.bao.forgePublisherClientKeyFile = "/etc/pki/forge-publisher-key.pem"; }; # The runner alone: no hive, no forge, no store, its store identity placed # by hand. `nix` is a label without a scheme. ciAlone = hive { deploy.hive-controller.enable = false; deploy.ci-runner.enable = true; deploy.ci-runner.labels = [ "hive-ci:host" "nix" ]; deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem"; deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem"; }; ciNoIdentity = hive { deploy.hive-controller.enable = false; deploy.ci-runner.enable = true; }; # A host config that still names the runner under the forge, every option. ciUnderForge = hive { deploy.forgejo.ci.enable = true; deploy.forgejo.ci.name = "old-runner"; deploy.forgejo.ci.concurrency = 3; deploy.forgejo.ci.labels = [ "nix:host" ]; deploy.forgejo.ci.package = pkgs.hello; deploy.forgejo.ci.jobTimeout = "2h"; deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem"; deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem"; }; # The old name alone, with no store identity placed and none defaulted. ciUnderForgeNoIdentity = hive { deploy.hive-controller.enable = false; deploy.forgejo.ci.enable = true; }; runsForge = m: m.services.hyperhive.deploy.forgejo.enable && m.containers ? hive-forge; forgeClientIds = m: map (c: c.id) ( lib.filter ( c: c.id == m.services.hyperhive.swarm.forge.sso.clientId ) m.services.hyperhive.swarm.authelia.oidc.clients ); # Matched on the option the message names, same reasoning as # ./grafana.nix's `grafanaRefusedFor`. refusedOver = m: needle: lib.any (a: !a.assertion && lib.hasInfix needle a.message) m.assertions; mint = m: m.containers.hive-forge.config.systemd.services.forgejo-swarm-controller-account; oidcFetch = m: m.systemd.services.swarm-bao-forge-oidc; forgeSettings = forgeHere.containers.hive-forge.config.services.forgejo.settings; runnerService = ciAlone.containers.swarm-ci-runner.config.systemd.services.gitea-runner-swarm; registerService = ciAlone.containers.swarm-ci-runner.config.systemd.services.swarm-ci-runner-register; cases = [ { # The absence the whole option exists for: a second forge in a swarm # is a split brain nobody notices, so a hive that has not been told it # is the forge's host runs none of its surface. name = "a hive that is not the forge's host runs no forge"; ok = !bare.services.hyperhive.deploy.forgejo.enable && !(bare.containers ? hive-forge) && !(lib.elem "forge.t.local" bare.services.hyperhive.gateway.localNames); } { # All three are published targets; ./swarm-otel-identity.nix covers them. name = "a host running the collector, authelia, grafana and nats scrapes none of them on loopback"; ok = let t = servicesHere.services.hyperhive.swarm.otel.scrapeTargets; in servicesHere.services.hyperhive.deploy.nats.enable && !(t ? authelia) && !(t ? grafana) && !(t ? nats); } { name = "hosting the swarm's shared services runs the forge"; ok = runsForge servicesHere; } { name = "the all-local mode runs the forge"; ok = runsForge allLocal && lib.elem "forge.t.local" allLocal.services.hyperhive.gateway.localNames; } { name = "an explicit deploy.forgejo.enable runs the forge on its own"; ok = runsForge forgeHere && !forgeHere.services.hyperhive.deploy.allSwarmServices; } { # `mkDefault`, not a plain assignment: the forge stays placeable on a # host of its own. `nats` is the control, so the case cannot pass on a # fixture where nothing came on. name = "placing the forge elsewhere survives the switch that would enable it"; ok = !(servicesForgeElsewhere.containers ? hive-forge) && servicesForgeElsewhere.services.hyperhive.deploy.nats.enable; } { # A client is a row in authelia's config, so it is declared where # authelia runs. With the forge's module gated, registering it from # there would leave a split swarm's forge unknown to its IdP. name = "the forge's OIDC client is registered wherever authelia runs, and only there"; ok = forgeClientIds servicesForgeElsewhere == [ "forgejo" ] && forgeClientIds allLocal == [ "forgejo" ] && forgeClientIds forgeAlone == [ ]; } { name = "the forge's OIDC callback is the one forgejo sends"; ok = servicesForgeElsewhere.services.hyperhive.swarm.forge.sso.redirectUri == "https://forge.t.local/user/oauth2/authelia/callback"; } { # The runner reaches the forge by name and reads its token with its own # certificate, so nothing about the host it runs on is asked. name = "the CI runner runs on a host with neither the forge nor a hive"; ok = ciAlone.containers ? swarm-ci-runner && !(ciAlone.containers ? hive-forge) && !ciAlone.services.hyperhive.deploy.hive-controller.enable && !(refusedOver ciAlone "ci-runner") && ciAlone.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT == "/etc/pki/bao-ci-runner.pem"; } { # The store's units exist only on its host, so elsewhere the fetch waits # on nothing local. The arm where the store is here is in # ./swarm-services-switch.nix. name = "the CI runner's token fetch names none of the store's units on a host without the store"; ok = let u = ciAlone.systemd.services.swarm-bao-ci-runner; storeUnits = [ "swarm-bao-pki.service" "container@${ciAlone.services.hyperhive.swarm.bao.machine}.service" ]; in !(lib.any (n: lib.elem n (u.after ++ u.wants ++ u.requires)) storeUnits); } { # Without the identity the token never arrives; the control is the # placed pair above. name = "the CI runner is refused without its store identity"; ok = refusedOver ciNoIdentity "deploy.bao.ciRunnerClientCertFile"; } { # systemd hands a credential to every process of the unit that loads # it, and a host job is one of the daemon's processes and inherits its # environment; a token in either lets any workflow register runners. name = "the CI runner daemon can read no registration token"; ok = let sc = runnerService.serviceConfig; in !(runnerService.environment ? TOKEN) && lib.toList (sc.EnvironmentFile or [ ]) == [ ] && !(lib.any (lib.hasPrefix "TOKEN=") (lib.toList (sc.Environment or [ ]))) && !(sc ? LoadCredential) && !(sc ? LoadCredentialEncrypted) && !(sc ? SetCredential); } { name = "the CI register unit alone loads the registration token, and runs before the daemon"; ok = let sc = registerService.serviceConfig; in sc.LoadCredential == [ "registration-token:/run/swarm-ci-runner/registration-token" ] && sc.Type == "oneshot" && sc.User == runnerService.serviceConfig.User && sc.DynamicUser && sc.StateDirectory == runnerService.serviceConfig.StateDirectory && lib.any ( s: lib.hasSuffix "/bin/systemctl start swarm-ci-runner-register.service" s ) runnerService.serviceConfig.ExecStartPre && !(lib.any ( s: lib.hasInfix "gitea-register-runner" (toString s) ) runnerService.serviceConfig.ExecStartPre); } { # A copy on persistent disk outlives the token's rotation and lands in # backups of the container's root. name = "the CI runner's registration token is written to the host's /run only, and the old copy removed"; ok = let script = ciAlone.systemd.services.swarm-bao-ci-runner.script; mount = ciAlone.containers.swarm-ci-runner.bindMounts."/run/swarm-ci-runner"; guestRules = ciAlone.containers.swarm-ci-runner.config.systemd.tmpfiles.rules; in lib.hasInfix "atomic_write_secret 0600 \"\" /run/swarm-ci-runner/registration-token" script && !(lib.hasInfix "/var/lib/nixos-containers" script) && mount.hostPath == "/run/swarm-ci-runner" && mount.isReadOnly && lib.elem "d /run/swarm-ci-runner 0700 root root -" ciAlone.systemd.tmpfiles.rules && lib.elem "r /var/lib/swarm-ci-runner/registration-token" guestRules && !(lib.any (lib.hasPrefix "f /var/lib/swarm-ci-runner/registration-token") guestRules); } { name = "the CI register unit registers from its credential, and only without a .runner"; ok = let text = registerService.script; beforeRegister = lib.head (lib.splitString "register --no-interactive" text); in lib.hasInfix "if [ -e .runner ]; then\n exit 0\nfi" beforeRegister && lib.hasInfix ''token="$(< "$CREDENTIALS_DIRECTORY/registration-token")"'' beforeRegister && lib.hasInfix ''--token "$token"'' text && !(lib.hasInfix "export" text) && !(lib.hasInfix "TOKEN" text); } { # The daemon rewrites `.runner`'s labels to this form on start; compared # against the configured strings verbatim, a scheme-less label would # discard `.runner` and register a new runner on every start. name = "the CI runner compares .runner's labels in the form the daemon writes them"; ok = registerService.environment.SWARM_CI_RUNNER_LABELS == "hive-ci:host,nix:host"; } { # The control is the container: it only exists if `enable` arrived. name = "a CI option under deploy.forgejo sets the same option under deploy.ci-runner"; ok = let r = ciUnderForge.services.hyperhive.deploy.ci-runner; in r.enable && r.name == "old-runner" && r.concurrency == 3 && r.labels == [ "nix:host" ] && r.package == pkgs.hello && r.jobTimeout == "2h" && ciUnderForge.containers ? swarm-ci-runner && lib.any (lib.hasInfix "services.hyperhive.deploy.forgejo.ci.enable") ciUnderForge.warnings && !(lib.any (a: !a.assertion) ciUnderForge.assertions); } { # The rename carries `enable` but not the identity: an old config with # only the old name is refused, and the message says what to set. The # control is the placed pair in `ciUnderForge` above. name = "an old CI enable without a store identity is refused naming the options to set"; ok = refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.ciRunnerClientCertFile" && refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.ciRunnerClientKeyFile" && refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.enable"; } { # The mint, in the forge's container, writes the controller's token to # the store under the forge host's leaf, loaded as a credential. On # stdin, so the token is no process's argument; the copy an earlier mint # kept on the container's disk goes once a token is published, and no # host unit reads into the container. name = "the forge's mint writes the controller's forge token straight to the store and keeps no copy"; ok = let u = mint allLocal; cred = id: "/run/credentials/forgejo-swarm-controller-account.service/${id}"; in lib.elem "forge-bao-cert" u.serviceConfig.LoadCredential && lib.elem "forge-bao-key" u.serviceConfig.LoadCredential && u.environment.BAO_CLIENT_CERT == cred "forge-bao-cert" && u.environment.BAO_CLIENT_KEY == cred "forge-bao-key" && lib.elem "--load-credential=forge-bao-cert:/var/lib/swarm-bao-pki/forge-publisher.pem" allLocal.containers.hive-forge.extraFlags && lib.elem "--load-credential=forge-bao-cert:/etc/pki/forge-publisher.pem" forgeAlone.containers.hive-forge.extraFlags && lib.hasInfix "printf '%s' \"$token\" | bao kv put secret/swarm/controller/swarm-controller/forge/token value=-" u.script && !(lib.hasInfix "value=@" u.script) && !(lib.hasInfix "atomic_write_secret" u.script) && lib.hasInfix "rm -f /var/lib/forgejo/swarm-controller-token\n" u.script && !(allLocal.systemd.services ? swarm-bao-forge-publisher) && !(allLocal.systemd.paths ? swarm-bao-forge-publisher) && !(forgeAlone.systemd.services ? swarm-bao-forge-publisher) && !(allLocal.systemd.services ? hive-forge-swarm-controller-token); } { # The order that keeps a token live only if the store has it: log in # before the mint, put, revoke every other `swarm-controller-boot-*` # token, and only then write the marker, which holds the name. A marker # present skips all of it. name = "the forge's mint writes its marker only after the put and the revocation of every earlier token"; ok = let s = (mint allLocal).script; at = needle: lib.stringLength (lib.head (lib.splitString needle s)); order = l: lib.length l < 2 || (at (lib.elemAt l 0) < at (lib.elemAt l 1) && order (lib.tail l)); steps = [ "if [ -s \"$name_path\" ]; then" "forgejo admin user list --admin" "bao login -method=cert" "forgejo admin user generate-access-token" "bao kv put" "forge GET \"/admin/users/swarm-controller/tokens?limit=50&page=$page\"" "forge DELETE \"/admin/users/swarm-controller/tokens/$n\"" "rm -f /var/lib/forgejo/swarm-controller-token\n" "mv -f \"$name_path.tmp\" \"$name_path\"" ]; in lib.all (step: lib.hasInfix step s) steps && order steps && lib.hasInfix "name_path=/var/lib/forgejo/swarm-controller-token-name" s && lib.hasInfix "printf '%s\\n' \"$name\" >\"$name_path.tmp\"" s && lib.hasInfix "swarm-controller-boot | swarm-controller-boot-*) stale+=(\"$n\") ;;" s && lib.hasInfix "-H @<(printf 'Authorization: token %s\\n' \"$token\")" s; } { # A store that is down retries the mint (../host-modules/lib/store-retry.nix), # and forgejo never waits for it. name = "the forge's mint retries on failure, and forgejo does not wait for it"; ok = let c = allLocal.containers.hive-forge.config.systemd.services; u = mint allLocal; in u.serviceConfig.Restart == "on-failure" && u.startLimitBurst == 2880 && u.serviceConfig.RemainAfterExit && lib.elem "forgejo.service" u.after && !(lib.elem "forgejo-swarm-controller-account.service" ( c.forgejo.after ++ c.forgejo.requires ++ c.forgejo.wants )) && lib.elem "forgejo-swarm-controller-account.service" c.swarm-bridge-dns.before; } { # The absence arm: a forge host without the store has no leaf unless one # is placed, and the mint says which options to set before it mints. name = "a forge host with no publisher leaf renders a mint that fails naming the options, minting nothing"; ok = let u = mint forgeHere; in !(u.environment ? BAO_CLIENT_CERT) && lib.hasInfix "forgePublisherClientCertFile" u.script && lib.hasInfix "exit 1" u.script && !(lib.hasInfix "forgejo admin user generate-access-token" u.script) && !(lib.hasInfix "bao kv put" u.script); } { # The forge's OIDC secret has one route, the store, under the forge # publisher's leaf. No unit copies it out of authelia's tree, even on a # host that runs authelia, and nothing defines the removed path option. name = "the forge reads its OIDC secret from the store under the forge publisher's leaf"; ok = let u = oidcFetch forgeAlone; d = forgeAlone.services.hyperhive.deploy; in !d.authelia.enable && !d.bao.enable && !(refusedOver forgeAlone "deploy.forgejo.sso.clientSecretFile") && !(refusedOver allLocal "deploy.forgejo.sso.clientSecretFile") && u.environment.BAO_CLIENT_CERT == "/etc/pki/forge-publisher.pem" && u.environment.BAO_CLIENT_KEY == "/etc/pki/forge-publisher-key.pem" && lib.hasInfix "bao kv get -field=value secret/swarm/services/forgejo/oidc/client" u.script && lib.hasInfix "refresh_consumer hive-forge forgejo-sso-source.service" u.script # An unpublished or empty secret fails the unit, so `storeRetry` retries it. && !(lib.hasInfix "exit 0" u.script) && u.serviceConfig.Restart == "on-failure" && lib.elem "container@hive-forge.service" u.before && lib.elem "multi-user.target" u.wantedBy && (oidcFetch allLocal).environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/forge-publisher.pem" && !(forgeAlone.systemd.services ? hive-forge-oidc-secret) && !(allLocal.systemd.services ? hive-forge-oidc-secret) && !(bare.systemd.services ? swarm-bao-forge-oidc); } { # The fetched secret lives in a host tmpfs directory bound read-only into # the container, and reaches the login source as a credential. Nothing # writes it under the container's persistent root, and the copy an # earlier fetch left there is removed. name = "the forge's OIDC secret reaches its login source from /run, not from persistent disk"; ok = let s = (oidcFetch forgeAlone).script; c = forgeAlone.containers.hive-forge; src = c.config.systemd.services.forgejo-sso-source; rules = forgeAlone.systemd.tmpfiles.rules; in lib.hasInfix "atomic_write_secret 0400" s && lib.hasInfix "/run/hive-forge-oidc/forgejo.secret" s && !(lib.hasInfix "/var/lib/nixos-containers/hive-forge" s) && c.bindMounts."/run/hive-forge-oidc".hostPath == "/run/hive-forge-oidc" && c.bindMounts."/run/hive-forge-oidc".isReadOnly && lib.elem "oidc-client-secret:/run/hive-forge-oidc/forgejo.secret" src.serviceConfig.LoadCredential && lib.hasInfix "$CREDENTIALS_DIRECTORY/oidc-client-secret" src.script && lib.elem "d /run/hive-forge-oidc 0700 root root -" rules && lib.elem "r /var/lib/nixos-containers/hive-forge/var/lib/forgejo-oidc/forgejo.secret" rules; } { # Beside the store, the fetch waits for the unit writing the forge # publisher's role, the leaf it logs in with. name = "the forge's OIDC secret fetch is ordered after the forge publisher's policy unit"; ok = let u = oidcFetch allLocal; in lib.elem "swarm-bao-forge-publisher-policy.service" u.after && lib.elem "swarm-bao-forge-publisher-policy.service" u.wants; } { # The absence arm: no publisher leaf, no secret, and the unit says which # options to set. name = "a forge host with no publisher leaf renders an OIDC secret fetch that fails naming the options"; ok = let u = oidcFetch forgeHere; in !(u.environment ? BAO_CLIENT_CERT) && lib.hasInfix "forgePublisherClientCertFile" u.script && lib.hasInfix "exit 1" u.script && !(lib.hasInfix "bao login" u.script); } { # The login source fetches the issuer's discovery document by name # wherever the forge runs, so it waits for the resolver on a forge host # without authelia too. name = "the login source waits for the container's resolver on a forge host without authelia"; ok = lib.elem "forgejo-sso-source.service" ( forgeAlone.containers.hive-forge.config.systemd.services.swarm-bridge-dns.before ); } { # The host copy is removed where the forge runs: nothing reads it. name = "the forge's host removes its old copy of the controller's forge token"; ok = lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" forgeHere.systemd.tmpfiles.rules && !(lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" bare.systemd.tmpfiles.rules); } { # The only thing that makes a human's forge account: nothing else # creates one. name = "a first authelia login creates the forge account, named by preferred_username"; ok = let o = forgeSettings.oauth2_client; in o.ENABLE_AUTO_REGISTRATION == true && o.USERNAME == "preferred_username"; } { # `auto` would give an SSO user whatever local account carries their # name — an agent's, or `core`'s. name = "an SSO login adopts an existing forge account only with that account's password"; ok = forgeSettings.oauth2_client.ACCOUNT_LINKING == "login"; } { # Both halves: local sign-up stays off, and nothing turns on the one # setting forgejo's auto-registration does check. name = "local sign-up stays off without blocking the SSO registration"; ok = forgeSettings.service.DISABLE_REGISTRATION == true && !(forgeSettings.service.ALLOW_ONLY_INTERNAL_REGISTRATION or false); } { # The runner's own store leaf, handed in as system credentials, reads the # forwarder's client secret; the host's token fetch keeps the same files. name = "the CI runner container forwards its journal to its own route as its own client"; ok = let h = ciAlone; container = h.containers.swarm-ci-runner; c = container.config; s = c.services.opentelemetry-collector.settings; fetch = c.systemd.services.swarm-journal-forwarder-oidc; clientId = h.services.hyperhive.swarm.ci-runner.otel.clientId; dir = "/run/credentials/swarm-journal-forwarder-oidc.service"; in c.services.opentelemetry-collector.enable && s.exporters.otlphttp.endpoint == "https://${h.services.hyperhive.swarm.otel.domain}/swarm-ci-runner" && s.extensions."oauth2client/swarm".client_id == clientId && s.extensions."oauth2client/swarm".endpoint_params.audience == clientId && h.services.hyperhive.swarm.otel.serviceProducers.swarm-ci-runner.clientId == clientId && (lib.findFirst (a: a.key == "service.name") null s.processors.resource.attributes).value == "ci-runner" && fetch.environment.BAO_CLIENT_CERT == "${dir}/ci-runner-bao-cert" && fetch.environment.BAO_CLIENT_KEY == "${dir}/ci-runner-bao-key" && lib.elem "ci-runner-bao-cert" fetch.serviceConfig.LoadCredential && lib.elem "ci-runner-bao-key" fetch.serviceConfig.LoadCredential && lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script && !(lib.elem "--link-journal=host" container.extraFlags) && lib.elem "--load-credential=ci-runner-bao-cert:/etc/pki/bao-ci-runner.pem" container.extraFlags && lib.elem "--load-credential=ci-runner-bao-key:/etc/pki/bao-ci-runner-key.pem" container.extraFlags && h.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT == "/etc/pki/bao-ci-runner.pem"; } { name = "the CI runner forwarder's metrics port is loopback"; ok = let c = ciAlone.containers.swarm-ci-runner.config; reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers; in c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895 && reader.pull.exporter.prometheus.host == "127.0.0.1" && reader.pull.exporter.prometheus.port == 8895; } ]; in runGroup "forge-placement" cases