# `checks.module-eval-nats-authelia` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, shared, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive runGroup ; # The queue's two old `swarm.nats` paths that still rename, `enable` and # `authPackage`, so dropping either shim fails the eval. No store identity, # which the first case below relies on. natsOldPath = hive { swarm.nats.enable = true; swarm.nats.authPackage = pkgs.emptyDirectory; }; # Seventh split slice, plus slice 10's two authelia packages. Of slice 7's # movers only `usersFile` has a rename entry — the other two are `readOnly`, # and a rename module contributes a definition, which a read-only option # refuses; see ./host-modules/deploy.nix. `package` and `bridgePackage` are # ordinary options, so they do carry one. The two arms # below have different jobs. `usersFile` tests the rename; the nats one tests # that a reader pointed at the `deploy` namespace still renders the derived # path — seven of those reads go through an alias a path-shaped grep # cannot see. autheliaOldPath = hive { deploy.authelia.enable = true; deploy.nats.enable = true; swarm.authelia.usersFile = "/var/lib/test-authelia/users.yml"; swarm.authelia.package = pkgs.emptyDirectory; swarm.authelia.bridgePackage = pkgs.emptyDirectory; # Renders the publisher, which reads the host client-secret dir. deploy.swarm-secret-publisher.baoClientCertFile = "/etc/pki/publisher.pem"; deploy.swarm-secret-publisher.baoClientKeyFile = "/etc/pki/publisher-key.pem"; }; # The queue with a store identity for its responder, placed by hand: the # queue host that is not the store's. natsWithStoreIdentity = hive { deploy.nats.enable = true; deploy.nats.authPackage = pkgs.emptyDirectory; deploy.nats.authBaoClientCertFile = "/etc/pki/nats-auth.pem"; deploy.nats.authBaoClientKeyFile = "/etc/pki/nats-auth-key.pem"; }; # The queue on the store's own host, where the PKI glue mints every leaf. natsOnStoreHost = hive { deploy.bao.enable = true; deploy.nats.enable = true; }; # The store's host with the hive's own reader pointed somewhere else. natsOnStoreHostReaderMoved = hive { deploy.bao.enable = true; deploy.nats.enable = true; deploy.bao.clientCertFile = "/etc/pki/h1.pem"; deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem"; }; # A queue host holding only the hive's own store reader, placed by hand: no # store here, and no responder leaf named. natsWithHiveReaderOnly = hive { deploy.nats.enable = true; deploy.nats.authPackage = pkgs.emptyDirectory; deploy.bao.clientCertFile = "/etc/pki/h1.pem"; deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem"; }; responderOf = h: h.containers.swarm-nats.config.systemd.services.swarm-nats-auth; refusedWithoutIdentity = m: lib.any ( a: !a.assertion && lib.hasInfix "deploy.nats.authBaoClientCertFile and" a.message ) m.assertions; # A hive running NOTHING of the swarm's own services — no IdP here, no # `swarm.authelia.url` set by hand. The whole point of the fixture is what it # does *not* say: no IdP address is given and none is derived from # co-location. autheliaNotColocated = shared.bare; # The same swarm's IdP host, for the arm that the two agree. autheliaColocated = shared.autheliaOnly; cases = [ { name = "authelia and nats declare no scrape target on a host with no collector"; ok = !(autheliaOldPath.services.hyperhive.deploy.swarm-otel.enable) && !(autheliaOldPath.services.hyperhive.swarm.otel.scrapeTargets ? authelia) && !(autheliaOldPath.services.hyperhive.swarm.otel.scrapeTargets ? nats); } { # The xkey and the responder's OIDC client secret are read only from # the store, so a queue with no responder leaf cannot start. Refused at # eval; the controls are a leaf named by hand and the store host's # minted one. name = "a queue with no responder store identity is refused at eval, one with a leaf is not"; ok = refusedWithoutIdentity natsOldPath && !(refusedWithoutIdentity natsWithStoreIdentity) && !(refusedWithoutIdentity natsOnStoreHost); } { # The hive's own reader is not the responder's identity, and no path is # derived from it: a queue host without the store that holds only the # hive's reader is refused like one holding nothing. name = "a queue host without the store is refused when it holds only the hive's reader"; ok = refusedWithoutIdentity natsWithHiveReaderOnly && natsWithHiveReaderOnly.services.hyperhive.deploy.nats.authBaoClientCertFile == null; } { # Reads the RENDERED settings, not the option: `calloutBlocks {…} // { # … }` is a shallow merge, and a future edit that dropped or shadowed # this key would still evaluate cleanly — the only reader that would # notice is a publisher whose row exceeds upstream's much smaller # default, and by then it is a dropped row, not an eval failure. # Piggybacks on the pre-rename nats fixture above, which already # renders this container's full config. name = "the queue's payload ceiling is set, not inherited from the server's default"; ok = natsOldPath.containers.swarm-nats.config.services.nats.settings.max_payload == 8388608; } { # What keeps a callout request inside one principal: the responder is # the callout account's only member, and no system account exists to # observe it from. The runtime fragment is rendered by the same # `calloutBlocks`, so the count is the same there. The account name is # the control that the assertions read the callout block at all. name = "the callout account has exactly one member and no system account is configured"; ok = lib.all ( h: let settings = h.containers.swarm-nats.config.services.nats.settings; in settings.authorization.auth_callout.account == "AUTH" && lib.length settings.accounts.AUTH.users == 1 && !(settings ? system_account) ) [ natsOldPath natsOnStoreHost ]; } { # The xkey exists only at runtime, so the server's config is rendered as # it starts: always the wrapper, and never the store-rendered settings # alone, which carry no xkey. `+` so the render may write `runtimeDir`. name = "the queue's server starts only through the config rendered with its xkey"; ok = lib.all ( h: let c = h.containers.swarm-nats.config; nats = c.systemd.services.nats.serviceConfig; pre = nats.ExecStartPre or ""; in lib.hasSuffix "/bin/nats-server -c /var/lib/nats-callout/nats.conf" nats.ExecStart && lib.hasPrefix "+/nix/store/" pre && lib.hasSuffix "-swarm-nats-render-config" pre && !(c.services.nats.settings.authorization.auth_callout ? xkey) ) [ natsOldPath natsOnStoreHost ]; } { # A server that starts late, once its xkey arrives, must bring the # responder with it: the responder requires the server, so a server that # failed to start left the responder inactive rather than failed. name = "starting the queue's server starts its responder"; ok = lib.elem "nats.service" (responderOf natsOldPath).wantedBy; } { # The keys are created before anything needs them, by the responder's # own verb under its own leaf, then each public half is read back from # the store and left where the server's render reads it. Before the # container and wanted by it, and a fetch that finds nothing fails so # `storeRetry` retries. name = "the callout keys are created, then their public halves fetched from the store before the queue's container starts"; ok = let u = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey; s = builtins.unsafeDiscardStringContext u.script; at = needle: lib.stringLength (lib.head (lib.splitString needle s)); ensure = "swarm-nats-auth ensure-callout-keys --store-cert-role swarm-nats-auth"; fetched = leaf: file: let fetch = "bao kv get -field=value secret/swarm/public/swarm-nats/${leaf}"; path = "/var/lib/nixos-containers/swarm-nats/var/lib/nats-callout/${file}"; in lib.hasInfix fetch s && at ensure < at fetch && lib.hasInfix "atomic_write_secret 0444 \"\" ${path}" s && lib.hasInfix "refresh_consumer swarm-nats nats.service ${path}" s; in lib.elem "container@swarm-nats.service" u.before && lib.elem "container@swarm-nats.service" u.wantedBy && u.environment.BAO_CLIENT_CERT == "/etc/pki/nats-auth.pem" && lib.hasInfix ensure s && fetched "callout-xkey" "callout-xkey.pub" && fetched "callout-issuer" "issuer.pub" && fetched "callout-user" "callout-user.pub" && !(lib.hasInfix "exit 0" s) && u.serviceConfig.Restart == "on-failure"; } { # The seeds a host kept on disk are what the verb stores when the store # holds none, so the keys keep their public halves; the verb removes # them itself. Nothing mints a seed on the host, and no # tmpfiles rule touches the host seeds, which could run before the # import does. name = "the callout keys import the host's seed files, which only the importing unit removes"; ok = let s = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey.script; rules = natsWithStoreIdentity.systemd.tmpfiles.rules; in lib.hasInfix "--import-issuer-seed-file /var/lib/swarm-nats-callout/issuer.seed" s && lib.hasInfix "--import-user-seed-file /var/lib/swarm-nats-callout/callout-user.seed" s && !(natsWithStoreIdentity.systemd.services ? swarm-nats-callout-keys) && !(lib.any (lib.hasInfix "/var/lib/swarm-nats-callout") rules); } { # Its creates are what the responder's role was granted, and the # publisher's policy is what denies the publisher the seeds' paths: a # seed found there is kept. On the store's host both units run first. name = "on the store's host the callout key fetch waits for the responder's role and the publisher's denial"; ok = let u = natsOnStoreHost.systemd.services.swarm-bao-nats-callout-xkey; waitsFor = p: lib.elem "${p}.service" u.after && lib.elem "${p}.service" u.wants; in u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats-auth.pem" && waitsFor "swarm-bao-nats-auth-policy" && waitsFor "swarm-bao-secret-publisher-policy" && natsOnStoreHost.systemd.services ? swarm-bao-secret-publisher-policy; } { # The control is the fixture above the store: off the store's host # there is no local policy unit to wait for. name = "off the store's host the callout key fetch waits for no policy unit"; ok = let u = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey; in !(lib.any (lib.hasSuffix "-policy.service") (u.after ++ u.wants)); } { # The responder refuses the hive's shared agent client and takes no flag # granting it subjects: an `--agent-publish-subject` fails the unit at # start. name = "the responder grants the hive's shared agent client nothing"; ok = let exec = natsOldPath.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart; in !(lib.hasInfix "--agent-publish-subject " exec); } { # The per-agent grant: the terminal and agent-state streams keyed on the # agent alone, with the doubled dollar, and the role the store writes for # it. name = "the responder grants a verified agent its own hive-free subjects"; ok = let exec = (responderOf natsOldPath).serviceConfig.ExecStart; in lib.hasInfix "--agent-token-publish-subject '$$SWARM.term.{agent}'" exec && lib.hasInfix "--agent-token-publish-subject '$$SWARM.agent-state.{agent}'" exec && lib.hasInfix "--store-cert-role swarm-nats-auth" exec; } { # Its own arm, like the two above: the icon write is a separate feature # and dropping it must not hide behind the streams still being granted. name = "the responder grants a verified agent its own icon key"; ok = let exec = (responderOf natsOldPath).serviceConfig.ExecStart; in lib.hasInfix "--agent-token-publish-subject '$$KV.agent-icons.{agent}'" exec; } { # The whole per-agent grant, compared as a list rather than by infix: a # wider subject added beside these (`$$JS.API.>`, another stream's name) # would pass every presence check above. name = "a verified agent's grant is exactly its own subjects, and its JetStream API subjects read its own schedule and inference keys"; ok = let args = lib.splitString " " (responderOf natsOldPath).serviceConfig.ExecStart; granted = lib.concatLists ( lib.imap0 ( i: a: lib.optional (a == "--agent-token-publish-subject" && i + 1 < lib.length args) ( lib.elemAt args (i + 1) ) ) args ); in granted == [ "'$$SWARM.term.{agent}'" "'$$SWARM.agent-state.{agent}'" "'$$KV.agent-icons.{agent}'" "'$$SWARM.term.{agent}.sub.>'" "'$$JS.API.DIRECT.GET.KV_agent-schedules.$$KV.agent-schedules.{agent}'" "'$$JS.API.DIRECT.GET.KV_agent-inference.$$KV.agent-inference.{agent}'" ]; } { # Each credential by `LoadCredential`, and the environment naming where # the unit sees it: a DynamicUser cannot read the copies directly. name = "a responder with a store identity loads it and is pointed at the store"; ok = let u = responderOf natsWithStoreIdentity; creds = u.serviceConfig.LoadCredential; in lib.elem "bao-client.pem:/var/lib/swarm-nats-auth/bao-client.pem" creds && lib.elem "bao-client-key.pem:/var/lib/swarm-nats-auth/bao-client-key.pem" creds && u.environment.BAO_CLIENT_CERT == "/run/credentials/swarm-nats-auth.service/bao-client.pem" && u.environment.BAO_CLIENT_KEY == "/run/credentials/swarm-nats-auth.service/bao-client-key.pem" && lib.hasPrefix "https://" u.environment.BAO_ADDR; } { # Copied so an absent leaf fails the unit rather than reaching the # responder as a file it cannot log in with. No seed is copied: the # responder reads every callout seed from the store. name = "the copy unit delivers the store identity and no seed, and nothing in its place when absent"; ok = let s = natsWithStoreIdentity.systemd.services.swarm-nats-auth-secrets.script; in lib.hasInfix "install -m 0400 /etc/pki/nats-auth.pem " s && lib.hasInfix "install -m 0400 /etc/pki/nats-auth-key.pem " s && !(lib.hasInfix "/dev/null" s) && !(lib.hasInfix ".seed" s); } { # The responder names no seed file, by flag or by credential, and the # copies a container may still hold are removed. The store identity is # the control that the credential list is read at all. name = "the responder takes no seed from a file, and the container's seed copies are removed"; ok = let u = responderOf natsWithStoreIdentity; creds = u.serviceConfig.LoadCredential; dir = "/var/lib/nixos-containers/swarm-nats/var/lib/swarm-nats-auth"; rules = natsWithStoreIdentity.systemd.tmpfiles.rules; in lib.elem "bao-client.pem:/var/lib/swarm-nats-auth/bao-client.pem" creds && !(lib.any (lib.hasInfix ".seed") creds) && !(lib.hasInfix "seed-file" u.serviceConfig.ExecStart) && lib.elem "r ${dir}/issuer.seed" rules && lib.elem "r ${dir}/callout-user.seed" rules; } { # Under either namespace, each removed option fails evaluation naming # itself and the store. The control is a queue host that sets none. name = "a definition of a removed callout option fails evaluation, naming the option and the secret store"; ok = let refused = m: path: lib.any ( a: !a.assertion && lib.hasInfix "services.hyperhive.${path}" a.message && lib.hasInfix "live in the swarm secret store" a.message ) m.assertions; removedOptions = { autoGenerateCallout = true; calloutUserPublicKey = "U"; calloutIssuerPublicKey = "A"; calloutUserSeedFile = "/run/u.seed"; calloutIssuerSeedFile = "/run/i.seed"; }; setting = lib.listToAttrs ( lib.concatMap ( ns: lib.mapAttrsToList (name: value: { name = "${ns}.nats.${name}"; value = hive { ${ns}.nats.${name} = value; }; }) removedOptions ) [ "deploy" "swarm" ] ); in lib.length (lib.attrNames setting) == 10 && lib.all (path: refused setting.${path} path) (lib.attrNames setting) && !(lib.any (path: refused natsWithStoreIdentity path) (lib.attrNames setting)); } { # The responder reads its OIDC secret from the store itself, so no host # unit fetches it, no credential or flag carries it, and a copy a # container still holds is removed, also on a host with the queue off. # On or off the store's host alike; the callout key fetch beside it is # the control that the unit set is read at all, and the store cert role # the control on the flags. name = "the responder's OIDC secret is fetched by no unit and carried by no credential or flag, and the container's copy is removed"; ok = let fetchedNowhere = h: !(h.systemd.services ? swarm-bao-nats-auth-oidc) && h.systemd.services ? swarm-bao-nats-callout-xkey; u = responderOf natsWithStoreIdentity; copy = natsWithStoreIdentity.systemd.services.swarm-nats-auth-secrets.script; removesCopy = h: lib.elem "r ${dir}/oidc-client.secret" h.systemd.tmpfiles.rules; dir = "/var/lib/nixos-containers/swarm-nats/var/lib/swarm-nats-auth"; in fetchedNowhere natsWithStoreIdentity && fetchedNowhere natsOnStoreHost && !(lib.any (lib.hasInfix "oidc-client") u.serviceConfig.LoadCredential) && !(lib.hasInfix "client-secret" u.serviceConfig.ExecStart) && lib.hasInfix "--store-cert-role swarm-nats-auth" u.serviceConfig.ExecStart && removesCopy natsWithStoreIdentity && removesCopy autheliaNotColocated && !(lib.hasInfix "oidc-client" copy) && !(lib.hasInfix "authelia" copy); } { # On the store's host the glue pairs the responder with a leaf of its # own, never the queue's TLS-issuing one or the hive's. name = "on the store's host the responder presents its own leaf"; ok = let n = natsOnStoreHost.services.hyperhive.deploy.nats; b = natsOnStoreHost.services.hyperhive.deploy.bao; in n.authBaoClientCertFile == "/var/lib/swarm-bao-pki/nats-auth.pem" && n.authBaoClientKeyFile == "/var/lib/swarm-bao-pki/nats-auth-key.pem" && n.authBaoClientCertFile != n.baoClientCertFile && n.authBaoClientCertFile != b.clientCertFile; } { # The default is the file the minting unit writes, wherever the hive's # own reader is pointed. name = "on the store's host the responder's leaf is the one minted there, not one beside the hive's reader"; ok = let n = natsOnStoreHostReaderMoved.services.hyperhive.deploy.nats; mint = natsOnStoreHostReaderMoved.systemd.services.swarm-bao-pki.script; in n.authBaoClientCertFile == "/var/lib/swarm-bao-pki/nats-auth.pem" && n.authBaoClientKeyFile == "/var/lib/swarm-bao-pki/nats-auth-key.pem" && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats-auth.pem ] ||" mint && lib.hasInfix "/var/lib/swarm-bao-pki nats-auth " mint; } { # Not a rename test. `hostClientSecretDir` is `readOnly`, so the fixture # cannot define it; what can break is a reader left pointing at # `swarm.authelia` instead of `deploy.authelia`. Several modules read this # through an `autheliaCfg` alias, where a path-shaped grep does not see it. name = "a consumer of authelia's host client-secret dir renders it from the deploy namespace"; ok = autheliaOldPath.systemd.paths.swarm-secret-publish.pathConfig.PathChanged == "/var/lib/nixos-containers/swarm-authelia/var/lib/authelia-swarm/oidc-clients"; } { # authelia exits at startup on `users: {}`, so an empty database, and a # file holding exactly that document, must be replaced by the disabled # placeholder rather than left for authelia to read. name = "an empty authelia users database is seeded with the disabled placeholder"; ok = let s = autheliaColocated.containers.swarm-authelia.config.systemd.services.authelia-swarm-secrets.script; in lib.hasInfix ''if [ ! -s "$users" ] || [ "$(cat "$users")" = "users: {}" ]; then'' s && lib.hasInfix "-authelia-users-placeholder.yml \"$users\"" s && !(lib.hasInfix ''echo "users: {}"'' s); } { # The bridge refuses every client but the one named here, so this has # to be the client the controller authenticates as. A drift is a # controller whose every bridge call is refused. name = "the authelia bridge answers the controller's client only"; ok = let env = autheliaColocated.containers.swarm-authelia.config.systemd.services.swarm-authelia-bridge.environment; in env.SWARM_AUTHELIA_BRIDGE_CONTROLLER_CLIENT_ID == autheliaColocated.services.hyperhive.swarm.controller.queueClientId && env.SWARM_AUTHELIA_BRIDGE_OPERATOR_GROUP == "admins"; } { # One address from every host, and a location the IdP host serves: a # controller off the authelia host has no other way to the bridge. name = "the auth bridge address is the authelia vhost's /bridge/ on every host"; ok = let url = h: h.services.hyperhive.deploy.swarm-controller.authBridgeUrl; authelia = autheliaColocated.services.hyperhive.swarm.authelia; location = autheliaColocated.services.nginx.virtualHosts.${authelia.domain}.locations."/bridge/"; in url autheliaNotColocated == "${autheliaNotColocated.services.hyperhive.swarm.authelia.url}/bridge" && url autheliaColocated == url autheliaNotColocated && location.proxyPass == "http://127.0.0.1:${toString authelia.bridgePort}/"; } { # Config NAMES the IdP; it never computes where the IdP is. Both arms # matter together: the address is the swarm's name on a hive that runs # nothing, and it is the SAME string on the hive that serves the vhost — # so a re-introduced co-location branch shows up as the two disagreeing # rather than as a value that merely looks plausible on one of them. name = "the swarm IdP address is its domain on every hive, co-located or not"; ok = autheliaNotColocated.services.hyperhive.swarm.authelia.url == "https://${autheliaNotColocated.services.hyperhive.swarm.authelia.domain}" && autheliaColocated.services.hyperhive.swarm.authelia.url == autheliaNotColocated.services.hyperhive.swarm.authelia.url; } { # …and the name it resolves through is a DOMAIN, not a host this config # picked. `127.0.0.1` is what a co-location-derived default renders on # the IdP host, the bridge address is what the queue's defaults render, # and neither is an address a client may be handed: the domain is # allowed to resolve differently in different places, which is the # whole property. name = "the swarm IdP address names no host address"; ok = let url = autheliaNotColocated.services.hyperhive.swarm.authelia.url; bridgeIp = autheliaNotColocated.services.hyperhive.network.bridgeIp; in !(lib.hasInfix "127.0.0.1" url) && !(lib.hasInfix "localhost" url) && !(lib.hasInfix bridgeIp url); } ]; in runGroup "nats-authelia" cases