hyperhive/nix/module-eval/nats-authelia.nix

# `checks.module-eval-nats-authelia` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    ;

  # The queue's two old `swarm.nats` paths that still rename, `enable` and
  # `authPackage`, so dropping either shim fails the eval. No store identity,
  # which the first case below relies on.
  natsOldPath = hive {
    swarm.nats.enable = true;
    swarm.nats.authPackage = pkgs.emptyDirectory;
  };

  # Seventh split slice, plus slice 10's two authelia packages. Of slice 7's
  # movers only `usersFile` has a rename entry — the other two are `readOnly`,
  # and a rename module contributes a definition, which a read-only option
  # refuses; see ./host-modules/deploy.nix. `package` and `bridgePackage` are
  # ordinary options, so they do carry one. The two arms
  # below have different jobs. `usersFile` tests the rename; the nats one tests
  # that a reader pointed at the `deploy` namespace still renders the derived
  # path — seven of those reads go through an alias a path-shaped grep
  # cannot see.
  autheliaOldPath = hive {
    deploy.authelia.enable = true;
    deploy.nats.enable = true;
    swarm.authelia.usersFile = "/var/lib/test-authelia/users.yml";
    swarm.authelia.package = pkgs.emptyDirectory;
    swarm.authelia.bridgePackage = pkgs.emptyDirectory;
  };

  # The queue with a store identity for its responder, placed by hand: the
  # queue host that is not the store's.
  natsWithStoreIdentity = hive {
    deploy.nats.enable = true;
    deploy.nats.authPackage = pkgs.emptyDirectory;
    deploy.nats.authBaoClientCertFile = "/etc/pki/nats-auth.pem";
    deploy.nats.authBaoClientKeyFile = "/etc/pki/nats-auth-key.pem";
  };

  # The queue on the store's own host, where the PKI glue mints every leaf.
  natsOnStoreHost = hive {
    deploy.bao.enable = true;
    deploy.nats.enable = true;
  };

  # The store's host with the hive's own reader pointed somewhere else.
  natsOnStoreHostReaderMoved = hive {
    deploy.bao.enable = true;
    deploy.nats.enable = true;
    deploy.bao.clientCertFile = "/etc/pki/h1.pem";
    deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
  };

  # A queue host holding only the hive's own store reader, placed by hand: no
  # store here, and no responder leaf named.
  natsWithHiveReaderOnly = hive {
    deploy.nats.enable = true;
    deploy.nats.authPackage = pkgs.emptyDirectory;
    deploy.bao.clientCertFile = "/etc/pki/h1.pem";
    deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
  };

  responderOf = h: h.containers.swarm-nats.config.systemd.services.swarm-nats-auth;

  refusedWithoutIdentity =
    m:
    lib.any (
      a: !a.assertion && lib.hasInfix "deploy.nats.authBaoClientCertFile and" a.message
    ) m.assertions;

  # A hive running NOTHING of the swarm's own services — no IdP here, no
  # `swarm.authelia.url` set by hand. The whole point of the fixture is what it
  # does *not* say: no IdP address is given and none is derived from
  # co-location.
  autheliaNotColocated = hive { };
  # The same swarm's IdP host, for the arm that the two agree.
  autheliaColocated = hive { deploy.authelia.enable = true; };

  cases = [
    {
      # The xkey and the responder's OIDC client secret are read only from
      # the store, so a queue with no responder leaf cannot start. Refused at
      # eval; the controls are a leaf named by hand and the store host's
      # minted one.
      name = "a queue with no responder store identity is refused at eval, one with a leaf is not";
      ok =
        refusedWithoutIdentity natsOldPath
        && !(refusedWithoutIdentity natsWithStoreIdentity)
        && !(refusedWithoutIdentity natsOnStoreHost);
    }
    {
      # The hive's own reader is not the responder's identity, and no path is
      # derived from it: a queue host without the store that holds only the
      # hive's reader is refused like one holding nothing.
      name = "a queue host without the store is refused when it holds only the hive's reader";
      ok =
        refusedWithoutIdentity natsWithHiveReaderOnly
        && natsWithHiveReaderOnly.services.hyperhive.deploy.nats.authBaoClientCertFile == null;
    }
    {
      # Reads the RENDERED settings, not the option: `calloutBlocks {…} // {
      # … }` is a shallow merge, and a future edit that dropped or shadowed
      # this key would still evaluate cleanly — the only reader that would
      # notice is a publisher whose row exceeds upstream's much smaller
      # default, and by then it is a dropped row, not an eval failure.
      # Piggybacks on the pre-rename nats fixture above, which already
      # renders this container's full config.
      name = "the queue's payload ceiling is set, not inherited from the server's default";
      ok = natsOldPath.containers.swarm-nats.config.services.nats.settings.max_payload == 8388608;
    }
    {
      # What keeps a callout request inside one principal: the responder is
      # the callout account's only member, and no system account exists to
      # observe it from. The runtime fragment is rendered by the same
      # `calloutBlocks`, so the count is the same there. The account name is
      # the control that the assertions read the callout block at all.
      name = "the callout account has exactly one member and no system account is configured";
      ok =
        lib.all
          (
            h:
            let
              settings = h.containers.swarm-nats.config.services.nats.settings;
            in
            settings.authorization.auth_callout.account == "AUTH"
            && lib.length settings.accounts.AUTH.users == 1
            && !(settings ? system_account)
          )
          [
            natsOldPath
            natsOnStoreHost
          ];
    }
    {
      # The xkey exists only at runtime, so the server's config is rendered as
      # it starts: always the wrapper, and never the store-rendered settings
      # alone, which carry no xkey. `+` so the render may write `runtimeDir`.
      name = "the queue's server starts only through the config rendered with its xkey";
      ok =
        lib.all
          (
            h:
            let
              c = h.containers.swarm-nats.config;
              nats = c.systemd.services.nats.serviceConfig;
              pre = nats.ExecStartPre or "";
            in
            lib.hasSuffix "/bin/nats-server -c /var/lib/nats-callout/nats.conf" nats.ExecStart
            && lib.hasPrefix "+/nix/store/" pre
            && lib.hasSuffix "-swarm-nats-render-config" pre
            && !(c.services.nats.settings.authorization.auth_callout ? xkey)
          )
          [
            natsOldPath
            natsOnStoreHost
          ];
    }
    {
      # A server that starts late, once its xkey arrives, must bring the
      # responder with it: the responder requires the server, so a server that
      # failed to start left the responder inactive rather than failed.
      name = "starting the queue's server starts its responder";
      ok = lib.elem "nats.service" (responderOf natsOldPath).wantedBy;
    }
    {
      # The keys are created before anything needs them, by the responder's
      # own verb under its own leaf, then each public half is read back from
      # the store and left where the server's render reads it. Before the
      # container and wanted by it, and a fetch that finds nothing fails so
      # `storeRetry` retries.
      name = "the callout keys are created, then their public halves fetched from the store before the queue's container starts";
      ok =
        let
          u = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey;
          s = builtins.unsafeDiscardStringContext u.script;
          at = needle: lib.stringLength (lib.head (lib.splitString needle s));
          ensure = "swarm-nats-auth ensure-callout-keys --store-cert-role swarm-nats-auth";
          fetched =
            leaf: file:
            let
              fetch = "bao kv get -field=value secret/swarm/public/swarm-nats/${leaf}";
              path = "/var/lib/nixos-containers/swarm-nats/var/lib/nats-callout/${file}";
            in
            lib.hasInfix fetch s
            && at ensure < at fetch
            && lib.hasInfix "atomic_write_secret 0444 \"\" ${path}" s
            && lib.hasInfix "refresh_consumer swarm-nats nats.service ${path}" s;
        in
        lib.elem "container@swarm-nats.service" u.before
        && lib.elem "container@swarm-nats.service" u.wantedBy
        && u.environment.BAO_CLIENT_CERT == "/etc/pki/nats-auth.pem"
        && lib.hasInfix ensure s
        && fetched "callout-xkey" "callout-xkey.pub"
        && fetched "callout-issuer" "issuer.pub"
        && fetched "callout-user" "callout-user.pub"
        && !(lib.hasInfix "exit 0" s)
        && u.serviceConfig.Restart == "on-failure";
    }
    {
      # The seeds a host kept on disk are what the verb stores when the store
      # holds none, so the keys keep their public halves; the verb removes
      # them itself. Nothing mints a seed on the host, and no
      # tmpfiles rule touches the host seeds, which could run before the
      # import does.
      name = "the callout keys import the host's seed files, which only the importing unit removes";
      ok =
        let
          s = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey.script;
          rules = natsWithStoreIdentity.systemd.tmpfiles.rules;
        in
        lib.hasInfix "--import-issuer-seed-file /var/lib/swarm-nats-callout/issuer.seed" s
        && lib.hasInfix "--import-user-seed-file /var/lib/swarm-nats-callout/callout-user.seed" s
        && !(natsWithStoreIdentity.systemd.services ? swarm-nats-callout-keys)
        && !(lib.any (lib.hasInfix "/var/lib/swarm-nats-callout") rules);
    }
    {
      # Its creates are what the responder's role was granted, and the
      # publisher's policy is what denies the publisher the seeds' paths: a
      # seed found there is kept. On the store's host both units run first.
      name = "on the store's host the callout key fetch waits for the responder's role and the publisher's denial";
      ok =
        let
          u = natsOnStoreHost.systemd.services.swarm-bao-nats-callout-xkey;
          waitsFor = p: lib.elem "${p}.service" u.after && lib.elem "${p}.service" u.wants;
        in
        u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats-auth.pem"
        && waitsFor "swarm-bao-nats-auth-policy"
        && waitsFor "swarm-bao-secret-publisher-policy"
        && natsOnStoreHost.systemd.services ? swarm-bao-secret-publisher-policy;
    }
    {
      # The control is the fixture above the store: off the store's host
      # there is no local policy unit to wait for.
      name = "off the store's host the callout key fetch waits for no policy unit";
      ok =
        let
          u = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey;
        in
        !(lib.any (lib.hasSuffix "-policy.service") (u.after ++ u.wants));
    }
    {
      # The responder refuses the hive's shared agent client and takes no flag
      # granting it subjects: an `--agent-publish-subject` fails the unit at
      # start.
      name = "the responder grants the hive's shared agent client nothing";
      ok =
        let
          exec =
            natsOldPath.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart;
        in
        !(lib.hasInfix "--agent-publish-subject " exec);
    }
    {
      # The per-agent grant: the terminal and agent-state streams keyed on the
      # agent alone, with the doubled dollar, and the role the store writes for
      # it.
      name = "the responder grants a verified agent its own hive-free subjects";
      ok =
        let
          exec = (responderOf natsOldPath).serviceConfig.ExecStart;
        in
        lib.hasInfix "--agent-token-publish-subject '$$SWARM.term.{agent}'" exec
        && lib.hasInfix "--agent-token-publish-subject '$$SWARM.agent-state.{agent}'" exec
        && lib.hasInfix "--store-cert-role swarm-nats-auth" exec;
    }
    {
      # Its own arm, like the two above: the icon write is a separate feature
      # and dropping it must not hide behind the streams still being granted.
      name = "the responder grants a verified agent its own icon key";
      ok =
        let
          exec = (responderOf natsOldPath).serviceConfig.ExecStart;
        in
        lib.hasInfix "--agent-token-publish-subject '$$KV.agent-icons.{agent}'" exec;
    }
    {
      # The whole per-agent grant, compared as a list rather than by infix: a
      # wider subject added beside these (`$$JS.API.>`, another stream's name)
      # would pass every presence check above.
      name = "a verified agent's grant is exactly its own subjects, and its one JetStream API subject reads its own schedule key";
      ok =
        let
          args = lib.splitString " " (responderOf natsOldPath).serviceConfig.ExecStart;
          granted = lib.concatLists (
            lib.imap0 (
              i: a:
              lib.optional (a == "--agent-token-publish-subject" && i + 1 < lib.length args) (
                lib.elemAt args (i + 1)
              )
            ) args
          );
        in
        granted == [
          "'$$SWARM.term.{agent}'"
          "'$$SWARM.agent-state.{agent}'"
          "'$$KV.agent-icons.{agent}'"
          "'$$SWARM.term.{agent}.sub.>'"
          "'$$JS.API.DIRECT.GET.KV_agent-schedules.$$KV.agent-schedules.{agent}'"
        ];
    }
    {
      # Each credential by `LoadCredential`, and the environment naming where
      # the unit sees it: a DynamicUser cannot read the copies directly.
      name = "a responder with a store identity loads it and is pointed at the store";
      ok =
        let
          u = responderOf natsWithStoreIdentity;
          creds = u.serviceConfig.LoadCredential;
        in
        lib.elem "bao-client.pem:/var/lib/swarm-nats-auth/bao-client.pem" creds
        && lib.elem "bao-client-key.pem:/var/lib/swarm-nats-auth/bao-client-key.pem" creds
        && u.environment.BAO_CLIENT_CERT == "/run/credentials/swarm-nats-auth.service/bao-client.pem"
        && u.environment.BAO_CLIENT_KEY == "/run/credentials/swarm-nats-auth.service/bao-client-key.pem"
        && lib.hasPrefix "https://" u.environment.BAO_ADDR;
    }
    {
      # Copied so an absent leaf fails the unit rather than reaching the
      # responder as a file it cannot log in with. No seed is copied: the
      # responder reads every callout seed from the store.
      name = "the copy unit delivers the store identity and no seed, and nothing in its place when absent";
      ok =
        let
          s = natsWithStoreIdentity.systemd.services.swarm-nats-auth-secrets.script;
        in
        lib.hasInfix "install -m 0400 /etc/pki/nats-auth.pem " s
        && lib.hasInfix "install -m 0400 /etc/pki/nats-auth-key.pem " s
        && !(lib.hasInfix "/dev/null" s)
        && !(lib.hasInfix ".seed" s);
    }
    {
      # The responder names no seed file, by flag or by credential, and the
      # copies a container may still hold are removed. The OIDC secret is the
      # control that the credential list is read at all.
      name = "the responder takes no seed from a file, and the container's seed copies are removed";
      ok =
        let
          u = responderOf natsWithStoreIdentity;
          creds = u.serviceConfig.LoadCredential;
          dir = "/var/lib/nixos-containers/swarm-nats/var/lib/swarm-nats-auth";
          rules = natsWithStoreIdentity.systemd.tmpfiles.rules;
        in
        lib.elem "oidc-client.secret:/var/lib/swarm-nats-auth/oidc-client.secret" creds
        && !(lib.any (lib.hasInfix ".seed") creds)
        && !(lib.hasInfix "seed-file" u.serviceConfig.ExecStart)
        && lib.elem "r ${dir}/issuer.seed" rules
        && lib.elem "r ${dir}/callout-user.seed" rules;
    }
    {
      # Under either namespace, each removed option fails evaluation naming
      # itself and the store. The control is a queue host that sets none.
      name = "a definition of a removed callout option fails evaluation, naming the option and the secret store";
      ok =
        let
          refused =
            m: path:
            lib.any (
              a:
              !a.assertion
              && lib.hasInfix "services.hyperhive.${path}" a.message
              && lib.hasInfix "live in the swarm secret store" a.message
            ) m.assertions;
          removedOptions = {
            autoGenerateCallout = true;
            calloutUserPublicKey = "U";
            calloutIssuerPublicKey = "A";
            calloutUserSeedFile = "/run/u.seed";
            calloutIssuerSeedFile = "/run/i.seed";
          };
          setting = lib.listToAttrs (
            lib.concatMap
              (
                ns:
                lib.mapAttrsToList (name: value: {
                  name = "${ns}.nats.${name}";
                  value = hive { ${ns}.nats.${name} = value; };
                }) removedOptions
              )
              [
                "deploy"
                "swarm"
              ]
          );
        in
        lib.length (lib.attrNames setting) == 10
        && lib.all (path: refused setting.${path} path) (lib.attrNames setting)
        && !(lib.any (path: refused natsWithStoreIdentity path) (lib.attrNames setting));
    }
    {
      # The responder's OIDC secret has one route, the store, under the
      # responder's own leaf. The copy unit's arms are the control: on a queue
      # host without authelia nothing reads authelia's tree, and nothing
      # copies the secret from any host file.
      name = "a queue host without authelia reads the responder's OIDC secret from the store, and copies none from a host file";
      ok =
        let
          services = natsWithStoreIdentity.systemd.services;
          fetch = services.swarm-bao-nats-auth-oidc;
          copy = services.swarm-nats-auth-secrets.script;
        in
        !natsWithStoreIdentity.services.hyperhive.deploy.authelia.enable
        && fetch.environment.BAO_CLIENT_CERT == "/etc/pki/nats-auth.pem"
        && fetch.environment.BAO_CLIENT_KEY == "/etc/pki/nats-auth-key.pem"
        && lib.hasInfix "bao kv get -field=value secret/swarm/services/swarm-nats/oidc/client" fetch.script
        && lib.hasInfix "/var/lib/nixos-containers/swarm-nats/var/lib/swarm-nats-auth/oidc-client.secret" fetch.script
        && lib.hasInfix "refresh_consumer swarm-nats swarm-nats-auth.service" fetch.script
        # An unpublished or empty secret fails the unit, so `storeRetry` retries it.
        && !(lib.hasInfix "exit 0" fetch.script)
        && lib.elem "container@swarm-nats.service" fetch.before
        && !(lib.hasInfix "oidc-client" copy)
        && !(lib.hasInfix "authelia" copy);
    }
    {
      # On the store's host the fetch logs in under the responder's leaf, so it
      # waits for the unit writing that role.
      name = "on the store's host the responder's OIDC secret fetch uses its leaf and waits for its role";
      ok =
        let
          u = natsOnStoreHost.systemd.services.swarm-bao-nats-auth-oidc;
        in
        u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats-auth.pem"
        && lib.elem "swarm-bao-nats-auth-policy.service" u.after
        && lib.elem "swarm-bao-nats-auth-policy.service" u.wants;
    }
    {
      # On the store's host the glue pairs the responder with a leaf of its
      # own, never the queue's TLS-issuing one or the hive's.
      name = "on the store's host the responder presents its own leaf";
      ok =
        let
          n = natsOnStoreHost.services.hyperhive.deploy.nats;
          b = natsOnStoreHost.services.hyperhive.deploy.bao;
        in
        n.authBaoClientCertFile == "/var/lib/swarm-bao-pki/nats-auth.pem"
        && n.authBaoClientKeyFile == "/var/lib/swarm-bao-pki/nats-auth-key.pem"
        && n.authBaoClientCertFile != n.baoClientCertFile
        && n.authBaoClientCertFile != b.clientCertFile;
    }
    {
      # The default is the file the minting unit writes, wherever the hive's
      # own reader is pointed.
      name = "on the store's host the responder's leaf is the one minted there, not one beside the hive's reader";
      ok =
        let
          n = natsOnStoreHostReaderMoved.services.hyperhive.deploy.nats;
          mint = natsOnStoreHostReaderMoved.systemd.services.swarm-bao-pki.script;
        in
        n.authBaoClientCertFile == "/var/lib/swarm-bao-pki/nats-auth.pem"
        && n.authBaoClientKeyFile == "/var/lib/swarm-bao-pki/nats-auth-key.pem"
        && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats-auth.pem ] ||" mint
        && lib.hasInfix "/var/lib/swarm-bao-pki nats-auth " mint;
    }
    {
      # Not a rename test. `hostClientSecretDir` is `readOnly`, so the fixture
      # cannot define it; what can break is a reader left pointing at
      # `swarm.authelia` instead of `deploy.authelia`. Several modules read this
      # through an `autheliaCfg` alias, where a path-shaped grep does not see it.
      name = "a consumer of authelia's host client-secret dir renders it from the deploy namespace";
      ok = lib.hasPrefix "/var/lib/nixos-containers/swarm-authelia/var/lib/authelia-swarm/oidc-clients/" autheliaOldPath.services.hyperhive.otel.clientSecretFile;
    }
    {
      # authelia exits at startup on `users: {}`, so an empty database, and a
      # file holding exactly that document, must be replaced by the disabled
      # placeholder rather than left for authelia to read.
      name = "an empty authelia users database is seeded with the disabled placeholder";
      ok =
        let
          s =
            autheliaColocated.containers.swarm-authelia.config.systemd.services.authelia-swarm-secrets.script;
        in
        lib.hasInfix ''if [ ! -s "$users" ] || [ "$(cat "$users")" = "users: {}" ]; then'' s
        && lib.hasInfix "-authelia-users-placeholder.yml \"$users\"" s
        && !(lib.hasInfix ''echo "users: {}"'' s);
    }
    {
      # The bridge refuses every client but the one named here, so this has
      # to be the client the controller authenticates as. A drift is a
      # controller whose every bridge call is refused.
      name = "the authelia bridge answers the controller's client only";
      ok =
        let
          env =
            autheliaColocated.containers.swarm-authelia.config.systemd.services.swarm-authelia-bridge.environment;
        in
        env.SWARM_AUTHELIA_BRIDGE_CONTROLLER_CLIENT_ID
        == autheliaColocated.services.hyperhive.swarm.controller.queueClientId
        && env.SWARM_AUTHELIA_BRIDGE_OPERATOR_GROUP == "admins";
    }
    {
      # One address from every host, and a location the IdP host serves: a
      # controller off the authelia host has no other way to the bridge.
      name = "the auth bridge address is the authelia vhost's /bridge/ on every host";
      ok =
        let
          url = h: h.services.hyperhive.deploy.swarm-controller.authBridgeUrl;
          authelia = autheliaColocated.services.hyperhive.swarm.authelia;
          location = autheliaColocated.services.nginx.virtualHosts.${authelia.domain}.locations."/bridge/";
        in
        url autheliaNotColocated == "${autheliaNotColocated.services.hyperhive.swarm.authelia.url}/bridge"
        && url autheliaColocated == url autheliaNotColocated
        && location.proxyPass == "http://127.0.0.1:${toString authelia.bridgePort}/";
    }
    {
      # Config NAMES the IdP; it never computes where the IdP is. Both arms
      # matter together: the address is the swarm's name on a hive that runs
      # nothing, and it is the SAME string on the hive that serves the vhost —
      # so a re-introduced co-location branch shows up as the two disagreeing
      # rather than as a value that merely looks plausible on one of them.
      name = "the swarm IdP address is its domain on every hive, co-located or not";
      ok =
        autheliaNotColocated.services.hyperhive.swarm.authelia.url
        == "https://${autheliaNotColocated.services.hyperhive.swarm.authelia.domain}"
        &&
          autheliaColocated.services.hyperhive.swarm.authelia.url
          == autheliaNotColocated.services.hyperhive.swarm.authelia.url;
    }
    {
      # …and the name it resolves through is a DOMAIN, not a host this config
      # picked. `127.0.0.1` is what a co-location-derived default renders on
      # the IdP host, the bridge address is what the queue's defaults render,
      # and neither is an address a client may be handed: the domain is
      # allowed to resolve differently in different places, which is the
      # whole property.
      name = "the swarm IdP address names no host address";
      ok =
        let
          url = autheliaNotColocated.services.hyperhive.swarm.authelia.url;
          bridgeIp = autheliaNotColocated.services.hyperhive.network.bridgeIp;
        in
        !(lib.hasInfix "127.0.0.1" url) && !(lib.hasInfix "localhost" url) && !(lib.hasInfix bridgeIp url);
    }
  ];
in
runGroup "nats-authelia" cases