hyperhive/nix/module-eval/bao-matrix-reader.nix

# `checks.module-eval-bao-matrix-reader` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    ;

  # The store and a service that reads from it, versus the store alone. The
  # pair is what makes the reader's absence arm mean anything.
  baoWithMatrix = hive {
    deploy.bao.enable = true;
    deploy.matrix.enable = true;
  };

  # A hive that reads from a store it does not run: no `deploy.bao.enable`, so
  # nothing here mints a leaf and the operator names one placed by hand. The
  # deployment this pairing exists to serve, and the one that was previously
  # inexpressible — the gate asked whether the store was a neighbour.
  #
  # 🩸 One leaf PER READER, and this fixture is where that cost is visible: the
  # two units below identify themselves to the store separately, so an operator
  # placing leaves by hand places one for each rather than one for both. Naming
  # only `clientCertFile` here would leave neither unit rendered — which is what
  # the arms below would then be asserting about.
  baoRemoteReader = hive {
    deploy.matrix.enable = true;
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
    deploy.bao.matrixTokenClientCertFile = "/etc/pki/bao-matrix-token.pem";
    deploy.bao.matrixTokenClientKeyFile = "/etc/pki/bao-matrix-token-key.pem";
    deploy.bao.queueAgentClientCertFile = "/etc/pki/bao-queue-agent.pem";
    deploy.bao.queueAgentClientKeyFile = "/etc/pki/bao-queue-agent-key.pem";
  };

  # A homeserver on a hive with NO store identity at all — neither a local
  # store nor a hand-placed leaf. The absence arm for the matrix-ctl cases below
  # needs it, and defining it here rather than importing keeps each group's
  # fixture set its own, as ./lib.nix asks.
  matrixNoBaoIdentity = hive { deploy.matrix.enable = true; };

  # 🩸 The hand-configured remote reader that named seven of the eight options.
  # `baoRemoteReader` above is the same deployment done right; this one holds
  # the hive's own leaf, so it demonstrably reads the store, and is missing both
  # per-principal pairs. Before the four-way split this host rendered both units
  # off `clientCertFile` alone, so what it has now is a silent regression rather
  # than any shape an operator chose — which is what the refusal arms below are
  # about. Kept as one fixture rather than two because both refusals fire on it
  # and each arm names which.
  baoRemoteReaderMissingLeaves = hive {
    deploy.matrix.enable = true;
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
  };

  # The same omission on a hive that does NOT run a homeserver. Separates the
  # matrix refusal's `deploy.matrix.enable` clause from the queue refusal, which
  # checks only the hive's own toggle — an arm below reads exactly one refusal
  # off it.
  remoteReaderNoMatrixMissingLeaves = hive {
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
  };

  # Did a module refuse this host, and over which option. An assertion is a
  # config VALUE until something forces it — `.config` never throws — so a
  # fixture in a state the module refuses stays evaluable and the refusal reads
  # back as data. Matched on the option name the message names rather than on
  # its prose, because the option name is the part an operator has to act on
  # and a message that stopped naming it would be the actual defect. Same shape
  # as ./grafana.nix's `grafanaRefusedFor`, which reads `swarm-grafana.nix`'s
  # own refusal.
  refusedOver = m: option: lib.any (a: !a.assertion && lib.hasInfix option a.message) m.assertions;
  cases = [
    {
      # A login failure is the store being unreachable, sealed, or not yet
      # holding this host's role — all of which a retry fixes. A read that
      # answers "nothing there" is not, so only the first is allowed to fail
      # the unit.
      name = "the matrix token reader retries a failed login and still degrades on an empty read";
      ok =
        let
          u = baoWithMatrix.systemd.services.swarm-bao-matrix-token;
          # Everything between the login's failure branch and the read's, which
          # is where the exit that decides "retry or give up" lives.
          afterLogin = lib.last (lib.splitString "bao login" u.script);
          loginBranch = lib.head (lib.splitString "bao kv get" afterLogin);
        in
        u.serviceConfig.Restart or null == "on-failure"
        && u.startLimitBurst or 0 > 0
        # The window has to outlast every attempt, or the burst is unreachable.
        && u.startLimitIntervalSec or 0 > (u.serviceConfig.RestartSec or 0) * (u.startLimitBurst or 0)
        && lib.hasInfix "exit 1" loginBranch
        && lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script));
    }
    {
      # The reader's own grant covers `swarm/hives/<this hive>/*` and
      # `swarm/agents/*`; a path outside those answers 403, not "no such key".
      # So the hive segment is what makes the read reachable, and a rename that
      # drops it looks correct and fails identically on every boot.
      name = "the matrix token path sits inside the prefix the reader is granted";
      ok =
        let
          s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script;
        in
        lib.hasInfix "secret/swarm/hives/" s
        && lib.hasInfix "/matrix/appservice-token" s
        # The shape it used to have: `matrix` where a principal kind belongs,
        # which no grant covers.
        && !(lib.hasInfix "secret/swarm/matrix/" s);
    }
    {
      # The store's second reader, and the gate that decides it exists is the
      # certificate rather than anything about agents: containers are created
      # at runtime, so there is no static "this hive runs agents" fact to ask.
      name = "a hive that names a client identity reads its agent queue credential";
      ok = baoRemoteReader.systemd.services ? swarm-bao-queue-agent;
    }
    {
      # Same 403-not-a-miss reason as the matrix arm above, against the path
      # `swarm_secret_client::queue::agent_client_path` builds from the same
      # pieces. The negative arm is the rename this one is exposed to: a
      # credential named for the queue rather than for the hive that presents
      # it reads as correct and is refused on every boot.
      name = "the agent queue credential path sits inside the prefix the reader is granted";
      ok =
        let
          s = baoRemoteReader.systemd.services.swarm-bao-queue-agent.script;
        in
        lib.hasInfix "secret/swarm/hives/h1/queue/agent" s && !(lib.hasInfix "secret/swarm/queue/" s);
    }
    {
      # The unit's output is the option's value, not a literal that agrees with
      # it today: an operator moving the directory has to move both files. The
      # prefix is asserted too because `hasInfix ""` is true — an option
      # renamed out from under this arm would otherwise read empty and pass.
      name = "the queue credential reader writes both files under the directory its option names";
      ok =
        let
          m = baoRemoteReader;
          dir = toString m.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir;
          s = m.systemd.services.swarm-bao-queue-agent.script;
        in
        lib.hasPrefix "/var/lib/" dir
        && lib.hasInfix "${dir}/secret" s
        && lib.hasInfix "${dir}/client_id" s;
    }
    {
      # No agent container may render before this unit has had its attempts,
      # and the edge that guarantees it must delay hive-c0re rather than sink
      # it: an unreachable store is this unit's `Restart=on-failure` window,
      # not a reason for the daemon that renders every agent to fail its own
      # start.
      name = "the queue credential reader orders before hive-c0re and is wanted, not required, by it";
      ok =
        let
          u = baoRemoteReader.systemd.services.swarm-bao-queue-agent;
        in
        builtins.elem "hive-c0re.service" (u.before or [ ])
        && builtins.elem "hive-c0re.service" (u.wantedBy or [ ])
        && !(builtins.elem "hive-c0re.service" (u.requiredBy or [ ]))
        && !(builtins.elem "hive-c0re.service" (u.requires or [ ]));
    }
    {
      # The store's first reader. Its unit belongs to the pairing, not to
      # either service: matrix must not learn the store exists, and the store
      # must not know who reads it.
      name = "a store deployed beside the homeserver fetches its appservice token";
      ok = baoWithMatrix.systemd.services ? swarm-bao-matrix-token;
    }
    {
      name = "a hive that names a client identity reads from a store it does not run";
      ok = baoRemoteReader.systemd.services ? swarm-bao-matrix-token;
    }
    {
      # `Requires=` on a unit that does not exist fails the job, and nothing
      # local mints certificates off-host — so this orders against nothing.
      # Eval cannot see that failure; only the empty list here stands in for it.
      name = "an off-host reader requires no unit the store's host would have provided";
      # Membership first, then the value: indexing a missing unit throws, and a
      # table that reports which property broke must not be the thing that dies.
      ok =
        let
          s = baoRemoteReader.systemd.services;
        in
        s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ ];
    }
    {
      # Presence control for the case above: the list is conditional, not gone.
      name = "a co-located reader still orders after the local pki unit";
      ok =
        let
          s = baoWithMatrix.systemd.services;
        in
        s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ "swarm-bao-pki.service" ];
    }
    {
      # Nothing asserted this script before, which is how it kept a branch that
      # named three states and threw away the only thing telling them apart. A
      # missing value, a refused identity and an unreachable host all end in the
      # same degraded mode here, correctly — what must survive is which one.
      name = "the matrix token reader carries the store's own diagnostic into the journal";
      ok =
        let
          s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script;
        in
        !(lib.hasInfix "2>/dev/null" s) && lib.hasInfix ''cat "''$err"'' s;
    }
    {
      # hive-c0re runs as hive-core and the client key is `0600` root-owned
      # inside a `0700` directory, so the identity reaches the daemon as a
      # systemd credential and the environment names `%d` rather than the
      # file. Both halves are asserted together because either alone is a
      # daemon that fails at the TLS handshake, naming neither.
      name = "a reader hands hive-c0re a store identity the daemon cannot open itself";
      ok =
        let
          s = baoRemoteReader.systemd.services;
        in
        s ? hive-c0re
        && (s.hive-c0re.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem"
        && (s.hive-c0re.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem"
        && builtins.elem "bao-client.pem:/etc/pki/bao-client.pem" s.hive-c0re.serviceConfig.LoadCredential
        && builtins.elem "bao-client-key.pem:/etc/pki/bao-client-key.pem" s.hive-c0re.serviceConfig.LoadCredential;
    }
    {
      # The CA is its own arm: absent means the system trust store, which is
      # right for a deployment with a real CA and wrong for a self-signed one.
      name = "a reader that names no store CA falls through to the system trust store";
      ok =
        let
          s = baoRemoteReader.systemd.services;
        in
        s ? hive-c0re && !(s.hive-c0re.environment ? BAO_CACERT);
    }
    {
      # Presence control for the arm above: the CA is conditional, not gone.
      # Co-located, ./host-modules/glue-bao-tls.nix mints one and names it.
      name = "a reader beside a self-signed store is given that store's CA";
      ok =
        let
          s = baoWithMatrix.systemd.services;
        in
        s ? hive-c0re
        && (s.hive-c0re.environment.BAO_CACERT or null) == "%d/bao-ca.pem"
        && lib.any (c: lib.hasPrefix "bao-ca.pem:" c) s.hive-c0re.serviceConfig.LoadCredential;
    }
    {
      # The same hole a third time, and the leaf whose absence is hardest to
      # see from outside: it is consumed by a unit INSIDE a container, so a
      # missing pairing renders as a container that comes up fine and publishes
      # nothing.
      name = "the store mints a leaf for matrix-ctl, and the container is pointed at it";
      ok =
        let
          m = baoWithMatrix;
          p = m.services.hyperhive.deploy.matrix;
        in
        lib.hasInfix "matrix-ctl.pem" m.systemd.services.swarm-bao-pki.script
        && p.ctlBaoClientCertFile == "/var/lib/swarm-bao-pki/matrix-ctl.pem"
        && p.ctlBaoClientKeyFile == "/var/lib/swarm-bao-pki/matrix-ctl-key.pem";
    }
    {
      # 🩸 The identity separation this whole arrangement buys, stated as the
      # one thing that would silently undo it. The container gets MATRIX-CTL's
      # leaf — whose grant is a single path — and not the hive's, which reads
      # every secret in the store. Both files exist in the same directory and
      # both would evaluate, deploy and work.
      name = "matrix-ctl presents its own leaf, never the hive's store-wide one";
      ok =
        let
          env =
            baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-publish.environment;
          hiveLeaf = baoWithMatrix.services.hyperhive.deploy.bao.clientCertFile;
        in
        env.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/matrix-ctl.pem" && env.BAO_CLIENT_CERT != hiveLeaf;
    }
    {
      # The bind mount is what makes the environment above resolvable: without
      # it the unit names two paths the container does not have, and fails at
      # the TLS handshake naming no cause. Read off the mount table rather than
      # the option, so a pairing that stops reaching `bindMounts` still fails.
      #
      # The second arm is the shape guard: `bindMounts` is one literal plus two
      # merges, and a rewrite that dropped the appservice registration would
      # take the homeserver's own credential with it.
      name = "the matrix container binds matrix-ctl's PKI read-only, without losing the appservice registration";
      ok =
        let
          mounts = baoWithMatrix.containers.hive-matrix.bindMounts;
        in
        mounts ? "/var/lib/swarm-bao-pki"
        && mounts."/var/lib/swarm-bao-pki".isReadOnly
        && mounts ? "/var/lib/hyperhive/matrix-appservice";
    }
    {
      # The cert role ./host-modules/swarm-bao.nix writes, which is the one
      # agreement the unit cannot get wrong, and the verb: a bare invocation
      # exits non-zero with clap's usage, a deploy-time failure with no local
      # signal.
      name = "the publish unit is handed the store role and invokes its verb";
      ok =
        let
          u = baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-publish;
        in
        u.environment.MATRIX_APPSERVICE_CERT_ROLE == "swarm-matrix-ctl"
        && lib.hasSuffix "/bin/swarm-matrix-ctl appservice publish" u.serviceConfig.ExecStart;
    }
    {
      # 🩸 A secret is a path, never a value. Every variable the unit is given
      # names a file or an address; the token itself is read out of the state
      # dir at runtime, so nothing here can be a token and an environment block
      # is world-readable through `systemctl show`.
      name = "the publish unit's environment carries paths and addresses, never a token";
      ok =
        let
          env =
            baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-publish.environment;
        in
        !(lib.any (v: lib.hasInfix "as_token" v || lib.hasInfix "syt_" v) (lib.attrValues env));
    }
    {
      # The absence arm, and the deployment it protects: a homeserver on a hive
      # with no store identity at all. Without it the mount would name `null`
      # as its source, which nixos renders as the literal string.
      name = "a matrix container with no store identity binds no PKI";
      ok = !(matrixNoBaoIdentity.containers.hive-matrix.bindMounts ? "/var/lib/swarm-bao-pki");
    }
    {
      # 🩸 The privilege arm: exactly one account is a homeserver admin, the
      # swarm appservice's sender, whose token only matrix-ctl and the
      # controller read. Read on the rendered settings rather than on an
      # option, because the grant is a boot command in `admin_execute`, and a
      # command list is exactly the shape a later edit extends without anything
      # noticing — so the list is compared whole, and a second entry fails.
      name = "the homeserver promotes exactly the swarm sender to admin at boot, and no hive's sender";
      ok =
        let
          g = baoWithMatrix.containers.hive-matrix.config.services.matrix-tuwunel.settings.global;
        in
        g.admin_execute == [ "users make-user-admin @swarm:${g.server_name}" ]
        && !(lib.any (c: lib.hasInfix "@hive-" c) g.admin_execute);
    }
    {
      # Load-bearing rather than lenient: tuwunel aborts startup on a failing
      # `admin_execute` command when this is false, and the homeserver must not
      # stay down over a promotion.
      name = "a failed admin promotion does not stop the homeserver";
      ok =
        baoWithMatrix.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.admin_execute_errors_ignore
          or false;
    }
    {
      # The swarm registration reaches tuwunel the way the hive's does: a
      # `.yaml` credential in the directory `appservice_dir` names.
      name = "tuwunel loads the swarm registration as a yaml credential";
      ok = lib.elem "swarm-appservice.yaml:/var/lib/swarm-matrix-appservice/swarm.yaml" (
        baoWithMatrix.containers.hive-matrix.config.systemd.services.tuwunel.serviceConfig.LoadCredential
      );
    }
    {
      # A missing `LoadCredential` source fails tuwunel, so the render has to
      # have run first — and it is local only, so it cannot fail on a store.
      name = "the swarm registration is rendered before tuwunel and required by it, without the store";
      ok =
        let
          r = baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-render;
        in
        lib.elem "tuwunel.service" r.before
        && lib.elem "tuwunel.service" r.requiredBy
        && lib.hasSuffix "swarm-matrix-ctl appservice render" r.serviceConfig.ExecStart
        && !(r.environment ? BAO_ADDR);
    }
    {
      # The absence arm for the four above: with no matrix-ctl identity there
      # is nobody to publish the swarm token, so no admin, no registration and
      # no render — rather than an admin credential nobody reads.
      name = "a matrix container with no store identity has no swarm appservice and promotes nobody";
      ok =
        let
          c = matrixNoBaoIdentity.containers.hive-matrix.config;
          g = c.services.matrix-tuwunel.settings.global;
        in
        !(g ? admin_execute)
        && !(c.systemd.services ? swarm-matrix-appservice-render)
        && !(c.systemd.services ? swarm-matrix-appservice-publish)
        && !(lib.any (lib.hasPrefix "swarm-appservice.yaml") c.systemd.services.tuwunel.serviceConfig.LoadCredential);
    }
    {
      # 🩸 The arm the whole refusal exists for. Both readers are gated on their
      # own leaf, so the way this regresses is the build going green and three
      # units rendering where four should — which no presence check on a
      # rendered unit can see, because the unit that is missing is the evidence.
      # Read as a refusal naming each option, so an operator acts on the message
      # without opening the nix.
      name = "a remote reader missing the per-principal leaves is refused, naming both options";
      ok =
        refusedOver baoRemoteReaderMissingLeaves "matrixTokenClientCertFile"
        && refusedOver baoRemoteReaderMissingLeaves "matrixTokenClientKeyFile"
        && refusedOver baoRemoteReaderMissingLeaves "queueAgentClientCertFile"
        && refusedOver baoRemoteReaderMissingLeaves "queueAgentClientKeyFile";
    }
    {
      # The matrix refusal's own gate, which the queue refusal does not have.
      # Without this arm the two are indistinguishable on the fixture above.
      name = "the queue refusal needs no homeserver, and the matrix refusal stays quiet without one";
      ok =
        refusedOver remoteReaderNoMatrixMissingLeaves "queueAgentClientCertFile"
        && !(refusedOver remoteReaderNoMatrixMissingLeaves "matrixTokenClientCertFile");
    }
    {
      # ⚠️ The arm that keeps the refusal from being worse than the silence it
      # replaced. A hive with NO store identity is the supported no-store
      # deployment and also the state an operator passes through bringing a hive
      # up — neither may fail to evaluate. Asserted as "no refusal names any of
      # the four options", not as "this one fixture is fine", because the way
      # this breaks is a gate widened to the principal's leaf alone.
      name = "a hive with no store identity at all is refused over none of the per-principal leaves";
      ok = lib.all (option: !(refusedOver matrixNoBaoIdentity option)) [
        "matrixTokenClientCertFile"
        "matrixTokenClientKeyFile"
        "queueAgentClientCertFile"
        "queueAgentClientKeyFile"
      ];
    }
    {
      # The other half of the same guard, and the one an operator meets far more
      # often: on the store's own host ./host-modules/glue-bao-tls.nix mkDefaults
      # all eight, so there is nothing to name and nothing to refuse. Paired with
      # the fully-named remote reader, which is the same deployment done by hand.
      name = "neither a store host nor a correctly-named remote reader is refused";
      ok =
        lib.all
          (
            m:
            lib.all (option: !(refusedOver m option)) [
              "matrixTokenClientCertFile"
              "queueAgentClientCertFile"
            ]
          )
          [
            baoWithMatrix
            baoRemoteReader
          ];
    }
  ];
in
runGroup "bao-matrix-reader" cases