hyperhive/nix/module-eval/bao-matrix-reader.nix

# `checks.module-eval-bao-matrix-reader` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    ;

  # The store and a service that reads from it, versus the store alone. The
  # pair is what makes the reader's absence arm mean anything.
  baoWithMatrix = hive {
    deploy.bao.enable = true;
    deploy.matrix.enable = true;
  };

  # A hive that reads from a store it does not run: no `deploy.bao.enable`, so
  # nothing here mints a leaf and the operator names one placed by hand. The
  # deployment this pairing exists to serve, and the one that was previously
  # inexpressible — the gate asked whether the store was a neighbour.
  #
  # 🩸 One leaf PER READER, and this fixture is where that cost is visible: the
  # two units below identify themselves to the store separately, so an operator
  # placing leaves by hand places one for each rather than one for both. Naming
  # only `clientCertFile` here would leave neither unit rendered — which is what
  # the arms below would then be asserting about.
  baoRemoteReader = hive {
    deploy.matrix.enable = true;
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
    deploy.bao.matrixTokenClientCertFile = "/etc/pki/bao-matrix-token.pem";
    deploy.bao.matrixTokenClientKeyFile = "/etc/pki/bao-matrix-token-key.pem";
    deploy.bao.queueAgentClientCertFile = "/etc/pki/bao-queue-agent.pem";
    deploy.bao.queueAgentClientKeyFile = "/etc/pki/bao-queue-agent-key.pem";
  };

  # A homeserver on a hive with NO store identity at all — neither a local
  # store nor a hand-placed leaf. The absence arm for the matrix-ctl cases below
  # needs it, and defining it here rather than importing keeps each group's
  # fixture set its own, as ./lib.nix asks.
  matrixNoBaoIdentity = hive { deploy.matrix.enable = true; };

  # 🩸 The hand-configured remote reader that named seven of the eight options.
  # `baoRemoteReader` above is the same deployment done right; this one holds
  # the hive's own leaf, so it demonstrably reads the store, and is missing both
  # per-principal pairs. Before the four-way split this host rendered both units
  # off `clientCertFile` alone, so what it has now is a silent regression rather
  # than any shape an operator chose — which is what the refusal arms below are
  # about. Kept as one fixture rather than two because both refusals fire on it
  # and each arm names which.
  baoRemoteReaderMissingLeaves = hive {
    deploy.matrix.enable = true;
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
  };

  # The same omission on a hive that does NOT run a homeserver. Separates the
  # matrix refusal's `deploy.matrix.enable` clause from the queue refusal, which
  # has no toggle to check — an arm below reads exactly one refusal off it.
  remoteReaderNoMatrixMissingLeaves = hive {
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
  };

  # Did a module refuse this host, and over which option. An assertion is a
  # config VALUE until something forces it — `.config` never throws — so a
  # fixture in a state the module refuses stays evaluable and the refusal reads
  # back as data. Matched on the option name the message names rather than on
  # its prose, because the option name is the part an operator has to act on
  # and a message that stopped naming it would be the actual defect. The same
  # helper ./grafana.nix uses for `swarm-grafana.nix`'s own refusal.
  refusedOver = m: option: lib.any (a: !a.assertion && lib.hasInfix option a.message) m.assertions;
  cases = [
    {
      # A login failure is the store being unreachable, sealed, or not yet
      # holding this host's role — all of which a retry fixes. A read that
      # answers "nothing there" is not, so only the first is allowed to fail
      # the unit.
      name = "the matrix token reader retries a failed login and still degrades on an empty read";
      ok =
        let
          u = baoWithMatrix.systemd.services.swarm-bao-matrix-token;
          # Everything between the login's failure branch and the read's, which
          # is where the exit that decides "retry or give up" lives.
          afterLogin = lib.last (lib.splitString "bao login" u.script);
          loginBranch = lib.head (lib.splitString "bao kv get" afterLogin);
        in
        u.serviceConfig.Restart or null == "on-failure"
        && u.startLimitBurst or 0 > 0
        # The window has to outlast every attempt, or the burst is unreachable.
        && u.startLimitIntervalSec or 0 > (u.serviceConfig.RestartSec or 0) * (u.startLimitBurst or 0)
        && lib.hasInfix "exit 1" loginBranch
        && lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script));
    }
    {
      # The reader's own grant covers `swarm/hives/<this hive>/*` and
      # `swarm/agents/*`; a path outside those answers 403, not "no such key".
      # So the hive segment is what makes the read reachable, and a rename that
      # drops it looks correct and fails identically on every boot.
      name = "the matrix token path sits inside the prefix the reader is granted";
      ok =
        let
          s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script;
        in
        lib.hasInfix "secret/swarm/hives/" s
        && lib.hasInfix "/matrix/appservice-token" s
        # The shape it used to have: `matrix` where a principal kind belongs,
        # which no grant covers.
        && !(lib.hasInfix "secret/swarm/matrix/" s);
    }
    {
      # The store's second reader, and the gate that decides it exists is the
      # certificate rather than anything about agents: containers are created
      # at runtime, so there is no static "this hive runs agents" fact to ask.
      name = "a hive that names a client identity reads its agent queue credential";
      ok = baoRemoteReader.systemd.services ? swarm-bao-queue-agent;
    }
    {
      # Same 403-not-a-miss reason as the matrix arm above, against the path
      # `swarm_secret_client::queue::agent_client_path` builds from the same
      # pieces. The negative arm is the rename this one is exposed to: a
      # credential named for the queue rather than for the hive that presents
      # it reads as correct and is refused on every boot.
      name = "the agent queue credential path sits inside the prefix the reader is granted";
      ok =
        let
          s = baoRemoteReader.systemd.services.swarm-bao-queue-agent.script;
        in
        lib.hasInfix "secret/swarm/hives/h1/queue/agent" s && !(lib.hasInfix "secret/swarm/queue/" s);
    }
    {
      # The unit's output is the option's value, not a literal that agrees with
      # it today: an operator moving the directory has to move both files. The
      # prefix is asserted too because `hasInfix ""` is true — an option
      # renamed out from under this arm would otherwise read empty and pass.
      name = "the queue credential reader writes both files under the directory its option names";
      ok =
        let
          m = baoRemoteReader;
          dir = toString m.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir;
          s = m.systemd.services.swarm-bao-queue-agent.script;
        in
        lib.hasPrefix "/var/lib/" dir
        && lib.hasInfix "${dir}/secret" s
        && lib.hasInfix "${dir}/client_id" s;
    }
    {
      # A reader off the store's host is a reader whose journal is the only
      # record of why a hive's agents never connected, so the collector has to
      # be told the unit exists. Nothing else can say it: the store's module
      # does not know who holds a certificate.
      name = "the queue credential reader's journal reaches the collector";
      ok = builtins.elem "swarm-bao-queue-agent" baoRemoteReader.services.hyperhive.swarm.otel.journaldUnits;
    }
    {
      # No agent container may render before this unit has had its attempts,
      # and the edge that guarantees it must delay hive-c0re rather than sink
      # it: an unreachable store is this unit's `Restart=on-failure` window,
      # not a reason for the daemon that renders every agent to fail its own
      # start.
      name = "the queue credential reader orders before hive-c0re and is wanted, not required, by it";
      ok =
        let
          u = baoRemoteReader.systemd.services.swarm-bao-queue-agent;
        in
        builtins.elem "hive-c0re.service" (u.before or [ ])
        && builtins.elem "hive-c0re.service" (u.wantedBy or [ ])
        && !(builtins.elem "hive-c0re.service" (u.requiredBy or [ ]))
        && !(builtins.elem "hive-c0re.service" (u.requires or [ ]));
    }
    {
      # The store's first reader. Its unit belongs to the pairing, not to
      # either service: matrix must not learn the store exists, and the store
      # must not know who reads it.
      name = "a store deployed beside the homeserver fetches its appservice token";
      ok = baoWithMatrix.systemd.services ? swarm-bao-matrix-token;
    }
    {
      name = "a hive that names a client identity reads from a store it does not run";
      ok = baoRemoteReader.systemd.services ? swarm-bao-matrix-token;
    }
    {
      # `Requires=` on a unit that does not exist fails the job, and nothing
      # local mints certificates off-host — so this orders against nothing.
      # Eval cannot see that failure; only the empty list here stands in for it.
      name = "an off-host reader requires no unit the store's host would have provided";
      # Membership first, then the value: indexing a missing unit throws, and a
      # table that reports which property broke must not be the thing that dies.
      ok =
        let
          s = baoRemoteReader.systemd.services;
        in
        s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ ];
    }
    {
      # Presence control for the case above: the list is conditional, not gone.
      name = "a co-located reader still orders after the local pki unit";
      ok =
        let
          s = baoWithMatrix.systemd.services;
        in
        s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ "swarm-bao-pki.service" ];
    }
    {
      # Nothing asserted this script before, which is how it kept a branch that
      # named three states and threw away the only thing telling them apart. A
      # missing value, a refused identity and an unreachable host all end in the
      # same degraded mode here, correctly — what must survive is which one.
      name = "the matrix token reader carries the store's own diagnostic into the journal";
      ok =
        let
          s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script;
        in
        !(lib.hasInfix "2>/dev/null" s) && lib.hasInfix ''cat "''$err"'' s;
    }
    {
      # hive-c0re runs as hive-core and the client key is `0600` root-owned
      # inside a `0700` directory, so the identity reaches the daemon as a
      # systemd credential and the environment names `%d` rather than the
      # file. Both halves are asserted together because either alone is a
      # daemon that fails at the TLS handshake, naming neither.
      name = "a reader hands hive-c0re a store identity the daemon cannot open itself";
      ok =
        let
          s = baoRemoteReader.systemd.services;
        in
        s ? hive-c0re
        && (s.hive-c0re.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem"
        && (s.hive-c0re.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem"
        && builtins.elem "bao-client.pem:/etc/pki/bao-client.pem" s.hive-c0re.serviceConfig.LoadCredential
        && builtins.elem "bao-client-key.pem:/etc/pki/bao-client-key.pem" s.hive-c0re.serviceConfig.LoadCredential;
    }
    {
      # The CA is its own arm: absent means the system trust store, which is
      # right for a deployment with a real CA and wrong for a self-signed one.
      name = "a reader that names no store CA falls through to the system trust store";
      ok =
        let
          s = baoRemoteReader.systemd.services;
        in
        s ? hive-c0re && !(s.hive-c0re.environment ? BAO_CACERT);
    }
    {
      # Presence control for the arm above: the CA is conditional, not gone.
      # Co-located, ./host-modules/glue-bao-tls.nix mints one and names it.
      name = "a reader beside a self-signed store is given that store's CA";
      ok =
        let
          s = baoWithMatrix.systemd.services;
        in
        s ? hive-c0re
        && (s.hive-c0re.environment.BAO_CACERT or null) == "%d/bao-ca.pem"
        && lib.any (c: lib.hasPrefix "bao-ca.pem:" c) s.hive-c0re.serviceConfig.LoadCredential;
    }
    {
      # The same hole a third time, and the leaf whose absence is hardest to
      # see from outside: it is consumed by a unit INSIDE a container, so a
      # missing pairing renders as a container that comes up fine and publishes
      # nothing.
      name = "the store mints a leaf for matrix-ctl, and the container is pointed at it";
      ok =
        let
          m = baoWithMatrix;
          p = m.services.hyperhive.deploy.matrix;
        in
        lib.hasInfix "matrix-ctl.pem" m.systemd.services.swarm-bao-pki.script
        && p.ctlBaoClientCertFile == "/var/lib/swarm-bao-pki/matrix-ctl.pem"
        && p.ctlBaoClientKeyFile == "/var/lib/swarm-bao-pki/matrix-ctl-key.pem";
    }
    {
      # 🩸 The identity separation this whole arrangement buys, stated as the
      # one thing that would silently undo it. The container gets MATRIX-CTL's
      # leaf — whose grant is a single path — and not the hive's, which reads
      # every secret in the store. Both files exist in the same directory and
      # both would evaluate, deploy and work.
      name = "matrix-ctl presents its own leaf, never the hive's store-wide one";
      ok =
        let
          env = baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-ctl.environment;
          hiveLeaf = baoWithMatrix.services.hyperhive.deploy.bao.clientCertFile;
        in
        env.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/matrix-ctl.pem" && env.BAO_CLIENT_CERT != hiveLeaf;
    }
    {
      # The bind mount is what makes the environment above resolvable: without
      # it the unit names two paths the container does not have, and fails at
      # the TLS handshake naming no cause. Read off the mount table rather than
      # the option, so a pairing that stops reaching `bindMounts` still fails.
      #
      # The second arm is the shape guard: `bindMounts` is one literal plus two
      # merges, and a rewrite that dropped the appservice registration would
      # take the homeserver's own credential with it.
      name = "the matrix container binds matrix-ctl's PKI read-only, without losing the appservice registration";
      ok =
        let
          mounts = baoWithMatrix.containers.hive-matrix.bindMounts;
        in
        mounts ? "/var/lib/swarm-bao-pki"
        && mounts."/var/lib/swarm-bao-pki".isReadOnly
        && mounts ? "/var/lib/hyperhive/matrix-appservice";
    }
    {
      # What the unit is for, read as the two agreements it cannot get wrong:
      # the cert role ./host-modules/swarm-bao.nix writes, and a homeserver
      # address that is loopback because the container shares the host netns. A
      # vhost here would be a request out through the gateway and back.
      name = "matrix-ctl is handed the store role and the loopback homeserver";
      ok =
        let
          m = baoWithMatrix;
          u = m.containers.hive-matrix.config.systemd.services.swarm-matrix-ctl;
          port = m.services.hyperhive.swarm.matrix.httpPort;
        in
        u.environment.MATRIX_MINT_CERT_ROLE == "swarm-matrix-ctl"
        && u.environment.MATRIX_MINT_API_URL == "http://127.0.0.1:${toString port}"
        && u.environment.MATRIX_MINT_REGISTRATION == "/var/lib/hyperhive/matrix-appservice/hyperhive.yaml"
        && u.serviceConfig.Type == "oneshot";
    }
    {
      # 🩸 The per-hive minting identity, read off the rendered unit rather than
      # off the option: the binary builds its store path out of
      # `MATRIX_MINT_HIVE` and logs in as `MATRIX_MINT_LOCALPART`, so a unit
      # that passed the old bare `hive` would publish one identity for the
      # whole swarm again and nothing in the Rust tests could see it. Both
      # spellings are pinned, and the localpart is pinned as *derived from* the
      # hive name rather than as a literal, which is the agreement
      # `swarm_secret_client::matrix::hive_localpart` owns.
      name = "matrix-ctl is told which hive it mints for, and acts as that hive's account";
      ok =
        let
          m = baoWithMatrix;
          u = m.containers.hive-matrix.config.systemd.services.swarm-matrix-ctl;
          hive = m.services.hyperhive.hiveName;
        in
        u.environment.MATRIX_MINT_HIVE == hive
        && u.environment.MATRIX_MINT_LOCALPART == "hive-${hive}"
        && u.environment.MATRIX_MINT_LOCALPART != "hive";
    }
    {
      # 🩸 The crate is a `*ctl` with subcommands, so the unit has to name a
      # VERB. This is the one end of that contract nix owns: the binary's own
      # test pins how `mint` is spelled, but only a rendered `ExecStart` can
      # say the unit actually passes it. A bare invocation exits non-zero with
      # clap's usage — which is a deploy-time failure with no local signal, and
      # exactly what the next verb added here is most likely to disturb.
      name = "the unit invokes a verb rather than the bare binary";
      ok =
        let
          exec =
            baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-ctl.serviceConfig.ExecStart;
        in
        lib.hasSuffix "/bin/swarm-matrix-ctl mint" exec;
    }
    {
      # 🩸 A secret is a path, never a value — checked on the one unit in this
      # tree whose whole job is an `as_token`. Every variable it is given names
      # a file or an address; the token itself is read out of the bind-mounted
      # registration at runtime, so nothing here can be a token and an
      # environment block is world-readable through `systemctl show`.
      name = "matrix-ctl's environment carries paths and addresses, never a token";
      ok =
        let
          env = baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-ctl.environment;
        in
        !(lib.any (v: lib.hasInfix "as_token" v || lib.hasInfix "syt_" v) (lib.attrValues env));
    }
    {
      # The absence arm, and the deployment it protects: a homeserver on a hive
      # with no store identity at all. Without it the unit would exist naming
      # `null` as its certificate, which nixos renders as the literal string.
      name = "a matrix container with no store identity runs no matrix-ctl and binds no PKI";
      ok =
        let
          units = matrixNoBaoIdentity.containers.hive-matrix.config.systemd.services;
        in
        !(units ? swarm-matrix-ctl)
        && !(matrixNoBaoIdentity.containers.hive-matrix.bindMounts ? "/var/lib/swarm-bao-pki");
    }
    {
      # 🩸 The privilege arm of the credential this slice publishes: the
      # account it belongs to must not be a homeserver admin. Read on the
      # rendered homeserver settings rather than on an option, because the
      # grant was never an option — it was a boot command in `admin_execute`,
      # and a command list is exactly the shape a later edit re-adds without
      # anything noticing.
      name = "the homeserver promotes no account to admin at boot";
      ok =
        let
          g = baoWithMatrix.containers.hive-matrix.config.services.matrix-tuwunel.settings.global;
        in
        !(g ? admin_execute) || g.admin_execute == [ ];
    }
    {
      # 🩸 The arm the whole refusal exists for. Both readers are gated on their
      # own leaf, so the way this regresses is the build going green and three
      # units rendering where four should — which no presence check on a
      # rendered unit can see, because the unit that is missing is the evidence.
      # Read as a refusal naming each option, so an operator acts on the message
      # without opening the nix.
      name = "a remote reader missing the per-principal leaves is refused, naming both options";
      ok =
        refusedOver baoRemoteReaderMissingLeaves "matrixTokenClientCertFile"
        && refusedOver baoRemoteReaderMissingLeaves "matrixTokenClientKeyFile"
        && refusedOver baoRemoteReaderMissingLeaves "queueAgentClientCertFile"
        && refusedOver baoRemoteReaderMissingLeaves "queueAgentClientKeyFile";
    }
    {
      # The matrix refusal's own gate, which the queue refusal does not have.
      # Without this arm the two are indistinguishable on the fixture above.
      name = "the queue refusal needs no homeserver, and the matrix refusal stays quiet without one";
      ok =
        refusedOver remoteReaderNoMatrixMissingLeaves "queueAgentClientCertFile"
        && !(refusedOver remoteReaderNoMatrixMissingLeaves "matrixTokenClientCertFile");
    }
    {
      # ⚠️ The arm that keeps the refusal from being worse than the silence it
      # replaced. A hive with NO store identity is the supported no-store
      # deployment and also the state an operator passes through bringing a hive
      # up — neither may fail to evaluate. Asserted as "no refusal names any of
      # the four options", not as "this one fixture is fine", because the way
      # this breaks is a gate widened to the principal's leaf alone.
      name = "a hive with no store identity at all is refused over none of the per-principal leaves";
      ok = lib.all (option: !(refusedOver matrixNoBaoIdentity option)) [
        "matrixTokenClientCertFile"
        "matrixTokenClientKeyFile"
        "queueAgentClientCertFile"
        "queueAgentClientKeyFile"
      ];
    }
    {
      # The other half of the same guard, and the one an operator meets far more
      # often: on the store's own host ./host-modules/glue-bao-tls.nix mkDefaults
      # all eight, so there is nothing to name and nothing to refuse. Paired with
      # the fully-named remote reader, which is the same deployment done by hand.
      name = "neither a store host nor a correctly-named remote reader is refused";
      ok =
        lib.all
          (
            m:
            lib.all (option: !(refusedOver m option)) [
              "matrixTokenClientCertFile"
              "queueAgentClientCertFile"
            ]
          )
          [
            baoWithMatrix
            baoRemoteReader
          ];
    }
  ];
in
runGroup "bao-matrix-reader" cases