# `checks.module-eval-bao-matrix-reader` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive runGroup ; # The store and a service that reads from it, versus the store alone. The # pair is what makes the reader's absence arm mean anything. baoWithMatrix = hive { deploy.bao.enable = true; deploy.matrix.enable = true; }; # A hive that reads from a store it does not run: no `deploy.bao.enable`, so # nothing here mints a leaf and the operator names one placed by hand. The # deployment this pairing exists to serve, and the one that was previously # inexpressible — the gate asked whether the store was a neighbour. baoRemoteReader = hive { deploy.matrix.enable = true; deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; }; cases = [ { # A login failure is the store being unreachable, sealed, or not yet # holding this host's role — all of which a retry fixes. A read that # answers "nothing there" is not, so only the first is allowed to fail # the unit. name = "the matrix token reader retries a failed login and still degrades on an empty read"; ok = let u = baoWithMatrix.systemd.services.swarm-bao-matrix-token; # Everything between the login's failure branch and the read's, which # is where the exit that decides "retry or give up" lives. afterLogin = lib.last (lib.splitString "bao login" u.script); loginBranch = lib.head (lib.splitString "bao kv get" afterLogin); in u.serviceConfig.Restart or null == "on-failure" && u.startLimitBurst or 0 > 0 # The window has to outlast every attempt, or the burst is unreachable. && u.startLimitIntervalSec or 0 > (u.serviceConfig.RestartSec or 0) * (u.startLimitBurst or 0) && lib.hasInfix "exit 1" loginBranch && lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script)); } { # The reader's own grant covers `swarm/hives//*` and # `swarm/agents/*`; a path outside those answers 403, not "no such key". # So the hive segment is what makes the read reachable, and a rename that # drops it looks correct and fails identically on every boot. name = "the matrix token path sits inside the prefix the reader is granted"; ok = let s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script; in lib.hasInfix "secret/swarm/hives/" s && lib.hasInfix "/matrix/appservice-token" s # The shape it used to have: `matrix` where a principal kind belongs, # which no grant covers. && !(lib.hasInfix "secret/swarm/matrix/" s); } { # The store's second reader, and the gate that decides it exists is the # certificate rather than anything about agents: containers are created # at runtime, so there is no static "this hive runs agents" fact to ask. name = "a hive that names a client identity reads its agent queue credential"; ok = baoRemoteReader.systemd.services ? swarm-bao-queue-agent; } { # Same 403-not-a-miss reason as the matrix arm above, against the path # `swarm_secret_client::queue::agent_client_path` builds from the same # pieces. The negative arm is the rename this one is exposed to: a # credential named for the queue rather than for the hive that presents # it reads as correct and is refused on every boot. name = "the agent queue credential path sits inside the prefix the reader is granted"; ok = let s = baoRemoteReader.systemd.services.swarm-bao-queue-agent.script; in lib.hasInfix "secret/swarm/hives/h1/queue/agent" s && !(lib.hasInfix "secret/swarm/queue/" s); } { # The unit's output is the option's value, not a literal that agrees with # it today: an operator moving the directory has to move both files. The # prefix is asserted too because `hasInfix ""` is true — an option # renamed out from under this arm would otherwise read empty and pass. name = "the queue credential reader writes both files under the directory its option names"; ok = let m = baoRemoteReader; dir = toString m.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir; s = m.systemd.services.swarm-bao-queue-agent.script; in lib.hasPrefix "/var/lib/" dir && lib.hasInfix "${dir}/secret" s && lib.hasInfix "${dir}/client_id" s; } { # A reader off the store's host is a reader whose journal is the only # record of why a hive's agents never connected, so the collector has to # be told the unit exists. Nothing else can say it: the store's module # does not know who holds a certificate. name = "the queue credential reader's journal reaches the collector"; ok = builtins.elem "swarm-bao-queue-agent" baoRemoteReader.services.hyperhive.swarm.otel.journaldUnits; } { # No agent container may render before this unit has had its attempts, # and the edge that guarantees it must delay hive-c0re rather than sink # it: an unreachable store is this unit's `Restart=on-failure` window, # not a reason for the daemon that renders every agent to fail its own # start. name = "the queue credential reader orders before hive-c0re and is wanted, not required, by it"; ok = let u = baoRemoteReader.systemd.services.swarm-bao-queue-agent; in builtins.elem "hive-c0re.service" (u.before or [ ]) && builtins.elem "hive-c0re.service" (u.wantedBy or [ ]) && !(builtins.elem "hive-c0re.service" (u.requiredBy or [ ])) && !(builtins.elem "hive-c0re.service" (u.requires or [ ])); } { # The store's first reader. Its unit belongs to the pairing, not to # either service: matrix must not learn the store exists, and the store # must not know who reads it. name = "a store deployed beside the homeserver fetches its appservice token"; ok = baoWithMatrix.systemd.services ? swarm-bao-matrix-token; } { name = "a hive that names a client identity reads from a store it does not run"; ok = baoRemoteReader.systemd.services ? swarm-bao-matrix-token; } { # `Requires=` on a unit that does not exist fails the job, and nothing # local mints certificates off-host — so this orders against nothing. # Eval cannot see that failure; only the empty list here stands in for it. name = "an off-host reader requires no unit the store's host would have provided"; # Membership first, then the value: indexing a missing unit throws, and a # table that reports which property broke must not be the thing that dies. ok = let s = baoRemoteReader.systemd.services; in s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ ]; } { # Presence control for the case above: the list is conditional, not gone. name = "a co-located reader still orders after the local pki unit"; ok = let s = baoWithMatrix.systemd.services; in s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ "swarm-bao-pki.service" ]; } { # Nothing asserted this script before, which is how it kept a branch that # named three states and threw away the only thing telling them apart. A # missing value, a refused identity and an unreachable host all end in the # same degraded mode here, correctly — what must survive is which one. name = "the matrix token reader carries the store's own diagnostic into the journal"; ok = let s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script; in !(lib.hasInfix "2>/dev/null" s) && lib.hasInfix ''cat "''$err"'' s; } { # hive-c0re runs as hive-core and the client key is `0600` root-owned # inside a `0700` directory, so the identity reaches the daemon as a # systemd credential and the environment names `%d` rather than the # file. Both halves are asserted together because either alone is a # daemon that fails at the TLS handshake, naming neither. name = "a reader hands hive-c0re a store identity the daemon cannot open itself"; ok = let s = baoRemoteReader.systemd.services; in s ? hive-c0re && (s.hive-c0re.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem" && (s.hive-c0re.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem" && builtins.elem "bao-client.pem:/etc/pki/bao-client.pem" s.hive-c0re.serviceConfig.LoadCredential && builtins.elem "bao-client-key.pem:/etc/pki/bao-client-key.pem" s.hive-c0re.serviceConfig.LoadCredential; } { # The CA is its own arm: absent means the system trust store, which is # right for a deployment with a real CA and wrong for a self-signed one. name = "a reader that names no store CA falls through to the system trust store"; ok = let s = baoRemoteReader.systemd.services; in s ? hive-c0re && !(s.hive-c0re.environment ? BAO_CACERT); } { # Presence control for the arm above: the CA is conditional, not gone. # Co-located, ./host-modules/glue-bao-tls.nix mints one and names it. name = "a reader beside a self-signed store is given that store's CA"; ok = let s = baoWithMatrix.systemd.services; in s ? hive-c0re && (s.hive-c0re.environment.BAO_CACERT or null) == "%d/bao-ca.pem" && lib.any (c: lib.hasPrefix "bao-ca.pem:" c) s.hive-c0re.serviceConfig.LoadCredential; } ]; in runGroup "bao-matrix-reader" cases