dc418a5223
The single module-eval derivation forced ~62 full nixosSystem fixtures live at once to compute its cases list: 10.6GB peak RSS / 5m25s to evaluate, by far the dominant cost in nix flake check. Splits it into 21 independent checks.module-eval-* derivations (1-7 fixtures each) sharing builders/helpers via module-eval/lib.nix, so no single derivation needs more than a handful of fixtures live at once. A few cases spanning two clusters carry a small duplicated fixture rather than threading shared state through lib.nix.
224 lines
10 KiB
Nix
224 lines
10 KiB
Nix
# `checks.module-eval-bao-matrix-reader` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
# The store and a service that reads from it, versus the store alone. The
|
|
# pair is what makes the reader's absence arm mean anything.
|
|
baoWithMatrix = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.matrix.enable = true;
|
|
};
|
|
|
|
# A hive that reads from a store it does not run: no `deploy.bao.enable`, so
|
|
# nothing here mints a leaf and the operator names one placed by hand. The
|
|
# deployment this pairing exists to serve, and the one that was previously
|
|
# inexpressible — the gate asked whether the store was a neighbour.
|
|
baoRemoteReader = hive {
|
|
deploy.matrix.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
};
|
|
cases = [
|
|
{
|
|
# A login failure is the store being unreachable, sealed, or not yet
|
|
# holding this host's role — all of which a retry fixes. A read that
|
|
# answers "nothing there" is not, so only the first is allowed to fail
|
|
# the unit.
|
|
name = "the matrix token reader retries a failed login and still degrades on an empty read";
|
|
ok =
|
|
let
|
|
u = baoWithMatrix.systemd.services.swarm-bao-matrix-token;
|
|
# Everything between the login's failure branch and the read's, which
|
|
# is where the exit that decides "retry or give up" lives.
|
|
afterLogin = lib.last (lib.splitString "bao login" u.script);
|
|
loginBranch = lib.head (lib.splitString "bao kv get" afterLogin);
|
|
in
|
|
u.serviceConfig.Restart or null == "on-failure"
|
|
&& u.startLimitBurst or 0 > 0
|
|
# The window has to outlast every attempt, or the burst is unreachable.
|
|
&& u.startLimitIntervalSec or 0 > (u.serviceConfig.RestartSec or 0) * (u.startLimitBurst or 0)
|
|
&& lib.hasInfix "exit 1" loginBranch
|
|
&& lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script));
|
|
}
|
|
{
|
|
# The reader's own grant covers `swarm/hives/<this hive>/*` and
|
|
# `swarm/agents/*`; a path outside those answers 403, not "no such key".
|
|
# So the hive segment is what makes the read reachable, and a rename that
|
|
# drops it looks correct and fails identically on every boot.
|
|
name = "the matrix token path sits inside the prefix the reader is granted";
|
|
ok =
|
|
let
|
|
s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/hives/" s
|
|
&& lib.hasInfix "/matrix/appservice-token" s
|
|
# The shape it used to have: `matrix` where a principal kind belongs,
|
|
# which no grant covers.
|
|
&& !(lib.hasInfix "secret/swarm/matrix/" s);
|
|
}
|
|
{
|
|
# The store's second reader, and the gate that decides it exists is the
|
|
# certificate rather than anything about agents: containers are created
|
|
# at runtime, so there is no static "this hive runs agents" fact to ask.
|
|
name = "a hive that names a client identity reads its agent queue credential";
|
|
ok = baoRemoteReader.systemd.services ? swarm-bao-queue-agent;
|
|
}
|
|
{
|
|
# Same 403-not-a-miss reason as the matrix arm above, against the path
|
|
# `swarm_secret_client::queue::agent_client_path` builds from the same
|
|
# pieces. The negative arm is the rename this one is exposed to: a
|
|
# credential named for the queue rather than for the hive that presents
|
|
# it reads as correct and is refused on every boot.
|
|
name = "the agent queue credential path sits inside the prefix the reader is granted";
|
|
ok =
|
|
let
|
|
s = baoRemoteReader.systemd.services.swarm-bao-queue-agent.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/hives/h1/queue/agent" s && !(lib.hasInfix "secret/swarm/queue/" s);
|
|
}
|
|
{
|
|
# The unit's output is the option's value, not a literal that agrees with
|
|
# it today: an operator moving the directory has to move both files. The
|
|
# prefix is asserted too because `hasInfix ""` is true — an option
|
|
# renamed out from under this arm would otherwise read empty and pass.
|
|
name = "the queue credential reader writes both files under the directory its option names";
|
|
ok =
|
|
let
|
|
m = baoRemoteReader;
|
|
dir = toString m.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir;
|
|
s = m.systemd.services.swarm-bao-queue-agent.script;
|
|
in
|
|
lib.hasPrefix "/var/lib/" dir
|
|
&& lib.hasInfix "${dir}/secret" s
|
|
&& lib.hasInfix "${dir}/client_id" s;
|
|
}
|
|
{
|
|
# A reader off the store's host is a reader whose journal is the only
|
|
# record of why a hive's agents never connected, so the collector has to
|
|
# be told the unit exists. Nothing else can say it: the store's module
|
|
# does not know who holds a certificate.
|
|
name = "the queue credential reader's journal reaches the collector";
|
|
ok = builtins.elem "swarm-bao-queue-agent" baoRemoteReader.services.hyperhive.swarm.otel.journaldUnits;
|
|
}
|
|
{
|
|
# No agent container may render before this unit has had its attempts,
|
|
# and the edge that guarantees it must delay hive-c0re rather than sink
|
|
# it: an unreachable store is this unit's `Restart=on-failure` window,
|
|
# not a reason for the daemon that renders every agent to fail its own
|
|
# start.
|
|
name = "the queue credential reader orders before hive-c0re and is wanted, not required, by it";
|
|
ok =
|
|
let
|
|
u = baoRemoteReader.systemd.services.swarm-bao-queue-agent;
|
|
in
|
|
builtins.elem "hive-c0re.service" (u.before or [ ])
|
|
&& builtins.elem "hive-c0re.service" (u.wantedBy or [ ])
|
|
&& !(builtins.elem "hive-c0re.service" (u.requiredBy or [ ]))
|
|
&& !(builtins.elem "hive-c0re.service" (u.requires or [ ]));
|
|
}
|
|
{
|
|
# The store's first reader. Its unit belongs to the pairing, not to
|
|
# either service: matrix must not learn the store exists, and the store
|
|
# must not know who reads it.
|
|
name = "a store deployed beside the homeserver fetches its appservice token";
|
|
ok = baoWithMatrix.systemd.services ? swarm-bao-matrix-token;
|
|
}
|
|
{
|
|
name = "a hive that names a client identity reads from a store it does not run";
|
|
ok = baoRemoteReader.systemd.services ? swarm-bao-matrix-token;
|
|
}
|
|
{
|
|
# `Requires=` on a unit that does not exist fails the job, and nothing
|
|
# local mints certificates off-host — so this orders against nothing.
|
|
# Eval cannot see that failure; only the empty list here stands in for it.
|
|
name = "an off-host reader requires no unit the store's host would have provided";
|
|
# Membership first, then the value: indexing a missing unit throws, and a
|
|
# table that reports which property broke must not be the thing that dies.
|
|
ok =
|
|
let
|
|
s = baoRemoteReader.systemd.services;
|
|
in
|
|
s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ ];
|
|
}
|
|
{
|
|
# Presence control for the case above: the list is conditional, not gone.
|
|
name = "a co-located reader still orders after the local pki unit";
|
|
ok =
|
|
let
|
|
s = baoWithMatrix.systemd.services;
|
|
in
|
|
s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ "swarm-bao-pki.service" ];
|
|
}
|
|
{
|
|
# Nothing asserted this script before, which is how it kept a branch that
|
|
# named three states and threw away the only thing telling them apart. A
|
|
# missing value, a refused identity and an unreachable host all end in the
|
|
# same degraded mode here, correctly — what must survive is which one.
|
|
name = "the matrix token reader carries the store's own diagnostic into the journal";
|
|
ok =
|
|
let
|
|
s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script;
|
|
in
|
|
!(lib.hasInfix "2>/dev/null" s) && lib.hasInfix ''cat "''$err"'' s;
|
|
}
|
|
{
|
|
# hive-c0re runs as hive-core and the client key is `0600` root-owned
|
|
# inside a `0700` directory, so the identity reaches the daemon as a
|
|
# systemd credential and the environment names `%d` rather than the
|
|
# file. Both halves are asserted together because either alone is a
|
|
# daemon that fails at the TLS handshake, naming neither.
|
|
name = "a reader hands hive-c0re a store identity the daemon cannot open itself";
|
|
ok =
|
|
let
|
|
s = baoRemoteReader.systemd.services;
|
|
in
|
|
s ? hive-c0re
|
|
&& (s.hive-c0re.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem"
|
|
&& (s.hive-c0re.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem"
|
|
&& builtins.elem "bao-client.pem:/etc/pki/bao-client.pem" s.hive-c0re.serviceConfig.LoadCredential
|
|
&& builtins.elem "bao-client-key.pem:/etc/pki/bao-client-key.pem" s.hive-c0re.serviceConfig.LoadCredential;
|
|
}
|
|
{
|
|
# The CA is its own arm: absent means the system trust store, which is
|
|
# right for a deployment with a real CA and wrong for a self-signed one.
|
|
name = "a reader that names no store CA falls through to the system trust store";
|
|
ok =
|
|
let
|
|
s = baoRemoteReader.systemd.services;
|
|
in
|
|
s ? hive-c0re && !(s.hive-c0re.environment ? BAO_CACERT);
|
|
}
|
|
{
|
|
# Presence control for the arm above: the CA is conditional, not gone.
|
|
# Co-located, ./host-modules/glue-bao-tls.nix mints one and names it.
|
|
name = "a reader beside a self-signed store is given that store's CA";
|
|
ok =
|
|
let
|
|
s = baoWithMatrix.systemd.services;
|
|
in
|
|
s ? hive-c0re
|
|
&& (s.hive-c0re.environment.BAO_CACERT or null) == "%d/bao-ca.pem"
|
|
&& lib.any (c: lib.hasPrefix "bao-ca.pem:" c) s.hive-c0re.serviceConfig.LoadCredential;
|
|
}
|
|
];
|
|
in
|
|
runGroup "bao-matrix-reader" cases
|