d39ad6c465
Six more comments and option descriptions across five modules said the hive's leaf or grant reads every secret in the store. It reads swarm/agents/* and swarm/hives/<hive>/* and nothing else, so they now say that, in the wording the clientCertFile option already used. Also the matrix-ctl check's label no longer calls the hive's leaf store-wide.
452 lines
22 KiB
Nix
452 lines
22 KiB
Nix
# `checks.module-eval-bao-matrix-reader` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
hiveWithController
|
|
runGroup
|
|
;
|
|
|
|
# The store and a service that reads from it, versus the store alone. The
|
|
# pair is what makes the reader's absence arm mean anything.
|
|
baoWithMatrix = hiveWithController {
|
|
deploy.bao.enable = true;
|
|
deploy.matrix.enable = true;
|
|
};
|
|
|
|
# A hive that reads from a store it does not run: no `deploy.bao.enable`, so
|
|
# nothing here mints a leaf and the operator names one placed by hand. The
|
|
# deployment this pairing exists to serve, and the one that was previously
|
|
# inexpressible — the gate asked whether the store was a neighbour.
|
|
#
|
|
# 🩸 One leaf PER READER, and this fixture is where that cost is visible: the
|
|
# queue-credential unit identifies itself to the store separately from the
|
|
# hive, so an operator placing leaves by hand places one for it as well as
|
|
# the hive's own. Naming only `clientCertFile` here would leave the unit
|
|
# unrendered — which is what the arms below would then be asserting about.
|
|
baoRemoteReader = hiveWithController {
|
|
deploy.matrix.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.queueAgentClientCertFile = "/etc/pki/bao-queue-agent.pem";
|
|
deploy.bao.queueAgentClientKeyFile = "/etc/pki/bao-queue-agent-key.pem";
|
|
};
|
|
|
|
# A homeserver on a hive with NO store identity at all — neither a local
|
|
# store nor a hand-placed leaf. The absence arm for the matrix-ctl cases below
|
|
# needs it, and defining it here rather than importing keeps each group's
|
|
# fixture set its own, as ./lib.nix asks.
|
|
matrixNoBaoIdentity = hiveWithController { deploy.matrix.enable = true; };
|
|
|
|
# 🩸 The hand-configured remote reader that named the hive's leaf and stopped.
|
|
# `baoRemoteReader` above is the same deployment done right; this one holds
|
|
# the hive's own leaf, so it demonstrably reads the store, and is missing the
|
|
# queue reader's pair. Without the refusal it would render no queue reader
|
|
# and say nothing — which is what the refusal arms below are about.
|
|
baoRemoteReaderMissingLeaves = hiveWithController {
|
|
deploy.matrix.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
};
|
|
|
|
# The same omission on a hive that does NOT run a homeserver: the queue
|
|
# refusal checks only the hive's own toggle.
|
|
remoteReaderNoMatrixMissingLeaves = hiveWithController {
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
};
|
|
|
|
# The store's host with the hive's own reader pointed somewhere else.
|
|
matrixOnStoreHostReaderMoved = hiveWithController {
|
|
deploy.bao.enable = true;
|
|
deploy.matrix.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
|
};
|
|
|
|
# Did a module refuse this host, and over which option. An assertion is a
|
|
# config VALUE until something forces it — `.config` never throws — so a
|
|
# fixture in a state the module refuses stays evaluable and the refusal reads
|
|
# back as data. Matched on the option name the message names rather than on
|
|
# its prose, because the option name is the part an operator has to act on
|
|
# and a message that stopped naming it would be the actual defect. Same shape
|
|
# as ./grafana.nix's `grafanaRefusedFor`, which reads `swarm-grafana.nix`'s
|
|
# own refusal.
|
|
refusedOver = m: option: lib.any (a: !a.assertion && lib.hasInfix option a.message) m.assertions;
|
|
cases = [
|
|
{
|
|
# A login failure is the store being unreachable, sealed, or not yet
|
|
# holding this host's role — all of which a retry fixes. A read that
|
|
# answers "nothing there" is not, so only the first is allowed to fail
|
|
# the unit.
|
|
name = "the queue credential reader retries a failed login and still degrades on an empty read";
|
|
ok =
|
|
let
|
|
u = baoWithMatrix.systemd.services.swarm-bao-queue-agent;
|
|
# Everything between the login's failure branch and the read's, which
|
|
# is where the exit that decides "retry or give up" lives.
|
|
afterLogin = lib.last (lib.splitString "bao login" u.script);
|
|
loginBranch = lib.head (lib.splitString "bao kv get" afterLogin);
|
|
in
|
|
u.serviceConfig.Restart or null == "on-failure"
|
|
&& u.startLimitBurst or 0 > 0
|
|
# The window has to outlast every attempt, or the burst is unreachable.
|
|
&& u.startLimitIntervalSec or 0 > (u.serviceConfig.RestartSec or 0) * (u.startLimitBurst or 0)
|
|
&& lib.hasInfix "exit 1" loginBranch
|
|
&& lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script));
|
|
}
|
|
{
|
|
# The gate that decides this reader exists is the certificate rather
|
|
# than anything about agents: containers are created
|
|
# at runtime, so there is no static "this hive runs agents" fact to ask.
|
|
name = "a hive that names a client identity reads its agent queue credential";
|
|
ok = baoRemoteReader.systemd.services ? swarm-bao-queue-agent;
|
|
}
|
|
{
|
|
# The reader's own grant covers `swarm/hives/<this hive>/*`; a path
|
|
# outside it answers 403, not "no such key". This is the path
|
|
# `swarm_secret_client::queue::agent_client_path` builds from the same
|
|
# pieces. The negative arm is the rename this one is exposed to: a
|
|
# credential named for the queue rather than for the hive that presents
|
|
# it reads as correct and is refused on every boot.
|
|
name = "the agent queue credential path sits inside the prefix the reader is granted";
|
|
ok =
|
|
let
|
|
s = baoRemoteReader.systemd.services.swarm-bao-queue-agent.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/hives/h1/queue/agent" s && !(lib.hasInfix "secret/swarm/queue/" s);
|
|
}
|
|
{
|
|
# The unit's output is the option's value, not a literal that agrees with
|
|
# it today: an operator moving the directory has to move both files. The
|
|
# prefix is asserted too because `hasInfix ""` is true — an option
|
|
# renamed out from under this arm would otherwise read empty and pass.
|
|
name = "the queue credential reader writes both files under the directory its option names";
|
|
ok =
|
|
let
|
|
m = baoRemoteReader;
|
|
dir = toString m.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir;
|
|
s = m.systemd.services.swarm-bao-queue-agent.script;
|
|
in
|
|
lib.hasPrefix "/var/lib/" dir
|
|
&& lib.hasInfix "${dir}/secret" s
|
|
&& lib.hasInfix "${dir}/client_id" s;
|
|
}
|
|
{
|
|
# No agent container may render before this unit has had its attempts,
|
|
# and the edge that guarantees it must delay hive-c0re rather than sink
|
|
# it: an unreachable store is this unit's `Restart=on-failure` window,
|
|
# not a reason for the daemon that renders every agent to fail its own
|
|
# start.
|
|
name = "the queue credential reader orders before hive-c0re and is wanted, not required, by it";
|
|
ok =
|
|
let
|
|
u = baoRemoteReader.systemd.services.swarm-bao-queue-agent;
|
|
in
|
|
builtins.elem "hive-c0re.service" (u.before or [ ])
|
|
&& builtins.elem "hive-c0re.service" (u.wantedBy or [ ])
|
|
&& !(builtins.elem "hive-c0re.service" (u.requiredBy or [ ]))
|
|
&& !(builtins.elem "hive-c0re.service" (u.requires or [ ]));
|
|
}
|
|
{
|
|
# `Requires=` on a unit that does not exist fails the job, and nothing
|
|
# local mints certificates off-host — so this orders against nothing.
|
|
# Eval cannot see that failure; only the empty list here stands in for it.
|
|
name = "an off-host reader requires no unit the store's host would have provided";
|
|
# Membership first, then the value: indexing a missing unit throws, and a
|
|
# table that reports which property broke must not be the thing that dies.
|
|
ok =
|
|
let
|
|
s = baoRemoteReader.systemd.services;
|
|
in
|
|
s ? swarm-bao-queue-agent && s.swarm-bao-queue-agent.requires == [ ];
|
|
}
|
|
{
|
|
# Presence control for the case above: the list is conditional, not gone.
|
|
name = "a co-located reader still orders after the local pki unit";
|
|
ok =
|
|
let
|
|
s = baoWithMatrix.systemd.services;
|
|
in
|
|
s ? swarm-bao-queue-agent && s.swarm-bao-queue-agent.requires == [ "swarm-bao-pki.service" ];
|
|
}
|
|
{
|
|
# Nothing asserted this script before, which is how it kept a branch that
|
|
# named three states and threw away the only thing telling them apart. A
|
|
# missing value, a refused identity and an unreachable host all end in the
|
|
# same degraded mode here, correctly — what must survive is which one.
|
|
name = "the queue credential reader carries the store's own diagnostic into the journal";
|
|
ok =
|
|
let
|
|
s = baoWithMatrix.systemd.services.swarm-bao-queue-agent.script;
|
|
in
|
|
!(lib.hasInfix "2>/dev/null" s) && lib.hasInfix ''cat "''$err"'' s;
|
|
}
|
|
{
|
|
# hive-c0re runs as hive-core and the client key is `0600` root-owned
|
|
# inside a `0700` directory, so the identity reaches the daemon as a
|
|
# systemd credential and the environment names `%d` rather than the
|
|
# file. Both halves are asserted together because either alone is a
|
|
# daemon that fails at the TLS handshake, naming neither.
|
|
name = "a reader hands hive-c0re a store identity the daemon cannot open itself";
|
|
ok =
|
|
let
|
|
s = baoRemoteReader.systemd.services;
|
|
in
|
|
s ? hive-c0re
|
|
&& (s.hive-c0re.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem"
|
|
&& (s.hive-c0re.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem"
|
|
&& builtins.elem "bao-client.pem:/etc/pki/bao-client.pem" s.hive-c0re.serviceConfig.LoadCredential
|
|
&& builtins.elem "bao-client-key.pem:/etc/pki/bao-client-key.pem" s.hive-c0re.serviceConfig.LoadCredential;
|
|
}
|
|
{
|
|
# The CA is its own arm: absent means the system trust store, which is
|
|
# right for a deployment with a real CA and wrong for a self-signed one.
|
|
name = "a reader that names no store CA falls through to the system trust store";
|
|
ok =
|
|
let
|
|
s = baoRemoteReader.systemd.services;
|
|
in
|
|
s ? hive-c0re && !(s.hive-c0re.environment ? BAO_CACERT);
|
|
}
|
|
{
|
|
# Presence control for the arm above: the CA is conditional, not gone.
|
|
# Co-located, ./host-modules/glue-bao-tls.nix mints one and names it.
|
|
name = "a reader beside a self-signed store is given that store's CA";
|
|
ok =
|
|
let
|
|
s = baoWithMatrix.systemd.services;
|
|
in
|
|
s ? hive-c0re
|
|
&& (s.hive-c0re.environment.BAO_CACERT or null) == "%d/bao-ca.pem"
|
|
&& lib.any (c: lib.hasPrefix "bao-ca.pem:" c) s.hive-c0re.serviceConfig.LoadCredential;
|
|
}
|
|
{
|
|
# The same hole a third time, and the leaf whose absence is hardest to
|
|
# see from outside: it is consumed by a unit INSIDE a container, so a
|
|
# missing pairing renders as a container that comes up fine and publishes
|
|
# nothing.
|
|
name = "the store mints a leaf for matrix-ctl, and the container is pointed at it";
|
|
ok =
|
|
let
|
|
m = baoWithMatrix;
|
|
p = m.services.hyperhive.deploy.matrix;
|
|
in
|
|
lib.hasInfix "matrix-ctl.pem" m.systemd.services.swarm-bao-pki.script
|
|
&& p.ctlBaoClientCertFile == "/var/lib/swarm-bao-pki/matrix-ctl.pem"
|
|
&& p.ctlBaoClientKeyFile == "/var/lib/swarm-bao-pki/matrix-ctl-key.pem";
|
|
}
|
|
{
|
|
# The hive's own reader is not matrix-ctl's identity, and no path is
|
|
# derived from it: a homeserver host without the store that holds only
|
|
# the hive's reader gets the secret fetch that fails naming the options,
|
|
# like one holding nothing.
|
|
name = "a homeserver host without the store is refused when it holds only the hive's reader";
|
|
ok =
|
|
let
|
|
m = baoRemoteReaderMissingLeaves;
|
|
u = m.systemd.services.swarm-bao-matrix-oidc;
|
|
in
|
|
m.services.hyperhive.deploy.matrix.ctlBaoClientCertFile == null
|
|
&& !(u.environment ? BAO_CLIENT_CERT)
|
|
&& lib.hasInfix "deploy.matrix.ctlBaoClientCertFile" u.script
|
|
&& !(lib.hasInfix "bao login" u.script);
|
|
}
|
|
{
|
|
# The default is the file the minting unit writes, wherever the hive's
|
|
# own reader is pointed.
|
|
name = "on the store's host matrix-ctl's leaf is the one minted there, not one beside the hive's reader";
|
|
ok =
|
|
let
|
|
p = matrixOnStoreHostReaderMoved.services.hyperhive.deploy.matrix;
|
|
mint = matrixOnStoreHostReaderMoved.systemd.services.swarm-bao-pki.script;
|
|
in
|
|
p.ctlBaoClientCertFile == "/var/lib/swarm-bao-pki/matrix-ctl.pem"
|
|
&& p.ctlBaoClientKeyFile == "/var/lib/swarm-bao-pki/matrix-ctl-key.pem"
|
|
&& lib.hasInfix "[ -s /var/lib/swarm-bao-pki/matrix-ctl.pem ] ||" mint
|
|
&& lib.hasInfix "/var/lib/swarm-bao-pki matrix-ctl " mint;
|
|
}
|
|
{
|
|
# 🩸 The identity separation this whole arrangement buys, stated as the
|
|
# one thing that would silently undo it. The container gets MATRIX-CTL's
|
|
# leaf — whose grant is a single path — and not the hive's, which reads
|
|
# every agent credential in the swarm and this hive's own secrets. Both files exist in the same directory and
|
|
# both would evaluate, deploy and work.
|
|
name = "matrix-ctl presents its own leaf, never the hive's";
|
|
ok =
|
|
let
|
|
env =
|
|
baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-publish.environment;
|
|
hiveLeaf = baoWithMatrix.services.hyperhive.deploy.bao.clientCertFile;
|
|
in
|
|
env.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/matrix-ctl.pem" && env.BAO_CLIENT_CERT != hiveLeaf;
|
|
}
|
|
{
|
|
# The bind mount is what makes the environment above resolvable: without
|
|
# it the unit names two paths the container does not have, and fails at
|
|
# the TLS handshake naming no cause. Read off the mount table rather than
|
|
# the option, so a pairing that stops reaching `bindMounts` still fails.
|
|
name = "the matrix container binds matrix-ctl's PKI read-only";
|
|
ok =
|
|
let
|
|
mounts = baoWithMatrix.containers.hive-matrix.bindMounts;
|
|
in
|
|
mounts ? "/var/lib/swarm-bao-pki" && mounts."/var/lib/swarm-bao-pki".isReadOnly;
|
|
}
|
|
{
|
|
# The cert role ./host-modules/swarm-bao.nix writes, which is the one
|
|
# agreement the unit cannot get wrong, and the verb: a bare invocation
|
|
# exits non-zero with clap's usage, a deploy-time failure with no local
|
|
# signal.
|
|
name = "the publish unit is handed the store role and invokes its verb";
|
|
ok =
|
|
let
|
|
u = baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-publish;
|
|
in
|
|
u.environment.MATRIX_APPSERVICE_CERT_ROLE == "swarm-matrix-ctl"
|
|
&& lib.hasSuffix "/bin/swarm-matrix-ctl appservice publish" u.serviceConfig.ExecStart;
|
|
}
|
|
{
|
|
# 🩸 A secret is a path, never a value. Every variable the unit is given
|
|
# names a file or an address; the token itself is read out of the state
|
|
# dir at runtime, so nothing here can be a token and an environment block
|
|
# is world-readable through `systemctl show`.
|
|
name = "the publish unit's environment carries paths and addresses, never a token";
|
|
ok =
|
|
let
|
|
env =
|
|
baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-publish.environment;
|
|
in
|
|
!(lib.any (v: lib.hasInfix "as_token" v || lib.hasInfix "syt_" v) (lib.attrValues env));
|
|
}
|
|
{
|
|
# The absence arm, and the deployment it protects: a homeserver on a hive
|
|
# with no store identity at all. Without it the mount would name `null`
|
|
# as its source, which nixos renders as the literal string.
|
|
name = "a matrix container with no store identity binds no PKI";
|
|
ok = !(matrixNoBaoIdentity.containers.hive-matrix.bindMounts ? "/var/lib/swarm-bao-pki");
|
|
}
|
|
{
|
|
# 🩸 The privilege arm: exactly one account is a homeserver admin, the
|
|
# swarm appservice's sender, whose token only matrix-ctl and the
|
|
# controller read. Read on the rendered settings rather than on an
|
|
# option, because the grant is a boot command in `admin_execute`, and a
|
|
# command list is exactly the shape a later edit extends without anything
|
|
# noticing — so the list is compared whole, and a second entry fails.
|
|
name = "the homeserver promotes exactly the swarm sender to admin at boot, and no hive's sender";
|
|
ok =
|
|
let
|
|
g = baoWithMatrix.containers.hive-matrix.config.services.matrix-tuwunel.settings.global;
|
|
in
|
|
g.admin_execute == [ "users make-user-admin @swarm:${g.server_name}" ]
|
|
&& !(lib.any (c: lib.hasInfix "@hive-" c) g.admin_execute);
|
|
}
|
|
{
|
|
# Load-bearing rather than lenient: tuwunel aborts startup on a failing
|
|
# `admin_execute` command when this is false, and the homeserver must not
|
|
# stay down over a promotion.
|
|
name = "a failed admin promotion does not stop the homeserver";
|
|
ok =
|
|
baoWithMatrix.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.admin_execute_errors_ignore
|
|
or false;
|
|
}
|
|
{
|
|
# The swarm registration reaches tuwunel as a `.yaml` credential in the
|
|
# directory `appservice_dir` names, and it is the only registration
|
|
# there: no client presents a per-hive one's token, so loading one would
|
|
# be a credential nobody reads.
|
|
name = "tuwunel loads the swarm registration as a yaml credential, and no other";
|
|
ok =
|
|
let
|
|
creds =
|
|
baoWithMatrix.containers.hive-matrix.config.systemd.services.tuwunel.serviceConfig.LoadCredential;
|
|
in
|
|
lib.elem "swarm-appservice.yaml:/var/lib/swarm-matrix-appservice/swarm.yaml" creds
|
|
&& lib.length (lib.filter (c: lib.hasInfix ".yaml:" c) creds) == 1;
|
|
}
|
|
{
|
|
# A missing `LoadCredential` source fails tuwunel, so the render has to
|
|
# have run first — and it is local only, so it cannot fail on a store.
|
|
name = "the swarm registration is rendered before tuwunel and required by it, without the store";
|
|
ok =
|
|
let
|
|
r = baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-render;
|
|
in
|
|
lib.elem "tuwunel.service" r.before
|
|
&& lib.elem "tuwunel.service" r.requiredBy
|
|
&& lib.hasSuffix "swarm-matrix-ctl appservice render" r.serviceConfig.ExecStart
|
|
&& !(r.environment ? BAO_ADDR);
|
|
}
|
|
{
|
|
# The absence arm for the four above: with no matrix-ctl identity there
|
|
# is nobody to publish the swarm token, so no admin, no registration and
|
|
# no render — rather than an admin credential nobody reads.
|
|
name = "a matrix container with no store identity has no swarm appservice and promotes nobody";
|
|
ok =
|
|
let
|
|
c = matrixNoBaoIdentity.containers.hive-matrix.config;
|
|
g = c.services.matrix-tuwunel.settings.global;
|
|
in
|
|
!(g ? admin_execute)
|
|
&& !(c.systemd.services ? swarm-matrix-appservice-render)
|
|
&& !(c.systemd.services ? swarm-matrix-appservice-publish)
|
|
&& !(lib.any (lib.hasPrefix "swarm-appservice.yaml") c.systemd.services.tuwunel.serviceConfig.LoadCredential);
|
|
}
|
|
{
|
|
# 🩸 The arm the whole refusal exists for. The reader is gated on its own
|
|
# leaf, so the way this regresses is the build going green and the unit
|
|
# not rendering — which no presence check on a rendered unit can see,
|
|
# because the unit that is missing is the evidence. Read as a refusal
|
|
# naming each option, so an operator acts on the message without opening
|
|
# the nix.
|
|
name = "a remote reader missing the queue reader's leaf is refused, naming both options";
|
|
ok =
|
|
refusedOver baoRemoteReaderMissingLeaves "queueAgentClientCertFile"
|
|
&& refusedOver baoRemoteReaderMissingLeaves "queueAgentClientKeyFile";
|
|
}
|
|
{
|
|
name = "the queue refusal needs no homeserver";
|
|
ok = refusedOver remoteReaderNoMatrixMissingLeaves "queueAgentClientCertFile";
|
|
}
|
|
{
|
|
# ⚠️ The arm that keeps the refusal from being worse than the silence it
|
|
# replaced. A hive with NO store identity is the supported no-store
|
|
# deployment and also the state an operator passes through bringing a hive
|
|
# up — neither may fail to evaluate. Asserted as "no refusal names either
|
|
# option", not as "this one fixture is fine", because the way this breaks
|
|
# is a gate widened to the principal's leaf alone.
|
|
name = "a hive with no store identity at all is refused over none of the per-principal leaves";
|
|
ok = lib.all (option: !(refusedOver matrixNoBaoIdentity option)) [
|
|
"queueAgentClientCertFile"
|
|
"queueAgentClientKeyFile"
|
|
];
|
|
}
|
|
{
|
|
# The other half of the same guard, and the one an operator meets far more
|
|
# often: on the store's own host ./host-modules/glue-bao-tls.nix mkDefaults
|
|
# every pair, so there is nothing to name and nothing to refuse. Paired with
|
|
# the fully-named remote reader, which is the same deployment done by hand.
|
|
name = "neither a store host nor a correctly-named remote reader is refused";
|
|
ok = lib.all (m: !(refusedOver m "queueAgentClientCertFile")) [
|
|
baoWithMatrix
|
|
baoRemoteReader
|
|
];
|
|
}
|
|
];
|
|
in
|
|
runGroup "bao-matrix-reader" cases
|