vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

nix: split module-eval into per-subsystem checks

Browse source 
The single module-eval derivation forced ~62 full nixosSystem
fixtures live at once to compute its cases list: 10.6GB peak RSS /
5m25s to evaluate, by far the dominant cost in nix flake check.
Splits it into 21 independent checks.module-eval-* derivations
(1-7 fixtures each) sharing builders/helpers via module-eval/lib.nix,
so no single derivation needs more than a handful of fixtures live
at once. A few cases spanning two clusters carry a small duplicated
fixture rather than threading shared state through lib.nix.
This commit is contained in:
müde 2026-09-20 04:25:54 +02:00
commit dc418a5223
24 changed files with 3760 additions and 2997 deletions
Download patch file Download diff file Expand all files Collapse all files

219
nix/module-eval/bao-basics.nix Normal file
View file

@ -0,0 +1,219 @@
# `checks.module-eval-bao-basics` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
pkgs,
lib,
self,
nixosSystem,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
hive
runGroup
baoNames
baoSettings
baoStream
bridgePorts
;
baoPkcs11 = hive {
deploy.bao.enable = true;
deploy.bao.seal = "pkcs11";
};
baoShamir = hive {
deploy.bao.enable = true;
deploy.bao.seal = "shamir";
};
baoExplicitCerts = hive {
deploy.bao.enable = true;
deploy.bao.serverCertFile = "/etc/pki/bao.pem";
deploy.bao.serverKeyFile = "/etc/pki/bao-key.pem";
};
# The store's units live inside its container, so the gates below have to
# look there rather than at the host's service set.
baoUnits = machine: machine.containers.swarm-bao.config.systemd.services;
cases = [
{
# The store's seal is spread over six gates — the stanza, the
# provisioning unit, two bind mounts, a device and an EnvironmentFile.
# Rendering only some of them is the dangerous state: a store that
# says hardware-backed and seals with a software key, which no
# assertion can catch because every value is individually valid.
name = "a shamir store renders no TPM provisioning unit";
ok = !(baoUnits baoShamir ? swarm-bao-token);
}
{
# Presence control for the case above. Without it, a typo in the
# option name would satisfy the absence arm forever. The second half is
# the fix itself: the unit has to run where openbao's `DynamicUser` is
# allocated, and a host unit writing the same bytes has no name to hand
# them to.
name = "a pkcs11 store provisions the token in the container, not on the host";
ok = (baoUnits baoPkcs11 ? swarm-bao-token) && !(baoPkcs11.systemd.services ? swarm-bao-token);
}
{
# `allowedDevices` renders `DeviceAllow=` and nothing else — nspawn
# mounts its own /dev and cannot create device nodes, so permission to
# use a device that was never bound in opens nothing. Neither half
# fails on its own, which is why they are asserted as a pair.
name = "a pkcs11 store gets the TPM device bound in, not merely allowed";
ok =
let
c = baoPkcs11.containers.swarm-bao;
in
(c.bindMounts ? "/dev/tpmrm0") && builtins.any (d: d.node == "/dev/tpmrm0") c.allowedDevices;
}
{
# The stanza's label and the label the unit creates are two literals that
# have to name one object, and the mechanism is asserted with them
# because it is valid only for an RSA key: openbao takes AES-GCM or
# RSA-OAEP and a TPM 2.0 has neither GCM nor an opinion about which the
# seal asked for. Every wrong combination renders and deploys, and
# surfaces as a pkcs11 error at `operator init`.
name = "the seal asks for the RSA key the provisioning unit creates";
ok =
let
p = (baoSettings baoPkcs11).seal.pkcs11;
in
(p.mechanism or "") == "CKM_RSA_PKCS_OAEP"
&& lib.hasInfix "--algorithm=rsa2048 --key-label=${p.key_label or ""}" (
(baoUnits baoPkcs11).swarm-bao-token.script or ""
);
}
{
# `DynamicUser` implies `ProtectSystem=strict`, so the token directory is
# read-only to the seal however it is owned, and the group is the only
# handle on a uid allocated at start. Dropping either surfaces as a
# pkcs11 error deep in a library, naming neither the mount nor the user.
name = "the store's seal may write the token directory, and is in both its groups";
ok =
let
sc = (baoUnits baoPkcs11).openbao.serviceConfig;
in
# `or [ ]` rather than a bare select: the interesting mutation is the
# key being gone, and a select would abort the whole run with a nix
# trace instead of failing this case by name.
builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ])
&& builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ])
&& builtins.elem "swarm-bao-tpm" (sc.SupplementaryGroups or [ ]);
}
{
# The device node belongs to the HOST and is matched by NUMBER, while the
# unit that opens it lives in the container — so the two sides holding
# the same gid is the entire mechanism. Letting either side auto-allocate
# renders cleanly, deploys cleanly, and leaves a 0660 node the seal
# cannot open. Compared rather than each checked against a literal: the
# property is that they AGREE, not what they agree on.
name = "the TPM group has the same gid on the host and inside the container";
ok =
let
host = baoPkcs11.users.groups.swarm-bao-tpm.gid or null;
inner = baoPkcs11.containers.swarm-bao.config.users.groups.swarm-bao-tpm.gid or null;
in
host != null && host == inner;
}
{
# Absence arm for the case above — a shamir store never opens a TPM, so
# it must not claim a device node's group. Without this, pinning the gid
# unconditionally would look identical.
name = "a shamir store claims no TPM device group";
ok = !(baoShamir.users.groups ? swarm-bao-tpm);
}
{
# The store's mTLS identity is a separate trust domain from both CAs in
# this tree, because it must not come from an authority the store will
# itself distribute. What supplies it is the glue, which mints a CA of
# the store's own — so an enabled store has all three paths, and if this
# ever reads null again the store stops coming up on its own.
name = "a deployed store is given its own certificate, key and client CA";
ok =
let
b = baoPkcs11.services.hyperhive.deploy.bao;
in
b.serverCertFile != null && b.serverKeyFile != null && b.clientCaFile != null;
}
{
# Everything the glue sets is `mkDefault`, and this is the case that
# says so: a deployment whose certificates come from somewhere the glue
# has never heard of must win. Also the presence control for the case
# above — a renamed option would read `null` on both and satisfy
# neither, but only this one names a value.
name = "an operator's own certificate path beats the glue's default";
ok = baoExplicitCerts.services.hyperhive.deploy.bao.serverCertFile == "/etc/pki/bao.pem";
}
{
# Absence arm. A store with nothing to serve renders no reader, so the
# unit is a function of the PAIRING rather than of the store — which is
# the property that makes it glue instead of a feature of either side.
name = "a store with no homeserver beside it renders no token reader";
ok = !(baoPkcs11.systemd.services ? swarm-bao-matrix-token);
}
{
# The name a reader dials has to resolve where the store runs; a
# multi-host swarm resolves it upstream instead.
name = "the store's host answers for the store's name";
ok = builtins.elem "bao.t.local" (baoNames baoPkcs11);
}
{
# What makes that name reachable from inside an agent container, and the
# single reason it is a stream server rather than a vhost: `ssl_preread`
# routes on the SNI without decrypting, so the handshake bao completes is
# still the client's own and the certificate it authenticates by arrives
# intact. Terminating here would hand the store one identity for the
# whole swarm.
name = "the store's host passes connections through without terminating TLS";
ok =
let
s = baoStream baoPkcs11;
in
lib.hasInfix "ssl_preread on;" s && lib.hasInfix "proxy_pass $swarm_bao_backend;" s;
}
{
# The address half, and it is bridge-only for a reason a wildcard would
# hide until deploy: the store already holds `127.0.0.1:8200` in this
# same netns, so `0.0.0.0:8200` is `EADDRINUSE` and nginx fails to start
# — taking every hive domain behind the gateway down with it.
name = "the passthrough listens on the bridge, not on every address";
ok = lib.hasInfix "listen 10.42.0.1:8200;" (baoStream baoPkcs11);
}
{
# The routing half: the SNI picks the backend and the only name that
# resolves to one is the store's own. A `default` that pointed anywhere
# would make this host a relay for whatever name a client invented.
name = "the passthrough routes only the store's name, to its loopback listener";
ok =
let
s = baoStream baoPkcs11;
in
lib.hasInfix "map $ssl_preread_server_name $swarm_bao_backend" s
&& lib.hasInfix "bao.t.local 127.0.0.1:8200;" s
&& lib.hasInfix ''default "";'' s;
}
{
# The listener is only half of reachable: the bridge firewall drops
# everything not named here, and a silent drop is the failure that reads
# as "the store is down" from inside a container.
name = "the store's port is open on the bridge where the store runs";
ok = builtins.elem 8200 (bridgePorts baoPkcs11);
}
{
# Raft refuses to start without it, and says so in a message that names
# neither the setting nor the stanza.
name = "the store advertises a cluster address";
ok = lib.hasPrefix "https://" ((baoSettings baoPkcs11).cluster_addr or "");
}
];
in
runGroup "bao-basics" cases