hyperhive/nix/module-eval.nix
atlas 694abf4439 swarm-bao: make the reader's identity declarable, not just the store's
`swarm-bao.nix` declared the store's half of the mTLS pair as options —
`serverCertFile`, `serverKeyFile`, `clientCaFile` — and left the reader's
half as a literal inside `glue-bao-tls.nix`, which only runs where
`deploy.bao.enable` is set. A hive that did not host the store therefore
could not read from it and could not be pointed at a certificate even
when one had been placed by hand.

Adds `clientCertFile`, `clientKeyFile` and `serverCaFile` beside their
three server siblings, `mkDefault`ed by the glue to the leaf it already
mints, and moves `glue-matrix-bao-token.nix` onto them. Its gate becomes
"this host holds an identity" rather than "the store is a neighbour",
and the unit ordering that names store-local units is now conditional --
`Requires=` on an absent unit fails the job.

`serverCaFile` is separate from `clientCaFile` on purpose: one is the
store choosing which readers to trust, the other a reader choosing which
store to trust. Self-signing collapses them to one file, which is a
property of that deployment and not of the pairing.

Closes #3855.
2026-08-31 19:54:56 +02:00

259 lines
12 KiB
Nix

# `checks.module-eval` — the flake check that covers **nix**.
#
# Why this exists: every other check in ./checks.nix is a Rust
# derivation, so a `.nix`-only diff moves no hash, every check is a
# cache hit, and `nix flake check` reports green **without evaluating
# what changed**. This one's derivation hash is a function of the
# evaluated *results* below, so a nix change that flips a property
# rebuilds it and the builder fails naming that property.
#
# ## What belongs here, and what does not
#
# Anything expressible as a module `assertion` **should be one instead**:
# an assertion fires at deploy time for a real operator, not only in CI.
# What cannot be an assertion is the **absence class** — "a hive that
# hasn't opted in renders exactly what it did before", "this unit does
# not exist unless X". Those are claims about the *rendered config*
# rather than about a config being invalid, so they need an evaluator.
#
# ⚠️ **Cases are named by the PROPERTY they defend, never by the ticket
# that prompted them.** A case named after a ticket has the ticket's
# lifetime; a case named after a property lives as long as the property.
#
# ⚠️ **This check evaluates. It does not execute.** Where the artifact is
# a command line, an HTTP request or a certificate, a value assertion
# cannot stand in — those need something that *runs* them. And a case
# that needs a **rendered file** must stub the packages that file drags
# in (`swarm.ui.package = pkgs.emptyDirectory`), or it costs a full
# frontend build to answer a question about a listen directive.
{
pkgs,
lib,
self,
nixosSystem,
}:
let
# Stub host, same shape ./docs/default.nix already uses: enough for a
# `nixosSystem` to evaluate, nothing that pulls a real disk or
# bootloader in.
hive =
extra:
(nixosSystem {
system = pkgs.stdenv.hostPlatform.system;
modules = [
self.nixosModules.default
{
fileSystems."/" = {
device = "/dev/null";
fsType = "tmpfs";
};
boot.loader.grub.enable = false;
system.stateVersion = "25.11";
# `recursiveUpdate`, not `//`: a plain `//` only merges the
# *top-level* keys of `extra` in, so an `extra` that touches a
# nested attr under an existing top-level key (e.g. `swarm.*`)
# silently drops every sibling under that key instead of merging
# into it — the same class of bug as the `services.hyperhive`
# double-nesting mistake this stub already had to dodge once,
# just one level down. `recursiveUpdate` merges nested attrsets
# all the way down instead.
services.hyperhive = lib.recursiveUpdate {
enable = true;
hiveName = "h1";
swarm.domain = "t.local";
swarm.hives.h1.domain = "h1.t.local";
} extra;
}
];
}).config;
allLocal = hive { deploy.singleHostSwarm = true; };
bare = hive { };
withCi = hive { deploy.forgejo.ci.enable = true; };
baoPkcs11 = hive {
deploy.bao.enable = true;
deploy.bao.seal = "pkcs11";
};
baoShamir = hive {
deploy.bao.enable = true;
deploy.bao.seal = "shamir";
};
baoExplicitCerts = hive {
deploy.bao.enable = true;
deploy.bao.serverCertFile = "/etc/pki/bao.pem";
deploy.bao.serverKeyFile = "/etc/pki/bao-key.pem";
};
# The store and a service that reads from it, versus the store alone. The
# pair is what makes the reader's absence arm mean anything.
baoWithMatrix = hive {
deploy.bao.enable = true;
deploy.matrix.enable = true;
};
# A hive that reads from a store it does not run: no `deploy.bao.enable`, so
# nothing here mints a leaf and the operator names one placed by hand. The
# deployment this pairing exists to serve, and the one that was previously
# inexpressible — the gate asked whether the store was a neighbour.
baoRemoteReader = hive {
deploy.matrix.enable = true;
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
};
# The same hive with the identity taken away, which separates "a homeserver
# is deployed" from "this host can authenticate to the store".
matrixNoBaoIdentity = hive { deploy.matrix.enable = true; };
# A priority collision is a property of the *option*, not
# of the merged value's interior — nix throws the moment the value is
# demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
# both necessary and sufficient. `deepSeq` over-specifies this: it keeps
# walking *into* the resulting value after the merge already succeeded,
# and a package/derivation-shaped value's `override`/`overrideAttrs`
# self-reference sends it into nixpkgs' fixpoint machinery and blows the
# stack (measured — this is not a hypothetical).
forceCiServiceConfigs =
let
svcs = withCi.containers.hive-ci.config.systemd.services;
vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs);
in
builtins.foldl' (acc: v: builtins.seq v acc) true vals;
# Each case: a name stating the property, and `ok`.
cases = [
{
name = "a hive that has not opted into all-local runs no swarm controller";
ok = !bare.services.hyperhive.deploy.swarm-controller.enable;
}
{
name = "the all-local mode turns the swarm controller on";
ok = allLocal.services.hyperhive.deploy.swarm-controller.enable;
}
{
# The gateway's per-name issuer choice. If this ever collapses to a
# constant, every swarm-service vhost serves a certificate its CA
# is name-constrained out of — which evaluates cleanly and fails in
# a browser.
name = "a swarm service name gets the swarm-services leaf and the default server does not";
ok =
let
l = allLocal.services.hyperhive.gateway.lib;
in
(l.tlsFor "t.local").sslCertificate != (l.tlsFor "_").sslCertificate;
}
{
# nixos asserts when a vhost declares both, so this is also a
# statement that the `removeAttrs` upstream of it still happens.
name = "the swarm UI vhost forces TLS instead of merely adding it";
ok =
let
v = allLocal.services.nginx.virtualHosts."t.local";
in
v.forceSSL && !(v.addSSL or false);
}
{
name = "a hive with matrix off serves no matrix discovery endpoint";
ok =
!(builtins.hasAttr "= /.well-known/matrix/client" bare.services.nginx.virtualHosts."_".locations);
}
{
# main got eval-borked twice by this exact class of bug (once on the
# unit's `Restart` key, once on `RestartSec`) — a nixpkgs bump to
# `gitea-actions-runner.nix` adds a plain `serviceConfig.*`
# definition that collides with one of ours, and nix refuses to
# merge two plain definitions at *host* eval. No other check
# instantiates a host with `containers.hive-ci` actually enabled, so
# the collision only surfaces on operator deploy, not in CI.
name = "the CI container's unit definitions merge without a priority collision";
ok = forceCiServiceConfigs;
}
{
# The store's seal is spread over five gates — the stanza, the
# provisioning unit, a bind mount, a device and an EnvironmentFile.
# Rendering only some of them is the dangerous state: a store that
# says hardware-backed and seals with a software key, which no
# assertion can catch because every value is individually valid.
name = "a shamir store renders no TPM provisioning unit";
ok = !(baoShamir.systemd.services ? swarm-bao-token);
}
{
# Presence control for the case above. Without it, a typo in the
# option name would satisfy the absence arm forever.
name = "a pkcs11 store renders the TPM provisioning unit";
ok = baoPkcs11.systemd.services ? swarm-bao-token;
}
{
# The store's mTLS identity is a separate trust domain from both CAs in
# this tree, because it must not come from an authority the store will
# itself distribute. What supplies it is the glue, which mints a CA of
# the store's own — so an enabled store has all three paths, and if this
# ever reads null again the store stops coming up on its own.
name = "a deployed store is given its own certificate, key and client CA";
ok =
let
b = baoPkcs11.services.hyperhive.deploy.bao;
in
b.serverCertFile != null && b.serverKeyFile != null && b.clientCaFile != null;
}
{
# Everything the glue sets is `mkDefault`, and this is the case that
# says so: a deployment whose certificates come from somewhere the glue
# has never heard of must win. Also the presence control for the case
# above — a renamed option would read `null` on both and satisfy
# neither, but only this one names a value.
name = "an operator's own certificate path beats the glue's default";
ok = baoExplicitCerts.services.hyperhive.deploy.bao.serverCertFile == "/etc/pki/bao.pem";
}
{
# The store's first reader. Its unit belongs to the pairing, not to
# either service: matrix must not learn the store exists, and the store
# must not know who reads it.
name = "a store deployed beside the homeserver fetches its registration token";
ok = baoWithMatrix.systemd.services ? swarm-bao-matrix-token;
}
{
# Absence arm. A store with nothing to serve renders no reader, so the
# unit is a function of the PAIRING rather than of the store — which is
# the property that makes it glue instead of a feature of either side.
name = "a store with no homeserver beside it renders no token reader";
ok = !(baoPkcs11.systemd.services ? swarm-bao-matrix-token);
}
{
name = "a hive that names a client identity reads from a store it does not run";
ok = baoRemoteReader.systemd.services ? swarm-bao-matrix-token;
}
{
# Absence arm for the one above, and the reason the gate is the identity
# rather than the homeserver: without it, deploying matrix anywhere would
# render a reader that cannot authenticate.
name = "a homeserver with no way to authenticate to the store renders no token reader";
ok = !(matrixNoBaoIdentity.systemd.services ? swarm-bao-matrix-token);
}
{
# `Requires=` on a unit that does not exist fails the job, and nothing
# local mints certificates off-host — so this orders against nothing.
# Eval cannot see that failure; only the empty list here stands in for it.
name = "an off-host reader requires no unit the store's host would have provided";
ok = baoRemoteReader.systemd.services.swarm-bao-matrix-token.requires == [ ];
}
{
# Presence control for the case above: the list is conditional, not gone.
name = "a co-located reader still orders after the local pki unit";
ok = baoWithMatrix.systemd.services.swarm-bao-matrix-token.requires == [ "swarm-bao-pki.service" ];
}
];
bad = builtins.filter (c: !c.ok) cases;
report = lib.concatMapStringsSep "\n" (c: " echo 'FAILED: ${c.name}' >&2") bad;
in
# The results are embedded in the builder text on purpose: that is what
# makes this derivation's hash depend on them, so a nix-only change that
# flips a case cannot be answered from cache.
pkgs.runCommand "hyperhive-module-eval" { } ''
${report}
${
if bad == [ ] then
"echo '${toString (builtins.length cases)} module properties hold' && touch $out"
else
"echo 'module-eval: ${toString (builtins.length bad)} of ${toString (builtins.length cases)} properties broke' >&2 && exit 1"
}
''