feat(nix): a flake check that actually covers nix

Every other check in nix/checks.nix is a Rust derivation, so a
.nix-only diff moves no hash, the whole set is cache hits, and
`nix flake check` reports green without evaluating what changed.

`checks.module-eval` is one derivation holding a table of cases, each
named by the PROPERTY it defends. Its builder text embeds the evaluated
results, so the derivation's hash is a function of them: a nix change
that flips a property rebuilds the check and fails in the builder,
naming that property.

PROVEN, not assumed — the mechanism was executed before the cases were
written. Same expression with one property true vs false:

  drvPath true  -> 5v11mnbv…-module-eval.drv
  drvPath false -> ivm3dvv8…-module-eval.drv        (differs)
  build false   -> FAILS, stderr names the property

and the table itself was mutation-tested: inverting one case's
expectation gives `FAILED: a hive that has not opted into all-local
runs no swarm controller / module-eval: 1 of 5 properties broke`. A
check that cannot go red on a broken tree is not evidence.

Cases are named by property and never by ticket: a case named after the
ticket that prompted it has that ticket's lifetime; one named after the
property lives as long as the property does.

⚠️ It evaluates, it does not execute. Where the artifact is a command
line, a request or a certificate, a value assertion cannot stand in —
that is written into the file's header, because the gap is exactly what
made two earlier outages evaluable-but-broken.
This commit is contained in:
atlas 2026-08-13 16:51:22 +02:00 committed by mara
commit fbffccbbb2
3 changed files with 133 additions and 2 deletions

View file

@ -204,6 +204,7 @@
system
treefmt-eval
;
inherit (nixpkgs.lib) nixosSystem;
}
);
};

View file

@ -1,6 +1,6 @@
# Flake checks: formatting, the clippy gate, the workspace test run,
# the nix-options docs eval, and the hivectl CLI-reference freshness
# check. Imported per system from flake.nix.
# the nix-options docs eval, the hivectl CLI-reference freshness check,
# and the module-eval property table. Imported per system from flake.nix.
{
pkgs,
craneLib,
@ -8,6 +8,7 @@
self,
system,
treefmt-eval,
nixosSystem,
}:
let
inherit (rust) cleanSrc cargoArtifacts nativeBuildInputs;
@ -15,6 +16,16 @@ in
{
formatting = treefmt-eval.config.build.check self;
# The only check here that covers **nix**. Every other one is a Rust
# derivation, so a `.nix`-only diff moves no hash and the whole set is
# cache hits — green without evaluating what changed. See the file's
# header for what belongs in it and what needs something that executes
# rather than evaluates.
module-eval = import ./module-eval.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;
};
# Clippy via crane's first-class `cargoClippy` builder. Reuses the
# shared `cargoArtifacts` (deps already built) and runs
# `cargo clippy --workspace --all-targets` directly.

119
nix/module-eval.nix Normal file
View file

@ -0,0 +1,119 @@
# `checks.module-eval` — the flake check that covers **nix**.
#
# Why this exists: every other check in ./checks.nix is a Rust
# derivation, so a `.nix`-only diff moves no hash, every check is a
# cache hit, and `nix flake check` reports green **without evaluating
# what changed**. This one's derivation hash is a function of the
# evaluated *results* below, so a nix change that flips a property
# rebuilds it and the builder fails naming that property.
#
# ## What belongs here, and what does not
#
# Anything expressible as a module `assertion` **should be one instead**:
# an assertion fires at deploy time for a real operator, not only in CI.
# What cannot be an assertion is the **absence class** — "a hive that
# hasn't opted in renders exactly what it did before", "this unit does
# not exist unless X". Those are claims about the *rendered config*
# rather than about a config being invalid, so they need an evaluator.
#
# ⚠️ **Cases are named by the PROPERTY they defend, never by the ticket
# that prompted them.** A case named after a ticket has the ticket's
# lifetime; a case named after a property lives as long as the property.
#
# ⚠️ **This check evaluates. It does not execute.** Where the artifact is
# a command line, an HTTP request or a certificate, a value assertion
# cannot stand in — those need something that *runs* them. And a case
# that needs a **rendered file** must stub the packages that file drags
# in (`swarm.ui.package = pkgs.emptyDirectory`), or it costs a full
# frontend build to answer a question about a listen directive.
{
pkgs,
lib,
self,
nixosSystem,
}:
let
# Stub host, same shape ./docs/default.nix already uses: enough for a
# `nixosSystem` to evaluate, nothing that pulls a real disk or
# bootloader in.
hive =
extra:
(nixosSystem {
system = pkgs.stdenv.hostPlatform.system;
modules = [
self.nixosModules.default
{
fileSystems."/" = {
device = "/dev/null";
fsType = "tmpfs";
};
boot.loader.grub.enable = false;
system.stateVersion = "25.11";
services.hyperhive = {
enable = true;
hiveName = "h1";
swarm.domain = "t.local";
swarm.hives.h1.domain = "h1.t.local";
}
// extra;
}
];
}).config;
allLocal = hive { enableAllLocalDefaults = true; };
bare = hive { };
# Each case: a name stating the property, and `ok`.
cases = [
{
name = "a hive that has not opted into all-local runs no swarm controller";
ok = !bare.services.hyperhive.swarm.controller.enable;
}
{
name = "the all-local mode turns the swarm controller on";
ok = allLocal.services.hyperhive.swarm.controller.enable;
}
{
# The gateway's per-name issuer choice. If this ever collapses to a
# constant, every swarm-service vhost serves a certificate its CA
# is name-constrained out of — which evaluates cleanly and fails in
# a browser.
name = "a swarm service name gets the swarm-services leaf and the default server does not";
ok =
let
l = allLocal.services.hyperhive.gateway.lib;
in
(l.tlsFor "t.local").sslCertificate != (l.tlsFor "_").sslCertificate;
}
{
# nixos asserts when a vhost declares both, so this is also a
# statement that the `removeAttrs` upstream of it still happens.
name = "the swarm UI vhost forces TLS instead of merely adding it";
ok =
let
v = allLocal.services.nginx.virtualHosts."t.local";
in
v.forceSSL && !(v.addSSL or false);
}
{
name = "a hive with matrix off serves no matrix discovery endpoint";
ok =
!(builtins.hasAttr "= /.well-known/matrix/client" bare.services.nginx.virtualHosts."_".locations);
}
];
bad = builtins.filter (c: !c.ok) cases;
report = lib.concatMapStringsSep "\n" (c: " echo 'FAILED: ${c.name}' >&2") bad;
in
# The results are embedded in the builder text on purpose: that is what
# makes this derivation's hash depend on them, so a nix-only change that
# flips a case cannot be answered from cache.
pkgs.runCommand "hyperhive-module-eval" { } ''
${report}
${
if bad == [ ] then
"echo '${toString (builtins.length cases)} module properties hold' && touch $out"
else
"echo 'module-eval: ${toString (builtins.length bad)} of ${toString (builtins.length cases)} properties broke' >&2 && exit 1"
}
''