dc418a5223
The single module-eval derivation forced ~62 full nixosSystem fixtures live at once to compute its cases list: 10.6GB peak RSS / 5m25s to evaluate, by far the dominant cost in nix flake check. Splits it into 21 independent checks.module-eval-* derivations (1-7 fixtures each) sharing builders/helpers via module-eval/lib.nix, so no single derivation needs more than a handful of fixtures live at once. A few cases spanning two clusters carry a small duplicated fixture rather than threading shared state through lib.nix.
422 lines
19 KiB
Nix
422 lines
19 KiB
Nix
# `checks.module-eval-core-toggle` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
baoNames
|
|
baoStream
|
|
bridgePorts
|
|
bridgePortsOrNone
|
|
swarmServiceEnables
|
|
;
|
|
|
|
allLocal = hive { deploy.singleHostSwarm = true; };
|
|
|
|
bare = hive { };
|
|
|
|
# The same stub with the central toggle off. Paired with `bare` below to pin
|
|
# the defaults that used to read `services.hyperhive.enable` and no longer
|
|
# do: each is asserted to hold the SAME literal in both, so a future edit
|
|
# that quietly re-introduces the dependency — or that changes what the
|
|
# default renders for a hive with the toggle on — fails here. Reading an
|
|
# option off this fixture forces that option only, not the config, so the
|
|
# toggle being off costs nothing. Also the "installs the modules and turns
|
|
# nothing on" host the swarm-service absences below read: none of the
|
|
# per-service deployment toggles derives from the hive being on, so it
|
|
# renders the same absences `bare` does.
|
|
centralToggleOff = hive { enable = false; };
|
|
|
|
withCi = hive { deploy.forgejo.ci.enable = true; };
|
|
|
|
# A priority collision is a property of the *option*, not
|
|
# of the merged value's interior — nix throws the moment the value is
|
|
# demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
|
|
# both necessary and sufficient. `deepSeq` over-specifies this: it keeps
|
|
# walking *into* the resulting value after the merge already succeeded,
|
|
# and a package/derivation-shaped value's `override`/`overrideAttrs`
|
|
# self-reference sends it into nixpkgs' fixpoint machinery and blows the
|
|
# stack (measured — this is not a hypothetical).
|
|
forceCiServiceConfigs =
|
|
let
|
|
svcs = withCi.containers.hive-ci.config.systemd.services;
|
|
vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs);
|
|
in
|
|
builtins.foldl' (acc: v: builtins.seq v acc) true vals;
|
|
cases = [
|
|
{
|
|
# Both halves matter. The equality is the "no longer consults the central
|
|
# toggle" half; the literal is the "and still renders what it always
|
|
# did" half, which an equality on its own would let drift to `false` in
|
|
# lockstep.
|
|
name = "the forge's behindGateway default is true regardless of the central toggle";
|
|
ok =
|
|
bare.services.hyperhive.deploy.forgejo.behindGateway == true
|
|
&& centralToggleOff.services.hyperhive.deploy.forgejo.behindGateway == true;
|
|
}
|
|
{
|
|
# Downstream of the one above — publicUrl reads `behindGateway`, so it
|
|
# tracked the central toggle transitively as well as directly. The domain
|
|
# is the stub's swarm domain, which both fixtures share.
|
|
name = "the forge's publicUrl default follows behindGateway alone, not the central toggle";
|
|
ok =
|
|
bare.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local"
|
|
&& centralToggleOff.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local";
|
|
}
|
|
{
|
|
# And that it still tracks `behindGateway` at all: without this arm the
|
|
# case above passes just as well for a default hardcoded to the URL.
|
|
name = "the forge's publicUrl default is still null with behindGateway off";
|
|
ok =
|
|
(hive { deploy.forgejo.behindGateway = false; }).services.hyperhive.swarm.forge.publicUrl == null;
|
|
}
|
|
{
|
|
# The controller's token path defaulted to forge's delivery path only on
|
|
# a host with the central toggle on, and to `null` otherwise. Forge
|
|
# deploys unconditionally, so the path is now unconditional too.
|
|
name = "the swarm controller's forgeTokenFile defaults to forge's delivery path regardless of the central toggle";
|
|
ok =
|
|
let
|
|
forgePath = "/var/lib/hyperhive-forge/swarm-controller.token";
|
|
in
|
|
bare.services.hyperhive.deploy.swarm-controller.forgeTokenFile == forgePath
|
|
&& centralToggleOff.services.hyperhive.deploy.swarm-controller.forgeTokenFile == forgePath;
|
|
}
|
|
{
|
|
name = "a hive that does not host the swarm's shared services runs none of them";
|
|
ok =
|
|
let
|
|
es = swarmServiceEnables bare;
|
|
in
|
|
lib.length (lib.attrNames es) == 9 && !lib.any lib.id (lib.attrValues es);
|
|
}
|
|
{
|
|
name = "a hive that has not opted into all-local runs no swarm controller";
|
|
ok = !bare.services.hyperhive.deploy.swarm-controller.enable;
|
|
}
|
|
{
|
|
name = "the all-local mode turns the swarm controller on";
|
|
ok = allLocal.services.hyperhive.deploy.swarm-controller.enable;
|
|
}
|
|
{
|
|
# Where the reader puts the files and where the daemon looks for them is
|
|
# one agreement spanning two modules. Asserted against the option rather
|
|
# than the literal so moving the directory moves both ends.
|
|
name = "hive-c0re is told where the agents' queue credential lands";
|
|
ok =
|
|
allLocal.systemd.services.hive-c0re.environment.HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR
|
|
== toString allLocal.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir;
|
|
}
|
|
{
|
|
# The one address in this file that must NOT be loopback. Both spellings
|
|
# sit in the same unit's environment and are correct for their own
|
|
# reader: hive-c0re shares the host netns, an agent container does not,
|
|
# so a copy-paste between them reaches the agent itself and the symptom
|
|
# is a connect that hangs.
|
|
name = "the agents' queue address is the bridge, not the loopback one the hive itself uses";
|
|
ok =
|
|
let
|
|
e = allLocal.systemd.services.hive-c0re.environment;
|
|
in
|
|
e.HIVE_AGENT_NATS_URL == "nats://${allLocal.services.hyperhive.network.bridgeIp}:4222"
|
|
&& !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL)
|
|
&& e.HIVE_AGENT_NATS_URL != e.HIVE_C0RE_NATS_URL;
|
|
}
|
|
{
|
|
# The agents mint against the swarm's IdP, the same endpoint the hive's
|
|
# own client uses — a hive-local guess would produce a token the queue
|
|
# would not accept.
|
|
name = "the agents' token endpoint is the swarm IdP's";
|
|
ok =
|
|
let
|
|
e = allLocal.systemd.services.hive-c0re.environment;
|
|
in
|
|
lib.hasSuffix "/api/oidc/token" e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT
|
|
&& e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT == e.HIVE_C0RE_OIDC_TOKEN_ENDPOINT;
|
|
}
|
|
{
|
|
# The absence arm, and what makes the two above able to fail: a hive
|
|
# with no queue address must forward neither coordinate, because half a
|
|
# pair reaches the harness as a partial configuration rather than as
|
|
# none.
|
|
name = "a hive with no swarm queue forwards no agent queue coordinates";
|
|
ok =
|
|
let
|
|
e = bare.systemd.services.hive-c0re.environment;
|
|
in
|
|
!(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT);
|
|
}
|
|
{
|
|
# Where the store is and where the agent is told it is, one agreement
|
|
# spanning two modules. Asserted against the hive's own `BAO_ADDR`
|
|
# rather than a literal, because an agent pointed at a different
|
|
# spelling of the same store presents a certificate to a listener whose
|
|
# name it cannot verify.
|
|
name = "an agent is told the same store address its hive uses";
|
|
ok =
|
|
let
|
|
e = allLocal.systemd.services.hive-c0re.environment;
|
|
in
|
|
e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR;
|
|
}
|
|
{
|
|
# A hive with no certificate of its own can collect no agent's identity,
|
|
# so forwarding an address would name a store nothing in the container
|
|
# can reach. The same gate the `BAO_*` pair beside it sits behind.
|
|
name = "a hive with no store identity forwards no store address to its agents";
|
|
ok = !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_ADDR);
|
|
}
|
|
{
|
|
# The gateway's per-name issuer choice. If this ever collapses to a
|
|
# constant, every swarm-service vhost serves a certificate its CA
|
|
# is name-constrained out of — which evaluates cleanly and fails in
|
|
# a browser.
|
|
name = "a swarm service name gets the swarm-services leaf and the default server does not";
|
|
ok =
|
|
let
|
|
l = allLocal.services.hyperhive.gateway.lib;
|
|
in
|
|
(l.tlsFor "t.local").sslCertificate != (l.tlsFor "_").sslCertificate;
|
|
}
|
|
{
|
|
# nixos asserts when a vhost declares both, so this is also a
|
|
# statement that the `removeAttrs` upstream of it still happens.
|
|
name = "the swarm UI vhost forces TLS instead of merely adding it";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts."t.local";
|
|
in
|
|
v.forceSSL && !(v.addSSL or false);
|
|
}
|
|
{
|
|
name = "a hive with matrix off serves no matrix discovery endpoint";
|
|
ok =
|
|
!(builtins.hasAttr "= /.well-known/matrix/client" bare.services.nginx.virtualHosts."_".locations);
|
|
}
|
|
{
|
|
# main got eval-borked twice by this exact class of bug (once on the
|
|
# unit's `Restart` key, once on `RestartSec`) — a nixpkgs bump to
|
|
# `gitea-actions-runner.nix` adds a plain `serviceConfig.*`
|
|
# definition that collides with one of ours, and nix refuses to
|
|
# merge two plain definitions at *host* eval. No other check
|
|
# instantiates a host with `containers.hive-ci` actually enabled, so
|
|
# the collision only surfaces on operator deploy, not in CI.
|
|
name = "the CI container's unit definitions merge without a priority collision";
|
|
ok = forceCiServiceConfigs;
|
|
}
|
|
{
|
|
# The collector reaches these routes through the gateway now, so each
|
|
# store needs an ingest location of its own. Without one the write rides
|
|
# the `/` catch-all: unauthenticated on the metrics store, and into a
|
|
# browser redirect on the log store.
|
|
name = "each store's vhost has an authenticated ingest location";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts;
|
|
m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push" or null;
|
|
l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs" or null;
|
|
in
|
|
m != null
|
|
&& l != null
|
|
&& lib.hasInfix "auth_request" m.extraConfig
|
|
&& lib.hasInfix "auth_request" l.extraConfig;
|
|
}
|
|
{
|
|
# The arm that actually protects something. A pusher handed
|
|
# `error_page 401 =302` FOLLOWS it and POSTs its batch at a login page,
|
|
# which answers 200 — ingest reporting healthy while storing nothing.
|
|
# The third clause is the positive control: the log store's browser
|
|
# location really does redirect, so this says the machine routes differ
|
|
# rather than that the string is absent from the whole file.
|
|
name = "the ingest locations answer 401 instead of redirecting a pusher";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts;
|
|
m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push".extraConfig;
|
|
l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs".extraConfig;
|
|
browser = v."logs.t.local".locations."/".extraConfig;
|
|
in
|
|
!(lib.hasInfix "error_page" m)
|
|
&& !(lib.hasInfix "error_page" l)
|
|
&& lib.hasInfix "error_page" browser;
|
|
}
|
|
{
|
|
# The read counterpart to the ingest location: an agent queries the log
|
|
# store with a bearer token, and the `/` catch-all is the browser's
|
|
# route. Riding it would mean inheriting the login redirect the next
|
|
# case is about, so the route has to exist separately to be gated
|
|
# separately.
|
|
name = "the log store's vhost has an authenticated machine query location";
|
|
ok =
|
|
let
|
|
q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/" or null;
|
|
in
|
|
q != null && lib.hasInfix "auth_request" q.extraConfig;
|
|
}
|
|
{
|
|
# Same trap as the ingest case, on the read side, where it is worse: a
|
|
# redirected pusher at least stores nothing visibly, while a redirected
|
|
# *reader* is handed a 200 carrying login HTML and records a query that
|
|
# succeeded and matched no logs. The browser clause is the positive
|
|
# control — that location really does redirect to the login host — so a
|
|
# pass means these two routes differ rather than that the strings are
|
|
# absent from the whole vhost.
|
|
name = "the machine query location answers 401 instead of redirecting to a login page";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts;
|
|
q = v."logs.t.local".locations."^~ /select/logsql/".extraConfig;
|
|
browser = v."logs.t.local".locations."/".extraConfig;
|
|
in
|
|
!(lib.hasInfix "error_page" q)
|
|
&& !(lib.hasInfix "auth.t.local" q)
|
|
&& lib.hasInfix "error_page" browser
|
|
&& lib.hasInfix "auth.t.local" browser;
|
|
}
|
|
{
|
|
# Read access is deliberately unscoped: an authenticated caller reads
|
|
# the whole swarm's logs until a permission system exists. Pinned so a
|
|
# scoping parameter arriving later is a visible diff here rather than a
|
|
# quiet change of rule — and pinned on `proxyPass` too, because
|
|
# VictoriaLogs takes its filters as request parameters, which ride an
|
|
# upstream URI as easily as a directive. The first clause is the
|
|
# control: it proves the location resolved and that `hasInfix` finds
|
|
# what is genuinely in this string, so the absences below mean absent
|
|
# rather than unreadable.
|
|
name = "the machine query location forwards the caller's query unmodified";
|
|
ok =
|
|
let
|
|
q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/";
|
|
in
|
|
lib.hasInfix "auth_request" q.extraConfig
|
|
&& !(lib.hasInfix "extra_filters" q.extraConfig)
|
|
&& !(lib.hasInfix "extra_stream_filters" q.extraConfig)
|
|
&& !(lib.hasInfix "$args" q.extraConfig)
|
|
&& !(lib.hasInfix "?" q.proxyPass);
|
|
}
|
|
{
|
|
# The absence arm for the case above, and the option's own rule — a
|
|
# service declares its entry under its own `enable` — made checkable.
|
|
# Without it, moving the assignment outside the collector's `mkIf`
|
|
# passes every arm above while handing a collector-less hive a scrape
|
|
# target for a port nothing binds.
|
|
name = "a hive with no collector declares no self-scrape target";
|
|
ok = bare.services.hyperhive.otel.scrapeTargets == { };
|
|
}
|
|
{
|
|
# Absence arm, and the one that matters: claiming a name this host does
|
|
# not serve points every local reader at the wrong machine.
|
|
name = "a hive that does not run the store claims no name for it";
|
|
ok = !(builtins.elem "bao.t.local" (baoNames bare));
|
|
}
|
|
{
|
|
# ⚠️ The absence arm that matters. `services.nginx.streamConfig` is a
|
|
# host-wide option, so a block rendered outside the store's own `mkIf`
|
|
# gives every hive in the swarm a listener — on the port the store
|
|
# answers on, in front of no store at all.
|
|
name = "a hive that does not run the store renders no stream passthrough";
|
|
ok = baoStream bare == "";
|
|
}
|
|
{
|
|
# Absence arm for the case above — a hive with no store has no reason to
|
|
# open the store's port, and opening it would point agents at a host that
|
|
# answers nothing.
|
|
name = "a hive that does not run the store opens no bridge port for it";
|
|
ok = !(builtins.elem 8200 (bridgePorts bare));
|
|
}
|
|
{
|
|
# The four blocks below used to be gated on the hive being enabled AND
|
|
# their own condition. The second half was always the load-bearing one —
|
|
# none of these conditions is derived from the hive toggle — so these
|
|
# arms pin what the conjunct was doing: nothing. Each is written against
|
|
# a host with the hive OFF as well as one with it on, because the way a
|
|
# dropped conjunct fails is by making something unconditional, and that
|
|
# shows up as a service appearing where nothing asked for it.
|
|
name = "no bridge port is opened for an exposeHostPorts nobody set";
|
|
ok =
|
|
!(builtins.elem 5432 (bridgePortsOrNone bare))
|
|
&& !(builtins.elem 5432 (bridgePortsOrNone centralToggleOff));
|
|
}
|
|
{
|
|
# `deploy.swarm-controller.enable`, which defaults false and is
|
|
# deliberately not derived from the hive toggle — a swarm has one
|
|
# controller, so the host that runs it says so itself.
|
|
#
|
|
# Probed by what the daemon needs in order to run, not by
|
|
# `? swarm-controller`: ./host-modules/hive-tls.nix defines an
|
|
# environment key on that unit name, which leaves the attr
|
|
# present-but-inert (no `ExecStart`, empty `wantedBy`) on every hive
|
|
# that has a CA — see the comment there. The credential oneshot has no
|
|
# second definer, so its absence is the unambiguous half.
|
|
name = "the swarm controller does not run unless this host is told to run it";
|
|
ok =
|
|
let
|
|
inert =
|
|
machine:
|
|
!(machine.systemd.services ? swarm-controller-credential)
|
|
&& !(
|
|
(machine.systemd.services.swarm-controller or { serviceConfig = { }; }).serviceConfig ? ExecStart
|
|
);
|
|
in
|
|
inert bare && inert centralToggleOff;
|
|
}
|
|
{
|
|
# `deploy.swarm-otel.enable`, same shape: the swarm's collector is one
|
|
# host's job, and the container is the whole of what it renders.
|
|
name = "the swarm collector container is absent unless this host is told to run it";
|
|
ok = !(bare.containers ? swarm-otel) && !(centralToggleOff.containers ? swarm-otel);
|
|
}
|
|
{
|
|
# `deploy.swarm-ui.enable`, which defaults to the controller's toggle —
|
|
# derived from a sibling deployment decision, still not from the hive
|
|
# toggle. `t.local` is the fixtures' swarm domain, which is the apex the
|
|
# UI claims; the arm below is what proves this vhost renders at all.
|
|
name = "the swarm UI vhost is absent unless this host is told to serve it";
|
|
ok =
|
|
!(bare.services.nginx.virtualHosts ? "t.local")
|
|
&& !(centralToggleOff.services.nginx.virtualHosts ? "t.local");
|
|
}
|
|
{
|
|
# The three infrastructure toggles are off by default and asserted by
|
|
# whoever needs them. With nothing on the host needing them, none of
|
|
# the three renders — which is also the control for the arm below.
|
|
name = "the gateway, resolver and bridge are absent where nothing on the host needs them";
|
|
ok =
|
|
!centralToggleOff.services.hyperhive.gateway.enable
|
|
&& !centralToggleOff.services.hyperhive.gateway.dns.enable
|
|
&& !centralToggleOff.services.hyperhive.network.enable
|
|
&& !(centralToggleOff.services.nginx.enable or false)
|
|
&& !(centralToggleOff.services.dnsmasq.enable or false)
|
|
&& !(centralToggleOff.networking.bridges ? hive-br0);
|
|
}
|
|
{
|
|
# hive-c0re asserts all three, and it follows the central toggle — so
|
|
# an ordinary hive keeps getting them with no opt-in, which is what
|
|
# this change must not break.
|
|
name = "an ordinary hive runs the gateway, resolver and bridge because its coordinator needs them";
|
|
ok =
|
|
bare.services.hyperhive.gateway.enable
|
|
&& bare.services.hyperhive.gateway.dns.enable
|
|
&& bare.services.hyperhive.network.enable
|
|
&& bare.services.nginx.enable
|
|
&& bare.services.dnsmasq.enable
|
|
&& bare.networking.bridges ? hive-br0;
|
|
}
|
|
];
|
|
in
|
|
runGroup "core-toggle" cases
|