Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/module-eval/core-toggle.nix
atlas 90e8158f37 remove the hive-side auto-spawn of ruth and the ruthless option
ruth is created from the swarm dashboard like any agent. An existing ruth container starts at boot through the boot reconcile, like every other agent. A definition of services.hyperhive.ruthless now fails evaluation with a removal message.
2026-10-16 21:42:26 +02:00

1141 lines
52 KiB
Nix

# `checks.module-eval-core-toggle` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
pkgs,
lib,
self,
nixosSystem,
shared,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
hive
hiveWithController
runGroup
baoNames
baoStream
bridgePorts
bridgePortsOrNone
swarmServiceEnables
otelSettings
;
inherit (shared) allLocal;
bare = shared.bareWithController;
# The same stub with the central toggle (`deploy.hive-controller.enable`)
# off. Paired with `bare` below to pin the defaults that do not read it:
# each is asserted to hold the SAME literal in both, so a future edit
# that quietly introduces the dependency — or that changes what the
# default renders for a hive with the toggle on — fails here. Reading an
# option off this fixture forces that option only, not the config, so the
# toggle being off costs nothing. Also the "installs the modules and turns
# nothing on" host the swarm-service absences below read: none of the
# per-service deployment toggles derives from the hive being on, so it
# renders the same absences `bare` does.
centralToggleOff = shared.bare;
inherit (shared) forgeHere;
# Every hive defaults the agents' queue address to the queue's name, so the
# only hive without one is a hive told to have none.
noAgentQueue = hiveWithController { deploy.hive-controller.queue.agentNatsUrl = null; };
# On the store's host, which mints the runner's store identity.
withCi = hive {
deploy.bao.enable = true;
deploy.ci-runner.enable = true;
};
# On the store's host, which mints the classifier's store identity.
withClassifier = hive {
deploy.bao.enable = true;
deploy.review-classifier.enable = true;
};
# A priority collision is a property of the *option*, not
# of the merged value's interior — nix throws the moment the value is
# demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
# both necessary and sufficient. `deepSeq` over-specifies this: it keeps
# walking *into* the resulting value after the merge already succeeded,
# and a package/derivation-shaped value's `override`/`overrideAttrs`
# self-reference sends it into nixpkgs' fixpoint machinery and blows the
# stack (measured — this is not a hypothetical).
forceCiServiceConfigs =
let
svcs = withCi.containers.swarm-ci-runner.config.systemd.services;
vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs);
in
builtins.foldl' (acc: v: builtins.seq v acc) true vals;
# The host fetches that restart their consumer when the store's value
# changes (../host-modules/lib/refresh-consumer.nix), each on a fixture
# that renders it.
storeRefreshed = [
{
unit = "swarm-bao-grafana-oidc";
machine = allLocal;
}
{
unit = "swarm-bao-otel-oidc";
machine = allLocal;
}
{
unit = "swarm-bao-matrix-oidc";
machine = allLocal;
}
{
unit = "swarm-bao-forge-oidc";
machine = allLocal;
}
{
unit = "swarm-bao-hive-otel-oidc";
machine = allLocal;
}
{
unit = "swarm-bao-forwarder-oidc";
machine = allLocal;
}
{
unit = "swarm-bao-ci-runner";
machine = withCi;
}
];
remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false;
cases = [
{
# The domain is the stub's swarm domain, which both fixtures share.
name = "the forge's publicUrl default does not consult the central toggle";
ok =
bare.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local"
&& centralToggleOff.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local";
}
{
# The controller's token mint follows where the forge runs
# (./forge-placement.nix), never the central toggle: a forge host with
# the toggle off still mints and publishes it.
name = "the forge's controller-token mint does not consult the central toggle";
ok = forgeHere.containers.hive-forge.config.systemd.services ? forgejo-swarm-controller-account;
}
{
name = "a hive that does not host the swarm's shared services runs none of them";
ok =
let
es = swarmServiceEnables bare;
in
lib.length (lib.attrNames es) == 9 && !lib.any lib.id (lib.attrValues es);
}
{
name = "a hive that has not opted into all-local runs no swarm controller";
ok = !bare.services.hyperhive.deploy.swarm-controller.enable;
}
{
name = "the all-local mode turns the swarm controller on";
ok = allLocal.services.hyperhive.deploy.swarm-controller.enable;
}
{
# ../host-modules/glue-controller-env.nix: the collector's coordinates
# follow swarm-otel running HERE. `controllerAlone` is the split case — a
# controller whose collector runs elsewhere.
name = "the controller exports to the collector only where the collector runs here";
ok =
let
env = m: m.systemd.services.swarm-controller.environment;
otel = allLocal.services.hyperhive.swarm.otel;
in
(env allLocal).OTEL_EXPORTER_OTLP_ENDPOINT or null == "https://${otel.domain}/${otel.producerName}"
&&
(env allLocal).SWARM_CONTROLLER_OTEL_AUDIENCE or null
== allLocal.services.hyperhive.swarm.controller.queueClientId
&& !(env shared.controllerAlone ? OTEL_EXPORTER_OTLP_ENDPOINT)
&& !(env shared.controllerAlone ? SWARM_CONTROLLER_OTEL_AUDIENCE);
}
{
# Same file: `swarmctl` is told where the configured authelia is only
# where authelia runs here.
name = "swarmctl is given the authelia binary and users file only where authelia runs here";
ok =
let
env = m: m.services.hyperhive.deploy.swarm-controller.swarmctlEnvironment;
d = allLocal.services.hyperhive.deploy.authelia;
in
env allLocal == {
SWARMCTL_AUTHELIA_BIN = "${d.package}/bin/authelia";
SWARMCTL_AUTHELIA_USERS_FILE = d.hostUsersFile;
}
&& env shared.controllerAlone == { };
}
{
# Where the reader puts the files and where the daemon looks for them is
# one agreement spanning two modules. Asserted against the option rather
# than the literal so moving the directory moves both ends.
name = "hive-c0re is told where the agents' queue credential lands";
ok =
allLocal.systemd.services.hive-c0re.environment.HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR
== toString allLocal.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir;
}
{
# Inside an agent container loopback is the agent itself, so the agents'
# address must never be one. By name it is the same string the hive
# dials, which ./nats-tls.nix pins for every client.
name = "the agents' queue address is the queue's name, never loopback";
ok =
let
e = allLocal.systemd.services.hive-c0re.environment;
in
e.HIVE_AGENT_NATS_URL == "tls://nats.t.local:4222"
&& !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL)
&& e.HIVE_AGENT_NATS_URL == e.HIVE_C0RE_NATS_URL;
}
{
# What agents are handed lands in their builds, so it has to be the same
# on every hive: a name, which this hive's resolver answers with the
# bridge IP its collector binds. hive-c0re itself keeps the address.
name = "the agents' collector address is a swarm-wide name this hive resolves to its bridge";
ok =
let
e = allLocal.systemd.services.hive-c0re.environment;
bridge = allLocal.services.hyperhive.network.bridgeIp;
in
e.HYPERHIVE_OTEL_ENDPOINT == "http://hive-collector.t.local:4318"
&& e.OTEL_EXPORTER_OTLP_ENDPOINT == "http://${bridge}:4318"
&& lib.elem "/hive-collector.t.local/${bridge}" allLocal.services.dnsmasq.settings.address;
}
{
# The agents mint against the swarm's IdP, the same endpoint the hive's
# own client uses — a hive-local guess would produce a token the queue
# would not accept.
name = "the agents' token endpoint is the swarm IdP's";
ok =
let
e = allLocal.systemd.services.hive-c0re.environment;
in
lib.hasSuffix "/api/oidc/token" e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT
&& e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT == e.HIVE_C0RE_OIDC_TOKEN_ENDPOINT;
}
{
# The absence arm, and what makes the two above able to fail: a hive
# with no queue address for its agents must forward neither coordinate,
# because half a pair reaches the harness as a partial configuration
# rather than as none.
name = "a hive with no agent queue address forwards no agent queue coordinates";
ok =
let
e = noAgentQueue.systemd.services.hive-c0re.environment;
in
!(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT);
}
{
# Where the store is and where the agent is told it is, one agreement
# spanning two modules. Asserted against the hive's own `BAO_ADDR`
# rather than a literal, because an agent pointed at a different
# spelling of the same store presents a certificate to a listener whose
# name it cannot verify.
name = "an agent is told the same store address its hive uses";
ok =
let
e = allLocal.systemd.services.hive-c0re.environment;
in
e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR;
}
{
# A hive with no certificate of its own can collect no agent's identity,
# so forwarding an address would name a store nothing in the container
# can reach. The same gate the `BAO_*` pair beside it sits behind.
name = "a hive with no store identity forwards no store address to its agents";
ok = !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_ADDR);
}
{
# The gateway's per-name issuer choice. If this ever collapses to a
# constant, every swarm-service vhost serves a certificate its CA
# is name-constrained out of — which evaluates cleanly and fails in
# a browser.
name = "a swarm service name gets the swarm-services leaf and the default server does not";
ok =
let
l = allLocal.services.hyperhive.gateway.lib;
in
(l.tlsFor "t.local").sslCertificate != (l.tlsFor "_").sslCertificate;
}
{
# A leaf is a key that can SERVE every name in it, so the names it
# asks for are that key's blast radius. The forge's host fronts forge
# and nothing else; the last conjunct is the control, since
# `auth.t.local` is in the swarm's set and an unnarrowed request would
# carry it here too.
name = "a gateway's services leaf asks for only the swarm names that host fronts";
ok =
let
s = forgeHere.services.hyperhive.swarm.serviceDomains;
u = forgeHere.systemd.services.swarm-services-cert.script;
in
lib.hasInfix "alt_names=forge.t.local" u
&& !(lib.hasInfix "auth.t.local" u)
&& lib.elem "auth.t.local" s;
}
{
# The store's issuing role is built from this list on the store's host,
# so it cannot depend on which services the evaluating host runs. The
# last conjunct is the control: grafana runs on neither `bare` nor the
# forge host, and is still listed.
name = "the swarm's service names are the same on hosts running different services";
ok =
let
s = h: h.services.hyperhive.swarm.serviceDomains;
in
s forgeHere == s bare && s allLocal == s bare && lib.elem "grafana.t.local" (s bare);
}
{
# The narrowing's floor: no names left means no request, rather than a
# request for an empty SAN set that the store would refuse. Its own
# vhosts are the hive leaf's, which this host still signs.
name = "a host fronting none of the swarm's service names requests no services leaf";
ok =
bare.services.hyperhive.swarm.localServiceDomains == [ ]
&& bare.services.hyperhive.swarm.serviceDomains != [ ]
&& lib.hasInfix "want_svc=0" bare.systemd.services.swarm-services-cert.script;
}
{
# nixos asserts when a vhost declares both, so this is also a
# statement that the `removeAttrs` upstream of it still happens.
name = "the swarm UI vhost forces TLS instead of merely adding it";
ok =
let
v = allLocal.services.nginx.virtualHosts."t.local";
in
v.forceSSL && !(v.addSSL or false);
}
{
# `swarmctl --swarm` reads a 401 as "log in again"; a 302 would hand it
# the login page instead. The `/` clause is the control: the browser
# route still redirects, so a pass means the two routes differ.
name = "the controller API answers a refused bearer caller 401 and a browser the login";
ok =
let
l = allLocal.services.nginx.virtualHosts."t.local".locations;
api = l."/api/".extraConfig;
refused = l."@swarm_api_unauthorized".extraConfig;
in
lib.hasInfix "error_page 401 = @swarm_api_unauthorized;" api
&& !(lib.hasInfix "=302" api)
&& lib.hasInfix ''proxy_set_header Authorization "";'' api
&& lib.hasInfix "if ($http_authorization) {\n return 401;\n}" refused
&& lib.hasInfix "return 302 https://auth.t.local/?rd=$target_url;" refused
&& lib.hasInfix "error_page 401 =302 https://auth.t.local/" l."/".extraConfig;
}
{
# authelia 4.39.20 refuses a bearer-scoped public client missing any of
# these at startup, after a green deploy; and a public client must not
# be handed a secret.
name = "the swarmctl login client is public, bearer-scoped to the controller API, and gets no secret";
ok =
let
s = allLocal.containers.swarm-authelia.config.systemd.services.authelia-swarm-secrets.script;
entry = lib.last (lib.splitString "client_id: %s\\n' swarmctl" s);
client = lib.head (lib.splitString "client_id: %s" entry);
in
lib.all (line: lib.hasInfix line client) [
"public: true"
"token_endpoint_auth_method: none"
"require_pkce: true"
"pkce_challenge_method: S256"
"require_pushed_authorization_requests: true"
"consent_mode: explicit"
"response_modes: [form_post]"
"grant_types: [authorization_code, refresh_token]"
"scopes: [offline_access, authelia.bearer.authz]"
"audience: [%s]\\n' https://t.local/api/"
"' - %s\\n' http://127.0.0.1/callback"
]
&& !(lib.hasInfix "client_secret" client)
&& !(lib.hasInfix "mint swarmctl" s)
&& lib.hasInfix "mint swarm-authelia-bridge" s;
}
{
# swarm-controller checks no caller itself: this location's authelia
# subrequest is the operator gate for every route under `/api/`. The
# webhook location is the control, the one path to the same socket that
# skips the gate, so the first clause reads a string that can lack it.
name = "the swarm UI's API location asks authelia before reaching the controller";
ok =
let
l = allLocal.services.nginx.virtualHosts."t.local".locations;
socket = allLocal.services.hyperhive.deploy.swarm-controller.socketPath;
in
lib.hasInfix "auth_request /__hive_authelia;" l."/api/".extraConfig
&& lib.hasInfix socket l."/api/".proxyPass
&& lib.hasInfix socket l."/webhook/forge/".proxyPass
&& !(lib.hasInfix "auth_request" l."/webhook/forge/".extraConfig);
}
{
# nginx serves a path from its longest matching prefix location, so that
# one must be the gated `/api/`, and no exact location may take a route
# out from under it.
name = "the claude login routes reach the controller only through a location that asks authelia";
ok =
let
l = allLocal.services.nginx.virtualHosts."t.local".locations;
route = "/api/agents/a/claude-login/code";
covering = lib.filter (
name: !(lib.hasPrefix "~" name) && !(lib.hasPrefix "= " name) && lib.hasPrefix (prefix name) route
) (lib.attrNames l);
prefix = lib.removePrefix "^~ ";
longest = lib.foldl' (
a: b: if lib.stringLength (prefix b) > lib.stringLength (prefix a) then b else a
) "" covering;
in
longest == "/api/"
&& lib.hasInfix "auth_request /__hive_authelia;" l.${longest}.extraConfig
&& !(lib.any (name: lib.hasPrefix "= /api/agents" name) (lib.attrNames l));
}
{
# `auth_request` does not inherit across sibling locations, so each
# gated one is read on its own. `/health/` is the control: the one
# c0re route that must stay open, so a pass means the block is
# per-location rather than everywhere.
name = "the dashboard answers on its own name behind authelia, and the hive domain redirects there";
ok =
let
vh = allLocal.services.nginx.virtualHosts;
l = vh."h1.hives.t.local".locations;
gated = loc: lib.hasInfix "auth_request /__hive_authelia;" loc.extraConfig;
in
allLocal.services.hyperhive.swarm.dashboardDomain == "h1.hives.t.local"
&& vh."h1.hives.t.local".forceSSL
&& lib.all (p: gated l.${p}) [
"/"
"/api/"
"/api/docs/"
]
&& lib.hasInfix "error_page 401 =302 https://auth.t.local/?rd=$target_url;" l."/".extraConfig
&& !(lib.hasInfix "auth_request" (l."/health/".extraConfig or ""))
&& l."= /__hive_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request"
&& lib.hasInfix "agents.conf" vh."h1.hives.t.local".extraConfig
&&
lib.hasInfix "return 301 https://h1.hives.t.local$request_uri;"
vh."h1.t.local".locations."/".extraConfig
&& !(vh."h1.t.local".locations ? "/api/")
&& lib.elem "h1.hives.t.local" allLocal.services.hyperhive.gateway.localNames
&& allLocal.systemd.services.hive-c0re.environment.HYPERHIVE_DASHBOARD_DOMAIN == "h1.hives.t.local";
}
{
# The gateway's own three vhosts are declared under `gateway.vhosts` like
# a service's, so the gateway's listen set and issuer choice reach them
# by the same path. The catch-all keeps its bare `return 444;` (no
# headers: it sends no response) and the dashboard keeps its exact
# location set.
name = "the gateway's own vhosts render with its listen set, issuer choice, headers and locations";
ok =
let
gw = allLocal.services.hyperhive.gateway;
vh = allLocal.services.nginx.virtualHosts;
listenOf = map (l: {
inherit (l) addr port;
ssl = l.ssl or false;
});
sameListen = v: listenOf v.listen == listenOf gw.lib.listen;
sameCert = name: v: v.sslCertificate == (gw.lib.tlsFor name).sslCertificate;
catchAll = vh."_";
dash = vh."h1.hives.t.local";
hive = vh."h1.t.local";
in
sameListen catchAll
&& sameCert "_" catchAll
&& catchAll.addSSL
&& catchAll.default
&& catchAll.extraConfig == "return 444;\n"
&& sameListen dash
&& sameCert "h1.hives.t.local" dash
&& dash.forceSSL
&& !(dash.addSSL or false)
&& dash.extraConfig == gw.lib.securityHeaders + "include /var/lib/hive-gateway/conf/agents.conf;\n"
&&
lib.attrNames dash.locations == [
"/"
"/agent/"
"/api/"
"/api/docs/"
"/health/"
"= /__hive_agent_not_found"
"= /__hive_agent_unreachable"
"= /__hive_authelia"
"= /api/docs"
]
&& sameListen hive
&& sameCert "h1.t.local" hive
&& hive.addSSL
&& hive.extraConfig == gw.lib.securityHeaders
&& hive.locations ? "/";
}
{
# The file held bcrypt hashes at 0644; a host carrying one from before
# gets it deleted, and nothing names it for nginx.
name = "the gateway reads no htpasswd, and tmpfiles removes a leftover one";
ok =
lib.elem "r /var/lib/hive-gateway/conf/gateway.htpasswd" allLocal.systemd.tmpfiles.rules
&& !(lib.any (
v:
lib.any (t: lib.hasInfix "auth_basic" t) (
[ (v.extraConfig or "") ] ++ map (loc: loc.extraConfig or "") (lib.attrValues (v.locations or { }))
)
) (lib.attrValues allLocal.services.nginx.virtualHosts));
}
{
name = "the swarm UI's hive roster carries each hive's dashboard name";
ok =
builtins.fromJSON allLocal.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_HIVES == [
{
name = "h1";
domain = "h1.t.local";
dashboard = "h1.hives.t.local";
}
];
}
{
name = "a hive with matrix off serves no matrix discovery endpoint";
ok =
!(builtins.hasAttr "= /.well-known/matrix/client" bare.services.nginx.virtualHosts."_".locations);
}
{
# main got eval-borked twice by this exact class of bug (once on the
# unit's `Restart` key, once on `RestartSec`) — a nixpkgs bump to
# `gitea-actions-runner.nix` adds a plain `serviceConfig.*`
# definition that collides with one of ours, and nix refuses to
# merge two plain definitions at *host* eval. No other check
# instantiates a host with `containers.swarm-ci-runner` actually enabled, so
# the collision only surfaces on operator deploy, not in CI.
name = "the CI container's unit definitions merge without a priority collision";
ok = forceCiServiceConfigs;
}
{
# It spends money on an operator's key, so no host runs it by default.
name = "the review classifier's container exists only where its host enables it";
ok =
!(bare.containers ? swarm-review-classifier)
&& !(centralToggleOff.containers ? swarm-review-classifier)
&& withClassifier.containers ? swarm-review-classifier;
}
{
# The daemon holds its store leaf, handed in by nspawn without a copy on
# disk, and nothing else from the host; a key is read per run and
# reaches only the model child's environment. Reading the claude path
# also evaluates the container's unfree allowance.
name = "the review classifier loads only its store leaf and carries no API or provider key in its unit";
ok =
let
svc =
withClassifier.containers.swarm-review-classifier.config.systemd.services.swarm-review-classifier;
env = svc.environment;
in
lib.elem "review-classifier-bao-key" svc.serviceConfig.LoadCredential
&&
env.BAO_CLIENT_KEY == "/run/credentials/swarm-review-classifier.service/review-classifier-bao-key"
&& env.SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID == "swarm-review-classifier"
&& lib.hasSuffix "/bin/claude" env.SWARM_REVIEW_CLASSIFIER_CLAUDE
&& lib.hasSuffix "/bin/opencode" env.SWARM_REVIEW_CLASSIFIER_OPENCODE
&& !(env ? ANTHROPIC_API_KEY)
&& !(env ? ACP_PROVIDER_API_KEY)
&& lib.elem "--load-credential=review-classifier-bao-key:/var/lib/swarm-bao-pki/review-classifier-key.pem" (
withClassifier.containers.swarm-review-classifier.extraFlags
)
# No host unit copies the leaf into the container's root.
&& !(lib.any (u: lib.hasInfix "nixos-containers/swarm-review-classifier" (u.script or "")) (
lib.attrValues withClassifier.systemd.services
));
}
{
# See the same case in ./builder.nix.
name = "the review classifier's container runs the leaf-minting unit before every start";
ok =
let
c = withClassifier.systemd.services."container@swarm-review-classifier";
in
lib.elem "swarm-bao-pki.service" c.wants && lib.elem "swarm-bao-pki.service" c.after;
}
{
# The collector reaches these routes through the gateway now, so each
# store needs an ingest location of its own. Without one the write rides
# the `/` catch-all: unauthenticated on the metrics store, and into a
# browser redirect on the log store.
name = "each store's vhost has an authenticated ingest location";
ok =
let
v = allLocal.services.nginx.virtualHosts;
m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push" or null;
l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs" or null;
in
m != null
&& l != null
&& lib.hasInfix "auth_request" m.extraConfig
&& lib.hasInfix "auth_request" l.extraConfig;
}
{
# The metrics vhost is default-deny; the collector's push only lands
# because its `metrics_endpoint` path is the vhost's one gated exact
# location. The two literals live in different modules, so renaming
# either alone breaks ingest while everything still evaluates.
name = "the collector's metrics push path is a gated location on the metrics vhost";
ok =
let
endpoint = (otelSettings allLocal).exporters."otlphttp/victoriametrics".metrics_endpoint;
# "https://host/path" -> "/path"
path = "/" + lib.concatStringsSep "/" (lib.drop 3 (lib.splitString "/" endpoint));
loc = allLocal.services.nginx.virtualHosts."metrics.t.local".locations."= ${path}" or null;
in
loc != null && lib.hasInfix "auth_request" loc.extraConfig;
}
{
# The arm that actually protects something. A pusher handed
# `error_page 401 =302` FOLLOWS it and POSTs its batch at a login page,
# which answers 200 — ingest reporting healthy while storing nothing.
# The third clause is the positive control: the log store's browser
# location really does redirect, so this says the machine routes differ
# rather than that the string is absent from the whole file.
name = "the ingest locations answer 401 instead of redirecting a pusher";
ok =
let
v = allLocal.services.nginx.virtualHosts;
m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push".extraConfig;
l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs".extraConfig;
browser = v."logs.t.local".locations."/".extraConfig;
in
!(lib.hasInfix "error_page" m)
&& !(lib.hasInfix "error_page" l)
&& lib.hasInfix "error_page" browser;
}
{
# The store authenticates nothing itself, so a location that forwards to
# it without `auth_request` is open to anything that can reach the
# domain. The query location is the control: it shows the filter finds
# store-forwarding locations at all.
name = "no location on the metrics vhost forwards to the store without auth_request";
ok =
let
locs = allLocal.services.nginx.virtualHosts."metrics.t.local".locations;
toStore = lib.filterAttrs (
_: l: l.proxyPass != null && lib.hasPrefix "http://unix:/run/swarm-victoriametrics/" l.proxyPass
) locs;
open = lib.filterAttrs (_: l: !(lib.hasInfix "auth_request " l.extraConfig)) toStore;
in
toStore ? "= /api/v1/query"
&& open == { }
&& locs."/".proxyPass == null
&& locs."/".return == "403";
}
{
# The read API's check, link by link: every read location names the
# switch with no login redirect (a machine reader would get login HTML
# as a 200), the switch rewrites to the map's pick, a bearer reaches
# the responder's socket, which the unit binds, and anything else goes
# to authelia. vmui is a browser's, so it redirects to the login.
name = "the metrics read API sends a bearer to swarm-metrics-auth and a session to authelia";
ok =
let
l = allLocal.services.nginx.virtualHosts."metrics.t.local".locations;
http = allLocal.services.nginx.appendHttpConfig;
unit = allLocal.systemd.services.swarm-metrics-auth;
reads = lib.filterAttrs (n: _: lib.hasPrefix "= /api/v1/" n || lib.hasPrefix "^~ /api/v1/" n) l;
in
reads ? "= /api/v1/query_range"
&& lib.all (
r:
lib.hasInfix "auth_request /__metrics_read_auth;" r.extraConfig
&& !(lib.hasInfix "error_page" r.extraConfig)
) (lib.attrValues reads)
&& lib.hasInfix "map \$http_authorization \$swarm_metrics_read_auth_target" http
&& lib.hasInfix "\"~^Bearer \" /__metrics_grafana_auth;" http
&& lib.hasInfix "default /__metrics_authelia;" http
&&
lib.hasInfix "rewrite ^ \$swarm_metrics_read_auth_target last;"
l."= /__metrics_read_auth".extraConfig
&& lib.hasInfix "internal;" l."= /__metrics_grafana_auth".extraConfig
&& l."= /__metrics_grafana_auth".proxyPass == "http://unix:/run/swarm-metrics-auth/auth.sock:/"
&& lib.hasInfix "--socket /run/swarm-metrics-auth/auth.sock" unit.serviceConfig.ExecStart
&& lib.hasInfix "auth_request /__metrics_authelia;" l."^~ /vmui/".extraConfig
&& lib.hasInfix "error_page 401 =302" l."^~ /vmui/".extraConfig
&& l."= /__metrics_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request";
}
{
# Grafana's host creates the bearer and its datasource sends it; the
# store's responder reads it. Three modules spell the path, and the
# store's grants name it twice: if any one drifts, Grafana sends a value
# the responder never reads, or one of them is refused by the store.
name = "Grafana's datasource sends the bearer swarm-metrics-auth checks, at one store path";
ok =
let
leaf = "swarm/services/${allLocal.services.hyperhive.swarm.grafana.oidc.clientId}/metrics/bearer";
ds =
lib.findFirst (d: d.uid == "swarm-victoriametrics") null
allLocal.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
bearerUnit = allLocal.systemd.services.swarm-bao-grafana-metrics-bearer;
s = allLocal.systemd.services;
in
ds.jsonData.httpHeaderName1 == "Authorization"
&& ds.secureJsonData.httpHeaderValue1 == "$__file{/run/swarm-grafana-secrets/metrics-bearer}"
&& lib.hasInfix "bao kv put -cas=0 secret/${leaf} value=-" bearerUnit.script
&& lib.hasInfix "/run/swarm-grafana-secrets/metrics-bearer \"Bearer $bearer\"" bearerUnit.script
&& lib.hasInfix "--store-path ${leaf}" s.swarm-metrics-auth.serviceConfig.ExecStart
&& lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"read\"]\n}" s.swarm-bao-metrics-auth-policy.script
&& lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"create\", \"read\"]\n}" s.swarm-bao-grafana-oidc-policy.script;
}
{
# The log store's second responder checks the very bearer the metrics
# datasource sends, at the same store path: a different file would be a
# bearer the responder never reads.
name = "Grafana's logs datasource sends the bearer the log store's responder checks, at one store path";
ok =
let
datasources =
allLocal.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
metrics = lib.findFirst (d: d.uid == "swarm-victoriametrics") null datasources;
logs = lib.findFirst (d: d.uid == "swarm-victorialogs") null datasources;
leaf = "swarm/services/${allLocal.services.hyperhive.swarm.grafana.oidc.clientId}/metrics/bearer";
in
logs.secureJsonData.httpHeaderValue1 == metrics.secureJsonData.httpHeaderValue1
&& lib.hasInfix "--store-path ${leaf}" allLocal.systemd.services.swarm-logs-grafana-auth.serviceConfig.ExecStart;
}
{
# The responder reads the store as itself, with the leaf the store's
# host mints for it.
name = "swarm-metrics-auth presents its own store leaf";
ok =
let
v = allLocal.services.hyperhive.deploy.victoriametrics;
lc = allLocal.systemd.services.swarm-metrics-auth.serviceConfig.LoadCredential;
in
lib.hasSuffix "/metrics-auth.pem" v.authBaoClientCertFile
&& lib.hasSuffix "/metrics-auth-key.pem" v.authBaoClientKeyFile
&& builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc
&& builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc
&& lib.hasInfix "/metrics-auth.pem ]" allLocal.systemd.services.swarm-bao-pki.script;
}
{
# The read counterpart to the ingest location: an agent queries the log
# store with a bearer token, and the `/` catch-all is the browser's
# route. Riding it would mean inheriting the login redirect the next
# case is about, so the route has to exist separately to be gated
# separately.
name = "the log store's vhost has an authenticated machine query location";
ok =
let
q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/" or null;
in
q != null && lib.hasInfix "auth_request" q.extraConfig;
}
{
# Same trap as the ingest case, on the read side, where it is worse: a
# redirected pusher at least stores nothing visibly, while a redirected
# *reader* is handed a 200 carrying login HTML and records a query that
# succeeded and matched no logs. The browser clause is the positive
# control — that location really does redirect to the login host — so a
# pass means these two routes differ rather than that the strings are
# absent from the whole vhost.
name = "the machine query location answers 401 instead of redirecting to a login page";
ok =
let
v = allLocal.services.nginx.virtualHosts;
q = v."logs.t.local".locations."^~ /select/logsql/".extraConfig;
browser = v."logs.t.local".locations."/".extraConfig;
in
!(lib.hasInfix "error_page" q)
&& !(lib.hasInfix "auth.t.local" q)
&& lib.hasInfix "error_page" browser
&& lib.hasInfix "auth.t.local" browser;
}
{
# Read access is deliberately unscoped: an authenticated caller reads
# the whole swarm's logs until a permission system exists. Pinned so a
# scoping parameter arriving later is a visible diff here rather than a
# quiet change of rule — and pinned on `proxyPass` too, because
# VictoriaLogs takes its filters as request parameters, which ride an
# upstream URI as easily as a directive. The first clause is the
# control: it proves the location resolved and that `hasInfix` finds
# what is genuinely in this string, so the absences below mean absent
# rather than unreadable.
name = "the machine query location forwards the caller's query unmodified";
ok =
let
q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/";
in
lib.hasInfix "auth_request" q.extraConfig
&& !(lib.hasInfix "extra_filters" q.extraConfig)
&& !(lib.hasInfix "extra_stream_filters" q.extraConfig)
&& !(lib.hasInfix "$args" q.extraConfig)
&& !(lib.hasInfix "?" q.proxyPass);
}
{
# `auth_request` takes a fixed URI, so the split between the agents'
# responder and authelia is a map plus a rewrite in an internal
# location. Every link of that chain is pinned: the map keys on the
# agent-token prefix and falls back to authelia, the query location
# names the switch, the switch rewrites to the map's pick, and the
# agent target reaches the responder's socket, which the unit binds.
name = "the machine query location sends agent bearers to swarm-logs-auth and the rest to authelia";
ok =
let
l = allLocal.services.nginx.virtualHosts."logs.t.local".locations;
http = allLocal.services.nginx.appendHttpConfig;
unit = allLocal.systemd.services.swarm-logs-auth;
grafanaUnit = allLocal.systemd.services.swarm-logs-grafana-auth;
in
lib.hasInfix "map \$http_authorization \$swarm_logs_auth_target" http
&& lib.hasInfix "\"~^Bearer swarm-agent\\.\" /__swarm_logs_agent_auth;" http
&& lib.hasInfix "\"~^Bearer [0-9a-f]{64}$\" /__swarm_logs_grafana_auth;" http
&& lib.hasInfix "default /__hive_authelia;" http
&& lib.hasInfix "auth_request /__swarm_logs_auth;" l."^~ /select/logsql/".extraConfig
&& lib.hasInfix "rewrite ^ \$swarm_logs_auth_target last;" l."= /__swarm_logs_auth".extraConfig
&& lib.hasInfix "internal;" l."= /__swarm_logs_agent_auth".extraConfig
&& l."= /__swarm_logs_agent_auth".proxyPass == "http://unix:/run/swarm-logs-auth/auth.sock:/"
&& lib.hasInfix "--socket /run/swarm-logs-auth/auth.sock" unit.serviceConfig.ExecStart
&& lib.hasInfix "internal;" l."= /__swarm_logs_grafana_auth".extraConfig
&&
l."= /__swarm_logs_grafana_auth".proxyPass == "http://unix:/run/swarm-logs-grafana-auth/auth.sock:/"
&& lib.hasInfix "--socket /run/swarm-logs-grafana-auth/auth.sock" grafanaUnit.serviceConfig.ExecStart
&& lib.hasInfix "swarm-metrics-auth" grafanaUnit.serviceConfig.ExecStart
&& lib.hasInfix "--store-cert-role swarm-logs-auth" grafanaUnit.serviceConfig.ExecStart
&& l ? "= /__hive_authelia";
}
{
# A `DynamicUser` unit's preserved runtime directory lives under
# `/run/private/` (0700 root:root), which nginx cannot traverse, so a
# socket there answers every proxied request with a 502. Each `unix:`
# upstream under `/run/<dir>/` is matched to the units creating `<dir>`.
# The first two clauses are the control: the match finds both responders.
name = "no unix socket nginx proxies to sits in a DynamicUser's preserved runtime directory";
ok =
let
prefix = "http://unix:/run/";
upstreamDirs = lib.unique (
map (p: lib.head (lib.splitString "/" (lib.removePrefix prefix p))) (
lib.filter (p: p != null && lib.hasPrefix prefix p) (
lib.concatMap (vh: lib.mapAttrsToList (_: loc: loc.proxyPass) vh.locations) (
lib.attrValues allLocal.services.nginx.virtualHosts
)
)
)
);
runtimeDirs =
sc:
let
r = sc.RuntimeDirectory or [ ];
in
if lib.isList r then r else lib.splitString " " r;
owners = lib.filterAttrs (
_: u: lib.any (d: lib.elem d upstreamDirs) (runtimeDirs u.serviceConfig)
) allLocal.systemd.services;
hidden =
sc:
lib.elem (sc.DynamicUser or false) [
true
"yes"
]
&& !(lib.elem (sc.RuntimeDirectoryPreserve or "no") [
false
"no"
]);
in
owners ? swarm-logs-auth
&& owners ? swarm-logs-grafana-auth
&& owners ? swarm-metrics-auth
&& !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners));
}
{
# Every agent bearer the responder has not recently granted costs a
# store login and read, so its location carries the per-address limit.
# The responder gets the grant TTL from the option. The query and
# authelia locations carry no limit of their own.
name = "agent bearer checks are rate-limited per address and granted for a TTL";
ok =
let
l = allLocal.services.nginx.virtualHosts."logs.t.local".locations;
http = allLocal.services.nginx.appendHttpConfig;
unit = allLocal.systemd.services.swarm-logs-auth;
v = allLocal.services.hyperhive.deploy.victorialogs;
in
lib.hasInfix "limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${v.authRate};" http
&&
lib.hasInfix "limit_req zone=swarm_logs_agent_auth burst=${toString v.authBurst} nodelay;"
l."= /__swarm_logs_agent_auth".extraConfig
&& lib.hasInfix "limit_req_status 403;" l."= /__swarm_logs_agent_auth".extraConfig
&& !(lib.hasInfix "limit_req" l."^~ /select/logsql/".extraConfig)
&& !(lib.hasInfix "limit_req" l."= /__hive_authelia".extraConfig)
&& lib.hasInfix "--verdict-ttl 60" unit.serviceConfig.ExecStart;
}
{
# The responder reads the store as itself, with the leaf the store's
# host mints for it.
name = "swarm-logs-auth presents its own store leaf";
ok =
let
v = allLocal.services.hyperhive.deploy.victorialogs;
lc = allLocal.systemd.services.swarm-logs-auth.serviceConfig.LoadCredential;
in
lib.hasSuffix "/logs-auth.pem" v.authBaoClientCertFile
&& lib.hasSuffix "/logs-auth-key.pem" v.authBaoClientKeyFile
&& builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc
&& builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc;
}
{
# The absence arm for the case above, and the option's own rule — a
# service declares its entry under its own `enable` — made checkable.
# Without it, moving the assignment outside the collector's `mkIf`
# passes every arm above while handing a collector-less hive a scrape
# target for a port nothing binds.
name = "a hive with no collector declares no self-scrape target";
ok = bare.services.hyperhive.otel.scrapeTargets == { };
}
{
# Absence arm, and the one that matters: claiming a name this host does
# not serve points every local reader at the wrong machine.
name = "a hive that does not run the store claims no name for it";
ok = !(builtins.elem "bao.t.local" (baoNames bare));
}
{
# ⚠️ The absence arm that matters. `services.nginx.streamConfig` is a
# host-wide option, so a block rendered outside the store's own `mkIf`
# gives every hive in the swarm a listener — on the port the store
# answers on, in front of no store at all.
name = "a hive that does not run the store renders no stream passthrough";
ok = baoStream bare == "";
}
{
# Absence arm for the case above — a hive with no store has no reason to
# open the store's port, and opening it would point agents at a host that
# answers nothing.
name = "a hive that does not run the store opens no bridge port for it";
ok = !(builtins.elem 8200 (bridgePorts bare));
}
{
# The four blocks below used to be gated on the hive being enabled AND
# their own condition. The second half was always the load-bearing one —
# none of these conditions is derived from the hive toggle — so these
# arms pin what the conjunct was doing: nothing. Each is written against
# a host with the hive OFF as well as one with it on, because the way a
# dropped conjunct fails is by making something unconditional, and that
# shows up as a service appearing where nothing asked for it.
name = "no bridge port is opened for an exposeHostPorts nobody set";
ok =
!(builtins.elem 5432 (bridgePortsOrNone bare))
&& !(builtins.elem 5432 (bridgePortsOrNone centralToggleOff));
}
{
# `deploy.swarm-controller.enable`, which defaults false and is
# deliberately not derived from the hive toggle — a swarm has one
# controller, so the host that runs it says so itself.
#
# Probed by what the daemon needs in order to run, not by
# `? swarm-controller`: ./host-modules/hive-tls.nix defines an
# environment key on that unit name, which leaves the attr
# present-but-inert (no `ExecStart`, empty `wantedBy`) on every hive
# that has a CA — see the comment there. The daemon's system user has no
# second definer, so its absence is the unambiguous half.
name = "the swarm controller does not run unless this host is told to run it";
ok =
let
inert =
machine:
!(machine.users.users ? swarm-controller)
&& !(
(machine.systemd.services.swarm-controller or { serviceConfig = { }; }).serviceConfig ? ExecStart
);
in
inert bare && inert centralToggleOff;
}
{
# `deploy.swarm-otel.enable`, same shape: the swarm's collector is one
# host's job, and the container is the whole of what it renders.
name = "the swarm collector container is absent unless this host is told to run it";
ok = !(bare.containers ? swarm-otel) && !(centralToggleOff.containers ? swarm-otel);
}
{
# The swarm UI follows `deploy.swarm-controller.enable` — a sibling
# deployment decision, not the hive toggle. `t.local` is the fixtures'
# swarm domain, which is the apex the UI claims; the arm below is what
# proves this vhost renders at all.
name = "the swarm UI vhost is absent unless this host runs the swarm controller";
ok =
!(bare.services.nginx.virtualHosts ? "t.local")
&& !(centralToggleOff.services.nginx.virtualHosts ? "t.local");
}
{
# The three infrastructure toggles are off by default and asserted by
# whoever needs them. With nothing on the host needing them, none of
# the three renders — which is also the control for the arm below.
name = "the gateway, resolver and bridge are absent where nothing on the host needs them";
ok =
!centralToggleOff.services.hyperhive.gateway.enable
&& !centralToggleOff.services.hyperhive.gateway.dns.enable
&& !centralToggleOff.services.hyperhive.network.enable
&& !(centralToggleOff.services.nginx.enable or false)
&& !(centralToggleOff.services.dnsmasq.enable or false)
&& !(centralToggleOff.networking.bridges ? hive-br0);
}
{
# hive-c0re asserts all three, and it follows the central toggle — so
# an ordinary hive keeps getting them with no opt-in, which is what
# this change must not break.
name = "an ordinary hive runs the gateway, resolver and bridge because its coordinator needs them";
ok =
bare.services.hyperhive.gateway.enable
&& bare.services.hyperhive.gateway.dns.enable
&& bare.services.hyperhive.network.enable
&& bare.services.nginx.enable
&& bare.services.dnsmasq.enable
&& bare.networking.bridges ? hive-br0;
}
{
# A host config written against either old spelling still runs its hive.
# The builder's own `true` is lowered to `mkDefault false`, so only the
# forwarded definition can turn it on.
name = "the old hive toggles forward to deploy.hive-controller.enable and warn";
ok =
let
renamedFrom =
old: m:
lib.any (
w: lib.hasInfix old w && lib.hasInfix "services.hyperhive.deploy.hive-controller.enable" w
) m.warnings;
viaOld =
extra:
hive (
extra
// {
deploy.hive-controller.enable = lib.mkDefault false;
}
);
oldEnable = viaOld { enable = true; };
oldC0re = viaOld { c0re.enable = true; };
in
oldEnable.services.hyperhive.deploy.hive-controller.enable
&& oldEnable.systemd.services ? hive-c0re
&& renamedFrom "services.hyperhive.enable" oldEnable
&& oldC0re.services.hyperhive.deploy.hive-controller.enable
&& oldC0re.systemd.services ? hive-c0re
&& renamedFrom "services.hyperhive.c0re.enable" oldC0re;
}
{
# Refused wherever something turns the name into an identifier, and only
# there. The store host is the control that the refusal is not the hive's
# alone; the host running neither is the control that it is gated at all.
name = "a missing hiveName is refused on a hive and on a store host, not elsewhere";
ok =
let
refusedHiveName =
extra:
lib.any (a: !a.assertion && lib.hasInfix "services.hyperhive.hiveName to be set" a.message)
(hiveWithController ({ hiveName = null; } // extra)).assertions;
in
refusedHiveName { }
&& refusedHiveName {
deploy.hive-controller.enable = false;
deploy.bao.enable = true;
}
&& !(refusedHiveName { deploy.hive-controller.enable = false; });
}
{
# The control is a host that defines no such option; the removal message
# must not appear for it.
name = "a definition of the removed services.hyperhive.ruthless option fails evaluation, pointing at the swarm dashboard";
ok =
let
refused =
m:
lib.any (
a:
!a.assertion
&& lib.hasInfix "services.hyperhive.ruthless" a.message
&& lib.hasInfix "swarm dashboard" a.message
) m.assertions;
in
refused (hive {
ruthless = true;
})
&& !(refused allLocal);
}
{
# Refused wherever any hyperhive service runs, the hive or a swarm
# service alone. The host enabling nothing is the control that it is
# gated at all, and has to pass every assertion: it is what an
# operator gets from importing the module.
name = "a missing swarm.domain is refused on a hive and on a swarm-service host, and a host enabling nothing evaluates";
ok =
let
noDomain = extra: hiveWithController ({ swarm.domain = null; } // extra);
refusedSwarmDomain =
extra:
lib.any (
a: !a.assertion && lib.hasInfix "services.hyperhive.swarm.domain to be set" a.message
) (noDomain extra).assertions;
in
refusedSwarmDomain { }
&& refusedSwarmDomain {
deploy.hive-controller.enable = false;
deploy.nats.enable = true;
}
&& lib.all (a: a.assertion) (noDomain { deploy.hive-controller.enable = false; }).assertions;
}
{
# Control: a store fetch with no timer still renders `RemainAfterExit`,
# so the cases below read a real value rather than a missing one.
name = "Grafana's admin-password fetch stays active after it exits and has no timer";
ok =
allLocal.systemd.services ? swarm-bao-grafana-admin
&& remainsAfterExit allLocal.systemd.services.swarm-bao-grafana-admin
&& !(allLocal.systemd.timers ? swarm-bao-grafana-admin);
}
{
# A timer whose unit does not exist fails to load, so each timer is
# gated exactly as its fetch is.
name = "a host running none of the store-refreshed fetches renders none of their timers";
ok = lib.all (
{ unit, ... }: !(bare.systemd.services ? ${unit}) && !(bare.systemd.timers ? ${unit})
) storeRefreshed;
}
{
# `or false` so a dropped key fails this case by name instead of
# aborting eval. Only the rendered unit is pinned; eval cannot show
# that hive-priv still works under the directives.
name = "hive-priv locks personality and denies real-time scheduling, kernel logs and module loading";
ok =
let
sc = allLocal.systemd.services.hive-priv.serviceConfig;
in
(sc.LockPersonality or false)
&& (sc.RestrictRealtime or false)
&& (sc.ProtectKernelLogs or false)
&& (sc.ProtectKernelModules or false);
}
]
# Starting a unit that stays `active (exited)` is a no-op, so a timer aimed
# at one fires and fetches nothing.
++ map (
{ unit, machine }:
{
name = "${unit} re-runs on a ten-minute timer and does not stay active after it exits";
ok =
let
s = machine.systemd.services;
t = machine.systemd.timers;
in
s ? ${unit}
&& t ? ${unit}
&& !(remainsAfterExit s.${unit})
&& lib.elem "timers.target" t.${unit}.wantedBy
&& t.${unit}.timerConfig.OnUnitInactiveSec == "10min"
&& (t.${unit}.timerConfig.Unit or "${unit}.service") == "${unit}.service";
}
) storeRefreshed;
in
runGroup "core-toggle" cases