37c58e79dc
The metrics vhost is default-deny; ingest works only because the collector's metrics_endpoint path equals the vhost's gated exact location. The literals live in separate modules, so renaming either broke ingest with CI green. Assert the endpoint path is a location on the metrics vhost carrying auth_request, in the existing core-toggle fixture. Refs #5186
778 lines
34 KiB
Nix
778 lines
34 KiB
Nix
# `checks.module-eval-core-toggle` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
hiveWithController
|
|
runGroup
|
|
baoNames
|
|
baoStream
|
|
bridgePorts
|
|
bridgePortsOrNone
|
|
swarmServiceEnables
|
|
otelSettings
|
|
;
|
|
|
|
inherit (shared) allLocal;
|
|
|
|
bare = shared.bareWithController;
|
|
|
|
# The same stub with the central toggle (`deploy.hive-controller.enable`)
|
|
# off. Paired with `bare` below to pin the defaults that do not read it:
|
|
# each is asserted to hold the SAME literal in both, so a future edit
|
|
# that quietly introduces the dependency — or that changes what the
|
|
# default renders for a hive with the toggle on — fails here. Reading an
|
|
# option off this fixture forces that option only, not the config, so the
|
|
# toggle being off costs nothing. Also the "installs the modules and turns
|
|
# nothing on" host the swarm-service absences below read: none of the
|
|
# per-service deployment toggles derives from the hive being on, so it
|
|
# renders the same absences `bare` does.
|
|
centralToggleOff = shared.bare;
|
|
|
|
inherit (shared) forgeHere;
|
|
|
|
# Every hive defaults the agents' queue address to the queue's name, so the
|
|
# only hive without one is a hive told to have none.
|
|
noAgentQueue = hiveWithController { deploy.hive-controller.queue.agentNatsUrl = null; };
|
|
|
|
# On the store's host, which mints the runner's store identity.
|
|
withCi = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.ci-runner.enable = true;
|
|
};
|
|
|
|
# On the store's host, which mints the classifier's store identity.
|
|
withClassifier = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.review-classifier.enable = true;
|
|
};
|
|
|
|
# A priority collision is a property of the *option*, not
|
|
# of the merged value's interior — nix throws the moment the value is
|
|
# demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
|
|
# both necessary and sufficient. `deepSeq` over-specifies this: it keeps
|
|
# walking *into* the resulting value after the merge already succeeded,
|
|
# and a package/derivation-shaped value's `override`/`overrideAttrs`
|
|
# self-reference sends it into nixpkgs' fixpoint machinery and blows the
|
|
# stack (measured — this is not a hypothetical).
|
|
forceCiServiceConfigs =
|
|
let
|
|
svcs = withCi.containers.swarm-ci-runner.config.systemd.services;
|
|
vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs);
|
|
in
|
|
builtins.foldl' (acc: v: builtins.seq v acc) true vals;
|
|
|
|
# The host fetches that restart their consumer when the store's value
|
|
# changes (../host-modules/lib/refresh-consumer.nix), each on a fixture
|
|
# that renders it.
|
|
storeRefreshed = [
|
|
{
|
|
unit = "swarm-bao-grafana-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-otel-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-matrix-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-forge-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-hive-otel-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-forwarder-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-ci-runner";
|
|
machine = withCi;
|
|
}
|
|
];
|
|
|
|
remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false;
|
|
|
|
cases = [
|
|
{
|
|
# The domain is the stub's swarm domain, which both fixtures share.
|
|
name = "the forge's publicUrl default does not consult the central toggle";
|
|
ok =
|
|
bare.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local"
|
|
&& centralToggleOff.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local";
|
|
}
|
|
{
|
|
# The controller's token publisher follows where the forge runs
|
|
# (./forge-placement.nix), never the central toggle: a forge host with
|
|
# the toggle off still publishes it.
|
|
name = "the forge's controller-token publisher does not consult the central toggle";
|
|
ok = forgeHere.systemd.services ? swarm-bao-forge-publisher;
|
|
}
|
|
{
|
|
name = "a hive that does not host the swarm's shared services runs none of them";
|
|
ok =
|
|
let
|
|
es = swarmServiceEnables bare;
|
|
in
|
|
lib.length (lib.attrNames es) == 9 && !lib.any lib.id (lib.attrValues es);
|
|
}
|
|
{
|
|
name = "a hive that has not opted into all-local runs no swarm controller";
|
|
ok = !bare.services.hyperhive.deploy.swarm-controller.enable;
|
|
}
|
|
{
|
|
name = "the all-local mode turns the swarm controller on";
|
|
ok = allLocal.services.hyperhive.deploy.swarm-controller.enable;
|
|
}
|
|
{
|
|
# Where the reader puts the files and where the daemon looks for them is
|
|
# one agreement spanning two modules. Asserted against the option rather
|
|
# than the literal so moving the directory moves both ends.
|
|
name = "hive-c0re is told where the agents' queue credential lands";
|
|
ok =
|
|
allLocal.systemd.services.hive-c0re.environment.HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR
|
|
== toString allLocal.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir;
|
|
}
|
|
{
|
|
# Inside an agent container loopback is the agent itself, so the agents'
|
|
# address must never be one. By name it is the same string the hive
|
|
# dials, which ./nats-tls.nix pins for every client.
|
|
name = "the agents' queue address is the queue's name, never loopback";
|
|
ok =
|
|
let
|
|
e = allLocal.systemd.services.hive-c0re.environment;
|
|
in
|
|
e.HIVE_AGENT_NATS_URL == "tls://nats.t.local:4222"
|
|
&& !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL)
|
|
&& e.HIVE_AGENT_NATS_URL == e.HIVE_C0RE_NATS_URL;
|
|
}
|
|
{
|
|
# The agents mint against the swarm's IdP, the same endpoint the hive's
|
|
# own client uses — a hive-local guess would produce a token the queue
|
|
# would not accept.
|
|
name = "the agents' token endpoint is the swarm IdP's";
|
|
ok =
|
|
let
|
|
e = allLocal.systemd.services.hive-c0re.environment;
|
|
in
|
|
lib.hasSuffix "/api/oidc/token" e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT
|
|
&& e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT == e.HIVE_C0RE_OIDC_TOKEN_ENDPOINT;
|
|
}
|
|
{
|
|
# The absence arm, and what makes the two above able to fail: a hive
|
|
# with no queue address for its agents must forward neither coordinate,
|
|
# because half a pair reaches the harness as a partial configuration
|
|
# rather than as none.
|
|
name = "a hive with no agent queue address forwards no agent queue coordinates";
|
|
ok =
|
|
let
|
|
e = noAgentQueue.systemd.services.hive-c0re.environment;
|
|
in
|
|
!(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT);
|
|
}
|
|
{
|
|
# Where the store is and where the agent is told it is, one agreement
|
|
# spanning two modules. Asserted against the hive's own `BAO_ADDR`
|
|
# rather than a literal, because an agent pointed at a different
|
|
# spelling of the same store presents a certificate to a listener whose
|
|
# name it cannot verify.
|
|
name = "an agent is told the same store address its hive uses";
|
|
ok =
|
|
let
|
|
e = allLocal.systemd.services.hive-c0re.environment;
|
|
in
|
|
e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR;
|
|
}
|
|
{
|
|
# A hive with no certificate of its own can collect no agent's identity,
|
|
# so forwarding an address would name a store nothing in the container
|
|
# can reach. The same gate the `BAO_*` pair beside it sits behind.
|
|
name = "a hive with no store identity forwards no store address to its agents";
|
|
ok = !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_ADDR);
|
|
}
|
|
{
|
|
# The gateway's per-name issuer choice. If this ever collapses to a
|
|
# constant, every swarm-service vhost serves a certificate its CA
|
|
# is name-constrained out of — which evaluates cleanly and fails in
|
|
# a browser.
|
|
name = "a swarm service name gets the swarm-services leaf and the default server does not";
|
|
ok =
|
|
let
|
|
l = allLocal.services.hyperhive.gateway.lib;
|
|
in
|
|
(l.tlsFor "t.local").sslCertificate != (l.tlsFor "_").sslCertificate;
|
|
}
|
|
{
|
|
# A leaf is a key that can SERVE every name in it, so the names it
|
|
# asks for are that key's blast radius. The forge's host fronts forge
|
|
# and nothing else; the last conjunct is the control, since
|
|
# `auth.t.local` is in the swarm's set and an unnarrowed request would
|
|
# carry it here too.
|
|
name = "a gateway's services leaf asks for only the swarm names that host fronts";
|
|
ok =
|
|
let
|
|
s = forgeHere.services.hyperhive.swarm.serviceDomains;
|
|
u = forgeHere.systemd.services.swarm-services-cert.script;
|
|
in
|
|
lib.hasInfix "alt_names=forge.t.local" u
|
|
&& !(lib.hasInfix "auth.t.local" u)
|
|
&& lib.elem "auth.t.local" s;
|
|
}
|
|
{
|
|
# The store's issuing role is built from this list on the store's host,
|
|
# so it cannot depend on which services the evaluating host runs. The
|
|
# last conjunct is the control: grafana runs on neither `bare` nor the
|
|
# forge host, and is still listed.
|
|
name = "the swarm's service names are the same on hosts running different services";
|
|
ok =
|
|
let
|
|
s = h: h.services.hyperhive.swarm.serviceDomains;
|
|
in
|
|
s forgeHere == s bare && s allLocal == s bare && lib.elem "grafana.t.local" (s bare);
|
|
}
|
|
{
|
|
# The narrowing's floor: no names left means no request, rather than a
|
|
# request for an empty SAN set that the store would refuse. Its own
|
|
# vhosts are the hive leaf's, which this host still signs.
|
|
name = "a host fronting none of the swarm's service names requests no services leaf";
|
|
ok =
|
|
bare.services.hyperhive.swarm.localServiceDomains == [ ]
|
|
&& bare.services.hyperhive.swarm.serviceDomains != [ ]
|
|
&& lib.hasInfix "want_svc=0" bare.systemd.services.swarm-services-cert.script;
|
|
}
|
|
{
|
|
# nixos asserts when a vhost declares both, so this is also a
|
|
# statement that the `removeAttrs` upstream of it still happens.
|
|
name = "the swarm UI vhost forces TLS instead of merely adding it";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts."t.local";
|
|
in
|
|
v.forceSSL && !(v.addSSL or false);
|
|
}
|
|
{
|
|
# `swarmctl --swarm` reads a 401 as "log in again"; a 302 would hand it
|
|
# the login page instead. The `/` clause is the control: the browser
|
|
# route still redirects, so a pass means the two routes differ.
|
|
name = "the controller API answers a refused bearer caller 401 and a browser the login";
|
|
ok =
|
|
let
|
|
l = allLocal.services.nginx.virtualHosts."t.local".locations;
|
|
api = l."/api/".extraConfig;
|
|
refused = l."@swarm_api_unauthorized".extraConfig;
|
|
in
|
|
lib.hasInfix "error_page 401 = @swarm_api_unauthorized;" api
|
|
&& !(lib.hasInfix "=302" api)
|
|
&& lib.hasInfix ''proxy_set_header Authorization "";'' api
|
|
&& lib.hasInfix "if ($http_authorization) {\n return 401;\n}" refused
|
|
&& lib.hasInfix "return 302 https://auth.t.local/?rd=$target_url;" refused
|
|
&& lib.hasInfix "error_page 401 =302 https://auth.t.local/" l."/".extraConfig;
|
|
}
|
|
{
|
|
# authelia 4.39.20 refuses a bearer-scoped public client missing any of
|
|
# these at startup, after a green deploy; and a public client must not
|
|
# be handed a secret.
|
|
name = "the swarmctl login client is public, bearer-scoped to the controller API, and gets no secret";
|
|
ok =
|
|
let
|
|
s = allLocal.containers.swarm-authelia.config.systemd.services.authelia-swarm-secrets.script;
|
|
entry = lib.last (lib.splitString "client_id: %s\\n' swarmctl" s);
|
|
client = lib.head (lib.splitString "client_id: %s" entry);
|
|
in
|
|
lib.all (line: lib.hasInfix line client) [
|
|
"public: true"
|
|
"token_endpoint_auth_method: none"
|
|
"require_pkce: true"
|
|
"pkce_challenge_method: S256"
|
|
"require_pushed_authorization_requests: true"
|
|
"consent_mode: explicit"
|
|
"response_modes: [form_post]"
|
|
"grant_types: [authorization_code, refresh_token]"
|
|
"scopes: [offline_access, authelia.bearer.authz]"
|
|
"audience: [%s]\\n' https://t.local/api/"
|
|
"' - %s\\n' http://127.0.0.1/callback"
|
|
]
|
|
&& !(lib.hasInfix "client_secret" client)
|
|
&& !(lib.hasInfix "mint swarmctl" s)
|
|
&& lib.hasInfix "mint swarm-authelia-bridge" s;
|
|
}
|
|
{
|
|
# swarm-controller checks no caller itself: this location's authelia
|
|
# subrequest is the operator gate for every route under `/api/`. The
|
|
# webhook location is the control, the one path to the same socket that
|
|
# skips the gate, so the first clause reads a string that can lack it.
|
|
name = "the swarm UI's API location asks authelia before reaching the controller";
|
|
ok =
|
|
let
|
|
l = allLocal.services.nginx.virtualHosts."t.local".locations;
|
|
socket = allLocal.services.hyperhive.deploy.swarm-controller.socketPath;
|
|
in
|
|
lib.hasInfix "auth_request /__hive_authelia;" l."/api/".extraConfig
|
|
&& lib.hasInfix socket l."/api/".proxyPass
|
|
&& lib.hasInfix socket l."/webhook/forge/".proxyPass
|
|
&& !(lib.hasInfix "auth_request" l."/webhook/forge/".extraConfig);
|
|
}
|
|
{
|
|
name = "a hive with matrix off serves no matrix discovery endpoint";
|
|
ok =
|
|
!(builtins.hasAttr "= /.well-known/matrix/client" bare.services.nginx.virtualHosts."_".locations);
|
|
}
|
|
{
|
|
# main got eval-borked twice by this exact class of bug (once on the
|
|
# unit's `Restart` key, once on `RestartSec`) — a nixpkgs bump to
|
|
# `gitea-actions-runner.nix` adds a plain `serviceConfig.*`
|
|
# definition that collides with one of ours, and nix refuses to
|
|
# merge two plain definitions at *host* eval. No other check
|
|
# instantiates a host with `containers.swarm-ci-runner` actually enabled, so
|
|
# the collision only surfaces on operator deploy, not in CI.
|
|
name = "the CI container's unit definitions merge without a priority collision";
|
|
ok = forceCiServiceConfigs;
|
|
}
|
|
{
|
|
# It spends money on an operator's key, so no host runs it by default.
|
|
name = "the review classifier's container exists only where its host enables it";
|
|
ok =
|
|
!(bare.containers ? swarm-review-classifier)
|
|
&& !(centralToggleOff.containers ? swarm-review-classifier)
|
|
&& withClassifier.containers ? swarm-review-classifier;
|
|
}
|
|
{
|
|
# The daemon holds its store leaf, handed in by nspawn without a copy on
|
|
# disk, and nothing else from the host; the key is read per run and
|
|
# reaches only `claude`'s environment. Reading the claude path also
|
|
# evaluates the container's unfree allowance.
|
|
name = "the review classifier loads only its store leaf and carries no API key in its unit";
|
|
ok =
|
|
let
|
|
svc =
|
|
withClassifier.containers.swarm-review-classifier.config.systemd.services.swarm-review-classifier;
|
|
env = svc.environment;
|
|
in
|
|
lib.elem "review-classifier-bao-key" svc.serviceConfig.LoadCredential
|
|
&&
|
|
env.BAO_CLIENT_KEY == "/run/credentials/swarm-review-classifier.service/review-classifier-bao-key"
|
|
&& env.SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID == "swarm-review-classifier"
|
|
&& lib.hasSuffix "/bin/claude" env.SWARM_REVIEW_CLASSIFIER_CLAUDE
|
|
&& !(env ? ANTHROPIC_API_KEY)
|
|
&& lib.elem "--load-credential=review-classifier-bao-key:/var/lib/swarm-bao-pki/review-classifier-key.pem" (
|
|
withClassifier.containers.swarm-review-classifier.extraFlags
|
|
)
|
|
# No host unit copies the leaf into the container's root.
|
|
&& !(lib.any (u: lib.hasInfix "nixos-containers/swarm-review-classifier" (u.script or "")) (
|
|
lib.attrValues withClassifier.systemd.services
|
|
));
|
|
}
|
|
{
|
|
# The collector reaches these routes through the gateway now, so each
|
|
# store needs an ingest location of its own. Without one the write rides
|
|
# the `/` catch-all: unauthenticated on the metrics store, and into a
|
|
# browser redirect on the log store.
|
|
name = "each store's vhost has an authenticated ingest location";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts;
|
|
m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push" or null;
|
|
l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs" or null;
|
|
in
|
|
m != null
|
|
&& l != null
|
|
&& lib.hasInfix "auth_request" m.extraConfig
|
|
&& lib.hasInfix "auth_request" l.extraConfig;
|
|
}
|
|
{
|
|
# The metrics vhost is default-deny; the collector's push only lands
|
|
# because its `metrics_endpoint` path is the vhost's one gated exact
|
|
# location. The two literals live in different modules, so renaming
|
|
# either alone breaks ingest while everything still evaluates.
|
|
name = "the collector's metrics push path is a gated location on the metrics vhost";
|
|
ok =
|
|
let
|
|
endpoint = (otelSettings allLocal).exporters."otlphttp/victoriametrics".metrics_endpoint;
|
|
# "https://host/path" -> "/path"
|
|
path = "/" + lib.concatStringsSep "/" (lib.drop 3 (lib.splitString "/" endpoint));
|
|
loc = allLocal.services.nginx.virtualHosts."metrics.t.local".locations."= ${path}" or null;
|
|
in
|
|
loc != null && lib.hasInfix "auth_request" loc.extraConfig;
|
|
}
|
|
{
|
|
# The arm that actually protects something. A pusher handed
|
|
# `error_page 401 =302` FOLLOWS it and POSTs its batch at a login page,
|
|
# which answers 200 — ingest reporting healthy while storing nothing.
|
|
# The third clause is the positive control: the log store's browser
|
|
# location really does redirect, so this says the machine routes differ
|
|
# rather than that the string is absent from the whole file.
|
|
name = "the ingest locations answer 401 instead of redirecting a pusher";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts;
|
|
m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push".extraConfig;
|
|
l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs".extraConfig;
|
|
browser = v."logs.t.local".locations."/".extraConfig;
|
|
in
|
|
!(lib.hasInfix "error_page" m)
|
|
&& !(lib.hasInfix "error_page" l)
|
|
&& lib.hasInfix "error_page" browser;
|
|
}
|
|
{
|
|
# The store authenticates nothing itself, so every location that
|
|
# forwards without `auth_request` is open to anything that can reach the
|
|
# domain. The query location is the positive control: it shows the
|
|
# filter finds forwarding locations at all.
|
|
name = "the metrics vhost forwards only read paths without a credential";
|
|
ok =
|
|
let
|
|
locs = allLocal.services.nginx.virtualHosts."metrics.t.local".locations;
|
|
open = lib.filterAttrs (
|
|
_: l:
|
|
l.proxyPass != null
|
|
&& !(lib.hasInfix "auth_request" l.extraConfig)
|
|
&& !(lib.hasInfix "internal;" l.extraConfig)
|
|
) locs;
|
|
writeWords = [
|
|
"import"
|
|
"write"
|
|
"admin"
|
|
"delete"
|
|
"insert"
|
|
"internal"
|
|
"push"
|
|
"prometheus"
|
|
"influx"
|
|
];
|
|
in
|
|
locs."/".proxyPass == null
|
|
&& locs."/".return == "403"
|
|
&& open ? "= /api/v1/query"
|
|
&& lib.all (n: n != "/" && !(lib.any (w: lib.hasInfix w n) writeWords)) (lib.attrNames open);
|
|
}
|
|
{
|
|
# The read counterpart to the ingest location: an agent queries the log
|
|
# store with a bearer token, and the `/` catch-all is the browser's
|
|
# route. Riding it would mean inheriting the login redirect the next
|
|
# case is about, so the route has to exist separately to be gated
|
|
# separately.
|
|
name = "the log store's vhost has an authenticated machine query location";
|
|
ok =
|
|
let
|
|
q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/" or null;
|
|
in
|
|
q != null && lib.hasInfix "auth_request" q.extraConfig;
|
|
}
|
|
{
|
|
# Same trap as the ingest case, on the read side, where it is worse: a
|
|
# redirected pusher at least stores nothing visibly, while a redirected
|
|
# *reader* is handed a 200 carrying login HTML and records a query that
|
|
# succeeded and matched no logs. The browser clause is the positive
|
|
# control — that location really does redirect to the login host — so a
|
|
# pass means these two routes differ rather than that the strings are
|
|
# absent from the whole vhost.
|
|
name = "the machine query location answers 401 instead of redirecting to a login page";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts;
|
|
q = v."logs.t.local".locations."^~ /select/logsql/".extraConfig;
|
|
browser = v."logs.t.local".locations."/".extraConfig;
|
|
in
|
|
!(lib.hasInfix "error_page" q)
|
|
&& !(lib.hasInfix "auth.t.local" q)
|
|
&& lib.hasInfix "error_page" browser
|
|
&& lib.hasInfix "auth.t.local" browser;
|
|
}
|
|
{
|
|
# Read access is deliberately unscoped: an authenticated caller reads
|
|
# the whole swarm's logs until a permission system exists. Pinned so a
|
|
# scoping parameter arriving later is a visible diff here rather than a
|
|
# quiet change of rule — and pinned on `proxyPass` too, because
|
|
# VictoriaLogs takes its filters as request parameters, which ride an
|
|
# upstream URI as easily as a directive. The first clause is the
|
|
# control: it proves the location resolved and that `hasInfix` finds
|
|
# what is genuinely in this string, so the absences below mean absent
|
|
# rather than unreadable.
|
|
name = "the machine query location forwards the caller's query unmodified";
|
|
ok =
|
|
let
|
|
q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/";
|
|
in
|
|
lib.hasInfix "auth_request" q.extraConfig
|
|
&& !(lib.hasInfix "extra_filters" q.extraConfig)
|
|
&& !(lib.hasInfix "extra_stream_filters" q.extraConfig)
|
|
&& !(lib.hasInfix "$args" q.extraConfig)
|
|
&& !(lib.hasInfix "?" q.proxyPass);
|
|
}
|
|
{
|
|
# `auth_request` takes a fixed URI, so the split between the agents'
|
|
# responder and authelia is a map plus a rewrite in an internal
|
|
# location. Every link of that chain is pinned: the map keys on the
|
|
# agent-token prefix and falls back to authelia, the query location
|
|
# names the switch, the switch rewrites to the map's pick, and the
|
|
# agent target reaches the responder's socket, which the unit binds.
|
|
name = "the machine query location sends agent bearers to swarm-logs-auth and the rest to authelia";
|
|
ok =
|
|
let
|
|
l = allLocal.services.nginx.virtualHosts."logs.t.local".locations;
|
|
http = allLocal.services.nginx.appendHttpConfig;
|
|
unit = allLocal.systemd.services.swarm-logs-auth;
|
|
in
|
|
lib.hasInfix "map \$http_authorization \$swarm_logs_auth_target" http
|
|
&& lib.hasInfix "\"~^Bearer swarm-agent\\.\" /__swarm_logs_agent_auth;" http
|
|
&& lib.hasInfix "default /__hive_authelia;" http
|
|
&& lib.hasInfix "auth_request /__swarm_logs_auth;" l."^~ /select/logsql/".extraConfig
|
|
&& lib.hasInfix "rewrite ^ \$swarm_logs_auth_target last;" l."= /__swarm_logs_auth".extraConfig
|
|
&& lib.hasInfix "internal;" l."= /__swarm_logs_agent_auth".extraConfig
|
|
&& l."= /__swarm_logs_agent_auth".proxyPass == "http://unix:/run/swarm-logs-auth/auth.sock:/"
|
|
&& lib.hasInfix "--socket /run/swarm-logs-auth/auth.sock" unit.serviceConfig.ExecStart
|
|
&& l ? "= /__hive_authelia";
|
|
}
|
|
{
|
|
# The responder reads the store as itself, with the leaf the store's
|
|
# host mints for it.
|
|
name = "swarm-logs-auth presents its own store leaf";
|
|
ok =
|
|
let
|
|
v = allLocal.services.hyperhive.deploy.victorialogs;
|
|
lc = allLocal.systemd.services.swarm-logs-auth.serviceConfig.LoadCredential;
|
|
in
|
|
lib.hasSuffix "/logs-auth.pem" v.authBaoClientCertFile
|
|
&& lib.hasSuffix "/logs-auth-key.pem" v.authBaoClientKeyFile
|
|
&& builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc
|
|
&& builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc;
|
|
}
|
|
{
|
|
# The absence arm for the case above, and the option's own rule — a
|
|
# service declares its entry under its own `enable` — made checkable.
|
|
# Without it, moving the assignment outside the collector's `mkIf`
|
|
# passes every arm above while handing a collector-less hive a scrape
|
|
# target for a port nothing binds.
|
|
name = "a hive with no collector declares no self-scrape target";
|
|
ok = bare.services.hyperhive.otel.scrapeTargets == { };
|
|
}
|
|
{
|
|
# Absence arm, and the one that matters: claiming a name this host does
|
|
# not serve points every local reader at the wrong machine.
|
|
name = "a hive that does not run the store claims no name for it";
|
|
ok = !(builtins.elem "bao.t.local" (baoNames bare));
|
|
}
|
|
{
|
|
# ⚠️ The absence arm that matters. `services.nginx.streamConfig` is a
|
|
# host-wide option, so a block rendered outside the store's own `mkIf`
|
|
# gives every hive in the swarm a listener — on the port the store
|
|
# answers on, in front of no store at all.
|
|
name = "a hive that does not run the store renders no stream passthrough";
|
|
ok = baoStream bare == "";
|
|
}
|
|
{
|
|
# Absence arm for the case above — a hive with no store has no reason to
|
|
# open the store's port, and opening it would point agents at a host that
|
|
# answers nothing.
|
|
name = "a hive that does not run the store opens no bridge port for it";
|
|
ok = !(builtins.elem 8200 (bridgePorts bare));
|
|
}
|
|
{
|
|
# The four blocks below used to be gated on the hive being enabled AND
|
|
# their own condition. The second half was always the load-bearing one —
|
|
# none of these conditions is derived from the hive toggle — so these
|
|
# arms pin what the conjunct was doing: nothing. Each is written against
|
|
# a host with the hive OFF as well as one with it on, because the way a
|
|
# dropped conjunct fails is by making something unconditional, and that
|
|
# shows up as a service appearing where nothing asked for it.
|
|
name = "no bridge port is opened for an exposeHostPorts nobody set";
|
|
ok =
|
|
!(builtins.elem 5432 (bridgePortsOrNone bare))
|
|
&& !(builtins.elem 5432 (bridgePortsOrNone centralToggleOff));
|
|
}
|
|
{
|
|
# `deploy.swarm-controller.enable`, which defaults false and is
|
|
# deliberately not derived from the hive toggle — a swarm has one
|
|
# controller, so the host that runs it says so itself.
|
|
#
|
|
# Probed by what the daemon needs in order to run, not by
|
|
# `? swarm-controller`: ./host-modules/hive-tls.nix defines an
|
|
# environment key on that unit name, which leaves the attr
|
|
# present-but-inert (no `ExecStart`, empty `wantedBy`) on every hive
|
|
# that has a CA — see the comment there. The daemon's system user has no
|
|
# second definer, so its absence is the unambiguous half.
|
|
name = "the swarm controller does not run unless this host is told to run it";
|
|
ok =
|
|
let
|
|
inert =
|
|
machine:
|
|
!(machine.users.users ? swarm-controller)
|
|
&& !(
|
|
(machine.systemd.services.swarm-controller or { serviceConfig = { }; }).serviceConfig ? ExecStart
|
|
);
|
|
in
|
|
inert bare && inert centralToggleOff;
|
|
}
|
|
{
|
|
# `deploy.swarm-otel.enable`, same shape: the swarm's collector is one
|
|
# host's job, and the container is the whole of what it renders.
|
|
name = "the swarm collector container is absent unless this host is told to run it";
|
|
ok = !(bare.containers ? swarm-otel) && !(centralToggleOff.containers ? swarm-otel);
|
|
}
|
|
{
|
|
# `deploy.swarm-ui.enable`, which defaults to the controller's toggle —
|
|
# derived from a sibling deployment decision, still not from the hive
|
|
# toggle. `t.local` is the fixtures' swarm domain, which is the apex the
|
|
# UI claims; the arm below is what proves this vhost renders at all.
|
|
name = "the swarm UI vhost is absent unless this host is told to serve it";
|
|
ok =
|
|
!(bare.services.nginx.virtualHosts ? "t.local")
|
|
&& !(centralToggleOff.services.nginx.virtualHosts ? "t.local");
|
|
}
|
|
{
|
|
# The three infrastructure toggles are off by default and asserted by
|
|
# whoever needs them. With nothing on the host needing them, none of
|
|
# the three renders — which is also the control for the arm below.
|
|
name = "the gateway, resolver and bridge are absent where nothing on the host needs them";
|
|
ok =
|
|
!centralToggleOff.services.hyperhive.gateway.enable
|
|
&& !centralToggleOff.services.hyperhive.gateway.dns.enable
|
|
&& !centralToggleOff.services.hyperhive.network.enable
|
|
&& !(centralToggleOff.services.nginx.enable or false)
|
|
&& !(centralToggleOff.services.dnsmasq.enable or false)
|
|
&& !(centralToggleOff.networking.bridges ? hive-br0);
|
|
}
|
|
{
|
|
# hive-c0re asserts all three, and it follows the central toggle — so
|
|
# an ordinary hive keeps getting them with no opt-in, which is what
|
|
# this change must not break.
|
|
name = "an ordinary hive runs the gateway, resolver and bridge because its coordinator needs them";
|
|
ok =
|
|
bare.services.hyperhive.gateway.enable
|
|
&& bare.services.hyperhive.gateway.dns.enable
|
|
&& bare.services.hyperhive.network.enable
|
|
&& bare.services.nginx.enable
|
|
&& bare.services.dnsmasq.enable
|
|
&& bare.networking.bridges ? hive-br0;
|
|
}
|
|
{
|
|
# A host config written against either old spelling still runs its hive.
|
|
# The builder's own `true` is lowered to `mkDefault false`, so only the
|
|
# forwarded definition can turn it on.
|
|
name = "the old hive toggles forward to deploy.hive-controller.enable and warn";
|
|
ok =
|
|
let
|
|
renamedFrom =
|
|
old: m:
|
|
lib.any (
|
|
w: lib.hasInfix old w && lib.hasInfix "services.hyperhive.deploy.hive-controller.enable" w
|
|
) m.warnings;
|
|
viaOld =
|
|
extra:
|
|
hive (
|
|
extra
|
|
// {
|
|
deploy.hive-controller.enable = lib.mkDefault false;
|
|
}
|
|
);
|
|
oldEnable = viaOld { enable = true; };
|
|
oldC0re = viaOld { c0re.enable = true; };
|
|
in
|
|
oldEnable.services.hyperhive.deploy.hive-controller.enable
|
|
&& oldEnable.systemd.services ? hive-c0re
|
|
&& renamedFrom "services.hyperhive.enable" oldEnable
|
|
&& oldC0re.services.hyperhive.deploy.hive-controller.enable
|
|
&& oldC0re.systemd.services ? hive-c0re
|
|
&& renamedFrom "services.hyperhive.c0re.enable" oldC0re;
|
|
}
|
|
{
|
|
# Refused wherever something turns the name into an identifier, and only
|
|
# there. The store host is the control that the refusal is not the hive's
|
|
# alone; the host running neither is the control that it is gated at all.
|
|
name = "a missing hiveName is refused on a hive and on a store host, not elsewhere";
|
|
ok =
|
|
let
|
|
refusedHiveName =
|
|
extra:
|
|
lib.any (a: !a.assertion && lib.hasInfix "services.hyperhive.hiveName to be set" a.message)
|
|
(hiveWithController ({ hiveName = null; } // extra)).assertions;
|
|
in
|
|
refusedHiveName { }
|
|
&& refusedHiveName {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.bao.enable = true;
|
|
}
|
|
&& !(refusedHiveName { deploy.hive-controller.enable = false; });
|
|
}
|
|
{
|
|
# Refused wherever any hyperhive service runs, the hive or a swarm
|
|
# service alone. The host enabling nothing is the control that it is
|
|
# gated at all, and has to pass every assertion: it is what an
|
|
# operator gets from importing the module.
|
|
name = "a missing swarm.domain is refused on a hive and on a swarm-service host, and a host enabling nothing evaluates";
|
|
ok =
|
|
let
|
|
noDomain = extra: hiveWithController ({ swarm.domain = null; } // extra);
|
|
refusedSwarmDomain =
|
|
extra:
|
|
lib.any (
|
|
a: !a.assertion && lib.hasInfix "services.hyperhive.swarm.domain to be set" a.message
|
|
) (noDomain extra).assertions;
|
|
in
|
|
refusedSwarmDomain { }
|
|
&& refusedSwarmDomain {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.nats.enable = true;
|
|
}
|
|
&& lib.all (a: a.assertion) (noDomain { deploy.hive-controller.enable = false; }).assertions;
|
|
}
|
|
{
|
|
# Control: a store fetch with no timer still renders `RemainAfterExit`,
|
|
# so the cases below read a real value rather than a missing one.
|
|
name = "Grafana's admin-password fetch stays active after it exits and has no timer";
|
|
ok =
|
|
allLocal.systemd.services ? swarm-bao-grafana-admin
|
|
&& remainsAfterExit allLocal.systemd.services.swarm-bao-grafana-admin
|
|
&& !(allLocal.systemd.timers ? swarm-bao-grafana-admin);
|
|
}
|
|
{
|
|
# A timer whose unit does not exist fails to load, so each timer is
|
|
# gated exactly as its fetch is.
|
|
name = "a host running none of the store-refreshed fetches renders none of their timers";
|
|
ok = lib.all (
|
|
{ unit, ... }: !(bare.systemd.services ? ${unit}) && !(bare.systemd.timers ? ${unit})
|
|
) storeRefreshed;
|
|
}
|
|
]
|
|
# Starting a unit that stays `active (exited)` is a no-op, so a timer aimed
|
|
# at one fires and fetches nothing.
|
|
++ map (
|
|
{ unit, machine }:
|
|
{
|
|
name = "${unit} re-runs on a ten-minute timer and does not stay active after it exits";
|
|
ok =
|
|
let
|
|
s = machine.systemd.services;
|
|
t = machine.systemd.timers;
|
|
in
|
|
s ? ${unit}
|
|
&& t ? ${unit}
|
|
&& !(remainsAfterExit s.${unit})
|
|
&& lib.elem "timers.target" t.${unit}.wantedBy
|
|
&& t.${unit}.timerConfig.OnUnitInactiveSec == "10min"
|
|
&& (t.${unit}.timerConfig.Unit or "${unit}.service") == "${unit}.service";
|
|
}
|
|
) storeRefreshed;
|
|
in
|
|
runGroup "core-toggle" cases
|