1216 lines
56 KiB
Nix
1216 lines
56 KiB
Nix
# `checks.module-eval-core-toggle` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
hiveWithController
|
|
runGroup
|
|
baoNames
|
|
baoStream
|
|
bridgePorts
|
|
bridgePortsOrNone
|
|
swarmServiceEnables
|
|
otelSettings
|
|
;
|
|
|
|
inherit (shared) allLocal;
|
|
|
|
bare = shared.bareWithController;
|
|
|
|
# `lib.hasInfix` matches by regex, which refuses a needle that refers to a
|
|
# store path. Both sides lose their string context here.
|
|
hasStorePath =
|
|
needle: hay:
|
|
lib.hasInfix (builtins.unsafeDiscardStringContext needle) (builtins.unsafeDiscardStringContext hay);
|
|
|
|
# The same stub with the central toggle (`deploy.hive-controller.enable`)
|
|
# off. Paired with `bare` below to pin the defaults that do not read it:
|
|
# each is asserted to hold the SAME literal in both, so a future edit
|
|
# that quietly introduces the dependency — or that changes what the
|
|
# default renders for a hive with the toggle on — fails here. Reading an
|
|
# option off this fixture forces that option only, not the config, so the
|
|
# toggle being off costs nothing. Also the "installs the modules and turns
|
|
# nothing on" host the swarm-service absences below read: none of the
|
|
# per-service deployment toggles derives from the hive being on, so it
|
|
# renders the same absences `bare` does.
|
|
centralToggleOff = shared.bare;
|
|
|
|
inherit (shared) forgeHere;
|
|
|
|
# Every hive defaults the agents' queue address to the queue's name, so the
|
|
# only hive without one is a hive told to have none. Also the host still
|
|
# setting the four removed host options; no case on its queue reads them.
|
|
noAgentQueue = hiveWithController {
|
|
deploy.hive-controller.queue.agentNatsUrl = null;
|
|
c0re.operatorPronouns = "they/them";
|
|
c0re.contextWindowTokens.sonnet = 900000;
|
|
otel.metricIntervalMs = 10000;
|
|
otel.debug = true;
|
|
};
|
|
|
|
# The controller behind a gateway on a non-default TLS port: the only
|
|
# fixture whose collector URL carries a port. Only the controller's
|
|
# environment is read off it.
|
|
controllerAltPort = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
gateway.httpsPort = 8443;
|
|
};
|
|
|
|
# On the store's host, which mints the runner's store identity.
|
|
withCi = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.ci-runner.enable = true;
|
|
};
|
|
|
|
# On the store's host, which mints the classifier's store identity.
|
|
withClassifier = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.review-classifier.enable = true;
|
|
};
|
|
|
|
# A priority collision is a property of the *option*, not
|
|
# of the merged value's interior — nix throws the moment the value is
|
|
# demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
|
|
# both necessary and sufficient. `deepSeq` over-specifies this: it keeps
|
|
# walking *into* the resulting value after the merge already succeeded,
|
|
# and a package/derivation-shaped value's `override`/`overrideAttrs`
|
|
# self-reference sends it into nixpkgs' fixpoint machinery and blows the
|
|
# stack (measured — this is not a hypothetical).
|
|
forceCiServiceConfigs =
|
|
let
|
|
svcs = withCi.containers.swarm-ci-runner.config.systemd.services;
|
|
vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs);
|
|
in
|
|
builtins.foldl' (acc: v: builtins.seq v acc) true vals;
|
|
|
|
# The host fetches that restart their consumer when the store's value
|
|
# changes (../host-modules/lib/refresh-consumer.nix), each on a fixture
|
|
# that renders it.
|
|
storeRefreshed = [
|
|
{
|
|
unit = "swarm-bao-grafana-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-otel-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-matrix-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-forge-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-hive-otel-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-forwarder-oidc";
|
|
machine = allLocal;
|
|
}
|
|
{
|
|
unit = "swarm-bao-ci-runner";
|
|
machine = withCi;
|
|
}
|
|
];
|
|
|
|
remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false;
|
|
|
|
cases = [
|
|
{
|
|
# The domain is the stub's swarm domain, which both fixtures share.
|
|
name = "the forge's publicUrl default does not consult the central toggle";
|
|
ok =
|
|
bare.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local"
|
|
&& centralToggleOff.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local";
|
|
}
|
|
{
|
|
# The controller's token mint follows where the forge runs
|
|
# (./forge-placement.nix), never the central toggle: a forge host with
|
|
# the toggle off still mints and publishes it.
|
|
name = "the forge's controller-token mint does not consult the central toggle";
|
|
ok = forgeHere.containers.hive-forge.config.systemd.services ? forgejo-swarm-controller-account;
|
|
}
|
|
{
|
|
name = "a hive that does not host the swarm's shared services runs none of them";
|
|
ok =
|
|
let
|
|
es = swarmServiceEnables bare;
|
|
in
|
|
lib.length (lib.attrNames es) == 9 && !lib.any lib.id (lib.attrValues es);
|
|
}
|
|
{
|
|
name = "the all-local mode turns the swarm controller on";
|
|
ok = allLocal.services.hyperhive.deploy.swarm-controller.enable;
|
|
}
|
|
{
|
|
# ../host-modules/glue-controller-env.nix: the controller always exports to
|
|
# the swarm's collector. `controllerAlone` runs no collector of its own
|
|
# (`allLocal` does, as the presence control); `centralToggleOff` has no
|
|
# controller, so no unit to set it on.
|
|
name = "the controller always exports to the swarm's collector";
|
|
ok =
|
|
let
|
|
env = m: m.systemd.services.swarm-controller.environment;
|
|
otel = allLocal.services.hyperhive.swarm.otel;
|
|
clientId = allLocal.services.hyperhive.swarm.controller.queueClientId;
|
|
# ../host-modules/glue-controller-oidc-client.nix: the collector's
|
|
# `oidc/swarm` reads only a signed access token.
|
|
client = lib.findFirst (
|
|
c: c.id == clientId
|
|
) null allLocal.services.hyperhive.swarm.authelia.oidc.clients;
|
|
in
|
|
(env allLocal).OTEL_EXPORTER_OTLP_ENDPOINT or null == "https://${otel.domain}/${otel.producerName}"
|
|
&& (env allLocal).SWARM_CONTROLLER_OTEL_AUDIENCE or null == clientId
|
|
&& client != null
|
|
&& lib.elem clientId client.audience
|
|
&& client.accessTokenSignedResponseAlg == "RS256"
|
|
&& allLocal.services.hyperhive.deploy.swarm-otel.enable
|
|
&& !shared.controllerAlone.services.hyperhive.deploy.swarm-otel.enable
|
|
&&
|
|
(env shared.controllerAlone).OTEL_EXPORTER_OTLP_ENDPOINT or null
|
|
== "https://${otel.domain}/${otel.producerName}"
|
|
&& (env shared.controllerAlone).SWARM_CONTROLLER_OTEL_AUDIENCE or null == clientId
|
|
&&
|
|
(env controllerAltPort).OTEL_EXPORTER_OTLP_ENDPOINT or null
|
|
== "https://${otel.domain}:8443/${otel.producerName}"
|
|
&& !(centralToggleOff.systemd.services ? swarm-controller);
|
|
}
|
|
{
|
|
# Same file: `swarmctl` is told where the configured authelia is only
|
|
# where authelia runs here.
|
|
name = "swarmctl is given the authelia binary and users file only where authelia runs here";
|
|
ok =
|
|
let
|
|
env = m: m.services.hyperhive.deploy.swarm-controller.swarmctlEnvironment;
|
|
d = allLocal.services.hyperhive.deploy.authelia;
|
|
in
|
|
env allLocal == {
|
|
SWARMCTL_AUTHELIA_BIN = "${d.package}/bin/authelia";
|
|
SWARMCTL_AUTHELIA_USERS_FILE = d.hostUsersFile;
|
|
}
|
|
&& env shared.controllerAlone == { };
|
|
}
|
|
{
|
|
# Where the reader puts the files and where the daemon looks for them is
|
|
# one agreement spanning two modules. Asserted against the option rather
|
|
# than the literal so moving the directory moves both ends.
|
|
name = "hive-c0re is told where the agents' queue credential lands";
|
|
ok =
|
|
allLocal.systemd.services.hive-c0re.environment.HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR
|
|
== toString allLocal.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir;
|
|
}
|
|
{
|
|
# Inside an agent container loopback is the agent itself, so the agents'
|
|
# address must never be one. By name it is the same string the hive
|
|
# dials, which ./nats-tls.nix pins for every client.
|
|
name = "the agents' queue address is the queue's name, never loopback";
|
|
ok =
|
|
let
|
|
e = allLocal.systemd.services.hive-c0re.environment;
|
|
in
|
|
e.HIVE_AGENT_NATS_URL == "tls://nats.t.local:4222"
|
|
&& !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL)
|
|
&& e.HIVE_AGENT_NATS_URL == e.HIVE_C0RE_NATS_URL;
|
|
}
|
|
{
|
|
# What agents are handed lands in their builds, so it has to be the same
|
|
# on every hive: a name, which this hive's resolver answers with the
|
|
# bridge IP its collector binds. hive-c0re itself keeps the address.
|
|
name = "the agents' collector address is a swarm-wide name this hive resolves to its bridge";
|
|
ok =
|
|
let
|
|
e = allLocal.systemd.services.hive-c0re.environment;
|
|
bridge = allLocal.services.hyperhive.network.bridgeIp;
|
|
in
|
|
e.HYPERHIVE_OTEL_ENDPOINT == "http://hive-collector.t.local:4318"
|
|
&& e.OTEL_EXPORTER_OTLP_ENDPOINT == "http://${bridge}:4318"
|
|
&& lib.elem "/hive-collector.t.local/${bridge}" allLocal.services.dnsmasq.settings.address;
|
|
}
|
|
{
|
|
# The agents mint against the swarm's IdP, the same endpoint the hive's
|
|
# own client uses — a hive-local guess would produce a token the queue
|
|
# would not accept.
|
|
name = "the agents' token endpoint is the swarm IdP's";
|
|
ok =
|
|
let
|
|
e = allLocal.systemd.services.hive-c0re.environment;
|
|
in
|
|
lib.hasSuffix "/api/oidc/token" e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT
|
|
&& e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT == e.HIVE_C0RE_OIDC_TOKEN_ENDPOINT;
|
|
}
|
|
{
|
|
# The absence arm, and what makes the two above able to fail: a hive
|
|
# with no queue address for its agents must forward neither coordinate,
|
|
# because half a pair reaches the harness as a partial configuration
|
|
# rather than as none.
|
|
name = "a hive with no agent queue address forwards no agent queue coordinates";
|
|
ok =
|
|
let
|
|
e = noAgentQueue.systemd.services.hive-c0re.environment;
|
|
in
|
|
!(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT);
|
|
}
|
|
{
|
|
# Control: `bare` sets none of them, and its serve.json carries the
|
|
# dashboard port, so the key lookup reads a real document.
|
|
name = "a definition of a removed host option evaluates and warns, and the c0re ones reach no serve.json";
|
|
ok =
|
|
let
|
|
warned =
|
|
m: option:
|
|
lib.any (w: lib.hasInfix "services.hyperhive.${option} is removed and ignored" w) m.warnings;
|
|
options = [
|
|
"c0re.operatorPronouns"
|
|
"c0re.contextWindowTokens"
|
|
"otel.metricIntervalMs"
|
|
"otel.debug"
|
|
];
|
|
hasKey = m: key: lib.hasInfix "\"${key}\":" m.environment.etc."hyperhive/serve.json".text;
|
|
noRemovedKeys =
|
|
m:
|
|
hasKey m "dashboard_port" && !(hasKey m "operator_pronouns") && !(hasKey m "context_window_tokens");
|
|
in
|
|
lib.all (warned noAgentQueue) options
|
|
&& !(lib.any (warned bare) options)
|
|
&& noRemovedKeys noAgentQueue
|
|
&& noRemovedKeys bare;
|
|
}
|
|
{
|
|
# Where the store is and where the agent is told it is, one agreement
|
|
# spanning two modules. Asserted against the hive's own `BAO_ADDR`
|
|
# rather than a literal, because an agent pointed at a different
|
|
# spelling of the same store presents a certificate to a listener whose
|
|
# name it cannot verify.
|
|
name = "an agent is told the same store address its hive uses";
|
|
ok =
|
|
let
|
|
e = allLocal.systemd.services.hive-c0re.environment;
|
|
in
|
|
e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR;
|
|
}
|
|
{
|
|
# A hive with no certificate of its own can collect no agent's identity,
|
|
# so forwarding an address would name a store nothing in the container
|
|
# can reach. The same gate the `BAO_*` pair beside it sits behind.
|
|
name = "a hive with no store identity forwards no store address to its agents";
|
|
ok = !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_ADDR);
|
|
}
|
|
{
|
|
# The gateway's per-name issuer choice. If this ever collapses to a
|
|
# constant, every swarm-service vhost serves a certificate its CA
|
|
# is name-constrained out of — which evaluates cleanly and fails in
|
|
# a browser.
|
|
name = "a swarm service name gets the swarm-services leaf and the default server does not";
|
|
ok =
|
|
let
|
|
l = allLocal.services.hyperhive.gateway.lib;
|
|
in
|
|
(l.tlsFor "t.local").sslCertificate != (l.tlsFor "_").sslCertificate;
|
|
}
|
|
{
|
|
# A leaf is a key that can SERVE every name in it, so the names it
|
|
# asks for are that key's blast radius. The forge's host fronts forge
|
|
# and nothing else; the last conjunct is the control, since
|
|
# `auth.t.local` is in the swarm's set and an unnarrowed request would
|
|
# carry it here too.
|
|
name = "a gateway's services leaf asks for only the swarm names that host fronts";
|
|
ok =
|
|
let
|
|
s = forgeHere.services.hyperhive.swarm.serviceDomains;
|
|
u = forgeHere.systemd.services.swarm-services-cert.script;
|
|
in
|
|
lib.hasInfix "alt_names=forge.t.local" u
|
|
&& !(lib.hasInfix "auth.t.local" u)
|
|
&& lib.elem "auth.t.local" s;
|
|
}
|
|
{
|
|
# The store's issuing role is built from this list on the store's host,
|
|
# so it cannot depend on which services the evaluating host runs. The
|
|
# last conjunct is the control: grafana runs on neither `bare` nor the
|
|
# forge host, and is still listed.
|
|
name = "the swarm's service names are the same on hosts running different services";
|
|
ok =
|
|
let
|
|
s = h: h.services.hyperhive.swarm.serviceDomains;
|
|
in
|
|
s forgeHere == s bare && s allLocal == s bare && lib.elem "grafana.t.local" (s bare);
|
|
}
|
|
{
|
|
# The narrowing's floor: no names left means no request, rather than a
|
|
# request for an empty SAN set that the store would refuse. Its own
|
|
# vhosts are the hive leaf's, which this host still signs.
|
|
name = "a host fronting none of the swarm's service names requests no services leaf";
|
|
ok =
|
|
bare.services.hyperhive.deploy.hive-controller.tls.localServiceDomains == [ ]
|
|
&& bare.services.hyperhive.swarm.serviceDomains != [ ]
|
|
&& lib.hasInfix "want_svc=0" bare.systemd.services.swarm-services-cert.script;
|
|
}
|
|
{
|
|
# nixos asserts when a vhost declares both, so this is also a
|
|
# statement that the `removeAttrs` upstream of it still happens.
|
|
name = "the swarm UI vhost forces TLS instead of merely adding it";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts."t.local";
|
|
in
|
|
v.forceSSL && !(v.addSSL or false);
|
|
}
|
|
{
|
|
# `swarmctl --swarm` reads a 401 as "log in again"; a 302 would hand it
|
|
# the login page instead. The `/` clause is the control: the browser
|
|
# route still redirects, so a pass means the two routes differ.
|
|
name = "the controller API answers a refused bearer caller 401 and a browser the login";
|
|
ok =
|
|
let
|
|
l = allLocal.services.nginx.virtualHosts."t.local".locations;
|
|
api = l."/api/".extraConfig;
|
|
refused = l."@swarm_api_unauthorized".extraConfig;
|
|
in
|
|
lib.hasInfix "error_page 401 = @swarm_api_unauthorized;" api
|
|
&& !(lib.hasInfix "=302" api)
|
|
&& lib.hasInfix ''proxy_set_header Authorization "";'' api
|
|
&& lib.hasInfix "if ($http_authorization) {\n return 401;\n}" refused
|
|
&& lib.hasInfix "return 302 https://auth.t.local/?rd=$target_url;" refused
|
|
&& lib.hasInfix "error_page 401 =302 https://auth.t.local/" l."/".extraConfig;
|
|
}
|
|
{
|
|
# authelia 4.39.20 refuses a bearer-scoped public client missing any of
|
|
# these at startup, after a green deploy; and a public client must not
|
|
# be handed a secret.
|
|
name = "the swarmctl login client is public, bearer-scoped to the controller API, and gets no secret";
|
|
ok =
|
|
let
|
|
s = allLocal.containers.swarm-authelia.config.systemd.services.authelia-swarm-secrets.script;
|
|
entry = lib.last (lib.splitString "client_id: %s\\n' swarmctl" s);
|
|
client = lib.head (lib.splitString "client_id: %s" entry);
|
|
in
|
|
lib.all (line: lib.hasInfix line client) [
|
|
"public: true"
|
|
"token_endpoint_auth_method: none"
|
|
"require_pkce: true"
|
|
"pkce_challenge_method: S256"
|
|
"require_pushed_authorization_requests: true"
|
|
"consent_mode: explicit"
|
|
"response_modes: [form_post]"
|
|
"grant_types: [authorization_code, refresh_token]"
|
|
"scopes: [offline_access, authelia.bearer.authz]"
|
|
"audience: [%s]\\n' https://t.local/api/"
|
|
"' - %s\\n' http://127.0.0.1/callback"
|
|
]
|
|
&& !(lib.hasInfix "client_secret" client)
|
|
&& !(lib.hasInfix "mint swarmctl" s)
|
|
&& lib.hasInfix "mint swarm-authelia-bridge" s;
|
|
}
|
|
{
|
|
# swarm-controller checks no caller itself: this location's authelia
|
|
# subrequest is the operator gate for every route under `/api/`. The
|
|
# webhook location is the control, the one path to the same socket that
|
|
# skips the gate, so the first clause reads a string that can lack it.
|
|
name = "the swarm UI's API location asks authelia before reaching the controller";
|
|
ok =
|
|
let
|
|
l = allLocal.services.nginx.virtualHosts."t.local".locations;
|
|
socket = allLocal.services.hyperhive.deploy.swarm-controller.socketPath;
|
|
in
|
|
lib.hasInfix "auth_request /__hive_authelia;" l."/api/".extraConfig
|
|
&& lib.hasInfix socket l."/api/".proxyPass
|
|
&& lib.hasInfix socket l."/webhook/forge/".proxyPass
|
|
&& !(lib.hasInfix "auth_request" l."/webhook/forge/".extraConfig);
|
|
}
|
|
{
|
|
# Only nginx (through the clients group) and root reach the controller
|
|
# socket: the runtime dir is 0750 with that group set. The last two
|
|
# clauses are the presence controls — the group exists and is not the
|
|
# daemon's own, so a misspelled name cannot satisfy the rest.
|
|
name = "the controller socket's directory is 0750 and group-owned by the group nginx joins";
|
|
ok =
|
|
let
|
|
sc = allLocal.systemd.services.swarm-controller.serviceConfig;
|
|
group = "swarm-controller-clients";
|
|
in
|
|
sc.RuntimeDirectoryMode == "0750"
|
|
&& lib.hasPrefix "+" sc.ExecStartPre
|
|
&& lib.hasInfix "chgrp ${group} /run/swarm-controller" sc.ExecStartPre
|
|
&& lib.elem group allLocal.systemd.services.nginx.serviceConfig.SupplementaryGroups
|
|
&& allLocal.users.groups ? ${group}
|
|
&& sc.Group != group;
|
|
}
|
|
{
|
|
# nginx serves a path from its longest matching prefix location, so that
|
|
# one must be the gated `/api/`, and no exact location may take a route
|
|
# out from under it.
|
|
name = "the claude login routes reach the controller only through a location that asks authelia";
|
|
ok =
|
|
let
|
|
l = allLocal.services.nginx.virtualHosts."t.local".locations;
|
|
route = "/api/agents/a/claude-login/code";
|
|
covering = lib.filter (
|
|
name: !(lib.hasPrefix "~" name) && !(lib.hasPrefix "= " name) && lib.hasPrefix (prefix name) route
|
|
) (lib.attrNames l);
|
|
prefix = lib.removePrefix "^~ ";
|
|
longest = lib.foldl' (
|
|
a: b: if lib.stringLength (prefix b) > lib.stringLength (prefix a) then b else a
|
|
) "" covering;
|
|
in
|
|
longest == "/api/"
|
|
&& lib.hasInfix "auth_request /__hive_authelia;" l.${longest}.extraConfig
|
|
&& !(lib.any (name: lib.hasPrefix "= /api/agents" name) (lib.attrNames l));
|
|
}
|
|
{
|
|
# `auth_request` does not inherit across sibling locations, so each
|
|
# gated one is read on its own. `/health/` is the control: the one
|
|
# c0re route that must stay open, so a pass means the block is
|
|
# per-location rather than everywhere.
|
|
name = "the dashboard answers on its own name behind authelia, and the hive domain redirects there";
|
|
ok =
|
|
let
|
|
vh = allLocal.services.nginx.virtualHosts;
|
|
l = vh."h1.hives.t.local".locations;
|
|
gated = loc: lib.hasInfix "auth_request /__hive_authelia;" loc.extraConfig;
|
|
in
|
|
allLocal.services.hyperhive.swarm.dashboardDomain == "h1.hives.t.local"
|
|
&& vh."h1.hives.t.local".forceSSL
|
|
&& lib.all (p: gated l.${p}) [
|
|
"/"
|
|
"/api/"
|
|
"/api/docs/"
|
|
]
|
|
&& lib.hasInfix "error_page 401 =302 https://auth.t.local/?rd=$target_url;" l."/".extraConfig
|
|
&& !(lib.hasInfix "auth_request" (l."/health/".extraConfig or ""))
|
|
&& l."= /__hive_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request"
|
|
&& lib.hasInfix "agents.conf" vh."h1.hives.t.local".extraConfig
|
|
&&
|
|
lib.hasInfix "return 301 https://h1.hives.t.local$request_uri;"
|
|
vh."h1.t.local".locations."/".extraConfig
|
|
&& !(vh."h1.t.local".locations ? "/api/")
|
|
&& lib.elem "h1.hives.t.local" allLocal.services.hyperhive.gateway.localNames
|
|
&& allLocal.systemd.services.hive-c0re.environment.HYPERHIVE_DASHBOARD_DOMAIN == "h1.hives.t.local";
|
|
}
|
|
{
|
|
# The gateway's catch-all and the hive's two vhosts (../host-modules/
|
|
# hive-c0re/vhosts.nix) are declared under `gateway.vhosts` like a
|
|
# service's, so the gateway's listen set and issuer choice reach them
|
|
# by the same path. The catch-all keeps its bare `return 444;` (no
|
|
# headers: it sends no response) and the dashboard keeps its exact
|
|
# location set.
|
|
name = "the catch-all and the hive's vhosts render with the gateway's listen set, issuer choice, headers and locations";
|
|
ok =
|
|
let
|
|
gw = allLocal.services.hyperhive.gateway;
|
|
vh = allLocal.services.nginx.virtualHosts;
|
|
listenOf = map (l: {
|
|
inherit (l) addr port;
|
|
ssl = l.ssl or false;
|
|
});
|
|
sameListen = v: listenOf v.listen == listenOf gw.lib.listen;
|
|
sameCert = name: v: v.sslCertificate == (gw.lib.tlsFor name).sslCertificate;
|
|
catchAll = vh."_";
|
|
dash = vh."h1.hives.t.local";
|
|
hive = vh."h1.t.local";
|
|
in
|
|
sameListen catchAll
|
|
&& sameCert "_" catchAll
|
|
&& catchAll.addSSL
|
|
&& catchAll.default
|
|
&& catchAll.extraConfig == "return 444;\n"
|
|
&& sameListen dash
|
|
&& sameCert "h1.hives.t.local" dash
|
|
&& dash.forceSSL
|
|
&& !(dash.addSSL or false)
|
|
&& dash.extraConfig == gw.lib.securityHeaders + "include /var/lib/hive-gateway/conf/agents.conf;\n"
|
|
&&
|
|
lib.attrNames dash.locations == [
|
|
"/"
|
|
"/agent/"
|
|
"/api/"
|
|
"/api/docs/"
|
|
"/health/"
|
|
"= /__hive_agent_not_found"
|
|
"= /__hive_agent_unreachable"
|
|
"= /__hive_authelia"
|
|
"= /api/docs"
|
|
]
|
|
&& sameListen hive
|
|
&& sameCert "h1.t.local" hive
|
|
&& hive.addSSL
|
|
&& hive.extraConfig == gw.lib.securityHeaders
|
|
&& hive.locations ? "/";
|
|
}
|
|
{
|
|
# The file held bcrypt hashes at 0644; a host carrying one from before
|
|
# gets it deleted, and nothing names it for nginx.
|
|
name = "the gateway reads no htpasswd, and tmpfiles removes a leftover one";
|
|
ok =
|
|
lib.elem "r /var/lib/hive-gateway/conf/gateway.htpasswd" allLocal.systemd.tmpfiles.rules
|
|
&& !(lib.any (
|
|
v:
|
|
lib.any (t: lib.hasInfix "auth_basic" t) (
|
|
[ (v.extraConfig or "") ] ++ map (loc: loc.extraConfig or "") (lib.attrValues (v.locations or { }))
|
|
)
|
|
) (lib.attrValues allLocal.services.nginx.virtualHosts));
|
|
}
|
|
{
|
|
name = "the swarm UI's hive roster carries each hive's dashboard name";
|
|
ok =
|
|
builtins.fromJSON allLocal.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_HIVES == [
|
|
{
|
|
name = "h1";
|
|
domain = "h1.t.local";
|
|
dashboard = "h1.hives.t.local";
|
|
}
|
|
];
|
|
}
|
|
{
|
|
name = "a hive with matrix off serves no matrix discovery endpoint";
|
|
ok =
|
|
!(builtins.hasAttr "= /.well-known/matrix/client" bare.services.nginx.virtualHosts."_".locations);
|
|
}
|
|
{
|
|
# main got eval-borked twice by this exact class of bug (once on the
|
|
# unit's `Restart` key, once on `RestartSec`) — a nixpkgs bump to
|
|
# `gitea-actions-runner.nix` adds a plain `serviceConfig.*`
|
|
# definition that collides with one of ours, and nix refuses to
|
|
# merge two plain definitions at *host* eval. No other check
|
|
# instantiates a host with `containers.swarm-ci-runner` actually enabled, so
|
|
# the collision only surfaces on operator deploy, not in CI.
|
|
name = "the CI container's unit definitions merge without a priority collision";
|
|
ok = forceCiServiceConfigs;
|
|
}
|
|
{
|
|
# It spends money on an operator's key, so no host runs it by default.
|
|
name = "the review classifier's container exists only where its host enables it";
|
|
ok =
|
|
!(bare.containers ? swarm-review-classifier)
|
|
&& !(centralToggleOff.containers ? swarm-review-classifier)
|
|
&& withClassifier.containers ? swarm-review-classifier;
|
|
}
|
|
{
|
|
# The daemon holds its store leaf, handed in by nspawn without a copy on
|
|
# disk, and nothing else from the host; a key is read per run and
|
|
# reaches only the model child's environment. Reading the claude path
|
|
# also evaluates the container's unfree allowance.
|
|
name = "the review classifier loads only its store leaf and carries no API or provider key in its unit";
|
|
ok =
|
|
let
|
|
svc =
|
|
withClassifier.containers.swarm-review-classifier.config.systemd.services.swarm-review-classifier;
|
|
env = svc.environment;
|
|
in
|
|
lib.elem "review-classifier-bao-key" svc.serviceConfig.LoadCredential
|
|
&&
|
|
env.BAO_CLIENT_KEY == "/run/credentials/swarm-review-classifier.service/review-classifier-bao-key"
|
|
&& env.SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID == "swarm-review-classifier"
|
|
&& lib.hasSuffix "/bin/claude" env.SWARM_REVIEW_CLASSIFIER_CLAUDE
|
|
&& lib.hasSuffix "/bin/opencode" env.SWARM_REVIEW_CLASSIFIER_OPENCODE
|
|
&& !(env ? ANTHROPIC_API_KEY)
|
|
&& !(env ? ACP_PROVIDER_API_KEY)
|
|
&& lib.elem "--load-credential=review-classifier-bao-key:/var/lib/swarm-bao-pki/review-classifier-key.pem" (
|
|
withClassifier.containers.swarm-review-classifier.extraFlags
|
|
)
|
|
# No host unit copies the leaf into the container's root.
|
|
&& !(lib.any (u: lib.hasInfix "nixos-containers/swarm-review-classifier" (u.script or "")) (
|
|
lib.attrValues withClassifier.systemd.services
|
|
));
|
|
}
|
|
{
|
|
# See the same case in ./builder.nix.
|
|
name = "the review classifier's container runs the leaf-minting unit before every start";
|
|
ok =
|
|
let
|
|
c = withClassifier.systemd.services."container@swarm-review-classifier";
|
|
in
|
|
lib.elem "swarm-bao-pki.service" c.wants && lib.elem "swarm-bao-pki.service" c.after;
|
|
}
|
|
{
|
|
# The collector reaches these routes through the gateway now, so each
|
|
# store needs an ingest location of its own. Without one the write rides
|
|
# the `/` catch-all: unauthenticated on the metrics store, and into a
|
|
# browser redirect on the log store.
|
|
name = "each store's vhost has an authenticated ingest location";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts;
|
|
m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push" or null;
|
|
l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs" or null;
|
|
in
|
|
m != null
|
|
&& l != null
|
|
&& lib.hasInfix "auth_request" m.extraConfig
|
|
&& lib.hasInfix "auth_request" l.extraConfig;
|
|
}
|
|
{
|
|
# The metrics vhost is default-deny; the collector's push only lands
|
|
# because its `metrics_endpoint` path is the vhost's one gated exact
|
|
# location. The two literals live in different modules, so renaming
|
|
# either alone breaks ingest while everything still evaluates.
|
|
name = "the collector's metrics push path is a gated location on the metrics vhost";
|
|
ok =
|
|
let
|
|
endpoint = (otelSettings allLocal).exporters."otlphttp/victoriametrics".metrics_endpoint;
|
|
# "https://host/path" -> "/path"
|
|
path = "/" + lib.concatStringsSep "/" (lib.drop 3 (lib.splitString "/" endpoint));
|
|
loc = allLocal.services.nginx.virtualHosts."metrics.t.local".locations."= ${path}" or null;
|
|
in
|
|
loc != null && lib.hasInfix "auth_request" loc.extraConfig;
|
|
}
|
|
{
|
|
# The arm that actually protects something. A pusher handed
|
|
# `error_page 401 =302` FOLLOWS it and POSTs its batch at a login page,
|
|
# which answers 200 — ingest reporting healthy while storing nothing.
|
|
# The third clause is the positive control: the log store's browser
|
|
# location really does redirect, so this says the machine routes differ
|
|
# rather than that the string is absent from the whole file.
|
|
name = "the ingest locations answer 401 instead of redirecting a pusher";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts;
|
|
m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push".extraConfig;
|
|
l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs".extraConfig;
|
|
browser = v."logs.t.local".locations."/".extraConfig;
|
|
in
|
|
!(lib.hasInfix "error_page" m)
|
|
&& !(lib.hasInfix "error_page" l)
|
|
&& lib.hasInfix "error_page" browser;
|
|
}
|
|
{
|
|
# The store authenticates nothing itself, so a location that forwards to
|
|
# it without `auth_request` is open to anything that can reach the
|
|
# domain. The query location is the control: it shows the filter finds
|
|
# store-forwarding locations at all.
|
|
name = "no location on the metrics vhost forwards to the store without auth_request";
|
|
ok =
|
|
let
|
|
locs = allLocal.services.nginx.virtualHosts."metrics.t.local".locations;
|
|
toStore = lib.filterAttrs (
|
|
_: l: l.proxyPass != null && lib.hasPrefix "http://unix:/run/swarm-victoriametrics/" l.proxyPass
|
|
) locs;
|
|
open = lib.filterAttrs (_: l: !(lib.hasInfix "auth_request " l.extraConfig)) toStore;
|
|
in
|
|
toStore ? "= /api/v1/query"
|
|
&& open == { }
|
|
&& locs."/".proxyPass == null
|
|
&& locs."/".return == "403";
|
|
}
|
|
{
|
|
# The read API's check, link by link: every read location names the
|
|
# switch with no login redirect (a machine reader would get login HTML
|
|
# as a 200), the switch rewrites to the map's pick, an agent bearer
|
|
# reaches the agents' responder (listed before the catch-all bearer arm,
|
|
# since a map takes the first matching regex), any other bearer reaches
|
|
# Grafana's, each socket bound by its unit, and anything else goes to
|
|
# authelia. vmui is a browser's, so it redirects to the login.
|
|
name = "the metrics read API sends an agent bearer to swarm-metrics-agent-auth, another bearer to swarm-metrics-auth and a session to authelia";
|
|
ok =
|
|
let
|
|
l = allLocal.services.nginx.virtualHosts."metrics.t.local".locations;
|
|
http = allLocal.services.nginx.appendHttpConfig;
|
|
unit = allLocal.systemd.services.swarm-metrics-auth;
|
|
agentUnit = allLocal.systemd.services.swarm-metrics-agent-auth;
|
|
agentArm = "\"~^Bearer swarm-agent\\.\" /__metrics_agent_auth;";
|
|
bearerArm = "\"~^Bearer \" /__metrics_grafana_auth;";
|
|
v = allLocal.services.hyperhive.deploy.victoriametrics;
|
|
logs = allLocal.services.hyperhive.deploy.victorialogs;
|
|
reads = lib.filterAttrs (n: _: lib.hasPrefix "= /api/v1/" n || lib.hasPrefix "^~ /api/v1/" n) l;
|
|
in
|
|
reads ? "= /api/v1/query_range"
|
|
&& lib.all (
|
|
r:
|
|
lib.hasInfix "auth_request /__metrics_read_auth;" r.extraConfig
|
|
&& !(lib.hasInfix "error_page" r.extraConfig)
|
|
) (lib.attrValues reads)
|
|
&& lib.hasInfix "map \$http_authorization \$swarm_metrics_read_auth_target" http
|
|
&& lib.hasInfix agentArm http
|
|
&& lib.hasInfix bearerArm http
|
|
&& !(lib.hasInfix bearerArm (lib.head (lib.splitString agentArm http)))
|
|
&& lib.hasInfix "default /__metrics_authelia;" http
|
|
&&
|
|
lib.hasInfix "rewrite ^ \$swarm_metrics_read_auth_target last;"
|
|
l."= /__metrics_read_auth".extraConfig
|
|
&& lib.hasInfix "internal;" l."= /__metrics_grafana_auth".extraConfig
|
|
&& l."= /__metrics_grafana_auth".proxyPass == "http://unix:/run/swarm-metrics-auth/auth.sock:/"
|
|
&& lib.hasInfix "--socket /run/swarm-metrics-auth/auth.sock" unit.serviceConfig.ExecStart
|
|
&& lib.hasInfix "internal;" l."= /__metrics_agent_auth".extraConfig
|
|
&& l."= /__metrics_agent_auth".proxyPass == "http://unix:/run/swarm-metrics-agent-auth/auth.sock:/"
|
|
&& lib.hasInfix "--socket /run/swarm-metrics-agent-auth/auth.sock" agentUnit.serviceConfig.ExecStart
|
|
&& hasStorePath "${v.agentAuthPackage}/bin/swarm-logs-auth " agentUnit.serviceConfig.ExecStart
|
|
&& !(hasStorePath "${logs.authPackage}" agentUnit.serviceConfig.ExecStart)
|
|
&& hasStorePath "${v.authPackage}/bin/swarm-metrics-auth " unit.serviceConfig.ExecStart
|
|
&& lib.hasInfix "--store-cert-role swarm-metrics-auth" agentUnit.serviceConfig.ExecStart
|
|
&& lib.hasInfix "--verdict-ttl ${toString v.agentAuthVerdictTtl}" agentUnit.serviceConfig.ExecStart
|
|
&& agentUnit.serviceConfig.LoadCredential == unit.serviceConfig.LoadCredential
|
|
&& lib.hasInfix "limit_req_zone $binary_remote_addr zone=swarm_metrics_agent_auth:1m rate=${v.agentAuthRate};" http
|
|
&&
|
|
lib.hasInfix "limit_req zone=swarm_metrics_agent_auth burst=${toString v.agentAuthBurst} nodelay;"
|
|
l."= /__metrics_agent_auth".extraConfig
|
|
&& !(lib.hasInfix "limit_req" l."= /__metrics_grafana_auth".extraConfig)
|
|
&& lib.hasInfix "auth_request /__metrics_authelia;" l."^~ /vmui/".extraConfig
|
|
&& lib.hasInfix "error_page 401 =302" l."^~ /vmui/".extraConfig
|
|
&& l."= /__metrics_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request";
|
|
}
|
|
{
|
|
# Grafana's host creates the bearer and its datasource sends it; the
|
|
# store's responder reads it. Three modules spell the path, and the
|
|
# store's grants name it twice: if any one drifts, Grafana sends a value
|
|
# the responder never reads, or one of them is refused by the store.
|
|
name = "Grafana's datasource sends the bearer swarm-metrics-auth checks, at one store path";
|
|
ok =
|
|
let
|
|
leaf = "swarm/services/${allLocal.services.hyperhive.swarm.grafana.oidc.clientId}/metrics/bearer";
|
|
ds =
|
|
lib.findFirst (d: d.uid == "swarm-victoriametrics") null
|
|
allLocal.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
|
|
bearerUnit = allLocal.systemd.services.swarm-bao-grafana-metrics-bearer;
|
|
s = allLocal.systemd.services;
|
|
in
|
|
ds.jsonData.httpHeaderName1 == "Authorization"
|
|
&& ds.secureJsonData.httpHeaderValue1 == "$__file{/run/swarm-grafana-secrets/metrics-bearer}"
|
|
&& lib.hasInfix "bao kv put -cas=0 secret/${leaf} value=-" bearerUnit.script
|
|
&& lib.hasInfix "/run/swarm-grafana-secrets/metrics-bearer \"Bearer $bearer\"" bearerUnit.script
|
|
&& lib.hasInfix "--store-path ${leaf}" s.swarm-metrics-auth.serviceConfig.ExecStart
|
|
&& lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"read\"]\n}" s.swarm-bao-metrics-auth-policy.script
|
|
&& lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"create\", \"read\"]\n}" s.swarm-bao-grafana-oidc-policy.script;
|
|
}
|
|
{
|
|
# The log store's second responder checks the very bearer the metrics
|
|
# datasource sends, at the same store path: a different file would be a
|
|
# bearer the responder never reads.
|
|
name = "Grafana's logs datasource sends the bearer the log store's responder checks, at one store path";
|
|
ok =
|
|
let
|
|
datasources =
|
|
allLocal.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
|
|
metrics = lib.findFirst (d: d.uid == "swarm-victoriametrics") null datasources;
|
|
logs = lib.findFirst (d: d.uid == "swarm-victorialogs") null datasources;
|
|
leaf = "swarm/services/${allLocal.services.hyperhive.swarm.grafana.oidc.clientId}/metrics/bearer";
|
|
in
|
|
logs.secureJsonData.httpHeaderValue1 == metrics.secureJsonData.httpHeaderValue1
|
|
&& lib.hasInfix "--store-path ${leaf}" allLocal.systemd.services.swarm-logs-grafana-auth.serviceConfig.ExecStart;
|
|
}
|
|
{
|
|
# The responder reads the store as itself, with the leaf the store's
|
|
# host mints for it.
|
|
name = "swarm-metrics-auth presents its own store leaf";
|
|
ok =
|
|
let
|
|
v = allLocal.services.hyperhive.deploy.victoriametrics;
|
|
lc = allLocal.systemd.services.swarm-metrics-auth.serviceConfig.LoadCredential;
|
|
in
|
|
lib.hasSuffix "/metrics-auth.pem" v.authBaoClientCertFile
|
|
&& lib.hasSuffix "/metrics-auth-key.pem" v.authBaoClientKeyFile
|
|
&& builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc
|
|
&& builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc
|
|
&& lib.hasInfix "/metrics-auth.pem ]" allLocal.systemd.services.swarm-bao-pki.script;
|
|
}
|
|
{
|
|
# The read counterpart to the ingest location: an agent queries the log
|
|
# store with a bearer token, and the `/` catch-all is the browser's
|
|
# route. Riding it would mean inheriting the login redirect the next
|
|
# case is about, so the route has to exist separately to be gated
|
|
# separately.
|
|
name = "the log store's vhost has an authenticated machine query location";
|
|
ok =
|
|
let
|
|
q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/" or null;
|
|
in
|
|
q != null && lib.hasInfix "auth_request" q.extraConfig;
|
|
}
|
|
{
|
|
# Same trap as the ingest case, on the read side, where it is worse: a
|
|
# redirected pusher at least stores nothing visibly, while a redirected
|
|
# *reader* is handed a 200 carrying login HTML and records a query that
|
|
# succeeded and matched no logs. The browser clause is the positive
|
|
# control — that location really does redirect to the login host — so a
|
|
# pass means these two routes differ rather than that the strings are
|
|
# absent from the whole vhost.
|
|
name = "the machine query location answers 401 instead of redirecting to a login page";
|
|
ok =
|
|
let
|
|
v = allLocal.services.nginx.virtualHosts;
|
|
q = v."logs.t.local".locations."^~ /select/logsql/".extraConfig;
|
|
browser = v."logs.t.local".locations."/".extraConfig;
|
|
in
|
|
!(lib.hasInfix "error_page" q)
|
|
&& !(lib.hasInfix "auth.t.local" q)
|
|
&& lib.hasInfix "error_page" browser
|
|
&& lib.hasInfix "auth.t.local" browser;
|
|
}
|
|
{
|
|
# Read access is deliberately unscoped: an authenticated caller reads
|
|
# the whole swarm's logs until a permission system exists. Pinned so a
|
|
# scoping parameter arriving later is a visible diff here rather than a
|
|
# quiet change of rule — and pinned on `proxyPass` too, because
|
|
# VictoriaLogs takes its filters as request parameters, which ride an
|
|
# upstream URI as easily as a directive. The first clause is the
|
|
# control: it proves the location resolved and that `hasInfix` finds
|
|
# what is genuinely in this string, so the absences below mean absent
|
|
# rather than unreadable.
|
|
name = "the machine query location forwards the caller's query unmodified";
|
|
ok =
|
|
let
|
|
q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/";
|
|
in
|
|
lib.hasInfix "auth_request" q.extraConfig
|
|
&& !(lib.hasInfix "extra_filters" q.extraConfig)
|
|
&& !(lib.hasInfix "extra_stream_filters" q.extraConfig)
|
|
&& !(lib.hasInfix "$args" q.extraConfig)
|
|
&& !(lib.hasInfix "?" q.proxyPass);
|
|
}
|
|
{
|
|
# `auth_request` takes a fixed URI, so the split between the agents'
|
|
# responder and authelia is a map plus a rewrite in an internal
|
|
# location. Every link of that chain is pinned: the map keys on the
|
|
# agent-token prefix and falls back to authelia, the query location
|
|
# names the switch, the switch rewrites to the map's pick, and the
|
|
# agent target reaches the responder's socket, which the unit binds.
|
|
name = "the machine query location sends agent bearers to swarm-logs-auth and the rest to authelia";
|
|
ok =
|
|
let
|
|
l = allLocal.services.nginx.virtualHosts."logs.t.local".locations;
|
|
http = allLocal.services.nginx.appendHttpConfig;
|
|
unit = allLocal.systemd.services.swarm-logs-auth;
|
|
grafanaUnit = allLocal.systemd.services.swarm-logs-grafana-auth;
|
|
logs = allLocal.services.hyperhive.deploy.victorialogs;
|
|
metrics = allLocal.services.hyperhive.deploy.victoriametrics;
|
|
in
|
|
lib.hasInfix "map \$http_authorization \$swarm_logs_auth_target" http
|
|
&& lib.hasInfix "\"~^Bearer swarm-agent\\.\" /__swarm_logs_agent_auth;" http
|
|
&& lib.hasInfix "\"~^Bearer [0-9a-f]{64}$\" /__swarm_logs_grafana_auth;" http
|
|
&& lib.hasInfix "default /__hive_authelia;" http
|
|
&& lib.hasInfix "auth_request /__swarm_logs_auth;" l."^~ /select/logsql/".extraConfig
|
|
&& lib.hasInfix "rewrite ^ \$swarm_logs_auth_target last;" l."= /__swarm_logs_auth".extraConfig
|
|
&& lib.hasInfix "internal;" l."= /__swarm_logs_agent_auth".extraConfig
|
|
&& l."= /__swarm_logs_agent_auth".proxyPass == "http://unix:/run/swarm-logs-auth/auth.sock:/"
|
|
&& lib.hasInfix "--socket /run/swarm-logs-auth/auth.sock" unit.serviceConfig.ExecStart
|
|
&& lib.hasInfix "internal;" l."= /__swarm_logs_grafana_auth".extraConfig
|
|
&&
|
|
l."= /__swarm_logs_grafana_auth".proxyPass == "http://unix:/run/swarm-logs-grafana-auth/auth.sock:/"
|
|
&& lib.hasInfix "--socket /run/swarm-logs-grafana-auth/auth.sock" grafanaUnit.serviceConfig.ExecStart
|
|
&& hasStorePath "${logs.grafanaAuthPackage}/bin/swarm-metrics-auth " grafanaUnit.serviceConfig.ExecStart
|
|
&& !(hasStorePath "${metrics.authPackage}" grafanaUnit.serviceConfig.ExecStart)
|
|
&& hasStorePath "${logs.authPackage}/bin/swarm-logs-auth " unit.serviceConfig.ExecStart
|
|
&& lib.hasInfix "--store-cert-role swarm-logs-auth" grafanaUnit.serviceConfig.ExecStart
|
|
&& l ? "= /__hive_authelia";
|
|
}
|
|
{
|
|
# A `DynamicUser` unit's preserved runtime directory lives under
|
|
# `/run/private/` (0700 root:root), which nginx cannot traverse, so a
|
|
# socket there answers every proxied request with a 502. Each `unix:`
|
|
# upstream under `/run/<dir>/` is matched to the units creating `<dir>`.
|
|
# The first two clauses are the control: the match finds both responders.
|
|
name = "no unix socket nginx proxies to sits in a DynamicUser's preserved runtime directory";
|
|
ok =
|
|
let
|
|
prefix = "http://unix:/run/";
|
|
upstreamDirs = lib.unique (
|
|
map (p: lib.head (lib.splitString "/" (lib.removePrefix prefix p))) (
|
|
lib.filter (p: p != null && lib.hasPrefix prefix p) (
|
|
lib.concatMap (vh: lib.mapAttrsToList (_: loc: loc.proxyPass) vh.locations) (
|
|
lib.attrValues allLocal.services.nginx.virtualHosts
|
|
)
|
|
)
|
|
)
|
|
);
|
|
runtimeDirs =
|
|
sc:
|
|
let
|
|
r = sc.RuntimeDirectory or [ ];
|
|
in
|
|
if lib.isList r then r else lib.splitString " " r;
|
|
owners = lib.filterAttrs (
|
|
_: u: lib.any (d: lib.elem d upstreamDirs) (runtimeDirs u.serviceConfig)
|
|
) allLocal.systemd.services;
|
|
hidden =
|
|
sc:
|
|
lib.elem (sc.DynamicUser or false) [
|
|
true
|
|
"yes"
|
|
]
|
|
&& !(lib.elem (sc.RuntimeDirectoryPreserve or "no") [
|
|
false
|
|
"no"
|
|
]);
|
|
in
|
|
owners ? swarm-logs-auth
|
|
&& owners ? swarm-logs-grafana-auth
|
|
&& owners ? swarm-metrics-auth
|
|
&& owners ? swarm-metrics-agent-auth
|
|
&& !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners));
|
|
}
|
|
{
|
|
# Every agent bearer the responder has not recently granted costs a
|
|
# store login and read, so its location carries the per-address limit.
|
|
# The responder gets the grant TTL from the option. The query and
|
|
# authelia locations carry no limit of their own.
|
|
name = "agent bearer checks are rate-limited per address and granted for a TTL";
|
|
ok =
|
|
let
|
|
l = allLocal.services.nginx.virtualHosts."logs.t.local".locations;
|
|
http = allLocal.services.nginx.appendHttpConfig;
|
|
unit = allLocal.systemd.services.swarm-logs-auth;
|
|
v = allLocal.services.hyperhive.deploy.victorialogs;
|
|
in
|
|
lib.hasInfix "limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${v.authRate};" http
|
|
&&
|
|
lib.hasInfix "limit_req zone=swarm_logs_agent_auth burst=${toString v.authBurst} nodelay;"
|
|
l."= /__swarm_logs_agent_auth".extraConfig
|
|
&& lib.hasInfix "limit_req_status 403;" l."= /__swarm_logs_agent_auth".extraConfig
|
|
&& !(lib.hasInfix "limit_req" l."^~ /select/logsql/".extraConfig)
|
|
&& !(lib.hasInfix "limit_req" l."= /__hive_authelia".extraConfig)
|
|
&& lib.hasInfix "--verdict-ttl 60" unit.serviceConfig.ExecStart;
|
|
}
|
|
{
|
|
# The responder reads the store as itself, with the leaf the store's
|
|
# host mints for it.
|
|
name = "swarm-logs-auth presents its own store leaf";
|
|
ok =
|
|
let
|
|
v = allLocal.services.hyperhive.deploy.victorialogs;
|
|
lc = allLocal.systemd.services.swarm-logs-auth.serviceConfig.LoadCredential;
|
|
in
|
|
lib.hasSuffix "/logs-auth.pem" v.authBaoClientCertFile
|
|
&& lib.hasSuffix "/logs-auth-key.pem" v.authBaoClientKeyFile
|
|
&& builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc
|
|
&& builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc;
|
|
}
|
|
{
|
|
# The absence arm for the case above, and the option's own rule — a
|
|
# service declares its entry under its own `enable` — made checkable.
|
|
# Without it, moving the assignment outside the collector's `mkIf`
|
|
# passes every arm above while handing a collector-less hive a scrape
|
|
# target for a port nothing binds.
|
|
name = "a hive with no collector declares no self-scrape target";
|
|
ok = bare.services.hyperhive.otel.scrapeTargets == { };
|
|
}
|
|
{
|
|
# Absence arm, and the one that matters: claiming a name this host does
|
|
# not serve points every local reader at the wrong machine.
|
|
name = "a hive that does not run the store claims no name for it";
|
|
ok = !(builtins.elem "bao.t.local" (baoNames bare));
|
|
}
|
|
{
|
|
# ⚠️ The absence arm that matters. `services.nginx.streamConfig` is a
|
|
# host-wide option, so a block rendered outside the store's own `mkIf`
|
|
# gives every hive in the swarm a listener — on the port the store
|
|
# answers on, in front of no store at all.
|
|
name = "a hive that does not run the store renders no stream passthrough";
|
|
ok = baoStream bare == "";
|
|
}
|
|
{
|
|
# Absence arm for the case above — a hive with no store has no reason to
|
|
# open the store's port, and opening it would point agents at a host that
|
|
# answers nothing.
|
|
name = "a hive that does not run the store opens no bridge port for it";
|
|
ok = !(builtins.elem 8200 (bridgePorts bare));
|
|
}
|
|
{
|
|
# The four blocks below used to be gated on the hive being enabled AND
|
|
# their own condition. The second half was always the load-bearing one —
|
|
# none of these conditions is derived from the hive toggle — so these
|
|
# arms pin what the conjunct was doing: nothing. Each is written against
|
|
# a host with the hive OFF as well as one with it on, because the way a
|
|
# dropped conjunct fails is by making something unconditional, and that
|
|
# shows up as a service appearing where nothing asked for it.
|
|
name = "no bridge port is opened for an exposeHostPorts nobody set";
|
|
ok =
|
|
!(builtins.elem 5432 (bridgePortsOrNone bare))
|
|
&& !(builtins.elem 5432 (bridgePortsOrNone centralToggleOff));
|
|
}
|
|
{
|
|
# `deploy.swarm-controller.enable`, which defaults false and is
|
|
# deliberately not derived from the hive toggle — a swarm has one
|
|
# controller, so the host that runs it says so itself.
|
|
#
|
|
# Probed by what the daemon needs in order to run, not by
|
|
# `? swarm-controller`: ./host-modules/glue-hive-tls-controllers.nix defines an
|
|
# environment key on that unit name, which leaves the attr
|
|
# present-but-inert (no `ExecStart`, empty `wantedBy`) on every hive
|
|
# that has a CA — see the comment there. The daemon's system user has no
|
|
# second definer, so its absence is the unambiguous half.
|
|
name = "the swarm controller does not run unless this host is told to run it";
|
|
ok =
|
|
let
|
|
inert =
|
|
machine:
|
|
!(machine.users.users ? swarm-controller)
|
|
&& !(
|
|
(machine.systemd.services.swarm-controller or { serviceConfig = { }; }).serviceConfig ? ExecStart
|
|
);
|
|
in
|
|
inert bare && inert centralToggleOff;
|
|
}
|
|
{
|
|
# `deploy.swarm-otel.enable`, same shape: the swarm's collector is one
|
|
# host's job, and the container is the whole of what it renders.
|
|
name = "the swarm collector container is absent unless this host is told to run it";
|
|
ok = !(bare.containers ? swarm-otel) && !(centralToggleOff.containers ? swarm-otel);
|
|
}
|
|
{
|
|
# The swarm UI follows `deploy.swarm-controller.enable` — a sibling
|
|
# deployment decision, not the hive toggle. `t.local` is the fixtures'
|
|
# swarm domain, which is the apex the UI claims; the arm below is what
|
|
# proves this vhost renders at all.
|
|
name = "the swarm UI vhost is absent unless this host runs the swarm controller";
|
|
ok =
|
|
!(bare.services.nginx.virtualHosts ? "t.local")
|
|
&& !(centralToggleOff.services.nginx.virtualHosts ? "t.local");
|
|
}
|
|
{
|
|
# The three infrastructure toggles are off by default and asserted by
|
|
# whoever needs them. With nothing on the host needing them, none of
|
|
# the three renders — which is also the control for the arm below.
|
|
name = "the gateway, resolver and bridge are absent where nothing on the host needs them";
|
|
ok =
|
|
!centralToggleOff.services.hyperhive.gateway.enable
|
|
&& !centralToggleOff.services.hyperhive.gateway.dns.enable
|
|
&& !centralToggleOff.services.hyperhive.network.enable
|
|
&& !(centralToggleOff.services.nginx.enable or false)
|
|
&& !(centralToggleOff.services.dnsmasq.enable or false)
|
|
&& !(centralToggleOff.networking.bridges ? hive-br0);
|
|
}
|
|
{
|
|
# hive-c0re asserts all three, and it follows the central toggle — so
|
|
# an ordinary hive keeps getting them with no opt-in, which is what
|
|
# this change must not break.
|
|
name = "an ordinary hive runs the gateway, resolver and bridge because its coordinator needs them";
|
|
ok =
|
|
bare.services.hyperhive.gateway.enable
|
|
&& bare.services.hyperhive.gateway.dns.enable
|
|
&& bare.services.hyperhive.network.enable
|
|
&& bare.services.nginx.enable
|
|
&& bare.services.dnsmasq.enable
|
|
&& bare.networking.bridges ? hive-br0;
|
|
}
|
|
{
|
|
# Refused wherever something turns the name into an identifier, and only
|
|
# there. The store host is the control that the refusal is not the hive's
|
|
# alone; the host running neither is the control that it is gated at all.
|
|
name = "a missing hiveName is refused on a hive and on a store host, not elsewhere";
|
|
ok =
|
|
let
|
|
refusedHiveName =
|
|
extra:
|
|
lib.any (a: !a.assertion && lib.hasInfix "services.hyperhive.hiveName to be set" a.message)
|
|
(hiveWithController ({ hiveName = null; } // extra)).assertions;
|
|
in
|
|
refusedHiveName { }
|
|
&& refusedHiveName {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.bao.enable = true;
|
|
}
|
|
&& !(refusedHiveName { deploy.hive-controller.enable = false; });
|
|
}
|
|
{
|
|
# The control is a host that defines no such option; the removal message
|
|
# must not appear for it.
|
|
name = "a definition of the removed services.hyperhive.ruthless option fails evaluation, pointing at the swarm dashboard";
|
|
ok =
|
|
let
|
|
refused =
|
|
m:
|
|
lib.any (
|
|
a:
|
|
!a.assertion
|
|
&& lib.hasInfix "services.hyperhive.ruthless" a.message
|
|
&& lib.hasInfix "swarm dashboard" a.message
|
|
) m.assertions;
|
|
in
|
|
refused (hive {
|
|
ruthless = true;
|
|
})
|
|
&& !(refused allLocal);
|
|
}
|
|
{
|
|
# Refused wherever any hyperhive service runs, the hive or a swarm
|
|
# service alone. The host enabling nothing is the control that it is
|
|
# gated at all, and has to pass every assertion: it is what an
|
|
# operator gets from importing the module.
|
|
name = "a missing swarm.domain is refused on a hive and on a swarm-service host, and a host enabling nothing evaluates";
|
|
ok =
|
|
let
|
|
noDomain = extra: hiveWithController ({ swarm.domain = null; } // extra);
|
|
refusedSwarmDomain =
|
|
extra:
|
|
lib.any (
|
|
a: !a.assertion && lib.hasInfix "services.hyperhive.swarm.domain to be set" a.message
|
|
) (noDomain extra).assertions;
|
|
in
|
|
refusedSwarmDomain { }
|
|
&& refusedSwarmDomain {
|
|
deploy.hive-controller.enable = false;
|
|
deploy.nats.enable = true;
|
|
}
|
|
&& lib.all (a: a.assertion) (noDomain { deploy.hive-controller.enable = false; }).assertions;
|
|
}
|
|
{
|
|
# Control: a store fetch with no timer still renders `RemainAfterExit`,
|
|
# so the cases below read a real value rather than a missing one.
|
|
name = "Grafana's admin-password fetch stays active after it exits and has no timer";
|
|
ok =
|
|
allLocal.systemd.services ? swarm-bao-grafana-admin
|
|
&& remainsAfterExit allLocal.systemd.services.swarm-bao-grafana-admin
|
|
&& !(allLocal.systemd.timers ? swarm-bao-grafana-admin);
|
|
}
|
|
{
|
|
# A timer whose unit does not exist fails to load, so each timer is
|
|
# gated exactly as its fetch is.
|
|
name = "a host running none of the store-refreshed fetches renders none of their timers";
|
|
ok = lib.all (
|
|
{ unit, ... }: !(bare.systemd.services ? ${unit}) && !(bare.systemd.timers ? ${unit})
|
|
) storeRefreshed;
|
|
}
|
|
{
|
|
# `or false` so a dropped key fails this case by name instead of
|
|
# aborting eval. Only the rendered unit is pinned; eval cannot show
|
|
# that hive-priv still works under the directives.
|
|
name = "hive-priv locks personality and denies real-time scheduling, kernel logs and module loading";
|
|
ok =
|
|
let
|
|
sc = allLocal.systemd.services.hive-priv.serviceConfig;
|
|
in
|
|
(sc.LockPersonality or false)
|
|
&& (sc.RestrictRealtime or false)
|
|
&& (sc.ProtectKernelLogs or false)
|
|
&& (sc.ProtectKernelModules or false);
|
|
}
|
|
]
|
|
# Starting a unit that stays `active (exited)` is a no-op, so a timer aimed
|
|
# at one fires and fetches nothing.
|
|
++ map (
|
|
{ unit, machine }:
|
|
{
|
|
name = "${unit} re-runs on a ten-minute timer and does not stay active after it exits";
|
|
ok =
|
|
let
|
|
s = machine.systemd.services;
|
|
t = machine.systemd.timers;
|
|
in
|
|
s ? ${unit}
|
|
&& t ? ${unit}
|
|
&& !(remainsAfterExit s.${unit})
|
|
&& lib.elem "timers.target" t.${unit}.wantedBy
|
|
&& t.${unit}.timerConfig.OnUnitInactiveSec == "10min"
|
|
&& (t.${unit}.timerConfig.Unit or "${unit}.service") == "${unit}.service";
|
|
}
|
|
) storeRefreshed;
|
|
in
|
|
runGroup "core-toggle" cases
|