2fc63d844b
Finishes the sweep of prose that said the publisher unit writes or carries the OIDC client secrets: rust docs, nix comments and option descriptions, test names and docs.
793 lines
40 KiB
Nix
793 lines
40 KiB
Nix
# `checks.module-eval-grafana` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
grafanaHere = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.datasourceUrl = "http://127.0.0.1:19999";
|
|
deploy.grafana.logsDatasourceUrl = "http://127.0.0.1:19998";
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
};
|
|
|
|
# A host that tries to move the socket directory; reading the option throws.
|
|
grafanaSocketDirSet = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.socketDir = "/run/test-grafana-sock";
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
};
|
|
|
|
# Same pairing `nix/module-eval/bao-otel-collector.nix` uses for its own
|
|
# cases: the store's container with its own collector on, so
|
|
# `.containers.swarm-bao.config` carries a real, evaluated
|
|
# `opentelemetry-collector.settings` rather than one this file would have
|
|
# to hand-assemble.
|
|
baoWithCollector = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.swarm-otel.enable = true;
|
|
};
|
|
|
|
# The metrics UI beside the IdP. It reads its secret out of the store like
|
|
# every other Grafana host, so it needs a store identity like every other
|
|
# Grafana host — the cert pair here is not scenery, it is the arm that would
|
|
# have caught the deleted co-located copy unit coming back.
|
|
grafanaWithAuthelia = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.grafanaOidcClientCertFile = "/etc/pki/bao-grafana-oidc.pem";
|
|
deploy.bao.grafanaOidcClientKeyFile = "/etc/pki/bao-grafana-oidc-key.pem";
|
|
};
|
|
|
|
# The same UI with the IdP on ANOTHER host and a store leaf placed by hand.
|
|
# Knowing an IdP is not running one: `swarm.authelia.url` is what says this
|
|
# swarm has SSO, and nothing about this host does. Identical to the fixture
|
|
# above in everything the delivery path reads, which is the point.
|
|
grafanaRemoteAuthelia = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.grafanaOidcClientCertFile = "/etc/pki/bao-grafana-oidc.pem";
|
|
deploy.bao.grafanaOidcClientKeyFile = "/etc/pki/bao-grafana-oidc-key.pem";
|
|
};
|
|
|
|
# A Grafana host holding no store identity. The module refuses it with an
|
|
# assertion naming the two options to set, rather than going QUIET: no OIDC
|
|
# block can render and the container's login form is off regardless, so
|
|
# there would be no way in and nothing would fail. Only the identity is
|
|
# missing, so an arm below can name which refusal fired.
|
|
grafanaNoIdentity = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
# Declares a vhost that binds its own `listen`, which the gateway refuses.
|
|
gateway.vhosts."own-listen.t.local".listen = [ ];
|
|
};
|
|
|
|
# Did ./host-modules/swarm-grafana.nix refuse this host, and for which of its
|
|
# two reasons. An assertion is a config VALUE until something forces it —
|
|
# `.config` never throws — so a fixture in a state the module refuses is
|
|
# evaluable and the refusal is readable as data. That is what lets a case
|
|
# check that a misconfiguration is REPORTED, rather than only that it is not
|
|
# silently accepted.
|
|
#
|
|
# Matched on the option name the message names, not on its prose, so the
|
|
# wording stays rewordable: the option name is the part an operator has to
|
|
# act on, and a message that stopped naming it would be the actual defect.
|
|
grafanaRefusedFor =
|
|
m: option:
|
|
lib.any (
|
|
a:
|
|
!a.assertion
|
|
&& lib.hasInfix "services.hyperhive.deploy.grafana.enable requires" a.message
|
|
&& lib.hasInfix option a.message
|
|
) m.assertions;
|
|
cases = [
|
|
{
|
|
# The socket's directory is its access control, so it must stay dedicated
|
|
# to this one socket. `readOnly` is what makes that inexpressible: a
|
|
# definition anywhere — here at the `deploy` path — has to fail the eval,
|
|
# and an unset host has to render the default into nginx's upstream.
|
|
name = "grafana's socket directory cannot be set and renders its default";
|
|
ok =
|
|
!(builtins.tryEval grafanaSocketDirSet.services.hyperhive.deploy.grafana.socketDir).success
|
|
&&
|
|
grafanaHere.services.nginx.virtualHosts.${grafanaHere.services.hyperhive.swarm.grafana.domain}.locations."= /metrics".proxyPass
|
|
== "http://unix:/run/swarm-grafana/grafana.sock:/metrics";
|
|
}
|
|
{
|
|
# Grafana serves `/metrics` unauthenticated, so the published name gates
|
|
# it on the collector's auth subrequest: no `error_page 401 =302` (a
|
|
# scraper would parse a login page as metrics) and no `/__hive_authelia`.
|
|
# `=` is what catches the normalised variants and still leaves the rest
|
|
# of Grafana proxied.
|
|
name = "grafana's published vhost serves /metrics only behind the collector's auth subrequest";
|
|
ok =
|
|
let
|
|
g = grafanaHere.services.hyperhive.swarm.grafana;
|
|
locs = grafanaHere.services.nginx.virtualHosts.${g.domain}.locations;
|
|
checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig);
|
|
authz = locs."= /__grafana_metrics_authz";
|
|
in
|
|
locs ? "= /metrics"
|
|
&& locs."= /metrics".return == null
|
|
&& checksOf locs."= /metrics" == [ "auth_request /__grafana_metrics_authz;" ]
|
|
&& !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
|
|
&& authz.proxyPass == "https://auth.t.local/api/authz/auth-request"
|
|
&& lib.hasInfix "internal;" authz.extraConfig
|
|
&& !(grafanaHere.services.nginx.virtualHosts ? "grafana-metrics")
|
|
&& lib.hasPrefix "http://unix:" locs."/".proxyPass;
|
|
}
|
|
{
|
|
# This fixture enables grafana and NOT authelia, so the login-form toggle
|
|
# has to follow "grafana requires SSO" rather than "both services are on
|
|
# this host". Its vhost is on the public gateway, so a password box there
|
|
# is a way in whatever the topology.
|
|
name = "grafana disables its local login form even where authelia is not on this host";
|
|
ok = grafanaHere.containers.swarm-grafana.config.services.grafana.settings.auth.disable_login_form;
|
|
}
|
|
{
|
|
# The absence class this whole file is for, and the reported defect in one
|
|
# arm: the OIDC block hung off "authelia is on this host", so the split
|
|
# deployment got a Grafana with no SSO settings and no login form — no way
|
|
# in at all. The block is emitted in every deployment now, so the negative
|
|
# arm is not "no block elsewhere" but "the two do not name the same IdP":
|
|
# each host's block has to point at the URL the SWARM names, and a block
|
|
# built from `deploy.authelia` rather than `swarm.authelia.url` would pass
|
|
# a presence check on both fixtures while sending one of them nowhere.
|
|
name = "grafana's OIDC block names the swarm's IdP, wherever that IdP runs";
|
|
ok =
|
|
let
|
|
oauth = m: m.containers.swarm-grafana.config.services.grafana.settings."auth.generic_oauth";
|
|
remote = oauth grafanaRemoteAuthelia;
|
|
local = oauth grafanaWithAuthelia;
|
|
in
|
|
remote.enabled
|
|
&& lib.hasInfix "https://auth.example.invalid/api/oidc/token" remote.token_url
|
|
&& local.enabled
|
|
&& lib.hasInfix "https://auth.t.local/api/oidc/token" local.token_url
|
|
&& !(lib.hasInfix "auth.example.invalid" local.token_url);
|
|
}
|
|
{
|
|
# 🩸 The arm that guards the ruling this slice landed under. There is ONE
|
|
# delivery route: the store reader, on every host that runs Grafana. The
|
|
# negative names the deleted unit rather than a generic absence, because
|
|
# the way this regresses is someone re-adding the co-located copy as an
|
|
# optimisation — a second writer of one path, and a second shape of "the
|
|
# secret is wrong" to debug.
|
|
name = "grafana's OIDC secret has exactly one delivery unit, the store reader, in both topologies";
|
|
ok =
|
|
let
|
|
local = grafanaWithAuthelia.systemd.services;
|
|
remote = grafanaRemoteAuthelia.systemd.services;
|
|
in
|
|
local ? swarm-bao-grafana-oidc
|
|
&& remote ? swarm-bao-grafana-oidc
|
|
&& !(local ? swarm-grafana-oidc-secret)
|
|
&& !(remote ? swarm-grafana-oidc-secret);
|
|
}
|
|
{
|
|
# What the deleted warning became. The shape is unchanged — a Grafana host
|
|
# holding no store leaf — but silence there is a container nobody can log
|
|
# into for a reason no log names, and a warning is read back by nothing.
|
|
# The second arm is what makes this a refusal about the IDENTITY: this
|
|
# fixture names an IdP, so a message about `swarm.authelia.url` here would
|
|
# mean the two assertions had been collapsed into one conjunction.
|
|
name = "a grafana host with no store identity is refused, naming the options to set";
|
|
ok =
|
|
grafanaRefusedFor grafanaNoIdentity "deploy.bao.grafanaOidcClientCertFile"
|
|
&& grafanaRefusedFor grafanaNoIdentity "deploy.bao.grafanaOidcClientKeyFile"
|
|
&& !(grafanaRefusedFor grafanaNoIdentity "swarm.authelia.url");
|
|
}
|
|
{
|
|
# Without this the arm above proves nothing: a refusal that fires on
|
|
# every host is not a check, and both of these are hosts a swarm is
|
|
# expected to have. Read through the same helper, so a message that
|
|
# stopped naming its option would fail the arms above rather than pass
|
|
# this one by accident.
|
|
name = "neither grafana refusal fires on a correctly configured host, co-located or not";
|
|
ok =
|
|
!(grafanaRefusedFor grafanaWithAuthelia "services.hyperhive.swarm.authelia.url")
|
|
&& !(grafanaRefusedFor grafanaWithAuthelia "deploy.bao.grafanaOidcClientCertFile")
|
|
&& !(grafanaRefusedFor grafanaRemoteAuthelia "services.hyperhive.swarm.authelia.url")
|
|
&& !(grafanaRefusedFor grafanaRemoteAuthelia "deploy.bao.grafanaOidcClientCertFile");
|
|
}
|
|
{
|
|
# Grafana's API takes basic auth whatever the login form says, so the
|
|
# built-in admin's password is the only thing between the public vhost
|
|
# and server-admin. Upstream sets it only when it creates the database,
|
|
# so both halves are pinned: the setting a new database reads, and the
|
|
# reset every start applies to whichever database exists. Without the
|
|
# reset, a database created before the setting keeps `admin`.
|
|
#
|
|
# The reset's `--config` must be the file the server reads, or it resets
|
|
# a different database: compared by store path against the server's own
|
|
# command line.
|
|
name = "grafana's admin password is minted into the store and applied to new and existing databases";
|
|
ok =
|
|
let
|
|
unit = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-admin;
|
|
grafana = grafanaRemoteAuthelia.containers.swarm-grafana.config.services.grafana;
|
|
service = grafanaRemoteAuthelia.containers.swarm-grafana.config.systemd.services.grafana;
|
|
resetConfig = lib.head (
|
|
lib.head (lib.filter lib.isList (builtins.split "--config ([^ ]+)" service.preStart))
|
|
);
|
|
in
|
|
lib.hasInfix "bao kv put -cas=0 secret/swarm/services/swarm-grafana/admin/password value=-" unit.script
|
|
&& lib.hasInfix "openssl rand -hex 32" unit.script
|
|
&& lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/admin-password" unit.script
|
|
&& lib.elem "container@swarm-grafana.service" unit.before
|
|
&& unit.environment.BAO_CLIENT_CERT == "/etc/pki/bao-grafana-oidc.pem"
|
|
&& grafana.settings.security.admin_password == "$__file{/run/swarm-grafana-secrets/admin-password}"
|
|
&& lib.hasInfix "admin reset-admin-password --password-from-stdin < /run/swarm-grafana-secrets/admin-password" service.preStart
|
|
&& lib.hasInfix "-config ${resetConfig}" service.script;
|
|
}
|
|
{
|
|
# Grafana's datasource-encryption key reaches Grafana from the store,
|
|
# seeded from the file an earlier release generated in the container,
|
|
# which stays on disk. A different key leaves Grafana unable to read its
|
|
# stored datasource secrets, so nothing may mint ahead of that seed, and
|
|
# the superseded in-container generator must be gone: left in place it
|
|
# would write a second key beside the delivered one. The last two arms
|
|
# are the presence controls for the absence arms: the guest's `grafana`
|
|
# unit is found by the same lookup, and the admin password's legacy file
|
|
# is among the removals.
|
|
name = "grafana's datasource key is seeded into the store from the earlier file, delivered from /run, and the earlier file is kept";
|
|
ok =
|
|
let
|
|
m = grafanaRemoteAuthelia;
|
|
unit = m.systemd.services.swarm-bao-grafana-secret-key;
|
|
guest = m.containers.swarm-grafana.config;
|
|
earlier = "/var/lib/nixos-containers/swarm-grafana/var/lib/grafana-secret/secret_key";
|
|
in
|
|
lib.hasInfix "bao kv put -cas=0 secret/swarm/services/swarm-grafana/secret-key value=-" unit.script
|
|
&& lib.hasInfix "tr -d '\\n' < ${earlier}" unit.script
|
|
&& lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/secret-key" unit.script
|
|
&& lib.elem "container@swarm-grafana.service" unit.before
|
|
&& unit.environment.BAO_CLIENT_CERT == "/etc/pki/bao-grafana-oidc.pem"
|
|
&&
|
|
guest.services.grafana.settings.security.secret_key
|
|
== "$__file{/run/swarm-grafana-secrets/secret-key}"
|
|
&& !(guest.systemd.services ? swarm-grafana-secret-key)
|
|
&& guest.systemd.services ? grafana
|
|
&& !(lib.any (lib.hasInfix "/var/lib/grafana-secret") m.systemd.tmpfiles.rules)
|
|
&& lib.any (lib.hasInfix "/var/lib/grafana-admin") m.systemd.tmpfiles.rules
|
|
&& lib.elem "r /var/lib/nixos-containers/swarm-grafana/var/lib/grafana-admin/password" m.systemd.tmpfiles.rules;
|
|
}
|
|
{
|
|
# swarm-controller reads alerts with this token, so the account it
|
|
# belongs to is created and kept a Viewer, the token lands at the path
|
|
# the controller's grant reads, and an attempt against a Grafana or a
|
|
# store that never answers ends and retries. The timer re-runs it, so
|
|
# the unit must not stay active.
|
|
name = "the controller's Grafana token is a Viewer account's, stored under Grafana's leaf, and each attempt is bounded";
|
|
ok =
|
|
let
|
|
unit = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-controller-token;
|
|
s = unit.script;
|
|
in
|
|
!(lib.elem (toString (unit.serviceConfig.TimeoutStartSec or "infinity")) [
|
|
"infinity"
|
|
"0"
|
|
])
|
|
&& lib.hasInfix "curl --silent --show-error --connect-timeout 5 --max-time 15" s
|
|
&& lib.hasInfix "--data '{\"name\":\"swarm-controller\",\"role\":\"Viewer\",\"isDisabled\":false}'" s
|
|
&& lib.hasInfix "--data '{\"role\":\"Viewer\",\"isDisabled\":false}'" s
|
|
&& lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/grafana/token value=-" s
|
|
&& lib.hasInfix "--unix-socket /run/swarm-grafana/grafana.sock" s
|
|
&& unit.environment.BAO_CLIENT_CERT == "/etc/pki/bao-grafana-oidc.pem"
|
|
&& unit.serviceConfig.Restart == "on-failure"
|
|
&& !(unit.serviceConfig.RemainAfterExit or false)
|
|
&& lib.elem "swarm-bao-grafana-admin.service" unit.after
|
|
&& grafanaRemoteAuthelia.systemd.timers ? swarm-bao-grafana-controller-token;
|
|
}
|
|
{
|
|
# Same 403-not-a-miss reason as the matrix and queue arms below: the
|
|
# reader's grant covers the `services` prefix, so a path outside it is
|
|
# refused rather than empty, however correct it reads. The negative arm is
|
|
# the rename this is exposed to — a secret filed under the hive that runs
|
|
# the service instead of under the service itself.
|
|
name = "grafana's OIDC secret is read from the prefix the mint writes";
|
|
ok =
|
|
let
|
|
s = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-oidc.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" s
|
|
&& !(lib.hasInfix "secret/swarm/hives/" s);
|
|
}
|
|
{
|
|
# A copy on persistent disk outlives the secret's rotation and lands in
|
|
# backups of the container's root.
|
|
name = "grafana's OIDC secret, admin password and metrics bearer are written to the host's /run only, and the old copies removed";
|
|
ok =
|
|
let
|
|
m = grafanaRemoteAuthelia;
|
|
services = m.systemd.services;
|
|
mount = m.containers.swarm-grafana.bindMounts."/run/swarm-grafana-secrets";
|
|
rules = m.systemd.tmpfiles.rules;
|
|
guest = m.containers.swarm-grafana.config.services.grafana;
|
|
persistent = s: lib.hasInfix "/var/lib/nixos-containers" s || lib.hasInfix "/var/lib/grafana-" s;
|
|
oidc = services.swarm-bao-grafana-oidc.script;
|
|
admin = services.swarm-bao-grafana-admin.script;
|
|
bearer = services.swarm-bao-grafana-metrics-bearer.script;
|
|
# The bind source must exist when the store refuses the login, so the
|
|
# directory is made ahead of it in every delivery script.
|
|
dirBeforeLogin =
|
|
s:
|
|
lib.hasInfix "install -d -m 0755 /run/swarm-grafana-secrets" (
|
|
lib.head (lib.splitString "bao login -method=cert" s)
|
|
);
|
|
in
|
|
lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/swarm-grafana.secret" oidc
|
|
&& lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/metrics-bearer" bearer
|
|
&& dirBeforeLogin oidc
|
|
&& dirBeforeLogin admin
|
|
&& dirBeforeLogin bearer
|
|
&& !(persistent oidc)
|
|
&& !(persistent admin)
|
|
&& !(persistent bearer)
|
|
&& !(persistent services.swarm-bao-grafana-controller-token.script)
|
|
&& lib.hasInfix "< /run/swarm-grafana-secrets/admin-password" services.swarm-bao-grafana-controller-token.script
|
|
&& mount.hostPath == "/run/swarm-grafana-secrets"
|
|
&& mount.isReadOnly
|
|
&&
|
|
guest.settings."auth.generic_oauth".client_secret
|
|
== "$__file{/run/swarm-grafana-secrets/swarm-grafana.secret}"
|
|
&& lib.any (
|
|
d: d.secureJsonData.httpHeaderValue1 or null == "$__file{/run/swarm-grafana-secrets/metrics-bearer}"
|
|
) guest.provision.datasources.settings.datasources
|
|
&& lib.elem "d /run/swarm-grafana-secrets 0755 root root -" rules
|
|
&& lib.elem "r /var/lib/nixos-containers/swarm-grafana/var/lib/grafana-oidc/swarm-grafana.secret" rules
|
|
&& lib.elem "r /var/lib/nixos-containers/swarm-grafana/var/lib/grafana-admin/password" rules
|
|
&& lib.elem "r /var/lib/nixos-containers/swarm-grafana/var/lib/grafana-metrics/bearer" rules
|
|
&& !(lib.any (lib.hasPrefix "f /run/swarm-grafana-secrets") rules);
|
|
}
|
|
{
|
|
# Both halves of the co-location assumption, which was one host's
|
|
# `deploy.*` answering a question about the whole swarm: the identities
|
|
# were minted only where the queue happened to run, and the token
|
|
# endpoint was known only where the IdP happened to run.
|
|
name = "hive identities and the token endpoint do not depend on which host runs what";
|
|
ok =
|
|
let
|
|
autheliaNoQueue = shared.autheliaOnly;
|
|
in
|
|
lib.elem "hive-h1" (map (c: c.id) autheliaNoQueue.services.hyperhive.swarm.authelia.oidc.clients)
|
|
&&
|
|
grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint
|
|
== "https://auth.example.invalid/api/oidc/token";
|
|
}
|
|
{
|
|
# The CI board is in the directory the provider scans (`bao.json` is the
|
|
# control that the entries are read at all), and every metric it queries
|
|
# is a name the poller pushes: a renamed metric on either side is a board
|
|
# of empty panels.
|
|
name = "the CI dashboard ships, and queries the metric names the CI run poller pushes";
|
|
ok =
|
|
let
|
|
provider = lib.head grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.dashboards.settings.providers;
|
|
shipped = lib.attrNames provider.options.path.entries;
|
|
board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/ci.json);
|
|
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
|
|
queried = lib.unique (
|
|
lib.concatMap (
|
|
e: map lib.head (lib.filter lib.isList (builtins.split ''"(hyperhive\.ci\.[a-z_.]+)"'' e))
|
|
) exprs
|
|
);
|
|
pushed = builtins.readFile ../../swarm-ci-metrics/src/sample.rs;
|
|
in
|
|
lib.elem "ci.json" shipped
|
|
&& lib.elem "bao.json" shipped
|
|
&&
|
|
lib.sort lib.lessThan queried == [
|
|
"hyperhive.ci.run.duration"
|
|
"hyperhive.ci.run.queue_wait"
|
|
]
|
|
&& lib.all (m: lib.hasInfix "\"${m}\"" pushed) queried
|
|
&& lib.any (v: v.name == "repo") board.templating.list;
|
|
}
|
|
{
|
|
# The review classifier board ships, and every metric it queries is a name
|
|
# the controller or the classifier service records: a renamed metric on
|
|
# either side is a board of empty panels. A histogram is queried by its
|
|
# `_bucket` series.
|
|
name = "the review classifier dashboard ships, and queries the metric names the controller and the classifier record";
|
|
ok =
|
|
let
|
|
provider = lib.head grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.dashboards.settings.providers;
|
|
shipped = lib.attrNames provider.options.path.entries;
|
|
board = builtins.fromJSON (
|
|
builtins.readFile ../host-modules/swarm-grafana/dashboards/review-classifier.json
|
|
);
|
|
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
|
|
queried = lib.unique (
|
|
lib.concatMap (
|
|
e:
|
|
map (m: lib.removeSuffix "_bucket" (lib.head m)) (
|
|
lib.filter lib.isList (builtins.split ''"(hyperhive\.review_classifier\.[a-z_.]+)"'' e)
|
|
)
|
|
) exprs
|
|
);
|
|
recorded =
|
|
builtins.readFile ../../swarm-controller/src/review_classifier/metrics.rs
|
|
+ builtins.readFile ../../swarm-review-classifier/src/cost.rs;
|
|
in
|
|
lib.elem "review-classifier.json" shipped
|
|
&& lib.elem "bao.json" shipped
|
|
&&
|
|
lib.sort lib.lessThan queried == [
|
|
"hyperhive.review_classifier.ask.retries"
|
|
"hyperhive.review_classifier.checks"
|
|
"hyperhive.review_classifier.requests"
|
|
"hyperhive.review_classifier.run.duration"
|
|
"hyperhive.review_classifier.verdicts"
|
|
]
|
|
&& lib.all (m: lib.hasInfix "\"${m}\"" recorded) queried;
|
|
}
|
|
{
|
|
# The egress board ships, and its queries read the two squid log lines as
|
|
# squid writes them: the would_deny line's field layout, and the default
|
|
# `squid` access format the overview and refused tabs parse. A changed
|
|
# `logformat` or `access_log` is a board of empty panels.
|
|
name = "the egress dashboard ships, and reads the field layout of squid's log lines";
|
|
ok =
|
|
let
|
|
provider = lib.head grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.dashboards.settings.providers;
|
|
shipped = lib.attrNames provider.options.path.entries;
|
|
board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/egress.json);
|
|
exprs = map (e: (lib.head e.spec.data.spec.queries).spec.query.spec.expr) (
|
|
lib.attrValues board.spec.elements
|
|
);
|
|
tabs = map (t: t.spec.title) board.spec.layout.spec.tabs;
|
|
wouldDeny = "would_deny user=<agent> method=<method> host=<dest> port=<port> result=<result>";
|
|
access = "(?P<result>TCP_[A-Z_]+)/(?P<status>[0-9]{3}) (?P<bytes>[0-9]+) (?P<method>[A-Z]+) (?P<url>\\\\S+) (?P<agent>\\\\S+)";
|
|
agentVar = lib.findFirst (v: v.spec.name == "agent") null board.spec.variables;
|
|
squid = builtins.readFile ../host-modules/hive-squid.nix;
|
|
in
|
|
lib.elem "egress.json" shipped
|
|
&&
|
|
tabs == [
|
|
"overview"
|
|
"refused"
|
|
"would-deny"
|
|
]
|
|
# every panel sits in a tab, and every tab item names a panel
|
|
&&
|
|
lib.sort (a: b: a < b) (
|
|
lib.concatMap (t: map (i: i.spec.element.name) t.spec.layout.spec.items) board.spec.layout.spec.tabs
|
|
) == lib.sort (a: b: a < b) (lib.attrNames board.spec.elements)
|
|
&& lib.length (lib.filter (e: lib.hasInfix wouldDeny e) exprs) == 6
|
|
&& lib.length (lib.filter (e: lib.hasInfix access e) exprs) == 14
|
|
&& agentVar != null
|
|
&& lib.hasInfix access agentVar.spec.query.spec.query
|
|
# `"TCP_"` alone is a word filter for the token TCP_, which no result
|
|
# code is (TCP_TUNNEL, TCP_MISS, ...): every panel reads zero. The prefix
|
|
# form `"TCP_"*` is the one that matches them.
|
|
&& lib.all (e: !(lib.hasInfix "\"TCP_\" " e)) (exprs ++ [ agentVar.spec.query.spec.query ])
|
|
&& lib.all (e: lib.hasInfix "\"TCP_\"*" e) (
|
|
lib.filter (e: lib.hasInfix access e) (exprs ++ [ agentVar.spec.query.spec.query ])
|
|
)
|
|
&& lib.hasInfix "logformat would_deny would_deny user=%[un method=%rm host=%>rd port=%>rP result=" squid
|
|
&& lib.hasInfix "access_log syslog:daemon.info squid !manager" squid;
|
|
}
|
|
{
|
|
# The standing check on the collector's PRIORITY mapping — whether a line
|
|
# ever reaches the log store carrying no severity. It is shipped config
|
|
# rather than something an operator imports by hand, which also means it
|
|
# can be deleted by hand. Read out of the dashboard the provisioner
|
|
# actually ships (`dashboardDir` in ../host-modules/swarm-grafana.nix
|
|
# names this file), and asserted on the QUERY rather than the title,
|
|
# because a panel that keeps its title and loses its expression renders
|
|
# an empty graph that looks exactly like zero prioless lines.
|
|
name = "the logs dashboard still counts lines that arrive with no severity";
|
|
ok =
|
|
let
|
|
board = builtins.fromJSON (
|
|
builtins.readFile ../host-modules/swarm-grafana/dashboards/logstore.json
|
|
);
|
|
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
|
|
counts = lib.filter (e: lib.hasInfix "severity_text" e && lib.hasInfix "stats count()" e) exprs;
|
|
in
|
|
# Both arms of the split: lines that HAD a priority and arrived without
|
|
# a severity anyway (the regression line, which must reach zero), and
|
|
# lines that never had one to map — Claude Code's own OTLP telemetry,
|
|
# which no mapping can reach. Folded into one number the second would
|
|
# keep the panel off zero forever and hide the first.
|
|
lib.any (e: lib.hasInfix "PRIORITY:*" e) counts
|
|
&& lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts;
|
|
}
|
|
{
|
|
# The logs datasource reaches the store through its swarm name — a
|
|
# loopback URL works only where Grafana and the store share a host — and
|
|
# sends the metrics datasource's bearer, which the store's `/select/logsql/`
|
|
# route checks.
|
|
name = "the logs datasource targets the log store's swarm name and sends the metrics bearer";
|
|
ok =
|
|
let
|
|
datasources =
|
|
grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
|
|
metrics = lib.findFirst (d: d.uid == "swarm-victoriametrics") null datasources;
|
|
logs = lib.findFirst (d: d.uid == "swarm-victorialogs") null datasources;
|
|
in
|
|
logs.url == "https://${grafanaWithAuthelia.services.hyperhive.swarm.victorialogs.domain}"
|
|
&& !(lib.hasInfix "127.0.0.1" logs.url)
|
|
&& logs.jsonData.httpHeaderName1 == "Authorization"
|
|
&& logs.secureJsonData.httpHeaderValue1 == metrics.secureJsonData.httpHeaderValue1;
|
|
}
|
|
{
|
|
# An external plugin inherits only Grafana's proxy variables unless its
|
|
# id is listed here, and without `SSL_CERT_FILE` every logs query fails
|
|
# `x509: unknown authority`. Compared against the provisioned
|
|
# datasource's `type`, so renaming one side without the other fails.
|
|
name = "the logs datasource's plugin is handed Grafana's environment";
|
|
ok =
|
|
let
|
|
grafana = grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana;
|
|
logs = lib.findFirst (
|
|
d: d.uid == "swarm-victorialogs"
|
|
) null grafana.provision.datasources.settings.datasources;
|
|
in
|
|
logs != null
|
|
&& lib.elem logs.type (lib.toList (grafana.settings.plugins.forward_host_env_vars or [ ]));
|
|
}
|
|
{
|
|
# The reader's half of the same mapping. The collector writes a severity
|
|
# TEXT; nothing downstream reads it as a level unless the datasource is
|
|
# told which field holds it, and that telling is a list of rules rather
|
|
# than a field name — so it goes stale one severity at a time. Asserted
|
|
# against ../journald-severity.nix rather than a literal list here,
|
|
# because the way this regresses is a severity added to the parser and
|
|
# not to Grafana: every line still arrives, the new one is just
|
|
# unfilterable, and no query errors to say so.
|
|
#
|
|
# `enabled` and the canonical `level` spelling are pinned alongside
|
|
# because both fail silently in the direction of "provisions fine,
|
|
# returns nothing" — see the comment on `logLevelRules` in
|
|
# ../host-modules/swarm-grafana.nix.
|
|
name = "the logs datasource maps every severity the collector can emit to a log level";
|
|
ok =
|
|
let
|
|
sources =
|
|
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
|
|
logs = lib.head (lib.filter (d: d.uid == "swarm-victorialogs") sources);
|
|
rules = logs.jsonData.logLevelRules or [ ];
|
|
# `overwrite_text` makes the stored text the OpenTelemetry short name
|
|
# for each mapped severity, which is the parser's own key uppercased.
|
|
emitted = map lib.toUpper (lib.attrNames (lib.head (import ../journald-severity.nix)).mapping);
|
|
canonical = [
|
|
"critical"
|
|
"error"
|
|
"warning"
|
|
"info"
|
|
"debug"
|
|
"trace"
|
|
];
|
|
in
|
|
rules != [ ]
|
|
&& lib.all (r: r.field == "severity_text" && r.enabled == true && lib.elem r.level canonical) rules
|
|
&& lib.all (text: lib.any (r: r.value == text) rules) emitted;
|
|
}
|
|
{
|
|
# Harness turn failures reach the operator only through this rule, so
|
|
# its absence is silent. Asserted on the query, not the title: the
|
|
# exact `_msg` is what keeps subagent turn failures out, and a phrase
|
|
# match would still provision and fire.
|
|
name = "the agent turn-failure alert rule is provisioned against the harness's log line";
|
|
ok =
|
|
let
|
|
groups =
|
|
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
|
|
rules = lib.concatMap (g: g.rules) groups;
|
|
rule = lib.findFirst (r: r.uid == "hh-agent-turn-failed") null rules;
|
|
expr = (lib.head rule.data).model.expr;
|
|
in
|
|
rule != null
|
|
&& lib.hasInfix ''_SYSTEMD_UNIT:="hive-agent.service"'' expr
|
|
&& lib.hasInfix ''_msg:="turn failed"'' expr
|
|
&& lib.hasInfix "stats by (hive, agent)" expr;
|
|
}
|
|
{
|
|
# The hive-health alerts query metric names that hive-c0re spells in
|
|
# Rust. A rename on either side provisions cleanly and the rule then
|
|
# reads nothing, so it never fires. Each name here is checked against
|
|
# the exporter's source rather than a copy of it, and each rule must
|
|
# query the metrics datasource with an instant query.
|
|
# 🔬 Invert-proof: rename a gauge in `otel_metrics.rs` and this fails.
|
|
name = "the hive-health alert rules query metrics hive-c0re exports";
|
|
ok =
|
|
let
|
|
groups =
|
|
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
|
|
rules = lib.concatMap (g: g.rules) groups;
|
|
source = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs;
|
|
expected = {
|
|
"hh-hive-degraded" = "hyperhive.hive.degraded";
|
|
"hh-hive-warning" = "hyperhive.hive.warning";
|
|
"hh-host-disk-high" = "hyperhive.hive.nix_store.utilization";
|
|
"hh-host-disk-critical" = "hyperhive.hive.nix_store.utilization";
|
|
};
|
|
check =
|
|
uid: metric:
|
|
let
|
|
rule = lib.findFirst (r: r.uid == uid) null rules;
|
|
query = (lib.head rule.data).model;
|
|
in
|
|
rule != null
|
|
&& lib.hasInfix ''{__name__="${metric}"'' query.expr
|
|
&& lib.hasInfix ''"${metric}"'' source
|
|
&& query.instant
|
|
&& (lib.head rule.data).datasourceUid == "swarm-victoriametrics";
|
|
in
|
|
lib.all (uid: check uid expected.${uid}) (builtins.attrNames expected);
|
|
}
|
|
{
|
|
# The cert-lifetime alert fires at a lifetime fraction over 0.8 and
|
|
# selects the two gauge names `swarm-secret-client` spells in Rust; a
|
|
# rename on either side provisions cleanly and the rule then reads
|
|
# nothing. The sibling disk rule is the presence control: it goes through
|
|
# the same lookup, so a misspelled attribute path fails both arms.
|
|
# 🔬 Invert-proof: change `threshold = 0.8` on `hh-bao-client-cert-lifetime`
|
|
# in swarm-grafana.nix, or a name in `swarm-secret-client/src/cert.rs`.
|
|
name = "the bao client-cert lifetime alert fires over 0.8 on the gauges the services export";
|
|
ok =
|
|
let
|
|
groups =
|
|
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
|
|
rules = lib.concatMap (g: g.rules) groups;
|
|
source = builtins.readFile ../../swarm-secret-client/src/cert.rs;
|
|
threshold =
|
|
uid:
|
|
let
|
|
rule = lib.findFirst (r: r.uid == uid) null rules;
|
|
condition = lib.findFirst (q: q.refId == "C") null rule.data;
|
|
in
|
|
lib.head (lib.head condition.model.conditions).evaluator.params;
|
|
query =
|
|
(lib.head (lib.findFirst (r: r.uid == "hh-bao-client-cert-lifetime") null rules).data).model;
|
|
in
|
|
threshold "hh-host-disk-high" == 0.85
|
|
&& threshold "hh-bao-client-cert-lifetime" == 0.8
|
|
&&
|
|
lib.all
|
|
(metric: lib.hasInfix ''{__name__="${metric}"'' query.expr && lib.hasInfix ''"${metric}"'' source)
|
|
[
|
|
"bao.client_cert.not_before"
|
|
"bao.client_cert.not_after"
|
|
]
|
|
&& query.instant;
|
|
}
|
|
{
|
|
# The held-deploys alert fires on any agent held for 30 minutes and
|
|
# selects the gauge name `held_deploys.rs` spells in Rust. The disk rule
|
|
# is the presence control: it goes through the same lookup.
|
|
# 🔬 Invert-proof: change `for = "30m"` or the selector on
|
|
# `hh-deploys-held` in swarm-grafana.nix, or `METRIC` in held_deploys.rs.
|
|
name = "the held-deploys alert fires after 30 minutes on the gauge swarm-controller exports";
|
|
ok =
|
|
let
|
|
groups =
|
|
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
|
|
rules = lib.concatMap (g: g.rules) groups;
|
|
source = builtins.readFile ../../swarm-controller/src/held_deploys.rs;
|
|
rule = uid: lib.findFirst (r: r.uid == uid) null rules;
|
|
held = rule "hh-deploys-held";
|
|
threshold =
|
|
r:
|
|
lib.head
|
|
(lib.head (lib.findFirst (q: q.refId == "C") null r.data).model.conditions).evaluator.params;
|
|
in
|
|
threshold held == 0
|
|
&& held.for == "30m"
|
|
&& threshold (rule "hh-host-disk-high") == 0.85
|
|
&& lib.hasInfix ''{__name__="swarm.deploys.held"}'' (lib.head held.data).model.expr
|
|
&& lib.hasInfix ''"swarm.deploys.held"'' source;
|
|
}
|
|
{
|
|
# 🩸 The regression this guards: moving bao's metrics scrape into its
|
|
# own in-container collector made that collector stamp every exported
|
|
# series `service.name=swarm-bao`, while every panel in the shipped
|
|
# bao.json still queried the literal `"service.name"="bao"` — delivery
|
|
# could be perfect and no panel would ever match. Fixed by renaming it
|
|
# in the collector rather than in the dashboard: `swarm-bao.nix`'s
|
|
# `collectorServiceName` now stamps `"bao"` directly. This case reads
|
|
# the collector's OWN evaluated config — not a hand-copied literal —
|
|
# so it fails again the moment that binding and the dashboard drift
|
|
# apart in either direction.
|
|
# 🔬 Invert-proof: change `collectorServiceName` in swarm-bao.nix back
|
|
# to `cfg.machine` and this case fails.
|
|
name = "the bao dashboard's selectors match the service.name bao's own collector stamps";
|
|
ok =
|
|
let
|
|
attrs =
|
|
baoWithCollector.containers.swarm-bao.config.services.opentelemetry-collector.settings.processors.resource.attributes;
|
|
stamped = lib.findFirst (a: a.key == "service.name") null attrs;
|
|
board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/bao.json);
|
|
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
|
|
selectors = lib.filter (e: lib.hasInfix "service.name" e) exprs;
|
|
in
|
|
stamped != null
|
|
&& builtins.length selectors == 12
|
|
&& lib.all (e: lib.hasInfix "\"service.name\"=\"${stamped.value}\"" e) selectors;
|
|
}
|
|
{
|
|
# The bao board's metrics panels can show a gap in the metrics, never a
|
|
# gap in the logs: "bao is quiet" and "bao's logs are not arriving" look
|
|
# the same there. These panels read the journal itself. Asserted on the
|
|
# query, as for the logs board, because a panel that keeps its title and
|
|
# loses its expression renders an empty graph.
|
|
name = "the bao dashboard reads openbao's journal from the log store";
|
|
ok =
|
|
let
|
|
board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/bao.json);
|
|
logPanels = lib.filter (
|
|
p: (p.datasource.type or "") == "victoriametrics-logs-datasource"
|
|
) board.panels;
|
|
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) logPanels;
|
|
in
|
|
builtins.length logPanels == 3
|
|
&& lib.all (e: lib.hasInfix "_SYSTEMD_UNIT:\"openbao.service\"" e) exprs
|
|
&& lib.any (p: p.type == "logs") logPanels;
|
|
}
|
|
{
|
|
# A module declares its vhost under `gateway.vhosts` and the gateway adds
|
|
# what it owns: the listen set, the cert for the name and the security
|
|
# headers ahead of the module's own `extraConfig`.
|
|
name = "grafana's vhost is rendered with the gateway's listen set, TLS and security headers";
|
|
ok =
|
|
let
|
|
gw = grafanaHere.services.hyperhive.gateway;
|
|
v = grafanaHere.services.nginx.virtualHosts.${grafanaHere.services.hyperhive.swarm.grafana.domain};
|
|
# nginx's `listen` entries come back with defaults filled in
|
|
# (`extraParameters`, `proxyProtocol`), so compare the fields set.
|
|
listenOf = map (l: {
|
|
inherit (l) addr port;
|
|
ssl = l.ssl or false;
|
|
});
|
|
in
|
|
listenOf v.listen == listenOf gw.lib.listen
|
|
&& lib.hasPrefix gw.lib.securityHeaders v.extraConfig
|
|
&& v.addSSL
|
|
&& !(v ? certificate);
|
|
}
|
|
{
|
|
# A vhost that sets `listen` itself would bind a port the gateway does
|
|
# not consider its own, so the eval refuses it and names the vhost.
|
|
name = "a gateway vhost that sets its own listen is refused";
|
|
ok = lib.any (
|
|
a:
|
|
!a.assertion
|
|
&& lib.hasInfix "sets attributes the gateway owns" a.message
|
|
&& lib.hasInfix "own-listen.t.local.listen" a.message
|
|
) grafanaNoIdentity.assertions;
|
|
}
|
|
];
|
|
in
|
|
runGroup "grafana" cases
|