cb176c7be7
bao's own in-container collector labelled every log line and metric it forwards `service.name=swarm-bao` (`processors.resource.attributes`, keyed off `swarm.bao.machine`), while every panel in the shipped Grafana bao dashboard queries the literal `service.name="bao"` — no panel has matched since the scrape moved into that collector. Rename it in the collector instead of templating the dashboard: a new `collectorServiceName` binding in swarm-bao.nix, deliberately not `cfg.machine` (that value names the container/receiver, not bao's display identity), stamps `service.name="bao"` directly. bao.json is back to its origin/main shape, unchanged. Adds a module-eval case to checks.module-eval-grafana that reads the collector's own evaluated config and asserts its service.name matches every selector the shipped dashboard uses; verified invert-proof by setting the value back to cfg.machine and confirming that specific case (and only it) fails.
303 lines
15 KiB
Nix
303 lines
15 KiB
Nix
# `checks.module-eval-grafana` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
grafanaOldPath = hive {
|
|
deploy.grafana.enable = true;
|
|
swarm.grafana.socketDir = "/run/test-grafana-sock";
|
|
swarm.grafana.datasourceUrl = "http://127.0.0.1:19999";
|
|
swarm.grafana.logsDatasourceUrl = "http://127.0.0.1:19998";
|
|
swarm.grafana.plugins = [ ];
|
|
swarm.grafana.package = pkgs.emptyDirectory;
|
|
};
|
|
|
|
# Same pairing `nix/module-eval/bao-otel-collector.nix` uses for its own
|
|
# cases: the store's container with its own collector on, so
|
|
# `.containers.swarm-bao.config` carries a real, evaluated
|
|
# `opentelemetry-collector.settings` rather than one this file would have
|
|
# to hand-assemble.
|
|
baoWithCollector = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.swarm-otel.enable = true;
|
|
};
|
|
|
|
# The metrics UI beside the IdP. It reads its secret out of the store like
|
|
# every other Grafana host, so it needs a store identity like every other
|
|
# Grafana host — the cert pair here is not scenery, it is the arm that would
|
|
# have caught the deleted co-located copy unit coming back.
|
|
grafanaWithAuthelia = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.grafanaOidcClientCertFile = "/etc/pki/bao-grafana-oidc.pem";
|
|
deploy.bao.grafanaOidcClientKeyFile = "/etc/pki/bao-grafana-oidc-key.pem";
|
|
};
|
|
|
|
# The same UI with the IdP on ANOTHER host and a store leaf placed by hand.
|
|
# Knowing an IdP is not running one: `swarm.authelia.url` is what says this
|
|
# swarm has SSO, and nothing about this host does. Identical to the fixture
|
|
# above in everything the delivery path reads, which is the point.
|
|
grafanaRemoteAuthelia = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.grafanaOidcClientCertFile = "/etc/pki/bao-grafana-oidc.pem";
|
|
deploy.bao.grafanaOidcClientKeyFile = "/etc/pki/bao-grafana-oidc-key.pem";
|
|
};
|
|
|
|
# A Grafana host holding no store identity. This used to be the shape the
|
|
# module went QUIET on — no OIDC block, a warning, and a container whose
|
|
# login form is off regardless, so no way in and nothing failed. It is kept
|
|
# rather than deleted because the shape is still reachable by an operator;
|
|
# what changed is the deliverable, from a warning nothing reads back to a
|
|
# refusal naming the two options to set. Only the identity is missing, so an
|
|
# arm below can name which refusal fired.
|
|
grafanaNoIdentity = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
};
|
|
|
|
# Did ./host-modules/swarm-grafana.nix refuse this host, and for which of its
|
|
# two reasons. An assertion is a config VALUE until something forces it —
|
|
# `.config` never throws — so a fixture in a state the module refuses is
|
|
# evaluable and the refusal is readable as data. That is what lets a case
|
|
# check that a misconfiguration is REPORTED, rather than only that it is not
|
|
# silently accepted.
|
|
#
|
|
# Matched on the option name the message names, not on its prose, so the
|
|
# wording stays rewordable: the option name is the part an operator has to
|
|
# act on, and a message that stopped naming it would be the actual defect.
|
|
grafanaRefusedFor =
|
|
m: option:
|
|
lib.any (
|
|
a:
|
|
!a.assertion
|
|
&& lib.hasInfix "services.hyperhive.deploy.grafana.enable requires" a.message
|
|
&& lib.hasInfix option a.message
|
|
) m.assertions;
|
|
cases = [
|
|
{
|
|
# This fixture enables grafana and NOT authelia, which is the shape the
|
|
# login form used to stay enabled in: the toggle read "both services are
|
|
# on this host" rather than "grafana requires SSO". Grafana ships an
|
|
# `admin`/`admin` account and its vhost is on the public gateway, so a
|
|
# password box there is a way in whatever the topology.
|
|
name = "grafana disables its local login form even where authelia is not on this host";
|
|
ok =
|
|
grafanaOldPath.containers.swarm-grafana.config.services.grafana.settings.auth.disable_login_form;
|
|
}
|
|
{
|
|
# The absence class this whole file is for, and the reported defect in one
|
|
# arm: the OIDC block hung off "authelia is on this host", so the split
|
|
# deployment got a Grafana with no SSO settings and no login form — no way
|
|
# in at all. The block is emitted in every deployment now, so the negative
|
|
# arm is not "no block elsewhere" but "the two do not name the same IdP":
|
|
# each host's block has to point at the URL the SWARM names, and a block
|
|
# built from `deploy.authelia` rather than `swarm.authelia.url` would pass
|
|
# a presence check on both fixtures while sending one of them nowhere.
|
|
name = "grafana's OIDC block names the swarm's IdP, wherever that IdP runs";
|
|
ok =
|
|
let
|
|
oauth = m: m.containers.swarm-grafana.config.services.grafana.settings."auth.generic_oauth";
|
|
remote = oauth grafanaRemoteAuthelia;
|
|
local = oauth grafanaWithAuthelia;
|
|
in
|
|
remote.enabled
|
|
&& lib.hasInfix "https://auth.example.invalid/api/oidc/token" remote.token_url
|
|
&& local.enabled
|
|
&& lib.hasInfix "https://auth.t.local/api/oidc/token" local.token_url
|
|
&& !(lib.hasInfix "auth.example.invalid" local.token_url);
|
|
}
|
|
{
|
|
# 🩸 The arm that guards the ruling this slice landed under. There is ONE
|
|
# delivery route: the store reader, on every host that runs Grafana. The
|
|
# negative names the deleted unit rather than a generic absence, because
|
|
# the way this regresses is someone re-adding the co-located copy as an
|
|
# optimisation — a second writer of one path, and a second shape of "the
|
|
# secret is wrong" to debug.
|
|
name = "grafana's OIDC secret has exactly one delivery unit, the store reader, in both topologies";
|
|
ok =
|
|
let
|
|
local = grafanaWithAuthelia.systemd.services;
|
|
remote = grafanaRemoteAuthelia.systemd.services;
|
|
in
|
|
local ? swarm-bao-grafana-oidc
|
|
&& remote ? swarm-bao-grafana-oidc
|
|
&& !(local ? swarm-grafana-oidc-secret)
|
|
&& !(remote ? swarm-grafana-oidc-secret);
|
|
}
|
|
{
|
|
# What the deleted warning became. The shape is unchanged — a Grafana host
|
|
# holding no store leaf — but silence there is a container nobody can log
|
|
# into for a reason no log names, and a warning is read back by nothing.
|
|
# The second arm is what makes this a refusal about the IDENTITY: this
|
|
# fixture names an IdP, so a message about `swarm.authelia.url` here would
|
|
# mean the two assertions had been collapsed into one conjunction.
|
|
name = "a grafana host with no store identity is refused, naming the options to set";
|
|
ok =
|
|
grafanaRefusedFor grafanaNoIdentity "deploy.bao.grafanaOidcClientCertFile"
|
|
&& grafanaRefusedFor grafanaNoIdentity "deploy.bao.grafanaOidcClientKeyFile"
|
|
&& !(grafanaRefusedFor grafanaNoIdentity "swarm.authelia.url");
|
|
}
|
|
{
|
|
# Without this the arm above proves nothing: a refusal that fires on
|
|
# every host is not a check, and both of these are hosts a swarm is
|
|
# expected to have. Read through the same helper, so a message that
|
|
# stopped naming its option would fail the arms above rather than pass
|
|
# this one by accident.
|
|
name = "neither grafana refusal fires on a correctly configured host, co-located or not";
|
|
ok =
|
|
!(grafanaRefusedFor grafanaWithAuthelia "services.hyperhive.swarm.authelia.url")
|
|
&& !(grafanaRefusedFor grafanaWithAuthelia "deploy.bao.grafanaOidcClientCertFile")
|
|
&& !(grafanaRefusedFor grafanaRemoteAuthelia "services.hyperhive.swarm.authelia.url")
|
|
&& !(grafanaRefusedFor grafanaRemoteAuthelia "deploy.bao.grafanaOidcClientCertFile");
|
|
}
|
|
{
|
|
# Same 403-not-a-miss reason as the matrix and queue arms below: the
|
|
# reader's grant covers the `services` prefix, so a path outside it is
|
|
# refused rather than empty, however correct it reads. The negative arm is
|
|
# the rename this is exposed to — a secret filed under the hive that runs
|
|
# the service instead of under the service itself.
|
|
name = "grafana's OIDC secret is read from the prefix the publisher writes";
|
|
ok =
|
|
let
|
|
s = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-oidc.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" s
|
|
&& !(lib.hasInfix "secret/swarm/hives/" s);
|
|
}
|
|
{
|
|
# Both halves of the co-location assumption, which was one host's
|
|
# `deploy.*` answering a question about the whole swarm: the identities
|
|
# were minted only where the queue happened to run, and the token
|
|
# endpoint was known only where the IdP happened to run.
|
|
name = "hive identities and the token endpoint do not depend on which host runs what";
|
|
ok =
|
|
let
|
|
autheliaNoQueue = hive { deploy.authelia.enable = true; };
|
|
in
|
|
lib.elem "hive-h1" (map (c: c.id) autheliaNoQueue.services.hyperhive.swarm.authelia.oidc.clients)
|
|
&&
|
|
grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint
|
|
== "https://auth.example.invalid/api/oidc/token";
|
|
}
|
|
{
|
|
# The standing check on the collector's PRIORITY mapping — whether a line
|
|
# ever reaches the log store carrying no severity. It is shipped config
|
|
# rather than something an operator imports by hand, which also means it
|
|
# can be deleted by hand. Read out of the dashboard the provisioner
|
|
# actually ships (`dashboardDir` in ../host-modules/swarm-grafana.nix
|
|
# names this file), and asserted on the QUERY rather than the title,
|
|
# because a panel that keeps its title and loses its expression renders
|
|
# an empty graph that looks exactly like zero prioless lines.
|
|
name = "the logs dashboard still counts lines that arrive with no severity";
|
|
ok =
|
|
let
|
|
board = builtins.fromJSON (
|
|
builtins.readFile ../host-modules/swarm-grafana/dashboards/logstore.json
|
|
);
|
|
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
|
|
counts = lib.filter (e: lib.hasInfix "severity_text" e && lib.hasInfix "stats count()" e) exprs;
|
|
in
|
|
# Both arms of the split: lines that HAD a priority and arrived without
|
|
# a severity anyway (the regression line, which must reach zero), and
|
|
# lines that never had one to map — Claude Code's own OTLP telemetry,
|
|
# which no mapping can reach. Folded into one number the second would
|
|
# keep the panel off zero forever and hide the first.
|
|
lib.any (e: lib.hasInfix "PRIORITY:*" e) counts
|
|
&& lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts;
|
|
}
|
|
{
|
|
# The reader's half of the same mapping. The collector writes a severity
|
|
# TEXT; nothing downstream reads it as a level unless the datasource is
|
|
# told which field holds it, and that telling is a list of rules rather
|
|
# than a field name — so it goes stale one severity at a time. Asserted
|
|
# against ../journald-severity.nix rather than a literal list here,
|
|
# because the way this regresses is a severity added to the parser and
|
|
# not to Grafana: every line still arrives, the new one is just
|
|
# unfilterable, and no query errors to say so.
|
|
#
|
|
# `enabled` and the canonical `level` spelling are pinned alongside
|
|
# because both fail silently in the direction of "provisions fine,
|
|
# returns nothing" — see the comment on `logLevelRules` in
|
|
# ../host-modules/swarm-grafana.nix.
|
|
name = "the logs datasource maps every severity the collector can emit to a log level";
|
|
ok =
|
|
let
|
|
sources =
|
|
grafanaOldPath.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
|
|
logs = lib.head (lib.filter (d: d.uid == "swarm-victorialogs") sources);
|
|
rules = logs.jsonData.logLevelRules or [ ];
|
|
# `overwrite_text` makes the stored text the OpenTelemetry short name
|
|
# for each mapped severity, which is the parser's own key uppercased.
|
|
emitted = map lib.toUpper (lib.attrNames (lib.head (import ../journald-severity.nix)).mapping);
|
|
canonical = [
|
|
"critical"
|
|
"error"
|
|
"warning"
|
|
"info"
|
|
"debug"
|
|
"trace"
|
|
];
|
|
in
|
|
rules != [ ]
|
|
&& lib.all (r: r.field == "severity_text" && r.enabled == true && lib.elem r.level canonical) rules
|
|
&& lib.all (text: lib.any (r: r.value == text) rules) emitted;
|
|
}
|
|
{
|
|
# 🩸 The regression this guards: moving bao's metrics scrape into its
|
|
# own in-container collector made that collector stamp every exported
|
|
# series `service.name=swarm-bao`, while every panel in the shipped
|
|
# bao.json still queried the literal `"service.name"="bao"` — delivery
|
|
# could be perfect and no panel would ever match. Fixed by renaming it
|
|
# in the collector rather than in the dashboard: `swarm-bao.nix`'s
|
|
# `collectorServiceName` now stamps `"bao"` directly. This case reads
|
|
# the collector's OWN evaluated config — not a hand-copied literal —
|
|
# so it fails again the moment that binding and the dashboard drift
|
|
# apart in either direction.
|
|
# 🔬 Invert-proof: change `collectorServiceName` in swarm-bao.nix back
|
|
# to `cfg.machine` and this case fails.
|
|
name = "the bao dashboard's selectors match the service.name bao's own collector stamps";
|
|
ok =
|
|
let
|
|
attrs =
|
|
baoWithCollector.containers.swarm-bao.config.services.opentelemetry-collector.settings.processors.resource.attributes;
|
|
stamped = lib.findFirst (a: a.key == "service.name") null attrs;
|
|
board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/bao.json);
|
|
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
|
|
selectors = lib.filter (e: lib.hasInfix "service.name" e) exprs;
|
|
in
|
|
stamped != null
|
|
&& builtins.length selectors == 12
|
|
&& lib.all (e: lib.hasInfix "\"service.name\"=\"${stamped.value}\"" e) selectors;
|
|
}
|
|
];
|
|
in
|
|
runGroup "grafana" cases
|