4b6214305f
`swarm.authelia.url` defaulted to `https://<domain>` only when this host ran the container, and to `null` otherwise — so the address a client is given was a statement about co-location rather than about the swarm. A swarm has one SSO provider; every hive addresses the same name and resolution decides which address that reaches, exactly as `swarm.otel.domain` already works. The option stays nullable: "this swarm has no IdP" is still expressible, it is just now something an operator states rather than something not running the container produces. The Grafana fixture that exercised the no-IdP refusal says it explicitly. Closes #4536
290 lines
14 KiB
Nix
290 lines
14 KiB
Nix
# `checks.module-eval-grafana` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
grafanaOldPath = hive {
|
|
deploy.grafana.enable = true;
|
|
swarm.grafana.socketDir = "/run/test-grafana-sock";
|
|
swarm.grafana.datasourceUrl = "http://127.0.0.1:19999";
|
|
swarm.grafana.logsDatasourceUrl = "http://127.0.0.1:19998";
|
|
swarm.grafana.plugins = [ ];
|
|
swarm.grafana.package = pkgs.emptyDirectory;
|
|
};
|
|
|
|
# The metrics UI beside the IdP. It reads its secret out of the store like
|
|
# every other Grafana host, so it needs a store identity like every other
|
|
# Grafana host — the cert pair here is not scenery, it is the arm that would
|
|
# have caught the deleted co-located copy unit coming back.
|
|
grafanaWithAuthelia = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
};
|
|
|
|
# The same UI with the IdP on ANOTHER host and a store leaf placed by hand.
|
|
# Knowing an IdP is not running one: `swarm.authelia.url` is what says this
|
|
# swarm has SSO, and nothing about this host does. Identical to the fixture
|
|
# above in everything the delivery path reads, which is the point.
|
|
grafanaRemoteAuthelia = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
};
|
|
|
|
# A Grafana host holding no store identity. This used to be the shape the
|
|
# module went QUIET on — no OIDC block, a warning, and a container whose
|
|
# login form is off regardless, so no way in and nothing failed. It is kept
|
|
# rather than deleted because the shape is still reachable by an operator;
|
|
# what changed is the deliverable, from a warning nothing reads back to a
|
|
# refusal naming the two options to set. Only the identity is missing, so an
|
|
# arm below can name which refusal fired.
|
|
grafanaNoIdentity = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
};
|
|
|
|
# The mirror image: the identity is placed, and the swarm names no IdP. The
|
|
# other half of "SSO must always be configured", and isolated the same way —
|
|
# exactly one thing wrong, so the arm reads one refusal.
|
|
#
|
|
# The null is now written out: `swarm.authelia.url` defaults to the swarm's
|
|
# IdP name on every hive, so "this swarm has no IdP" is a thing an operator
|
|
# states rather than a thing not running the container produces.
|
|
grafanaNoSso = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
swarm.authelia.url = null;
|
|
};
|
|
|
|
# Did ./host-modules/swarm-grafana.nix refuse this host, and for which of its
|
|
# two reasons. An assertion is a config VALUE until something forces it —
|
|
# `.config` never throws — so a fixture in a state the module refuses is
|
|
# evaluable and the refusal is readable as data. That is what lets a case
|
|
# check that a misconfiguration is REPORTED, rather than only that it is not
|
|
# silently accepted.
|
|
#
|
|
# Matched on the option name the message names, not on its prose, so the
|
|
# wording stays rewordable: the option name is the part an operator has to
|
|
# act on, and a message that stopped naming it would be the actual defect.
|
|
grafanaRefusedFor =
|
|
m: option:
|
|
lib.any (
|
|
a:
|
|
!a.assertion
|
|
&& lib.hasInfix "services.hyperhive.deploy.grafana.enable requires" a.message
|
|
&& lib.hasInfix option a.message
|
|
) m.assertions;
|
|
cases = [
|
|
{
|
|
# This fixture enables grafana and NOT authelia, which is the shape the
|
|
# login form used to stay enabled in: the toggle read "both services are
|
|
# on this host" rather than "grafana requires SSO". Grafana ships an
|
|
# `admin`/`admin` account and its vhost is on the public gateway, so a
|
|
# password box there is a way in whatever the topology.
|
|
name = "grafana disables its local login form even where authelia is not on this host";
|
|
ok =
|
|
grafanaOldPath.containers.swarm-grafana.config.services.grafana.settings.auth.disable_login_form;
|
|
}
|
|
{
|
|
# The absence class this whole file is for, and the reported defect in one
|
|
# arm: the OIDC block hung off "authelia is on this host", so the split
|
|
# deployment got a Grafana with no SSO settings and no login form — no way
|
|
# in at all. The block is emitted in every deployment now, so the negative
|
|
# arm is not "no block elsewhere" but "the two do not name the same IdP":
|
|
# each host's block has to point at the URL the SWARM names, and a block
|
|
# built from `deploy.authelia` rather than `swarm.authelia.url` would pass
|
|
# a presence check on both fixtures while sending one of them nowhere.
|
|
name = "grafana's OIDC block names the swarm's IdP, wherever that IdP runs";
|
|
ok =
|
|
let
|
|
oauth = m: m.containers.swarm-grafana.config.services.grafana.settings."auth.generic_oauth";
|
|
remote = oauth grafanaRemoteAuthelia;
|
|
local = oauth grafanaWithAuthelia;
|
|
in
|
|
remote.enabled
|
|
&& lib.hasInfix "https://auth.example.invalid/api/oidc/token" remote.token_url
|
|
&& local.enabled
|
|
&& lib.hasInfix "https://auth.t.local/api/oidc/token" local.token_url
|
|
&& !(lib.hasInfix "auth.example.invalid" local.token_url);
|
|
}
|
|
{
|
|
# 🩸 The arm that guards the ruling this slice landed under. There is ONE
|
|
# delivery route: the store reader, on every host that runs Grafana. The
|
|
# negative names the deleted unit rather than a generic absence, because
|
|
# the way this regresses is someone re-adding the co-located copy as an
|
|
# optimisation — a second writer of one path, and a second shape of "the
|
|
# secret is wrong" to debug.
|
|
name = "grafana's OIDC secret has exactly one delivery unit, the store reader, in both topologies";
|
|
ok =
|
|
let
|
|
local = grafanaWithAuthelia.systemd.services;
|
|
remote = grafanaRemoteAuthelia.systemd.services;
|
|
in
|
|
local ? swarm-bao-grafana-oidc
|
|
&& remote ? swarm-bao-grafana-oidc
|
|
&& !(local ? swarm-grafana-oidc-secret)
|
|
&& !(remote ? swarm-grafana-oidc-secret);
|
|
}
|
|
{
|
|
# What the deleted warning became. The shape is unchanged — a Grafana host
|
|
# holding no store leaf — but silence there is a container nobody can log
|
|
# into for a reason no log names, and a warning is read back by nothing.
|
|
# The second arm is what makes this a refusal about the IDENTITY: this
|
|
# fixture names an IdP, so a message about `swarm.authelia.url` here would
|
|
# mean the two assertions had been collapsed into one conjunction.
|
|
name = "a grafana host with no store identity is refused, naming the options to set";
|
|
ok =
|
|
grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientCertFile"
|
|
&& grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientKeyFile"
|
|
&& !(grafanaRefusedFor grafanaNoIdentity "swarm.authelia.url");
|
|
}
|
|
{
|
|
# "SSO must always be configured", as an eval-time refusal rather than a
|
|
# gate. A null URL used to drop the OIDC block silently, and
|
|
# `disable_login_form` is unconditional a hundred lines below it, so that
|
|
# combination produced a Grafana with no SSO and no password box — an
|
|
# outage whose cause is a boolean that evaluated to false at build time
|
|
# and left no trace. Same isolation as the arm above, mirrored.
|
|
name = "a grafana host in a swarm with no IdP is refused, naming swarm.authelia.url";
|
|
ok =
|
|
grafanaRefusedFor grafanaNoSso "services.hyperhive.swarm.authelia.url"
|
|
&& !(grafanaRefusedFor grafanaNoSso "deploy.bao.clientCertFile");
|
|
}
|
|
{
|
|
# Without this the two arms above prove nothing: a refusal that fires on
|
|
# every host is not a check, and both of these are hosts a swarm is
|
|
# expected to have. Read through the same helper, so a message that
|
|
# stopped naming its option would fail the arms above rather than pass
|
|
# this one by accident.
|
|
name = "neither grafana refusal fires on a correctly configured host, co-located or not";
|
|
ok =
|
|
!(grafanaRefusedFor grafanaWithAuthelia "services.hyperhive.swarm.authelia.url")
|
|
&& !(grafanaRefusedFor grafanaWithAuthelia "deploy.bao.clientCertFile")
|
|
&& !(grafanaRefusedFor grafanaRemoteAuthelia "services.hyperhive.swarm.authelia.url")
|
|
&& !(grafanaRefusedFor grafanaRemoteAuthelia "deploy.bao.clientCertFile");
|
|
}
|
|
{
|
|
# Same 403-not-a-miss reason as the matrix and queue arms below: the
|
|
# reader's grant covers the `services` prefix, so a path outside it is
|
|
# refused rather than empty, however correct it reads. The negative arm is
|
|
# the rename this is exposed to — a secret filed under the hive that runs
|
|
# the service instead of under the service itself.
|
|
name = "grafana's OIDC secret is read from the prefix the publisher writes";
|
|
ok =
|
|
let
|
|
s = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-oidc.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" s
|
|
&& !(lib.hasInfix "secret/swarm/hives/" s);
|
|
}
|
|
{
|
|
# Both halves of the co-location assumption, which was one host's
|
|
# `deploy.*` answering a question about the whole swarm: the identities
|
|
# were minted only where the queue happened to run, and the token
|
|
# endpoint was known only where the IdP happened to run.
|
|
name = "hive identities and the token endpoint do not depend on which host runs what";
|
|
ok =
|
|
let
|
|
autheliaNoQueue = hive { deploy.authelia.enable = true; };
|
|
in
|
|
lib.elem "hive-h1" (map (c: c.id) autheliaNoQueue.services.hyperhive.swarm.authelia.oidc.clients)
|
|
&&
|
|
grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint
|
|
== "https://auth.example.invalid/api/oidc/token";
|
|
}
|
|
{
|
|
# The standing check on the collector's PRIORITY mapping — whether a line
|
|
# ever reaches the log store carrying no severity. It is shipped config
|
|
# rather than something an operator imports by hand, which also means it
|
|
# can be deleted by hand. Read out of the dashboard the provisioner
|
|
# actually ships (`dashboardDir` in ../host-modules/swarm-grafana.nix
|
|
# names this file), and asserted on the QUERY rather than the title,
|
|
# because a panel that keeps its title and loses its expression renders
|
|
# an empty graph that looks exactly like zero prioless lines.
|
|
name = "the logs dashboard still counts lines that arrive with no severity";
|
|
ok =
|
|
let
|
|
board = builtins.fromJSON (
|
|
builtins.readFile ../host-modules/swarm-grafana/dashboards/logstore.json
|
|
);
|
|
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
|
|
counts = lib.filter (e: lib.hasInfix "severity_text" e && lib.hasInfix "stats count()" e) exprs;
|
|
in
|
|
# Both arms of the split: lines that HAD a priority and arrived without
|
|
# a severity anyway (the regression line, which must reach zero), and
|
|
# lines that never had one to map — Claude Code's own OTLP telemetry,
|
|
# which no mapping can reach. Folded into one number the second would
|
|
# keep the panel off zero forever and hide the first.
|
|
lib.any (e: lib.hasInfix "PRIORITY:*" e) counts
|
|
&& lib.any (e: lib.hasInfix "PRIORITY:\"\"" e) counts;
|
|
}
|
|
{
|
|
# The reader's half of the same mapping. The collector writes a severity
|
|
# TEXT; nothing downstream reads it as a level unless the datasource is
|
|
# told which field holds it, and that telling is a list of rules rather
|
|
# than a field name — so it goes stale one severity at a time. Asserted
|
|
# against ../journald-severity.nix rather than a literal list here,
|
|
# because the way this regresses is a severity added to the parser and
|
|
# not to Grafana: every line still arrives, the new one is just
|
|
# unfilterable, and no query errors to say so.
|
|
#
|
|
# `enabled` and the canonical `level` spelling are pinned alongside
|
|
# because both fail silently in the direction of "provisions fine,
|
|
# returns nothing" — see the comment on `logLevelRules` in
|
|
# ../host-modules/swarm-grafana.nix.
|
|
name = "the logs datasource maps every severity the collector can emit to a log level";
|
|
ok =
|
|
let
|
|
sources =
|
|
grafanaOldPath.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources;
|
|
logs = lib.head (lib.filter (d: d.uid == "swarm-victorialogs") sources);
|
|
rules = logs.jsonData.logLevelRules or [ ];
|
|
# `overwrite_text` makes the stored text the OpenTelemetry short name
|
|
# for each mapped severity, which is the parser's own key uppercased.
|
|
emitted = map lib.toUpper (lib.attrNames (lib.head (import ../journald-severity.nix)).mapping);
|
|
canonical = [
|
|
"critical"
|
|
"error"
|
|
"warning"
|
|
"info"
|
|
"debug"
|
|
"trace"
|
|
];
|
|
in
|
|
rules != [ ]
|
|
&& lib.all (r: r.field == "severity_text" && r.enabled == true && lib.elem r.level canonical) rules
|
|
&& lib.all (text: lib.any (r: r.value == text) rules) emitted;
|
|
}
|
|
];
|
|
in
|
|
runGroup "grafana" cases
|