9407db69b5
Each store reader (forge's OIDC fetch and token publisher, matrix's OIDC fetch, the hive collector's fetch, Grafana's OIDC fetch and admin password, the queue's callout-xkey unit, the CI runner's token fetch, the swarm collector's OIDC fetch) used to read `deploy.bao.enable` in its own module to add `after`/`wants` on `container@<store machine>` and `after`/`requires` on `swarm-bao-pki.service`. Per the #3742 ruling, that cross-service read moves into glue-bao-readers-store-order.nix, which sets the same edges when the store and the reader are both on this host and restates each reader's own gate so no stub unit is rendered. Rendered `after`/`wants`/`requires` are identical as sets to main on 16 fixtures, store here and elsewhere. New module-eval cases in existing fixtures pin both arms. Part of #3742.
237 lines
11 KiB
Nix
237 lines
11 KiB
Nix
# `checks.module-eval-swarm-otel-identity` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
otelSettings
|
|
;
|
|
|
|
# A collector holding no store identity at all. Unlike Grafana's mirror
|
|
# image, this is not a refused shape: the collector still receives every
|
|
# hive's telemetry with nothing to push authenticated with, which is the
|
|
# already-supported degrade `haveCollectorSecret` names above the module's
|
|
# `let`. What this fixture is for is checking the reading unit itself does
|
|
# not render, rather than rendering with an env var nothing filled in.
|
|
otelNoIdentity = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
|
|
};
|
|
|
|
# 🩸 The shape the degrade above must not swallow: a collector on a host that
|
|
# demonstrably READS the store — it holds the hive's own leaf — and is missing
|
|
# only the collector's own pair. Before the four-way split this host rendered
|
|
# the reading unit off `clientCertFile` alone, so the silence it gets now is a
|
|
# regression rather than the supported degrade `otelNoIdentity` stands for. The
|
|
# two fixtures differ in exactly that one pair, which is what lets the arms
|
|
# below separate a refusal from a degrade.
|
|
inherit (shared) otelReaderMissingLeaf;
|
|
|
|
# The same deployment named in full, which must stay buildable. Pairs with the
|
|
# fixture above so the refusal is pinned to the omission and not to reading a
|
|
# remote store at all.
|
|
inherit (shared) otelReaderNamedInFull;
|
|
|
|
# Did the module refuse this host, and over which option. An assertion is a
|
|
# config VALUE until something forces it — `.config` never throws — so a
|
|
# fixture in a refused state stays evaluable and the refusal reads back as
|
|
# data. Matched on the option name the message names rather than on its prose:
|
|
# the option name is the part an operator has to act on, and a message that
|
|
# stopped naming it would be the actual defect.
|
|
refusedOver = m: option: lib.any (a: !a.assertion && lib.hasInfix option a.message) m.assertions;
|
|
|
|
# authelia somewhere else, the credential delivered by hand. Whether this
|
|
# collector authenticates must follow the credential, never another
|
|
# service's placement.
|
|
#
|
|
# The `swarm.otel.clientSecretFile` below is the PRE-RENAME path. It predates
|
|
# the split and is deliberately left spelled that way: it makes this fixture
|
|
# the old-path case for that option too, so dropping its rename entry fails
|
|
# the eval here rather than only in a real operator's config.
|
|
otelRemoteAuthelia = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
deploy.authelia.enable = false;
|
|
# Where that elsewhere IS. Running no IdP does not mean knowing no IdP:
|
|
# the authenticator this fixture exists to render puts this address in its
|
|
# `token_url`, so a hive with a secret and no URL has a credential it can
|
|
# present nowhere.
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
swarm.otel.clientSecretFile = "/var/lib/swarm-otel-oidc/by-hand.secret";
|
|
};
|
|
|
|
# A collector whose ONLY scrape work is published: loopback targets forced
|
|
# empty, one published job declared. Unreachable in a real deploy today —
|
|
# the module seeds `scrapeTargets.collector` under its own `enable`, so the
|
|
# loopback set is never empty on its own — which is exactly why the arm
|
|
# below needs a fixture that takes that seeding away. `mkForce` is what
|
|
# does it, and it leaves the collector itself enabled: the state under test
|
|
# is a running collector with no self-scrape, not an absent one.
|
|
otelOnlyPublished = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
swarm.otel.scrapeTargets = lib.mkForce { };
|
|
swarm.otel.publishedScrapeTargets.remote = "https://remote.t.local/metrics";
|
|
};
|
|
|
|
# Two hives in the roster, which no other fixture here has: every one of
|
|
# them declares `swarm.hives.h1` alone, so a per-hive arm written against
|
|
# one of those passes on a hardcoded literal.
|
|
otelTwoHives = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
deploy.authelia.enable = true;
|
|
swarm.hives.h2.domain = "h2.t.local";
|
|
};
|
|
cases = [
|
|
{
|
|
# The collector's non-assertion, the deliberate mirror of Grafana's
|
|
# assertion two cases up: a host with no store identity is a supported,
|
|
# merely degraded shape here, so the reading unit simply does not exist
|
|
# rather than refusing the build. `haveCollectorSecret` is what the
|
|
# degrade already reads, unchanged by this slice.
|
|
name = "a collector with no store identity renders no reading unit, and is not refused";
|
|
ok =
|
|
!(otelNoIdentity.systemd.services ? swarm-bao-otel-oidc)
|
|
&& otelNoIdentity.services.hyperhive.deploy.swarm-otel.clientSecretFile == null
|
|
&& !(lib.any (a: !a.assertion) otelNoIdentity.assertions);
|
|
}
|
|
{
|
|
# The collector's half of the same split, and a different arm from the
|
|
# authenticator case below: this one reads the PATH the unit loads, so a
|
|
# reader left on a source that is non-null but wrong still fails. The
|
|
# fixture spells the option its pre-rename way, so it covers the rename
|
|
# entry at the same time.
|
|
name = "a config written against the pre-rename otel secret path still loads it as a credential";
|
|
ok =
|
|
lib.any (c: lib.hasInfix "/var/lib/swarm-otel-oidc/by-hand.secret" c)
|
|
otelRemoteAuthelia.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential;
|
|
}
|
|
{
|
|
# The collector authenticates because it HOLDS a credential, not because
|
|
# authelia happens to share its host. Gating on the other service's
|
|
# placement renders a collector that pushes unauthenticated wherever
|
|
# authelia lives elsewhere — one of the supported shapes.
|
|
name = "a collector with a hand-delivered secret authenticates without authelia beside it";
|
|
ok =
|
|
let
|
|
s = otelSettings otelRemoteAuthelia;
|
|
in
|
|
(s.exporters."otlphttp/victoriametrics" ? auth)
|
|
&& builtins.elem "oauth2client/victoriametrics" s.service.extensions;
|
|
}
|
|
{
|
|
# Defining a receiver and attaching it are two separate lists, and the
|
|
# two gates were spelled differently: the receiver appeared for either
|
|
# scrape option, the pipeline only for the loopback one. A published-
|
|
# only collector therefore rendered scrape configs that reached no
|
|
# pipeline — requested, parsed, delivered nowhere, and valid enough to
|
|
# deploy. The receiver clause is what stops the arm passing for the
|
|
# wrong reason, by an empty `prometheus` never rendering at all.
|
|
name = "a published-only collector attaches its prometheus receiver to the swarm pipeline";
|
|
ok =
|
|
let
|
|
s = otelSettings otelOnlyPublished;
|
|
in
|
|
otelOnlyPublished.services.hyperhive.swarm.otel.scrapeTargets == { }
|
|
&& otelOnlyPublished.services.hyperhive.swarm.otel.publishedScrapeTargets != { }
|
|
&& (s.receivers ? prometheus)
|
|
&& builtins.elem "prometheus" s.service.pipelines."metrics/swarm".receivers;
|
|
}
|
|
{
|
|
# Read against the roster the fixture declares rather than against
|
|
# names spelled here: an arm naming `h1` passes on a single-hive
|
|
# config however the mapping is written. The length clause is what
|
|
# makes the `all` mean anything — over an empty roster it holds
|
|
# vacuously.
|
|
name = "the swarm collector routes every hive's logs, not just one";
|
|
ok =
|
|
let
|
|
p = (otelSettings otelTwoHives).service.pipelines;
|
|
hives = lib.attrNames otelTwoHives.services.hyperhive.swarm.hives;
|
|
in
|
|
lib.length hives == 2
|
|
&& lib.all (h: (p ? "logs/${h}") && p."logs/${h}".receivers == [ "otlp/${h}" ]) hives;
|
|
}
|
|
{
|
|
# The same split as the metrics case above — defining an exporter and
|
|
# naming it are two lists — plus the half one shared list cannot have:
|
|
# the metrics store's exporter renders perfectly well inside a logs
|
|
# pipeline and posts journal records at an ingest route that is not
|
|
# for them.
|
|
name = "every logs pipeline sends to the log store and to no metrics one";
|
|
ok =
|
|
let
|
|
s = otelSettings otelTwoHives;
|
|
logPipes = lib.filterAttrs (n: _: lib.hasPrefix "logs/" n) s.service.pipelines;
|
|
used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues logPipes));
|
|
in
|
|
logPipes != { }
|
|
&& builtins.elem "otlphttp/victorialogs" used
|
|
&& !(builtins.elem "otlphttp/victoriametrics" used)
|
|
&& lib.all (e: s.exporters ? ${e}) used;
|
|
}
|
|
{
|
|
# 🩸 The refusal, and the only way this module's omission is visible at
|
|
# all: the reading unit is gated on its own leaf, so the regression is a
|
|
# green build with the unit absent — and the missing unit is the evidence.
|
|
# Read as a refusal naming both options, so an operator acts on the
|
|
# message without opening the nix.
|
|
name = "a store reader missing the collector's own leaf is refused, naming both options";
|
|
ok =
|
|
refusedOver otelReaderMissingLeaf "otelOidcClientCertFile"
|
|
&& refusedOver otelReaderMissingLeaf "otelOidcClientKeyFile";
|
|
}
|
|
{
|
|
# ⚠️ The arm that keeps the refusal from eating the degrade beside it. A
|
|
# collector on a host with NO store identity still receives every hive's
|
|
# telemetry and is a supported deployment — `otelNoIdentity` is that
|
|
# fixture, and it differs from the refused one only in the hive's own
|
|
# leaf. Widening the gate to the collector's leaf alone would reject it,
|
|
# which is what this pins.
|
|
name = "a collector with no store identity at all is not refused over the collector's leaf";
|
|
ok =
|
|
!(refusedOver otelNoIdentity "otelOidcClientCertFile")
|
|
&& !(refusedOver otelNoIdentity "otelOidcClientKeyFile");
|
|
}
|
|
{
|
|
# The other two shapes that must stay buildable: the same remote reader
|
|
# named in full, and a host that mints its own PKI and therefore has all
|
|
# eight as defaults from ./host-modules/glue-bao-tls.nix.
|
|
name = "neither a fully-named remote reader nor a store host is refused";
|
|
ok =
|
|
!(refusedOver otelReaderNamedInFull "otelOidcClientCertFile")
|
|
&& !(refusedOver otelRemoteAuthelia "otelOidcClientCertFile");
|
|
}
|
|
{
|
|
# The store's units exist only on its host; `Requires=` on an absent one
|
|
# fails the fetch's start. The arm where the store is here is in
|
|
# ./swarm-services-switch.nix.
|
|
name = "the remote reader's fetch names none of the store's units";
|
|
ok =
|
|
let
|
|
u = otelReaderNamedInFull.systemd.services.swarm-bao-otel-oidc;
|
|
storeUnits = [
|
|
"swarm-bao-pki.service"
|
|
"container@${otelReaderNamedInFull.services.hyperhive.swarm.bao.machine}.service"
|
|
];
|
|
in
|
|
!(lib.any (n: lib.elem n (u.after ++ u.wants ++ u.requires)) storeUnits);
|
|
}
|
|
];
|
|
in
|
|
runGroup "swarm-otel-identity" cases
|