bb0afcd256
bao's metrics were scraped by the SWARM collector over loopback, via a `swarm.otel.scrapeTargets.bao` entry gated on `deploy.swarm-otel.enable` — "does the swarm's collector run on THIS host". It had to be: loopback only reaches a reader that landed on the same host. What that rendered everywhere else was nothing at all. Off that host the metrics listener was not emitted, so the store's metrics reached the store nowhere, and a host with no entry is indistinguishable from a host nobody asked to scrape. Moves the scrape into the collector this container already runs, per mara on #4537: "move the existing scraper to the local collector". The container shares the host netns (privateNetwork = false), so the scrape still dials 127.0.0.1 — the listener keeps its address, its `metrics_only` narrowing and its loopback-only bind, and the API listener's `tls_require_and_verify_client_cert` is untouched. The listener and its `prometheus_retention_time` lose their gate: the reader ships with the store now, so there is no host where the endpoint has none. The metrics pipeline reuses the logs pipeline's `resource` processor and `otlphttp` exporter, so both signals carry the same `service.name` and leave by the one hop. Logs are unaffected: `journaldUnits` and --link-journal=host stay until every sibling swarm container has a collector of its own. The module-eval absence arm "a store with no collector beside it serves no metrics" is inverted rather than dropped — the condition it asserted is the bug. Three cases join it: the job is in swarm-bao AND gone from swarm-otel (a move, not a copy), the scrape target and listener are both pinned to loopback, and the metrics pipeline shares its exporter with the logs one.
436 lines
20 KiB
Nix
436 lines
20 KiB
Nix
# `checks.module-eval-bao-otel-collector` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
baoSettings
|
|
;
|
|
|
|
baoTwoAddresses = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
|
|
# Pinned, not incidental: the case counting these listeners is about the
|
|
# declared API addresses. ⚠️ Since the store got its own collector, this no longer keeps the METRICS
|
|
# listener out of the count — that one is a function of the store, not of
|
|
# a collector's placement — so the case below subtracts it by name rather
|
|
# than by leaving the swarm tier off.
|
|
deploy.swarm-otel.enable = false;
|
|
};
|
|
|
|
# The store with and without a collector on the same host. `scrapeTargets`
|
|
# is only ever read by a local collector, so the metrics endpoint is a
|
|
# function of the pairing rather than of the store.
|
|
baoWithCollector = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.swarm-otel.enable = true;
|
|
# A second job declared as bare `host:port`, so the pair of cases below
|
|
# reads one rendered scrape list: the store's entry carries a path, this
|
|
# one carries none.
|
|
swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
|
|
};
|
|
|
|
# The SPLIT topology, and the reason it is not called `baoNoCollector` any
|
|
# more: a swarm always runs every swarm-level service somewhere, so this host
|
|
# having `deploy.swarm-otel.enable = false` means the collector is on ANOTHER
|
|
# host, never that the swarm has none. Both of those read the same locally,
|
|
# and only one of them is a real deployment — which is what the two gates
|
|
# this PR removed each mistook for "there is nothing to forward to".
|
|
baoOtelElsewhere = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.swarm-otel.enable = false;
|
|
};
|
|
|
|
# The same split, plus a HIVE collector on the store's host — the pairing the
|
|
# first gate read as permission to forward. It is a third collector with a
|
|
# different job, so it changes neither whether the store forwards nor where.
|
|
# `clientSecretFile` is what ../host-modules/otel.nix's identity assertion
|
|
# demands of any hive with the tier on.
|
|
baoWithHiveOtel = hive {
|
|
deploy.bao.enable = true;
|
|
otel.enable = true;
|
|
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
|
|
};
|
|
|
|
# The forwarder INSIDE the store's container, not the host's collector and
|
|
# not the swarm tier's — three collectors in this tree, and only this one can
|
|
# see the store's journal.
|
|
baoForwarder = machine: machine.containers.swarm-bao.config.services.opentelemetry-collector;
|
|
|
|
# The scrape list prometheus is handed, not the option a service declared:
|
|
# the address, the path and the query are one string on the way in and three
|
|
# fields on the way out, and only the second shape is what gets requested.
|
|
#
|
|
# ⚠️ Two of these now, aimed at two different containers, and the split is
|
|
# the point: the store's own job left the swarm collector for the
|
|
# one inside its container, so a case that looked in the old place would
|
|
# find `null` and a case that only looked in the new one would not notice
|
|
# the store still being declared to both.
|
|
scrapeJobIn =
|
|
container: machine: job:
|
|
lib.findFirst (c: c.job_name == job) null (
|
|
machine.containers.${container}.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs
|
|
or [ ]
|
|
);
|
|
scrapeJob = scrapeJobIn "swarm-otel";
|
|
baoScrapeJob = scrapeJobIn "swarm-bao";
|
|
cases = [
|
|
{
|
|
# Same gap one tier up, and it needs its own arm: this collector
|
|
# already had six scrape targets, so a pass here is about the seventh
|
|
# rather than about the receiver existing at all.
|
|
name = "the swarm collector scrapes its own telemetry endpoint";
|
|
ok =
|
|
let
|
|
j = scrapeJob baoWithCollector "collector";
|
|
in
|
|
j != null && j.static_configs == [ { targets = [ "127.0.0.1:8889" ]; } ];
|
|
}
|
|
{
|
|
# The store stays behind the passthrough rather than beside it: loopback
|
|
# plus whatever was declared, never the bridge. A store that also bound
|
|
# the bridge itself would collide with the listener above, and the
|
|
# colliding one is nginx — the whole gateway, not just this port.
|
|
name = "the store binds loopback and its declared addresses, never the bridge";
|
|
ok =
|
|
let
|
|
l = (baoSettings baoTwoAddresses).listener;
|
|
in
|
|
l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200";
|
|
}
|
|
{
|
|
# Control for the case above: these settings are rendered per deployment,
|
|
# not constants a passing case could be indifferent to. The metrics
|
|
# listener is excluded by name — it answers on a port of its own and is
|
|
# not one of the API addresses this case is counting.
|
|
name = "a declared extra address renders a second listener beside loopback";
|
|
ok =
|
|
builtins.length (
|
|
builtins.filter (n: n != "metrics") (builtins.attrNames (baoSettings baoTwoAddresses).listener)
|
|
) == 2;
|
|
}
|
|
{
|
|
# Retention is what serves the endpoint at all, so the listener alone
|
|
# would be a port that answers 404.
|
|
name = "a store beside a collector serves metrics on its own listener";
|
|
ok =
|
|
let
|
|
s = baoSettings baoWithCollector;
|
|
in
|
|
s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
|
|
}
|
|
{
|
|
# openbao serves no `/metrics` at all, so a scrape of the default path
|
|
# 404s: the store looks like a dead exporter, and every panel built on
|
|
# it renders empty rather than erroring. The query is the other half —
|
|
# `/v1/sys/metrics` answers JSON without it, which parses as no series
|
|
# rather than as an error.
|
|
name = "the store's scrape asks for the path openbao serves";
|
|
ok =
|
|
let
|
|
j = baoScrapeJob baoWithCollector "bao";
|
|
in
|
|
j != null
|
|
&& (j.metrics_path or "") == "/v1/sys/metrics"
|
|
&& (j.params.format or [ ]) == [ "prometheus" ];
|
|
}
|
|
{
|
|
# 🎯 This move's deliverable, and it needs BOTH halves or it is not a move:
|
|
# the job is in the store's own container, and it is no longer declared
|
|
# to the swarm collector. Only the second half can catch a "move" that
|
|
# left the original behind — two scrapers of one endpoint, double-counted
|
|
# and neither obviously wrong.
|
|
name = "the store's metrics are scraped by its own collector, not the swarm's";
|
|
ok = baoScrapeJob baoWithCollector "bao" != null && scrapeJob baoWithCollector "bao" == null;
|
|
}
|
|
{
|
|
# The scrape target is the loopback listener, not the API one: dialing
|
|
# the API listener would need a client certificate the collector has no
|
|
# way to present, and the fix for that would be relaxing
|
|
# `tls_require_and_verify_client_cert` — which is the trade this address
|
|
# exists to avoid. Pinned so a later edit has to argue with it.
|
|
name = "the store's collector scrapes loopback, never a wider address";
|
|
ok =
|
|
let
|
|
j = baoScrapeJob baoWithCollector "bao";
|
|
s = baoSettings baoWithCollector;
|
|
in
|
|
j.static_configs == [ { targets = [ "127.0.0.1:8202" ]; } ]
|
|
&& s.listener.metrics.address == "127.0.0.1:8202";
|
|
}
|
|
{
|
|
# A pipeline is typed: a prometheus receiver named in `logs` is rejected
|
|
# at startup, and metrics with no pipeline at all are collected and
|
|
# dropped. The exporter is pinned alongside because "no otel bypass"
|
|
# means both signals leave by the one hop.
|
|
name = "the store's collector has a metrics pipeline out the same hop as its logs";
|
|
ok =
|
|
let
|
|
s = (baoForwarder baoWithCollector).settings;
|
|
m = s.service.pipelines.metrics;
|
|
in
|
|
m.receivers == [ "prometheus" ]
|
|
&& m.exporters == [ "otlphttp" ]
|
|
&& m.exporters == s.service.pipelines.logs.exporters;
|
|
}
|
|
{
|
|
# Presence control for the case above: both fields are omitted rather
|
|
# than defaulted, so a target declared as bare `host:port` renders what
|
|
# it rendered before the path grammar existed.
|
|
name = "a target with no path renders neither metrics_path nor params";
|
|
ok =
|
|
let
|
|
j = scrapeJob baoWithCollector "plain";
|
|
in
|
|
j != null && !(j ? metrics_path) && !(j ? params);
|
|
}
|
|
{
|
|
# 🩸 This case USED to be the absence arm — "a store with no collector
|
|
# beside it serves no metrics" — and this inverts it deliberately
|
|
# rather than deleting it, because the condition it asserted is the bug.
|
|
#
|
|
# The old reasoning was sound for the old reader: the endpoint is
|
|
# unauthenticated, so it should not exist where nothing reads it, and
|
|
# the only reader was a swarm collector that had to be on this host.
|
|
# What that rendered on every OTHER host was a store whose metrics went
|
|
# nowhere, indistinguishably from a store nobody had asked to scrape.
|
|
#
|
|
# The reader now ships inside the container, so "no collector beside it"
|
|
# describes no host that exists. The endpoint is still not exposed to
|
|
# anything new: it is loopback in a netns whose only other occupants are
|
|
# the store and its collector.
|
|
name = "a store whose swarm collector is elsewhere still serves and scrapes its metrics";
|
|
ok =
|
|
let
|
|
s = baoSettings baoOtelElsewhere;
|
|
in
|
|
s.listener.metrics.address == "127.0.0.1:8202"
|
|
&& (s.telemetry.prometheus_retention_time or "0s") != "0s"
|
|
&& baoScrapeJob baoOtelElsewhere "bao" != null;
|
|
}
|
|
{
|
|
# The store's journal reaches a reader through a collector of its own,
|
|
# and every one of these fields is silent when wrong: the runtime journal
|
|
# is the receiver's own default and is empty here, an unlisted extension
|
|
# is inert so the cursor silently stops persisting, and a pipeline is
|
|
# free to name none of it.
|
|
name = "the store's container forwards its own journal";
|
|
ok =
|
|
let
|
|
s = (baoForwarder baoWithCollector).settings;
|
|
p = s.service.pipelines.logs;
|
|
in
|
|
s.receivers.journald.directory == "/var/log/journal"
|
|
&& s.receivers.journald.storage == "file_storage"
|
|
&&
|
|
s.service.extensions == [
|
|
"file_storage"
|
|
"oauth2client/swarm-bao"
|
|
]
|
|
&& p.receivers == [ "journald" ]
|
|
&& p.exporters == [ "otlphttp" ]
|
|
&& s ? exporters.otlphttp;
|
|
}
|
|
{
|
|
# The whole journal, which is what the shared collector's unit allowlist
|
|
# is not. A `units` list here would render and deploy perfectly while
|
|
# shipping only the units someone remembered to name — the failure this
|
|
# forwarder exists to end.
|
|
name = "the store's forwarder filters no units";
|
|
ok = !((baoForwarder baoWithCollector).settings.receivers.journald ? units);
|
|
}
|
|
{
|
|
# Both ends of the hop, because a mismatch between them is silent in both
|
|
# directions: the exporter retries into a 404 and the receiver never
|
|
# hears from it. Not "on one host" any more — the far end is the gateway
|
|
# vhost the swarm collector serves its name on, which is the whole point
|
|
# of addressing it by name rather than by a bridge address.
|
|
name = "the store's forwarder exports to the swarm collector's own route";
|
|
ok =
|
|
let
|
|
otel = baoWithCollector.services.hyperhive.swarm.otel;
|
|
vhost = baoWithCollector.services.nginx.virtualHosts.${otel.domain};
|
|
in
|
|
(baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint
|
|
== "https://${otel.domain}/${otel.storeProducerName}"
|
|
&& vhost.locations ? "/${otel.storeProducerName}/";
|
|
}
|
|
{
|
|
# The case the old hive-tier gate got wrong: this host runs the swarm's
|
|
# collector and no hive collector at all, and the forwarder still exists
|
|
# with an address that resolves. Gated on `otel.enable` it rendered
|
|
# nothing here, so the store's journal left no trace anywhere.
|
|
name = "a store on a host with no hive collector still forwards";
|
|
ok =
|
|
!baoWithCollector.services.hyperhive.otel.enable
|
|
&& (baoForwarder baoWithCollector).enable
|
|
&& (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != "";
|
|
}
|
|
{
|
|
# 🎯 The split topology, which is what both removed gates got wrong and
|
|
# neither earlier test could see: the swarm's collector runs on ANOTHER
|
|
# host, so nothing local says it exists — and it exists anyway, because
|
|
# every swarm-level service runs somewhere in the swarm. The forwarder
|
|
# has to render here and has to be addressed by the collector's swarm
|
|
# name, which is reachable from a host that serves no vhost for it.
|
|
#
|
|
# Both halves matter. Without `enable` this is the old bug; without the
|
|
# endpoint it is a forwarder that renders unconditionally and points at
|
|
# nothing. The `!(vhost ? domain)` conjunct is what makes the fixture
|
|
# genuinely remote rather than a co-located host in disguise — the
|
|
# premise the co-located fixtures cannot test.
|
|
name = "a store forwards by name from a host that does not run the swarm collector";
|
|
ok =
|
|
let
|
|
otel = baoOtelElsewhere.services.hyperhive.swarm.otel;
|
|
route = "https://${otel.domain}/${otel.storeProducerName}";
|
|
in
|
|
!(baoOtelElsewhere.services.nginx.virtualHosts ? ${otel.domain})
|
|
&& (baoForwarder baoOtelElsewhere).enable
|
|
&& (baoForwarder baoOtelElsewhere).settings.exporters.otlphttp.endpoint == route
|
|
# …and a hive collector standing beside the store neither supplies the
|
|
# reason to forward nor changes the addressee: same route, same tier.
|
|
&& (baoForwarder baoWithHiveOtel).enable
|
|
&& (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint == route;
|
|
}
|
|
{
|
|
# 🎯 The credential half of this move, and the arm that says the hop
|
|
# DELIVERS rather than merely resolves. The receiver at the far end
|
|
# takes an OIDC token and nothing else, so an exporter with no
|
|
# authenticator is a forwarder that retries into 401s while every other
|
|
# case here passes. Both halves: the extension has to exist AND the
|
|
# exporter has to name it.
|
|
name = "the store's forwarder presents a credential on the hop it exports over";
|
|
ok =
|
|
let
|
|
s = (baoForwarder baoWithCollector).settings;
|
|
auth = "oauth2client/swarm-bao";
|
|
in
|
|
s.exporters.otlphttp.auth.authenticator == auth
|
|
&& s.extensions ? ${auth}
|
|
&& lib.elem auth s.service.extensions;
|
|
}
|
|
{
|
|
# Its OWN client, never the swarm collector's and never the
|
|
# controller's: one identity per principal, and this one runs on a host
|
|
# neither of those two does. The audience is the same id because the
|
|
# receiver checks exactly that — a token minted without ASKING for it
|
|
# carries `aud: []` and is refused with a config that reads correctly at
|
|
# both ends.
|
|
name = "the forwarder authenticates as a principal of its own";
|
|
ok =
|
|
let
|
|
bao = baoWithCollector.services.hyperhive.swarm.bao;
|
|
e = (baoForwarder baoWithCollector).settings.extensions."oauth2client/swarm-bao";
|
|
in
|
|
e.client_id == bao.otel.clientId
|
|
&& e.endpoint_params.audience == bao.otel.clientId
|
|
&& bao.otel.clientId != baoWithCollector.services.hyperhive.swarm.otel.clientId
|
|
&& bao.otel.clientId != baoWithCollector.services.hyperhive.swarm.controller.queueClientId;
|
|
}
|
|
{
|
|
# A path systemd resolves at runtime, never a value and never a store
|
|
# path: the collector runs under `DynamicUser` and opens this file
|
|
# itself, so the credential arrives through `LoadCredential` and the
|
|
# config names only the directory systemd exports.
|
|
name = "the forwarder's client secret reaches it as a credential, not a value";
|
|
ok =
|
|
let
|
|
container = baoWithCollector.containers.swarm-bao.config;
|
|
e = (baoForwarder baoWithCollector).settings.extensions."oauth2client/swarm-bao";
|
|
unit = container.systemd.services.opentelemetry-collector.serviceConfig;
|
|
in
|
|
e.client_secret_file == "\${env:CREDENTIALS_DIRECTORY}/oidc-client-secret"
|
|
&&
|
|
unit.LoadCredential == [
|
|
"oidc-client-secret:/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret"
|
|
];
|
|
}
|
|
{
|
|
# Same 403-not-a-miss property the grafana and matrix readers are pinned
|
|
# for: the reader's grant covers the `services` prefix, so a secret
|
|
# filed under the hive that happens to run the store would be refused
|
|
# rather than missing, however correct the path reads.
|
|
name = "the forwarder's secret is read from the prefix the publisher writes";
|
|
ok =
|
|
let
|
|
s = baoWithCollector.systemd.services.swarm-bao-forwarder-oidc.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" s
|
|
&& !(lib.hasInfix "secret/swarm/hives/" s);
|
|
}
|
|
{
|
|
# The delivery unit is ordered AFTER the store's container, not before
|
|
# it: the store it reads lives in the container it delivers into, so a
|
|
# `before` edge is a wait on a process that cannot start until this
|
|
# finishes. Pinned because the unit it was copied from does the
|
|
# opposite, and copying that line too would deadlock every cold boot.
|
|
name = "the delivery unit waits for the store rather than blocking it";
|
|
ok =
|
|
let
|
|
u = baoWithCollector.systemd.services.swarm-bao-forwarder-oidc;
|
|
in
|
|
lib.elem "container@swarm-bao.service" u.after
|
|
&& !(lib.elem "container@swarm-bao.service" (u.before or [ ]))
|
|
&& lib.elem "swarm-bao-pki.service" u.requires;
|
|
}
|
|
{
|
|
# The far end of the credential. An authenticator checks ONE audience,
|
|
# so the store's forwarder gets a receiver of its own rather than a
|
|
# second sender into the route whose audience belongs to
|
|
# `swarm-controller` — and every one of these four is silent when
|
|
# wrong: an unlisted extension is inert, a receiver in no pipeline
|
|
# drops what it accepts, and a mismatched audience is a healthy 401.
|
|
name = "the swarm collector admits this principal on a receiver of its own";
|
|
ok =
|
|
let
|
|
otel = baoWithCollector.services.hyperhive.swarm.otel;
|
|
s = baoWithCollector.containers.swarm-otel.config.services.opentelemetry-collector.settings;
|
|
name = otel.storeProducerName;
|
|
in
|
|
s.receivers."otlp/${name}".protocols.http.auth.authenticator == "oidc/${name}"
|
|
&&
|
|
s.extensions."oidc/${name}".audience == baoWithCollector.services.hyperhive.swarm.bao.otel.clientId
|
|
&& lib.elem "oidc/${name}" s.service.extensions
|
|
&& lib.elem "otlp/${name}" s.service.pipelines."metrics/${otel.producerName}".receivers
|
|
&& lib.elem "otlp/${name}" s.service.pipelines."logs/${otel.producerName}".receivers;
|
|
}
|
|
{
|
|
# A client authelia has never heard of mints nothing, so the publisher
|
|
# has nothing to copy and the delivery unit reads an empty path
|
|
# forever. Registered where AUTHELIA runs, which is why it is a glue
|
|
# module and not a line in the store's own config — this fixture runs
|
|
# no store at all and still has to register it.
|
|
name = "the forwarder's client is registered wherever authelia runs";
|
|
ok =
|
|
let
|
|
autheliaOnly = hive { deploy.authelia.enable = true; };
|
|
clients = autheliaOnly.services.hyperhive.swarm.authelia.oidc.clients;
|
|
c = lib.findFirst (c: c.id == "swarm-bao-collector") null clients;
|
|
in
|
|
!autheliaOnly.services.hyperhive.deploy.bao.enable
|
|
&& c != null
|
|
&& c.audience == [ "swarm-bao-collector" ]
|
|
# ⚠️ What makes the token READABLE by the receiver at all: authelia's
|
|
# default is an opaque handle, and an `oidc` extension verifies
|
|
# offline against the provider's JWKS.
|
|
&& c.accessTokenSignedResponseAlg == "RS256";
|
|
}
|
|
];
|
|
in
|
|
runGroup "bao-otel-collector" cases
|