d4313fc34d
Both earlier versions asked the wrong host. `hyperhive.otel.enable` asked whether this host runs a HIVE collector; `deploy.swarm-otel.enable` asked whether this host runs the SWARM one. Neither answers the question the forwarder actually has — "is there a collector to forward to" — and that question cannot be false: a swarm always runs at least one instance of every swarm-level service. So the forwarder renders under the condition already enclosing it, that the store is deployed here, and nothing else. `scrapeHere` deliberately keeps its `deploy.*` gate one line up. It is a loopback metrics listener, which genuinely only works where the scraper is — the two are different tiers, and the name says so. The module-eval case that pins it is the split topology: the swarm collector on another host, nothing local naming it, and the forwarder still enabled and still addressed at `swarm.otel.domain`'s route. Both removed gates render nothing in that fixture, which the co-located ones they shipped with could not show.
234 lines
10 KiB
Nix
234 lines
10 KiB
Nix
# `checks.module-eval-bao-otel-collector` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
baoSettings
|
|
;
|
|
|
|
baoTwoAddresses = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
|
|
# Pinned, not incidental: the case counting these listeners is about the
|
|
# declared addresses, and a collector on this host would add one of its own.
|
|
deploy.swarm-otel.enable = false;
|
|
};
|
|
|
|
# The store with and without a collector on the same host. `scrapeTargets`
|
|
# is only ever read by a local collector, so the metrics endpoint is a
|
|
# function of the pairing rather than of the store.
|
|
baoWithCollector = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.swarm-otel.enable = true;
|
|
# A second job declared as bare `host:port`, so the pair of cases below
|
|
# reads one rendered scrape list: the store's entry carries a path, this
|
|
# one carries none.
|
|
swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
|
|
};
|
|
|
|
# The SPLIT topology, and the reason it is not called `baoNoCollector` any
|
|
# more: a swarm always runs every swarm-level service somewhere, so this host
|
|
# having `deploy.swarm-otel.enable = false` means the collector is on ANOTHER
|
|
# host, never that the swarm has none. Both of those read the same locally,
|
|
# and only one of them is a real deployment — which is what the two gates
|
|
# this PR removed each mistook for "there is nothing to forward to".
|
|
baoOtelElsewhere = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.swarm-otel.enable = false;
|
|
};
|
|
|
|
# The same split, plus a HIVE collector on the store's host — the pairing the
|
|
# first gate read as permission to forward. It is a third collector with a
|
|
# different job, so it changes neither whether the store forwards nor where.
|
|
# `clientSecretFile` is what ../host-modules/otel.nix's identity assertion
|
|
# demands of any hive with the tier on.
|
|
baoWithHiveOtel = hive {
|
|
deploy.bao.enable = true;
|
|
otel.enable = true;
|
|
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
|
|
};
|
|
|
|
# The forwarder INSIDE the store's container, not the host's collector and
|
|
# not the swarm tier's — three collectors in this tree, and only this one can
|
|
# see the store's journal.
|
|
baoForwarder = machine: machine.containers.swarm-bao.config.services.opentelemetry-collector;
|
|
|
|
# The scrape list prometheus is handed, not the option a service declared:
|
|
# the address, the path and the query are one string on the way in and three
|
|
# fields on the way out, and only the second shape is what gets requested.
|
|
scrapeJob =
|
|
machine: job:
|
|
lib.findFirst (c: c.job_name == job) null
|
|
machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs;
|
|
cases = [
|
|
{
|
|
# Same gap one tier up, and it needs its own arm: this collector
|
|
# already had six scrape targets, so a pass here is about the seventh
|
|
# rather than about the receiver existing at all.
|
|
name = "the swarm collector scrapes its own telemetry endpoint";
|
|
ok =
|
|
let
|
|
j = scrapeJob baoWithCollector "collector";
|
|
in
|
|
j != null && j.static_configs == [ { targets = [ "127.0.0.1:8889" ]; } ];
|
|
}
|
|
{
|
|
# The store stays behind the passthrough rather than beside it: loopback
|
|
# plus whatever was declared, never the bridge. A store that also bound
|
|
# the bridge itself would collide with the listener above, and the
|
|
# colliding one is nginx — the whole gateway, not just this port.
|
|
name = "the store binds loopback and its declared addresses, never the bridge";
|
|
ok =
|
|
let
|
|
l = (baoSettings baoTwoAddresses).listener;
|
|
in
|
|
l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200";
|
|
}
|
|
{
|
|
# Control for the case above: these settings are rendered per deployment,
|
|
# not constants a passing case could be indifferent to.
|
|
name = "a declared extra address renders a second listener beside loopback";
|
|
ok = builtins.length (builtins.attrNames (baoSettings baoTwoAddresses).listener) == 2;
|
|
}
|
|
{
|
|
# Retention is what serves the endpoint at all, so the listener alone
|
|
# would be a port that answers 404.
|
|
name = "a store beside a collector serves metrics on its own listener";
|
|
ok =
|
|
let
|
|
s = baoSettings baoWithCollector;
|
|
in
|
|
s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
|
|
}
|
|
{
|
|
# openbao serves no `/metrics` at all, so a scrape of the default path
|
|
# 404s: the store looks like a dead exporter, and every panel built on
|
|
# it renders empty rather than erroring.
|
|
name = "the store's scrape asks for the path openbao serves";
|
|
ok =
|
|
let
|
|
j = scrapeJob baoWithCollector "bao";
|
|
in
|
|
(j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ];
|
|
}
|
|
{
|
|
# Presence control for the case above: both fields are omitted rather
|
|
# than defaulted, so a target declared as bare `host:port` renders what
|
|
# it rendered before the path grammar existed.
|
|
name = "a target with no path renders neither metrics_path nor params";
|
|
ok =
|
|
let
|
|
j = scrapeJob baoWithCollector "plain";
|
|
in
|
|
j != null && !(j ? metrics_path) && !(j ? params);
|
|
}
|
|
{
|
|
# Absence arm. Unauthenticated by design, so it must not exist where
|
|
# nothing reads it.
|
|
name = "a store with no collector beside it serves no metrics";
|
|
ok =
|
|
let
|
|
s = baoSettings baoOtelElsewhere;
|
|
in
|
|
!(s.listener ? metrics) && !(s ? telemetry);
|
|
}
|
|
{
|
|
# The store's journal reaches a reader through a collector of its own,
|
|
# and every one of these fields is silent when wrong: the runtime journal
|
|
# is the receiver's own default and is empty here, an unlisted extension
|
|
# is inert so the cursor silently stops persisting, and a pipeline is
|
|
# free to name none of it.
|
|
name = "the store's container forwards its own journal";
|
|
ok =
|
|
let
|
|
s = (baoForwarder baoWithCollector).settings;
|
|
p = s.service.pipelines.logs;
|
|
in
|
|
s.receivers.journald.directory == "/var/log/journal"
|
|
&& s.receivers.journald.storage == "file_storage"
|
|
&& s.service.extensions == [ "file_storage" ]
|
|
&& p.receivers == [ "journald" ]
|
|
&& p.exporters == [ "otlphttp" ]
|
|
&& s ? exporters.otlphttp;
|
|
}
|
|
{
|
|
# The whole journal, which is what the shared collector's unit allowlist
|
|
# is not. A `units` list here would render and deploy perfectly while
|
|
# shipping only the units someone remembered to name — the failure this
|
|
# forwarder exists to end.
|
|
name = "the store's forwarder filters no units";
|
|
ok = !((baoForwarder baoWithCollector).settings.receivers.journald ? units);
|
|
}
|
|
{
|
|
# Both ends of the hop, because a mismatch between them is silent in both
|
|
# directions: the exporter retries into a 404 and the receiver never
|
|
# hears from it. Not "on one host" any more — the far end is the gateway
|
|
# vhost the swarm collector serves its name on, which is the whole point
|
|
# of addressing it by name rather than by a bridge address.
|
|
name = "the store's forwarder exports to the swarm collector's own route";
|
|
ok =
|
|
let
|
|
otel = baoWithCollector.services.hyperhive.swarm.otel;
|
|
vhost = baoWithCollector.services.nginx.virtualHosts.${otel.domain};
|
|
in
|
|
(baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint
|
|
== "https://${otel.domain}/${otel.producerName}"
|
|
&& vhost.locations ? "/${otel.producerName}/";
|
|
}
|
|
{
|
|
# The case the old hive-tier gate got wrong: this host runs the swarm's
|
|
# collector and no hive collector at all, and the forwarder still exists
|
|
# with an address that resolves. Gated on `otel.enable` it rendered
|
|
# nothing here, so the store's journal left no trace anywhere.
|
|
name = "a store on a host with no hive collector still forwards";
|
|
ok =
|
|
!baoWithCollector.services.hyperhive.otel.enable
|
|
&& (baoForwarder baoWithCollector).enable
|
|
&& (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != "";
|
|
}
|
|
{
|
|
# 🎯 The split topology, which is what both removed gates got wrong and
|
|
# neither earlier test could see: the swarm's collector runs on ANOTHER
|
|
# host, so nothing local says it exists — and it exists anyway, because
|
|
# every swarm-level service runs somewhere in the swarm. The forwarder
|
|
# has to render here and has to be addressed by the collector's swarm
|
|
# name, which is reachable from a host that serves no vhost for it.
|
|
#
|
|
# Both halves matter. Without `enable` this is the old bug; without the
|
|
# endpoint it is a forwarder that renders unconditionally and points at
|
|
# nothing. The `!(vhost ? domain)` conjunct is what makes the fixture
|
|
# genuinely remote rather than a co-located host in disguise — the
|
|
# premise the co-located fixtures cannot test.
|
|
name = "a store forwards by name from a host that does not run the swarm collector";
|
|
ok =
|
|
let
|
|
otel = baoOtelElsewhere.services.hyperhive.swarm.otel;
|
|
route = "https://${otel.domain}/${otel.producerName}";
|
|
in
|
|
!(baoOtelElsewhere.services.nginx.virtualHosts ? ${otel.domain})
|
|
&& (baoForwarder baoOtelElsewhere).enable
|
|
&& (baoForwarder baoOtelElsewhere).settings.exporters.otlphttp.endpoint == route
|
|
# …and a hive collector standing beside the store neither supplies the
|
|
# reason to forward nor changes the addressee: same route, same tier.
|
|
&& (baoForwarder baoWithHiveOtel).enable
|
|
&& (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint == route;
|
|
}
|
|
];
|
|
in
|
|
runGroup "bao-otel-collector" cases
|