# `checks.module-eval-bao-otel-collector` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive runGroup baoSettings ; baoTwoAddresses = hive { deploy.bao.enable = true; deploy.bao.extraListenAddresses = [ "10.0.0.1" ]; # Pinned, not incidental: the case counting these listeners is about the # declared addresses, and a collector on this host would add one of its own. deploy.swarm-otel.enable = false; }; # The store with and without a collector on the same host. `scrapeTargets` # is only ever read by a local collector, so the metrics endpoint is a # function of the pairing rather than of the store. baoWithCollector = hive { deploy.bao.enable = true; deploy.swarm-otel.enable = true; # A second job declared as bare `host:port`, so the pair of cases below # reads one rendered scrape list: the store's entry carries a path, this # one carries none. swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; }; baoNoCollector = hive { deploy.bao.enable = true; deploy.swarm-otel.enable = false; }; # The store on a host whose HIVE collector is running and whose SWARM one is # not. The forwarder is a function of the swarm tier, so this fixture pins # the negative: a hive collector beside the store buys it nothing. # `clientSecretFile` is what ../host-modules/otel.nix's identity assertion # demands of any hive with the tier on. baoWithHiveOtel = hive { deploy.bao.enable = true; otel.enable = true; otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; }; # The forwarder INSIDE the store's container, not the host's collector and # not the swarm tier's — three collectors in this tree, and only this one can # see the store's journal. baoForwarder = machine: machine.containers.swarm-bao.config.services.opentelemetry-collector; # The scrape list prometheus is handed, not the option a service declared: # the address, the path and the query are one string on the way in and three # fields on the way out, and only the second shape is what gets requested. scrapeJob = machine: job: lib.findFirst (c: c.job_name == job) null machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs; cases = [ { # Same gap one tier up, and it needs its own arm: this collector # already had six scrape targets, so a pass here is about the seventh # rather than about the receiver existing at all. name = "the swarm collector scrapes its own telemetry endpoint"; ok = let j = scrapeJob baoWithCollector "collector"; in j != null && j.static_configs == [ { targets = [ "127.0.0.1:8889" ]; } ]; } { # The store stays behind the passthrough rather than beside it: loopback # plus whatever was declared, never the bridge. A store that also bound # the bridge itself would collide with the listener above, and the # colliding one is nginx — the whole gateway, not just this port. name = "the store binds loopback and its declared addresses, never the bridge"; ok = let l = (baoSettings baoTwoAddresses).listener; in l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200"; } { # Control for the case above: these settings are rendered per deployment, # not constants a passing case could be indifferent to. name = "a declared extra address renders a second listener beside loopback"; ok = builtins.length (builtins.attrNames (baoSettings baoTwoAddresses).listener) == 2; } { # Retention is what serves the endpoint at all, so the listener alone # would be a port that answers 404. name = "a store beside a collector serves metrics on its own listener"; ok = let s = baoSettings baoWithCollector; in s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s"; } { # openbao serves no `/metrics` at all, so a scrape of the default path # 404s: the store looks like a dead exporter, and every panel built on # it renders empty rather than erroring. name = "the store's scrape asks for the path openbao serves"; ok = let j = scrapeJob baoWithCollector "bao"; in (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ]; } { # Presence control for the case above: both fields are omitted rather # than defaulted, so a target declared as bare `host:port` renders what # it rendered before the path grammar existed. name = "a target with no path renders neither metrics_path nor params"; ok = let j = scrapeJob baoWithCollector "plain"; in j != null && !(j ? metrics_path) && !(j ? params); } { # Absence arm. Unauthenticated by design, so it must not exist where # nothing reads it. name = "a store with no collector beside it serves no metrics"; ok = let s = baoSettings baoNoCollector; in !(s.listener ? metrics) && !(s ? telemetry); } { # The store's journal reaches a reader through a collector of its own, # and every one of these fields is silent when wrong: the runtime journal # is the receiver's own default and is empty here, an unlisted extension # is inert so the cursor silently stops persisting, and a pipeline is # free to name none of it. name = "the store's container forwards its own journal"; ok = let s = (baoForwarder baoWithCollector).settings; p = s.service.pipelines.logs; in s.receivers.journald.directory == "/var/log/journal" && s.receivers.journald.storage == "file_storage" && s.service.extensions == [ "file_storage" ] && p.receivers == [ "journald" ] && p.exporters == [ "otlphttp" ] && s ? exporters.otlphttp; } { # The whole journal, which is what the shared collector's unit allowlist # is not. A `units` list here would render and deploy perfectly while # shipping only the units someone remembered to name — the failure this # forwarder exists to end. name = "the store's forwarder filters no units"; ok = !((baoForwarder baoWithCollector).settings.receivers.journald ? units); } { # Both ends of the hop, because a mismatch between them is silent in both # directions: the exporter retries into a 404 and the receiver never # hears from it. Not "on one host" any more — the far end is the gateway # vhost the swarm collector serves its name on, which is the whole point # of addressing it by name rather than by a bridge address. name = "the store's forwarder exports to the swarm collector's own route"; ok = let otel = baoWithCollector.services.hyperhive.swarm.otel; vhost = baoWithCollector.services.nginx.virtualHosts.${otel.domain}; in (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint == "https://${otel.domain}/${otel.producerName}" && vhost.locations ? "/${otel.producerName}/"; } { # The case the old hive-tier gate got wrong: this host runs the swarm's # collector and no hive collector at all, and the forwarder still exists # with an address that resolves. Gated on `otel.enable` it rendered # nothing here, so the store's journal left no trace anywhere. name = "a store on a host with no hive collector still forwards"; ok = !baoWithCollector.services.hyperhive.otel.enable && (baoForwarder baoWithCollector).enable && (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != ""; } { # Absence arm — no swarm collector, so nothing this side of the gateway # serves that name. A forwarder rendered anyway would start, find nothing # listening, and retry forever while reporting healthy. `baoWithHiveOtel` # is the pointed half: a hive collector beside the store is not a reason # to forward, which is exactly what the old gate assumed. name = "a store with no swarm collector renders no forwarder"; ok = !(baoForwarder baoNoCollector).enable && !(baoForwarder baoWithHiveOtel).enable; } ]; in runGroup "bao-otel-collector" cases