From 98945d4c5a08fa8056ba53142323c40b6e0fa524 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 12 Sep 2026 09:18:30 +0200 Subject: [PATCH] otel: scrape each collector's own loss counters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A collector's `refused` / `failed` / queue-depth counters are the only signal that says telemetry is being dropped, and nothing read them at any tier — so a collector losing records looked exactly like a quiet system. The hive tier could not be scraped without first naming its port. 8888 is the collector's built-in default and appeared in no config, which is also why nothing comparing configured ports could see it clash with a co-located collector — swarm-otel.nix sidesteps 8888 by hand for that reason, and says so. Declaring the port and binding it explicitly makes the value comparable; wiring the scrape is then one entry per tier. Extending the port-collision assertion to cover it is deliberately left out: that belongs with the other port work, and coupling a collision fix to a scraping fix makes both harder to review. Gate: 101 module properties hold, was 95. The six cases pin the rendered scrape job rather than the option; the metrics pipeline naming the prometheus receiver, a path never emitted on any hive before this since the hive tier's scrapeTargets was empty everywhere; the `readers` spelling, with a control so a missing telemetry block cannot pass the port check vacuously; the swarm tier's own entry; the two tiers not claiming the same port; and the absence arm, a hive with no collector declaring no target. scrapeTargets' description said "Empty by default, and that is the shipped case". This makes that false, so the paragraph moves with it. --- nix/host-modules/otel.nix | 53 +++++++++++++++++++++-- nix/host-modules/swarm-otel.nix | 7 ++++ nix/module-eval.nix | 74 +++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 4 deletions(-) diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 65b90c3d..77df0175 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -193,6 +193,24 @@ in ''; }; + telemetryPort = lib.mkOption { + type = lib.types.port; + default = 8888; + description = '' + Port this collector serves its **own** metrics on — queue depth, + refused and dropped samples, exporter failures. How you find out + that telemetry is being lost. + + 8888 is the collector's built-in default; declaring it here makes + the value *configured* rather than implied, which is the point of + the option. An undeclared port is invisible to anything comparing + ports, so a second collector on the same host picking the same + number fails at `bind()` rather than at eval — + `services.hyperhive.swarm.otel.telemetryPort` sidesteps 8888 by + hand for that reason. + ''; + }; + scrapeTargets = lib.mkOption { type = lib.types.attrsOf lib.types.str; default = { }; @@ -221,10 +239,11 @@ in onward, which is one hop for the whole hive rather than one per target. - Empty by default, and that is the shipped case — no scrape - receiver is emitted at all, because a `prometheus` receiver with - nothing to scrape is a config that renders, starts and collects - nothing. + No longer empty on a default hive: this collector's own endpoint + is always in here, so the scrape receiver is emitted everywhere. + The `!= { }` guards around it stay meaningful — the set can still + be forced empty, and a `prometheus` receiver with nothing to + scrape is a config that renders, starts and collects nothing. ''; }; @@ -317,6 +336,15 @@ in # the port on the bridge interface only. services.hyperhive.network.exposeHostPorts = [ otel.collector.port ]; + # The collector scrapes itself. Its own counters — refused, failed, + # queue depth — are the only signal that says telemetry is being + # dropped, and they reach no store unless something reads them. + # + # Loopback, so this deliberately does NOT go through + # `exposeHostPorts`: the endpoint is for the collector beside it, + # not for agent containers. + services.hyperhive.otel.scrapeTargets.collector = "127.0.0.1:${toString otel.telemetryPort}"; + services.opentelemetry-collector = { enable = true; # Contrib, matching the swarm tier (./swarm-otel.nix). The upstream @@ -556,6 +584,23 @@ in processors = [ "resourcedetection" ]; exporters = [ swarmName ]; }; + + # Bound explicitly rather than left to the collector's built-in + # default, so `telemetryPort` is a value something can compare + # instead of an assumption. + # + # ⚠️ `metrics.address` is the spelling that looks right and is + # rejected by this collector version — ./swarm-otel.nix documents + # the exact error it produces. `readers` is the schema it accepts, + # and the difference is a startup failure rather than a warning. + service.telemetry.metrics.readers = [ + { + pull.exporter.prometheus = { + host = "127.0.0.1"; + port = otel.telemetryPort; + }; + } + ]; } // lib.optionalAttrs senderAuth { extensions.${authName} = { diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index f9de162f..32ba6eba 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -533,6 +533,13 @@ in # single hive's receiver refusing pushes, not the process dying. services.hyperhive.swarm.otel.journaldUnits = [ "opentelemetry-collector" ]; + # The metrics counterpart to the journal line above, closing the same + # gap from the other side: the journal says the process is alive, these + # counters say whether it is dropping what it receives. Loopback works + # here because `privateNetwork = false` — this container shares the + # host's netns, so `127.0.0.1` is where `telemetryPort` is bound. + services.hyperhive.swarm.otel.scrapeTargets.collector = "127.0.0.1:${toString cfg.telemetryPort}"; + # OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as # every sibling swarm service: TLS terminates here, then plain http to # the co-located container over loopback (shared netns, like the store diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 7a2c81d9..dea14c02 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -447,6 +447,16 @@ let otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; }; hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines; + hiveOtelSettings = hiveOtel.services.opentelemetry-collector.settings; + + # The hive tier's rendered scrape list. Same reasoning as `scrapeJob` for + # the swarm tier — the option is one string, what prometheus is handed is a + # job — but this collector is a host service, so the path to it differs. + hiveScrapeJob = + job: + lib.findFirst ( + c: c.job_name == job + ) null hiveOtelSettings.receivers.prometheus.config.scrape_configs; # Two hives in the roster, which no other fixture here has: every one of # them declares `swarm.hives.h1` alone, so a per-hive arm written against @@ -1368,6 +1378,70 @@ let !(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors) && builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors; } + { + # The counters that say telemetry is being LOST — refused, failed, + # queue depth — are served on loopback and reach no store unless + # something reads them. Read off the rendered job rather than the + # option: only the job is what prometheus actually requests. + name = "the hive collector scrapes its own telemetry endpoint"; + ok = + let + j = hiveScrapeJob "collector"; + in + j != null && j.static_configs == [ { targets = [ "127.0.0.1:8888" ]; } ]; + } + { + # A `prometheus` receiver no pipeline names collects nothing while + # rendering and starting perfectly, so the scrape above is inert + # without this. The path is newly reachable: until the hive tier had a + # target of its own, this receiver was never emitted on any hive. + name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs"; + ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers; + } + { + # `metrics.address` is the spelling that looks right and is rejected by + # this collector version. The first clause is the control: without it a + # missing telemetry block would pass the port check vacuously. + name = "the hive collector binds its telemetry port through readers, not address"; + ok = + let + m = hiveOtelSettings.service.telemetry.metrics; + in + !(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888; + } + { + # Same gap one tier up, and it needs its own arm: this collector + # already had six scrape targets, so a pass here is about the seventh + # rather than about the receiver existing at all. + name = "the swarm collector scrapes its own telemetry endpoint"; + ok = + let + j = scrapeJob baoWithCollector "collector"; + in + j != null && j.static_configs == [ { targets = [ "127.0.0.1:8889" ]; } ]; + } + { + # Both collectors share a network namespace whenever they are + # co-located, and this port appears in no config the port-collision + # assertion can read — so equal defaults mean the second to start dies + # at `bind()`. Pinned as a case rather than an assertion: enforcing it + # belongs with the other port checks, not here. + name = "the two collector tiers do not claim the same self-telemetry port"; + ok = + let + portOf = s: (lib.head s.service.telemetry.metrics.readers).pull.exporter.prometheus.port; + in + portOf hiveOtelSettings != portOf (otelSettings baoWithCollector); + } + { + # The absence arm for the case above, and the option's own rule — a + # service declares its entry under its own `enable` — made checkable. + # Without it, moving the assignment outside the collector's `mkIf` + # passes every arm above while handing a collector-less hive a scrape + # target for a port nothing binds. + name = "a hive with no collector declares no self-scrape target"; + ok = bare.services.hyperhive.otel.scrapeTargets == { }; + } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive