diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index bcbfee5a..b6a2156d 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -181,6 +181,33 @@ in addresses its siblings by name. ''; }; + + scrapeTargets = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + example = lib.literalExpression ''{ forgejo = "127.0.0.1:3000"; }''; + description = '' + Prometheus exposition endpoints this collector scrapes, as + ` = ":"`. + + **A service declares its own entry, from its own module, under its + own `enable`.** That is what puts the scraper and the target on the + same host by construction rather than by luck: an entry exists only + where the service that named it runs. Do not assemble the list here. + Every swarm service being co-located is a property of the all-local + deployment, not a guarantee — and that is precisely the case where + the difference is invisible until a swarm splits across hosts. + + Samples land in a swarm-level pipeline that stamps `swarm` and + **never** `hive`: a swarm service does not belong to a hive, and + `hive` stays a property of which authenticated receiver accepted a + push, not something a scrape can acquire. + + Empty by default, in which case no scrape receiver, processor or + pipeline is emitted at all — an enabled scraper with nothing to + scrape is the inert configuration this option exists to avoid. + ''; + }; }; config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) { @@ -387,15 +414,32 @@ in # an auth claim yields nothing — silently, with a healthy # startup), and one receiver holding many credentials never # reveals which one matched. - receivers = lib.mapAttrs' ( - h: p: - lib.nameValuePair "otlp/${h}" { - protocols.http = { - endpoint = "127.0.0.1:${toString p}"; - auth.authenticator = "oidc/${h}"; - }; - } - ) hivePorts; + receivers = + lib.mapAttrs' ( + h: p: + lib.nameValuePair "otlp/${h}" { + protocols.http = { + endpoint = "127.0.0.1:${toString p}"; + auth.authenticator = "oidc/${h}"; + }; + } + ) hivePorts + # MERGED with the per-hive receivers, never assigned over + # them. A plain assignment here would drop every hive's + # receiver and still render a valid config that starts + # cleanly — the collector has no opinion about how many + # pipelines it was supposed to have. + # + # Only emitted when a service has actually declared a + # target: a `prometheus` receiver with nothing to scrape is + # the shape this whole issue is about, a config that renders + # and deploys perfectly while adding no data. + // lib.optionalAttrs (cfg.scrapeTargets != { }) { + prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: { + job_name = job; + static_configs = [ { targets = [ target ]; } ]; + }) cfg.scrapeTargets; + }; exporters = lib.optionalAttrs vmCfg.enable { @@ -452,14 +496,28 @@ in # `exporterNames` is shared by every pipeline — where a # sample goes is a property of this tier, not of the hive # that sent it. - service.pipelines = lib.mapAttrs' ( - h: _: - lib.nameValuePair "metrics/${h}" { - receivers = [ "otlp/${h}" ]; - processors = [ "resource/${h}" ]; - exporters = exporterNames; - } - ) hivePorts; + service.pipelines = + lib.mapAttrs' ( + h: _: + lib.nameValuePair "metrics/${h}" { + receivers = [ "otlp/${h}" ]; + processors = [ "resource/${h}" ]; + exporters = exporterNames; + } + ) hivePorts + # Its OWN pipeline, and that separation is the ruling, not a + # tidiness choice: every `resource/` above UPSERTS a + # `hive` key, so a scraped swarm sample routed through any of + # them would acquire the one label a swarm-level service must + # not have. Keeping it out of them makes the absence + # structural rather than something to remember to strip. + // lib.optionalAttrs (cfg.scrapeTargets != { }) { + "metrics/swarm" = { + receivers = [ "prometheus" ]; + processors = [ "resource/swarm" ]; + exporters = exporterNames; + }; + }; } // { extensions = lib.mapAttrs' ( @@ -507,36 +565,51 @@ in # processor is the whole attribution boundary — the value # is a constant per receiver, so it says which hive # authenticated, not which hive claimed to be sending. - processors = lib.mapAttrs' ( - h: _: - lib.nameValuePair "resource/${h}" { - attributes = [ - { - key = "hive"; - value = h; - action = "upsert"; - } - # Stamped here rather than on a separate upstream-only - # pipeline, which would double the pipeline count to - # withhold one constant label from the local store. It - # is redundant there — one VictoriaMetrics per swarm, so - # every series in it already belongs to this swarm — but - # a constant label multiplies no series, and it means - # what LEAVES and what STAYS have the same shape. - # - # Upstream is where it stops being redundant: that is the - # one hop where several swarms can land in one store, and - # samples that cannot name their swarm collide there - # exactly as hives collided here before per-hive - # receivers existed. + processors = + lib.mapAttrs' ( + h: _: + lib.nameValuePair "resource/${h}" { + attributes = [ + { + key = "hive"; + value = h; + action = "upsert"; + } + # Stamped here rather than on a separate upstream-only + # pipeline, which would double the pipeline count to + # withhold one constant label from the local store. It + # is redundant there — one VictoriaMetrics per swarm, so + # every series in it already belongs to this swarm — but + # a constant label multiplies no series, and it means + # what LEAVES and what STAYS have the same shape. + # + # Upstream is where it stops being redundant: that is the + # one hop where several swarms can land in one store, and + # samples that cannot name their swarm collide there + # exactly as hives collided here before per-hive + # receivers existed. + { + key = "swarm"; + value = swarmDisplayName; + action = "upsert"; + } + ]; + } + ) hivePorts + # The swarm tier's own stamp: `swarm` and deliberately NO + # `hive`. A scraped swarm service belongs to the swarm, not to + # any one hive, so there is no honest value to put there — and + # an invented one (a sentinel, the local hive's name) would be + # queried as though it meant something. + // lib.optionalAttrs (cfg.scrapeTargets != { }) { + "resource/swarm".attributes = [ { key = "swarm"; value = swarmDisplayName; action = "upsert"; } ]; - } - ) hivePorts; + }; }; }; diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index 533dff8b..25c1f328 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -113,6 +113,21 @@ in } ]; + # This store publishes its own health as prometheus metrics on the same + # listener it serves queries on, so the swarm's collector can scrape it + # with no exporter and no extra port. + # + # Declared here rather than in the collector's module because that is the + # rule the option carries: an entry exists only where the service that + # named it runs, which is what keeps scraper and target on one host by + # construction instead of by luck. + # + # The loopback literal introduces no new assumption — it is the address + # this module already pins the listener to, and the same one the + # collector's `otlphttp/victoriametrics` exporter already writes to. If + # that reach is ever wrong, it is wrong for the write path first. + services.hyperhive.swarm.otel.scrapeTargets.victoriametrics = "127.0.0.1:${toString cfg.port}"; + services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders;