diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index d10996a4..d2fa786e 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -41,6 +41,17 @@ let networkCfg = hyperhiveCfg.network; swarmDomain = hyperhiveCfg.swarm.domain; + # What this container's own collector stamps as `service.name` on every + # log line and metric it forwards, and the value the shipped Grafana + # dashboard's (`swarm-grafana/dashboards/bao.json`) panels all select on. + # Deliberately NOT `cfg.machine`: that value names the *container* — it is + # also the swarm-tier receiver/authenticator component name + # (`otlp/${cfg.machine}` in ./swarm-otel.nix) and could change for reasons + # that have nothing to do with what the dashboard queries. One literal, + # bound once, so the collector and the dashboard can't drift apart — + # `nix/module-eval/grafana.nix` pins it against the shipped dashboard file. + collectorServiceName = "bao"; + # Upstream's own default, kept so its documentation matches. The raft data # lives INSIDE the container on `ephemeral = false`, the same way # ./swarm-grafana.nix keeps its sqlite database — no sibling service binds @@ -2782,10 +2793,14 @@ in # `hive` or `swarm` label — the swarm tier upserts `hive` from # the receiver that accepted the record, precisely so it comes # from something the sender cannot write. + # + # `collectorServiceName`, not `cfg.machine`: what the + # dashboard selects on is a display identity, not the + # container/receiver name — see that binding's own comment. processors.resource.attributes = [ { key = "service.name"; - value = cfg.machine; + value = collectorServiceName; action = "upsert"; } ]; diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index fe3cd97e..9fe2f86b 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -30,6 +30,16 @@ let swarm.grafana.package = pkgs.emptyDirectory; }; + # Same pairing `nix/module-eval/bao-otel-collector.nix` uses for its own + # cases: the store's container with its own collector on, so + # `.containers.swarm-bao.config` carries a real, evaluated + # `opentelemetry-collector.settings` rather than one this file would have + # to hand-assemble. + baoWithCollector = hive { + deploy.bao.enable = true; + deploy.swarm-otel.enable = true; + }; + # The metrics UI beside the IdP. It reads its secret out of the store like # every other Grafana host, so it needs a store identity like every other # Grafana host — the cert pair here is not scenery, it is the arm that would @@ -261,6 +271,33 @@ let && lib.all (r: r.field == "severity_text" && r.enabled == true && lib.elem r.level canonical) rules && lib.all (text: lib.any (r: r.value == text) rules) emitted; } + { + # 🩸 The regression this guards: moving bao's metrics scrape into its + # own in-container collector made that collector stamp every exported + # series `service.name=swarm-bao`, while every panel in the shipped + # bao.json still queried the literal `"service.name"="bao"` — delivery + # could be perfect and no panel would ever match. Fixed by renaming it + # in the collector rather than in the dashboard: `swarm-bao.nix`'s + # `collectorServiceName` now stamps `"bao"` directly. This case reads + # the collector's OWN evaluated config — not a hand-copied literal — + # so it fails again the moment that binding and the dashboard drift + # apart in either direction. + # 🔬 Invert-proof: change `collectorServiceName` in swarm-bao.nix back + # to `cfg.machine` and this case fails. + name = "the bao dashboard's selectors match the service.name bao's own collector stamps"; + ok = + let + attrs = + baoWithCollector.containers.swarm-bao.config.services.opentelemetry-collector.settings.processors.resource.attributes; + stamped = lib.findFirst (a: a.key == "service.name") null attrs; + board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/bao.json); + exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels; + selectors = lib.filter (e: lib.hasInfix "service.name" e) exprs; + in + stamped != null + && builtins.length selectors == 12 + && lib.all (e: lib.hasInfix "\"service.name\"=\"${stamped.value}\"" e) selectors; + } ]; in runGroup "grafana" cases