From d4313fc34d30cab3821e114ff06105b85bff7437 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 19 Sep 2026 20:23:43 +0200 Subject: [PATCH] nix: the store's journal forwarder has no gate to have MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both earlier versions asked the wrong host. `hyperhive.otel.enable` asked whether this host runs a HIVE collector; `deploy.swarm-otel.enable` asked whether this host runs the SWARM one. Neither answers the question the forwarder actually has — "is there a collector to forward to" — and that question cannot be false: a swarm always runs at least one instance of every swarm-level service. So the forwarder renders under the condition already enclosing it, that the store is deployed here, and nothing else. `scrapeHere` deliberately keeps its `deploy.*` gate one line up. It is a loopback metrics listener, which genuinely only works where the scraper is — the two are different tiers, and the name says so. The module-eval case that pins it is the split topology: the swarm collector on another host, nothing local naming it, and the forwarder still enabled and still addressed at `swarm.otel.domain`'s route. Both removed gates render nothing in that fixture, which the co-located ones they shipped with could not show. --- nix/host-modules/swarm-bao.nix | 27 +++++++++------ nix/module-eval/bao-otel-collector.nix | 48 +++++++++++++++++++------- 2 files changed, 53 insertions(+), 22 deletions(-) diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 3f6e956b..5f88a6c0 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -455,12 +455,19 @@ let scrapeHere = deployCfg.swarm-otel.enable; - # Whether this container forwards its own journal, gated exactly like - # `scrapeHere` above: the store is a swarm-tier component, so what decides - # this is the swarm collector existing, never which hive happens to stand - # beside it. On the hive flag it rendered no forwarder at all on a host that - # runs the swarm's own collector but not a hive's. - shipJournal = deployCfg.swarm-otel.enable; + # ⚠️ The forwarder below has NO condition of its own, and `scrapeHere` above + # is not a template for one — the two are different tiers on purpose. A + # loopback metrics listener works only where the scraper is, which is what + # `Here` in that name says; forwarding reaches the swarm's collector BY NAME, + # from wherever this store runs. + # + # There is nothing left to gate on. "Does the swarm have a collector" is not + # a question that can be false (mara: "the swarm always has at least one + # instance of all the swarm level services"), so both earlier gates — + # `otel.enable`, then `deploy.swarm-otel.enable` — asked about the wrong + # host and rendered no forwarder wherever the answer happened to be no. The + # only condition that belongs here is the one already enclosing this block: + # the store is deployed on this host. # The swarm collector's own name, served by the gateway — the address # `swarm.otel.domain` exists to be, resolved by dnsmasq on a co-located host @@ -1635,7 +1642,7 @@ in # one reads only openbao and the two oneshots beside it, so there # is nothing foreign to separate out — and a list of unit names is # a thing to get wrong, which ships nothing while looking healthy. - assertions = lib.optionals shipJournal [ + assertions = [ { # Sibling of the agent forwarder's identical assertion, and it # exists because the failure is silent at every layer: with a @@ -1648,8 +1655,8 @@ in "none" ]); message = '' - services.hyperhive.deploy.swarm-otel.enable is on, so ${cfg.machine} forwards its - own journal — but services.journald.storage is + ${cfg.machine} forwards its own journal to the swarm's + collector — but services.journald.storage is "${config.services.journald.storage}" in this container. The forwarder reads /var/log/journal, which journald only writes @@ -1660,7 +1667,7 @@ in } ]; - services.opentelemetry-collector = lib.mkIf shipJournal { + services.opentelemetry-collector = { enable = true; # Contrib, and not a preference: `journald` is a contrib # receiver. The upstream default build has no way to read a diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index cae6b220..f6d9b83f 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -42,14 +42,20 @@ let swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; }; - baoNoCollector = hive { + # The SPLIT topology, and the reason it is not called `baoNoCollector` any + # more: a swarm always runs every swarm-level service somewhere, so this host + # having `deploy.swarm-otel.enable = false` means the collector is on ANOTHER + # host, never that the swarm has none. Both of those read the same locally, + # and only one of them is a real deployment — which is what the two gates + # this PR removed each mistook for "there is nothing to forward to". + baoOtelElsewhere = hive { deploy.bao.enable = true; deploy.swarm-otel.enable = false; }; - # The store on a host whose HIVE collector is running and whose SWARM one is - # not. The forwarder is a function of the swarm tier, so this fixture pins - # the negative: a hive collector beside the store buys it nothing. + # The same split, plus a HIVE collector on the store's host — the pairing the + # first gate read as permission to forward. It is a third collector with a + # different job, so it changes neither whether the store forwards nor where. # `clientSecretFile` is what ../host-modules/otel.nix's identity assertion # demands of any hive with the tier on. baoWithHiveOtel = hive { @@ -138,7 +144,7 @@ let name = "a store with no collector beside it serves no metrics"; ok = let - s = baoSettings baoNoCollector; + s = baoSettings baoOtelElsewhere; in !(s.listener ? metrics) && !(s ? telemetry); } @@ -197,13 +203,31 @@ let && (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != ""; } { - # Absence arm — no swarm collector, so nothing this side of the gateway - # serves that name. A forwarder rendered anyway would start, find nothing - # listening, and retry forever while reporting healthy. `baoWithHiveOtel` - # is the pointed half: a hive collector beside the store is not a reason - # to forward, which is exactly what the old gate assumed. - name = "a store with no swarm collector renders no forwarder"; - ok = !(baoForwarder baoNoCollector).enable && !(baoForwarder baoWithHiveOtel).enable; + # 🎯 The split topology, which is what both removed gates got wrong and + # neither earlier test could see: the swarm's collector runs on ANOTHER + # host, so nothing local says it exists — and it exists anyway, because + # every swarm-level service runs somewhere in the swarm. The forwarder + # has to render here and has to be addressed by the collector's swarm + # name, which is reachable from a host that serves no vhost for it. + # + # Both halves matter. Without `enable` this is the old bug; without the + # endpoint it is a forwarder that renders unconditionally and points at + # nothing. The `!(vhost ? domain)` conjunct is what makes the fixture + # genuinely remote rather than a co-located host in disguise — the + # premise the co-located fixtures cannot test. + name = "a store forwards by name from a host that does not run the swarm collector"; + ok = + let + otel = baoOtelElsewhere.services.hyperhive.swarm.otel; + route = "https://${otel.domain}/${otel.producerName}"; + in + !(baoOtelElsewhere.services.nginx.virtualHosts ? ${otel.domain}) + && (baoForwarder baoOtelElsewhere).enable + && (baoForwarder baoOtelElsewhere).settings.exporters.otlphttp.endpoint == route + # …and a hive collector standing beside the store neither supplies the + # reason to forward nor changes the addressee: same route, same tier. + && (baoForwarder baoWithHiveOtel).enable + && (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint == route; } ]; in