diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index fc5aa664..3f6e956b 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -455,20 +455,31 @@ let scrapeHere = deployCfg.swarm-otel.enable; - # Whether this container forwards its own journal, and it is gated on the - # HIVE collector rather than the swarm one: the forwarder below pushes to the - # first hop, and with `otel.enable` off nothing listens at that address at - # all — a collector aimed at it would retry forever while looking healthy. - shipJournal = hyperhiveCfg.otel.enable; + # Whether this container forwards its own journal, gated exactly like + # `scrapeHere` above: the store is a swarm-tier component, so what decides + # this is the swarm collector existing, never which hive happens to stand + # beside it. On the hive flag it rendered no forwarder at all on a host that + # runs the swarm's own collector but not a hive's. + shipJournal = deployCfg.swarm-otel.enable; - # The first hop for anything running on this host, byte for byte what - # hive-c0re hands every agent (`firstHop` in ./hive-c0re/environment.nix). - # Plain http with no credential is the tier boundary working as designed, not - # an omission: the hive's collector is the only thing here that holds one, - # and presenting it to the swarm is its job. This container shares the host - # netns (`privateNetwork = false` below), so the bridge address is reachable - # from inside it without the firewall hole an agent container needs. - otelFirstHop = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.otel.collector.port}"; + # The swarm collector's own name, served by the gateway — the address + # `swarm.otel.domain` exists to be, resolved by dnsmasq on a co-located host + # and over the real network otherwise, with no loopback-vs-remote knob to get + # wrong. The path selects WHICH receiver: this container is a swarm-level + # producer and not a hive, so it is the swarm tier's own route, the same one + # ./swarm-controller.nix exports to. `/` on that vhost answers 404. + # + # The port suffix follows ./hive-forge/default.nix: elided on the canonical + # 443 so this renders byte for byte like the tier's other two consumers of + # this name, present when an operator moved the gateway's TLS port. + otelFirstHop = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${swarmOtelCfg.producerName}"; + + swarmOtelCfg = hyperhiveCfg.swarm.otel; + gatewayPortSuffix = + let + p = hyperhiveCfg.gateway.httpsPort; + in + lib.optionalString (p != 443) ":${toString p}"; # Non-zero is what SERVES the endpoint at all — the switch is a duration, not # a boolean, so a zero here is an openbao that answers 404 on a listener @@ -1637,7 +1648,7 @@ in "none" ]); message = '' - services.hyperhive.otel.enable is on, so ${cfg.machine} forwards its + services.hyperhive.deploy.swarm-otel.enable is on, so ${cfg.machine} forwards its own journal — but services.journald.storage is "${config.services.journald.storage}" in this container. @@ -1705,8 +1716,8 @@ in ]; # `otlphttp` unconditionally, unlike the agent forwarder's - # protocol-derived exporter name: the first hop's receiver - # speaks OTLP/HTTP protobuf whatever the hive's *upstream* + # protocol-derived exporter name: the swarm receiver speaks + # OTLP/HTTP protobuf whatever this tier's *upstream* # protocol is, which is the same reason # ./hive-c0re/environment.nix pins the protocol it hands out. # `endpoint` is a BASE the exporter appends `/v1/logs` to. diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index 8345f2a8..cae6b220 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -47,13 +47,11 @@ let deploy.swarm-otel.enable = false; }; - # The store on a host whose HIVE collector is running — the only state in - # which the container forwards its own journal, since that collector is the - # hop it forwards to. `clientSecretFile` is what ../host-modules/otel.nix's - # identity assertion demands of any hive with the tier on. `swarm-otel` is - # deliberately left off: the forwarder is a function of the first hop - # existing, and pairing the two fixtures would make a case that passed on the - # wrong condition. + # The store on a host whose HIVE collector is running and whose SWARM one is + # not. The forwarder is a function of the swarm tier, so this fixture pins + # the negative: a hive collector beside the store buys it nothing. + # `clientSecretFile` is what ../host-modules/otel.nix's identity assertion + # demands of any hive with the tier on. baoWithHiveOtel = hive { deploy.bao.enable = true; otel.enable = true; @@ -153,7 +151,7 @@ let name = "the store's container forwards its own journal"; ok = let - s = (baoForwarder baoWithHiveOtel).settings; + s = (baoForwarder baoWithCollector).settings; p = s.service.pipelines.logs; in s.receivers.journald.directory == "/var/log/journal" @@ -169,23 +167,43 @@ let # shipping only the units someone remembered to name — the failure this # forwarder exists to end. name = "the store's forwarder filters no units"; - ok = !((baoForwarder baoWithHiveOtel).settings.receivers.journald ? units); + ok = !((baoForwarder baoWithCollector).settings.receivers.journald ? units); } { - # Both ends of the first hop, on ONE host, because a mismatch between - # them is silent in both directions: the exporter retries into nothing - # and the receiver never hears from it. - name = "the store's forwarder exports to this hive's own collector"; + # Both ends of the hop, because a mismatch between them is silent in both + # directions: the exporter retries into a 404 and the receiver never + # hears from it. Not "on one host" any more — the far end is the gateway + # vhost the swarm collector serves its name on, which is the whole point + # of addressing it by name rather than by a bridge address. + name = "the store's forwarder exports to the swarm collector's own route"; ok = - (baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint - == "http://${baoWithHiveOtel.services.opentelemetry-collector.settings.receivers.otlp.protocols.http.endpoint}"; + let + otel = baoWithCollector.services.hyperhive.swarm.otel; + vhost = baoWithCollector.services.nginx.virtualHosts.${otel.domain}; + in + (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint + == "https://${otel.domain}/${otel.producerName}" + && vhost.locations ? "/${otel.producerName}/"; } { - # Absence arm — `baoNoCollector` runs neither tier, so there is no first - # hop on this host at all. A forwarder rendered anyway would start, find - # nothing listening, and retry forever while reporting healthy. - name = "a store with no hive collector renders no forwarder"; - ok = !(baoForwarder baoNoCollector).enable; + # The case the old hive-tier gate got wrong: this host runs the swarm's + # collector and no hive collector at all, and the forwarder still exists + # with an address that resolves. Gated on `otel.enable` it rendered + # nothing here, so the store's journal left no trace anywhere. + name = "a store on a host with no hive collector still forwards"; + ok = + !baoWithCollector.services.hyperhive.otel.enable + && (baoForwarder baoWithCollector).enable + && (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != ""; + } + { + # Absence arm — no swarm collector, so nothing this side of the gateway + # serves that name. A forwarder rendered anyway would start, find nothing + # listening, and retry forever while reporting healthy. `baoWithHiveOtel` + # is the pointed half: a hive collector beside the store is not a reason + # to forward, which is exactly what the old gate assumed. + name = "a store with no swarm collector renders no forwarder"; + ok = !(baoForwarder baoNoCollector).enable && !(baoForwarder baoWithHiveOtel).enable; } ]; in