vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

nix: address swarm-bao's journal forwarder by swarm name

Browse source 
The forwarder pointed at the hive bridge address and was gated on the
hive's `otel.enable`, so it existed only where a hive collector stood
beside it. It now exports to `swarm.otel.domain` — the gateway-served
name that resolves locally when co-located and over the network
otherwise — on the swarm tier's own producer route, and is gated on
`deploy.swarm-otel.enable` like its sibling `scrapeHere`.

Refs #4526
This commit is contained in:
atlas 2026-09-19 18:32:13 +02:00 • committed by mara
commit ca8fc4ca64
2 changed files with 66 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

43
nix/host-modules/swarm-bao.nix
View file

@ -455,20 +455,31 @@ let
scrapeHere = deployCfg.swarm-otel.enable; scrapeHere = deployCfg.swarm-otel.enable;
# Whether this container forwards its own journal, and it is gated on the # Whether this container forwards its own journal, gated exactly like
# HIVE collector rather than the swarm one: the forwarder below pushes to the # `scrapeHere` above: the store is a swarm-tier component, so what decides
# first hop, and with `otel.enable` off nothing listens at that address at # this is the swarm collector existing, never which hive happens to stand
# all — a collector aimed at it would retry forever while looking healthy. # beside it. On the hive flag it rendered no forwarder at all on a host that
shipJournal = hyperhiveCfg.otel.enable; # runs the swarm's own collector but not a hive's.
shipJournal = deployCfg.swarm-otel.enable;
# The first hop for anything running on this host, byte for byte what # The swarm collector's own name, served by the gateway — the address
# hive-c0re hands every agent (`firstHop` in ./hive-c0re/environment.nix). # `swarm.otel.domain` exists to be, resolved by dnsmasq on a co-located host
# Plain http with no credential is the tier boundary working as designed, not # and over the real network otherwise, with no loopback-vs-remote knob to get
# an omission: the hive's collector is the only thing here that holds one, # wrong. The path selects WHICH receiver: this container is a swarm-level
# and presenting it to the swarm is its job. This container shares the host # producer and not a hive, so it is the swarm tier's own route, the same one
# netns (`privateNetwork = false` below), so the bridge address is reachable # ./swarm-controller.nix exports to. `/` on that vhost answers 404.
# from inside it without the firewall hole an agent container needs. #
otelFirstHop = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.otel.collector.port}"; # The port suffix follows ./hive-forge/default.nix: elided on the canonical
# 443 so this renders byte for byte like the tier's other two consumers of
# this name, present when an operator moved the gateway's TLS port.
otelFirstHop = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${swarmOtelCfg.producerName}";
swarmOtelCfg = hyperhiveCfg.swarm.otel;
gatewayPortSuffix =
let
p = hyperhiveCfg.gateway.httpsPort;
in
lib.optionalString (p != 443) ":${toString p}";
# Non-zero is what SERVES the endpoint at all — the switch is a duration, not # Non-zero is what SERVES the endpoint at all — the switch is a duration, not
# a boolean, so a zero here is an openbao that answers 404 on a listener # a boolean, so a zero here is an openbao that answers 404 on a listener
@ -1637,7 +1648,7 @@ in
"none" "none"
]); ]);
message = '' message = ''
services.hyperhive.otel.enable is on, so ${cfg.machine} forwards its services.hyperhive.deploy.swarm-otel.enable is on, so ${cfg.machine} forwards its
own journal — but services.journald.storage is own journal — but services.journald.storage is
"${config.services.journald.storage}" in this container. "${config.services.journald.storage}" in this container.
@ -1705,8 +1716,8 @@ in
]; ];
# `otlphttp` unconditionally, unlike the agent forwarder's # `otlphttp` unconditionally, unlike the agent forwarder's
# protocol-derived exporter name: the first hop's receiver # protocol-derived exporter name: the swarm receiver speaks
# speaks OTLP/HTTP protobuf whatever the hive's *upstream* # OTLP/HTTP protobuf whatever this tier's *upstream*
# protocol is, which is the same reason # protocol is, which is the same reason
# ./hive-c0re/environment.nix pins the protocol it hands out. # ./hive-c0re/environment.nix pins the protocol it hands out.
# `endpoint` is a BASE the exporter appends `/v1/logs` to. # `endpoint` is a BASE the exporter appends `/v1/logs` to.

58
nix/module-eval/bao-otel-collector.nix
View file

@ -47,13 +47,11 @@ let
deploy.swarm-otel.enable = false; deploy.swarm-otel.enable = false;
}; };
# The store on a host whose HIVE collector is running — the only state in # The store on a host whose HIVE collector is running and whose SWARM one is
# which the container forwards its own journal, since that collector is the # not. The forwarder is a function of the swarm tier, so this fixture pins
# hop it forwards to. `clientSecretFile` is what ../host-modules/otel.nix's # the negative: a hive collector beside the store buys it nothing.
# identity assertion demands of any hive with the tier on. `swarm-otel` is # `clientSecretFile` is what ../host-modules/otel.nix's identity assertion
# deliberately left off: the forwarder is a function of the first hop # demands of any hive with the tier on.
# existing, and pairing the two fixtures would make a case that passed on the
# wrong condition.
baoWithHiveOtel = hive { baoWithHiveOtel = hive {
deploy.bao.enable = true; deploy.bao.enable = true;
otel.enable = true; otel.enable = true;
@ -153,7 +151,7 @@ let
name = "the store's container forwards its own journal"; name = "the store's container forwards its own journal";
ok = ok =
let let
s = (baoForwarder baoWithHiveOtel).settings; s = (baoForwarder baoWithCollector).settings;
p = s.service.pipelines.logs; p = s.service.pipelines.logs;
in in
s.receivers.journald.directory == "/var/log/journal" s.receivers.journald.directory == "/var/log/journal"
@ -169,23 +167,43 @@ let
# shipping only the units someone remembered to name — the failure this # shipping only the units someone remembered to name — the failure this
# forwarder exists to end. # forwarder exists to end.
name = "the store's forwarder filters no units"; name = "the store's forwarder filters no units";
ok = !((baoForwarder baoWithHiveOtel).settings.receivers.journald ? units); ok = !((baoForwarder baoWithCollector).settings.receivers.journald ? units);
} }
{ {
# Both ends of the first hop, on ONE host, because a mismatch between # Both ends of the hop, because a mismatch between them is silent in both
# them is silent in both directions: the exporter retries into nothing # directions: the exporter retries into a 404 and the receiver never
# and the receiver never hears from it. # hears from it. Not "on one host" any more — the far end is the gateway
name = "the store's forwarder exports to this hive's own collector"; # vhost the swarm collector serves its name on, which is the whole point
# of addressing it by name rather than by a bridge address.
name = "the store's forwarder exports to the swarm collector's own route";
ok = ok =
(baoForwarder baoWithHiveOtel).settings.exporters.otlphttp.endpoint let
== "http://${baoWithHiveOtel.services.opentelemetry-collector.settings.receivers.otlp.protocols.http.endpoint}"; otel = baoWithCollector.services.hyperhive.swarm.otel;
vhost = baoWithCollector.services.nginx.virtualHosts.${otel.domain};
in
(baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint
== "https://${otel.domain}/${otel.producerName}"
&& vhost.locations ? "/${otel.producerName}/";
} }
{ {
# Absence arm — `baoNoCollector` runs neither tier, so there is no first # The case the old hive-tier gate got wrong: this host runs the swarm's
# hop on this host at all. A forwarder rendered anyway would start, find # collector and no hive collector at all, and the forwarder still exists
# nothing listening, and retry forever while reporting healthy. # with an address that resolves. Gated on `otel.enable` it rendered
name = "a store with no hive collector renders no forwarder"; # nothing here, so the store's journal left no trace anywhere.
ok = !(baoForwarder baoNoCollector).enable; name = "a store on a host with no hive collector still forwards";
ok =
!baoWithCollector.services.hyperhive.otel.enable
&& (baoForwarder baoWithCollector).enable
&& (baoForwarder baoWithCollector).settings.exporters.otlphttp.endpoint != "";
}
{
# Absence arm — no swarm collector, so nothing this side of the gateway
# serves that name. A forwarder rendered anyway would start, find nothing
# listening, and retry forever while reporting healthy. `baoWithHiveOtel`
# is the pointed half: a hive collector beside the store is not a reason
# to forward, which is exactly what the old gate assumed.
name = "a store with no swarm collector renders no forwarder";
ok = !(baoForwarder baoNoCollector).enable && !(baoForwarder baoWithHiveOtel).enable;
} }
]; ];
in in