From d4e20803ec1fab601779ee0ffcbf04f6347ea087 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 16 Oct 2026 02:26:04 +0200 Subject: [PATCH 1/2] otel: agents reach their hive's collector by a swarm-wide name Agents are handed `http://hive-collector.:` as their otel endpoint instead of `http://:`. Every hive running its collector contributes the name to `gateway.localNames`, so its own dnsmasq answers it with its bridge IP, the way the queue's name already works. The rendered meta flake then carries no hive-specific address, which a swarm-level render needs (#4857, operator ruling 100744: "b"). hive-c0re's own exporter keeps the bridge address: on the host the name may resolve to loopback through `gateway.localHostsEntry`, where the receiver does not listen. The name sits under the swarm domain and starts with `hive`, a fragment no hive name may contain, so no hive's default domain can take it. --- docs/networking/network.md | 7 +++--- docs/scheduler/observability.md | 6 ++++-- hive-c0re/src/stats/otel_metrics.rs | 3 ++- nix/host-modules/hive-c0re/environment.nix | 16 ++++++++------ nix/host-modules/otel.nix | 25 ++++++++++++++++++++++ nix/module-eval/core-toggle.nix | 14 ++++++++++++ 6 files changed, 58 insertions(+), 13 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index f2f828c3..3fd990d8 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -116,9 +116,10 @@ schemes pick their own. dnsmasq is **authoritative** for the hive's own zone (``) plus whatever swarm-service names this host contributes via `gateway.localNames` — `forge.` and `chat.` -(matrix) when this host runs those services, and `auth.` -when it runs authelia — answering each with the bridge IP (where nginx -is reachable). Note forge and matrix are swarm-domain names, not +(matrix) when this host runs those services, `auth.` +when it runs authelia, and `hive-collector.` on every hive +with `otel.enable` — answering each with the bridge IP (where nginx +is reachable, and where the hive's own collector listens). Note forge and matrix are swarm-domain names, not sub-domains of ``: a swarm runs one forge and one homeserver, so their names belong to the swarm rather than to whichever hive happens to host them. dnsmasq forwards everything else to the host's diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 41c4922e..52ed90ce 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -42,8 +42,10 @@ destination in its own right. With both, telemetry goes to both. See **Telemetry leaves a hive exactly one way: through the collector that `enable` starts on the host.** Agents never talk to `endpoint` themselves — they export unauthenticated to a bridge address only their own containers can -reach (`http://:`; the otel module opens that port -on the bridge itself). That collector forwards to the swarm's +reach. They dial it as `http://hive-collector.:`, +a name identical on every hive, which each hive's own dnsmasq +answers with its bridge IP, so an agent's build carries no hive-specific +address. The otel module opens that port on the bridge itself. That collector forwards to the swarm's ([`swarm/services.md`](../swarm/services.md#telemetry-collector-otel)), the single process holding the upstream credential and the only writer to the swarm's stores. No agent holds a copy, and neither does the hive. diff --git a/hive-c0re/src/stats/otel_metrics.rs b/hive-c0re/src/stats/otel_metrics.rs index d5c394a3..9fd8ec90 100644 --- a/hive-c0re/src/stats/otel_metrics.rs +++ b/hive-c0re/src/stats/otel_metrics.rs @@ -2,7 +2,8 @@ //! sampled in [`super::container_stats`] feeds: it rides those gauges out to the //! hive's own collector — the same first hop the agents use, named by the //! standard `OTEL_EXPORTER_OTLP_ENDPOINT` (see `nix/host-modules/hive-c0re`, -//! which sets it from the same binding it hands agents). No toggle of +//! which sets it to the collector's bridge address where agents get its +//! swarm-wide name). No toggle of //! its own, and no credential: a hive's collector takes unauthenticated OTLP //! on the bridge, and the only hop that presents anything is the swarm tier's, //! which is the one that leaves the swarm. diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 2696bf66..549078e5 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -83,11 +83,13 @@ in # don't render no-op env lines. let otel = config.services.hyperhive.otel; - # The first hop, bound once and consumed twice below: what agents are - # handed, and where hive-c0re's own exporter sends. One binding so the - # address a hive tells its agents about and the one it uses itself - # cannot drift apart. - firstHop = "http://${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}"; + # The first hop, this hive's own collector, under the two spellings its + # two kinds of caller need. Agents get the swarm-wide name, so their + # builds carry nothing hive-specific; this daemon gets the bridge + # address, because on the host the name may resolve to loopback + # (`gateway.localHostsEntry`), where the receiver does not listen. + agentFirstHop = "http://${otel.collector.host}:${toString otel.collector.port}"; + hostFirstHop = "http://${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}"; in { # `otel.endpoint` means "where telemetry ultimately goes" and keeps @@ -95,7 +97,7 @@ in # always this hive's own collector. Deriving it rather than # redefining `endpoint` is what lets every existing deployment keep # its configured value untouched. - HYPERHIVE_OTEL_ENDPOINT = firstHop; + HYPERHIVE_OTEL_ENDPOINT = agentFirstHop; # hive-c0re's OWN container-resource exporter (stats/otel_metrics.rs) # reads the STANDARD OTLP variable — the same one hive-metric and every # agent read — and lets the SDK resolve the URL, which appends the @@ -108,7 +110,7 @@ in # replace this one: # it is the agent-config transport meta.rs reads, and the SDK does not # know that name. - OTEL_EXPORTER_OTLP_ENDPOINT = firstHop; + OTEL_EXPORTER_OTLP_ENDPOINT = hostFirstHop; # The first hop is the collector's OTLP/HTTP receiver, which speaks # protobuf regardless of what the upstream wants — `otel.protocol` # describes the *upstream* link, and the collector's own exporter is diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index a87b81d2..0550ede6 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -203,6 +203,28 @@ in ''; }; + collector.host = lib.mkOption { + type = lib.types.str; + readOnly = true; + # Total on a null swarm domain, as in ./swarm-otel-service.nix: the + # required-domain assertion should be what an operator sees. + default = "hive-collector.${ + if config.services.hyperhive.swarm.domain == null then + "invalid" + else + config.services.hyperhive.swarm.domain + }"; + defaultText = lib.literalExpression ''"hive-collector.''${services.hyperhive.swarm.domain}"''; + description = '' + The name agents reach this hive's collector on. The same string on + every hive: each hive's own resolver answers it with its own bridge + IP, so an agent's build carries no hive-specific address. + + No hive can claim this name as its domain: + `nix/reserved-hive-fragments.nix` forbids `hive` in any hive name. + ''; + }; + telemetryPort = lib.mkOption { type = lib.types.port; default = 8888; @@ -357,6 +379,9 @@ in # the port on the bridge interface only. services.hyperhive.network.exposeHostPorts = [ otel.collector.port ]; + # The name agents dial, answered with the bridge IP the receiver binds. + services.hyperhive.gateway.localNames = [ otel.collector.host ]; + # …and the receiver binds that bridge address, so the interface has # to exist. `exposeHostPorts` is a firewall hole, not a demand. services.hyperhive.network.enable = lib.mkDefault true; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index ecc4f2ee..00e0c0a4 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -199,6 +199,20 @@ let && !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL) && e.HIVE_AGENT_NATS_URL == e.HIVE_C0RE_NATS_URL; } + { + # What agents are handed lands in their builds, so it has to be the same + # on every hive: a name, which this hive's resolver answers with the + # bridge IP its collector binds. hive-c0re itself keeps the address. + name = "the agents' collector address is a swarm-wide name this hive resolves to its bridge"; + ok = + let + e = allLocal.systemd.services.hive-c0re.environment; + bridge = allLocal.services.hyperhive.network.bridgeIp; + in + e.HYPERHIVE_OTEL_ENDPOINT == "http://hive-collector.t.local:4318" + && e.OTEL_EXPORTER_OTLP_ENDPOINT == "http://${bridge}:4318" + && lib.elem "/hive-collector.t.local/${bridge}" allLocal.services.dnsmasq.settings.address; + } { # The agents mint against the swarm's IdP, the same endpoint the hive's # own client uses — a hive-local guess would produce a token the queue From b66a09f65bd46a4bc1cd098dd4c7caafa03a420d Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 16 Oct 2026 02:36:33 +0200 Subject: [PATCH 2/2] otel: finish the agent collector name in option and network docs The otel enable description and the exposeHostPorts example in network.md still described agents dialing a bridge address. Rewrap the two long lines the earlier edit left. --- docs/networking/network.md | 14 ++++++++------ docs/scheduler/observability.md | 11 ++++++----- nix/host-modules/otel.nix | 6 ++++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/networking/network.md b/docs/networking/network.md index 3fd990d8..e6546918 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -119,10 +119,10 @@ plus whatever swarm-service names this host contributes via (matrix) when this host runs those services, `auth.` when it runs authelia, and `hive-collector.` on every hive with `otel.enable` — answering each with the bridge IP (where nginx -is reachable, and where the hive's own collector listens). Note forge and matrix are swarm-domain names, not -sub-domains of ``: a swarm runs one forge and one -homeserver, so their names belong to the swarm rather than to whichever -hive happens to host them. dnsmasq forwards everything else to the host's +is reachable, and where the hive's own collector listens). Note forge and +matrix are swarm-domain names, not sub-domains of ``: a swarm +runs one forge and one homeserver, so their names belong to the swarm rather +than to whichever hive happens to host them. dnsmasq forwards everything else to the host's own resolvers: it runs on the host and reads the host's `/etc/resolv.conf` directly. Containers don't need to know the upstream — they query the bridge IP and dnsmasq does the right thing @@ -188,8 +188,10 @@ that `otel.endpoint` is the _upstream_, which no agent ever dials. See `services.hyperhive.network.exposeHostPorts = [ 4318 ];` opens each listed TCP port `P` on the bridge-interface `allowedTCPPorts`, so an -agent can connect to `:P` (point the collector endpoint at -`http://:4318`, default `http://10.42.0.1:4318`). +agent can connect to `:P` (for a dev OTLP collector, point its +endpoint at `http://:4318`, default `http://10.42.0.1:4318`). +The hive's own collector needs none of this: agents dial it by +`otel.collector.host`. diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 52ed90ce..c57328cb 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -41,11 +41,12 @@ destination in its own right. With both, telemetry goes to both. See **Telemetry leaves a hive exactly one way: through the collector that `enable` starts on the host.** Agents never talk to `endpoint` themselves — -they export unauthenticated to a bridge address only their own containers can -reach. They dial it as `http://hive-collector.:`, -a name identical on every hive, which each hive's own dnsmasq -answers with its bridge IP, so an agent's build carries no hive-specific -address. The otel module opens that port on the bridge itself. That collector forwards to the swarm's +they export unauthenticated to a collector address only their own containers +can reach. They dial it as +`http://hive-collector.:`, a name identical on +every hive, which each hive's own dnsmasq answers with its bridge IP, so an +agent's build carries no hive-specific address. The otel module opens that +port on the bridge itself. That collector forwards to the swarm's ([`swarm/services.md`](../swarm/services.md#telemetry-collector-otel)), the single process holding the upstream credential and the only writer to the swarm's stores. No agent holds a copy, and neither does the hive. diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 0550ede6..df609ea0 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -67,8 +67,10 @@ in Enabling this also runs this hive's collector on the host: there is exactly one way telemetry leaves this hive, and it is through that - collector. Agents export unauthenticated to a bridge address only - their own containers can reach. + collector. Agents export unauthenticated to + {option}`services.hyperhive.otel.collector.host`, a name this hive's + resolver answers with a bridge address only its own containers can + reach. That collector forwards to the swarm's ({option}`services.hyperhive.deploy.swarm-otel.enable`), which holds the