diff --git a/docs/networking/network.md b/docs/networking/network.md index f2f828c3..e6546918 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -116,12 +116,13 @@ schemes pick their own. dnsmasq is **authoritative** for the hive's own zone (``) plus whatever swarm-service names this host contributes via `gateway.localNames` — `forge.` and `chat.` -(matrix) when this host runs those services, and `auth.` -when it runs authelia — answering each with the bridge IP (where nginx -is reachable). Note forge and matrix are swarm-domain names, not -sub-domains of ``: a swarm runs one forge and one -homeserver, so their names belong to the swarm rather than to whichever -hive happens to host them. dnsmasq forwards everything else to the host's +(matrix) when this host runs those services, `auth.` +when it runs authelia, and `hive-collector.` on every hive +with `otel.enable` — answering each with the bridge IP (where nginx +is reachable, and where the hive's own collector listens). Note forge and +matrix are swarm-domain names, not sub-domains of ``: a swarm +runs one forge and one homeserver, so their names belong to the swarm rather +than to whichever hive happens to host them. dnsmasq forwards everything else to the host's own resolvers: it runs on the host and reads the host's `/etc/resolv.conf` directly. Containers don't need to know the upstream — they query the bridge IP and dnsmasq does the right thing @@ -187,8 +188,10 @@ that `otel.endpoint` is the _upstream_, which no agent ever dials. See `services.hyperhive.network.exposeHostPorts = [ 4318 ];` opens each listed TCP port `P` on the bridge-interface `allowedTCPPorts`, so an -agent can connect to `:P` (point the collector endpoint at -`http://:4318`, default `http://10.42.0.1:4318`). +agent can connect to `:P` (for a dev OTLP collector, point its +endpoint at `http://:4318`, default `http://10.42.0.1:4318`). +The hive's own collector needs none of this: agents dial it by +`otel.collector.host`. diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 41c4922e..c57328cb 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -41,9 +41,12 @@ destination in its own right. With both, telemetry goes to both. See **Telemetry leaves a hive exactly one way: through the collector that `enable` starts on the host.** Agents never talk to `endpoint` themselves — -they export unauthenticated to a bridge address only their own containers can -reach (`http://:`; the otel module opens that port -on the bridge itself). That collector forwards to the swarm's +they export unauthenticated to a collector address only their own containers +can reach. They dial it as +`http://hive-collector.:`, a name identical on +every hive, which each hive's own dnsmasq answers with its bridge IP, so an +agent's build carries no hive-specific address. The otel module opens that +port on the bridge itself. That collector forwards to the swarm's ([`swarm/services.md`](../swarm/services.md#telemetry-collector-otel)), the single process holding the upstream credential and the only writer to the swarm's stores. No agent holds a copy, and neither does the hive. diff --git a/hive-c0re/src/stats/otel_metrics.rs b/hive-c0re/src/stats/otel_metrics.rs index d5c394a3..9fd8ec90 100644 --- a/hive-c0re/src/stats/otel_metrics.rs +++ b/hive-c0re/src/stats/otel_metrics.rs @@ -2,7 +2,8 @@ //! sampled in [`super::container_stats`] feeds: it rides those gauges out to the //! hive's own collector — the same first hop the agents use, named by the //! standard `OTEL_EXPORTER_OTLP_ENDPOINT` (see `nix/host-modules/hive-c0re`, -//! which sets it from the same binding it hands agents). No toggle of +//! which sets it to the collector's bridge address where agents get its +//! swarm-wide name). No toggle of //! its own, and no credential: a hive's collector takes unauthenticated OTLP //! on the bridge, and the only hop that presents anything is the swarm tier's, //! which is the one that leaves the swarm. diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 2696bf66..549078e5 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -83,11 +83,13 @@ in # don't render no-op env lines. let otel = config.services.hyperhive.otel; - # The first hop, bound once and consumed twice below: what agents are - # handed, and where hive-c0re's own exporter sends. One binding so the - # address a hive tells its agents about and the one it uses itself - # cannot drift apart. - firstHop = "http://${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}"; + # The first hop, this hive's own collector, under the two spellings its + # two kinds of caller need. Agents get the swarm-wide name, so their + # builds carry nothing hive-specific; this daemon gets the bridge + # address, because on the host the name may resolve to loopback + # (`gateway.localHostsEntry`), where the receiver does not listen. + agentFirstHop = "http://${otel.collector.host}:${toString otel.collector.port}"; + hostFirstHop = "http://${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}"; in { # `otel.endpoint` means "where telemetry ultimately goes" and keeps @@ -95,7 +97,7 @@ in # always this hive's own collector. Deriving it rather than # redefining `endpoint` is what lets every existing deployment keep # its configured value untouched. - HYPERHIVE_OTEL_ENDPOINT = firstHop; + HYPERHIVE_OTEL_ENDPOINT = agentFirstHop; # hive-c0re's OWN container-resource exporter (stats/otel_metrics.rs) # reads the STANDARD OTLP variable — the same one hive-metric and every # agent read — and lets the SDK resolve the URL, which appends the @@ -108,7 +110,7 @@ in # replace this one: # it is the agent-config transport meta.rs reads, and the SDK does not # know that name. - OTEL_EXPORTER_OTLP_ENDPOINT = firstHop; + OTEL_EXPORTER_OTLP_ENDPOINT = hostFirstHop; # The first hop is the collector's OTLP/HTTP receiver, which speaks # protobuf regardless of what the upstream wants — `otel.protocol` # describes the *upstream* link, and the collector's own exporter is diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index a87b81d2..df609ea0 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -67,8 +67,10 @@ in Enabling this also runs this hive's collector on the host: there is exactly one way telemetry leaves this hive, and it is through that - collector. Agents export unauthenticated to a bridge address only - their own containers can reach. + collector. Agents export unauthenticated to + {option}`services.hyperhive.otel.collector.host`, a name this hive's + resolver answers with a bridge address only its own containers can + reach. That collector forwards to the swarm's ({option}`services.hyperhive.deploy.swarm-otel.enable`), which holds the @@ -203,6 +205,28 @@ in ''; }; + collector.host = lib.mkOption { + type = lib.types.str; + readOnly = true; + # Total on a null swarm domain, as in ./swarm-otel-service.nix: the + # required-domain assertion should be what an operator sees. + default = "hive-collector.${ + if config.services.hyperhive.swarm.domain == null then + "invalid" + else + config.services.hyperhive.swarm.domain + }"; + defaultText = lib.literalExpression ''"hive-collector.''${services.hyperhive.swarm.domain}"''; + description = '' + The name agents reach this hive's collector on. The same string on + every hive: each hive's own resolver answers it with its own bridge + IP, so an agent's build carries no hive-specific address. + + No hive can claim this name as its domain: + `nix/reserved-hive-fragments.nix` forbids `hive` in any hive name. + ''; + }; + telemetryPort = lib.mkOption { type = lib.types.port; default = 8888; @@ -357,6 +381,9 @@ in # the port on the bridge interface only. services.hyperhive.network.exposeHostPorts = [ otel.collector.port ]; + # The name agents dial, answered with the bridge IP the receiver binds. + services.hyperhive.gateway.localNames = [ otel.collector.host ]; + # …and the receiver binds that bridge address, so the interface has # to exist. `exposeHostPorts` is a firewall hole, not a demand. services.hyperhive.network.enable = lib.mkDefault true; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index ecc4f2ee..00e0c0a4 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -199,6 +199,20 @@ let && !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL) && e.HIVE_AGENT_NATS_URL == e.HIVE_C0RE_NATS_URL; } + { + # What agents are handed lands in their builds, so it has to be the same + # on every hive: a name, which this hive's resolver answers with the + # bridge IP its collector binds. hive-c0re itself keeps the address. + name = "the agents' collector address is a swarm-wide name this hive resolves to its bridge"; + ok = + let + e = allLocal.systemd.services.hive-c0re.environment; + bridge = allLocal.services.hyperhive.network.bridgeIp; + in + e.HYPERHIVE_OTEL_ENDPOINT == "http://hive-collector.t.local:4318" + && e.OTEL_EXPORTER_OTLP_ENDPOINT == "http://${bridge}:4318" + && lib.elem "/hive-collector.t.local/${bridge}" allLocal.services.dnsmasq.settings.address; + } { # The agents mint against the swarm's IdP, the same endpoint the hive's # own client uses — a hive-local guess would produce a token the queue