From d2fb4bff7914901674085795deab352b355d8e79 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 18 Aug 2026 20:45:13 +0200 Subject: [PATCH] feat(#3125): reshape the hive-to-swarm OTEL hop by domain Drops swarm.otel.url (a loopback default an operator had to override on a split host) in favor of swarm.otel.domain -- the same gateway.localNames + nginx-vhost-through-the-gateway shape every other swarm service (authelia, grafana, victoriametrics, ui) already uses. The hive tier's exporter now reaches it as https:// unconditionally, resolved locally by dnsmasq on a co-located host and over the real network otherwise, instead of a config knob nobody sets until they hit the silent drop. Costs CA trust on the hive tier: otel.nix wires lib/hive-ca-trust.nix's trustBundle with hostUnit = true on the opentelemetry-collector host unit, the same flag #3441/#3442 added for swarm-controller and hive-c0re. mara, #3125 comment 58363: "go c". --- docs/observability.md | 15 ++--- docs/swarm/services.md | 9 ++- nix/host-modules/otel.nix | 47 ++++++++++++--- nix/host-modules/swarm-otel.nix | 60 ++++++++++++++------ nix/host-modules/swarm-required-services.nix | 2 +- 5 files changed, 96 insertions(+), 37 deletions(-) diff --git a/docs/observability.md b/docs/observability.md index f2ec11d4..19daedc6 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -30,16 +30,11 @@ reach. That collector forwards to the swarm's the single process holding the upstream credential and the only writer to the swarm's store. No agent holds a copy, and neither does this hive. -⚠️ **On a hive that does not run the swarm's services, say where that swarm -collector is:** - -```nix -services.hyperhive.swarm.otel.url = "http://services-host.example:4319"; -``` - -Left unset it points at this host, where nothing is listening — the collector -starts, agents export happily, and the samples go nowhere. The service host -itself needs no such line. +The hive collector reaches the swarm collector by its gateway name +(`swarm.otel.domain`, default `otel.`) — the same DNS-and-CA-trust +shape every hive-to-swarm-service hop uses, not a URL an operator has to point +anywhere. A hive that does not run the swarm's services still resolves that +name through the gateway; nothing here needs setting for the split-host case. ⚠️ **The collector is therefore in the path of all telemetry.** It runs on the same host as the agents and restarts on failure, and telemetry is not the diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 23742b27..bb5dc93e 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -113,9 +113,16 @@ than OTLP's usual `4318`, which the hive tier already uses — swarm containers share the host's network namespace, so two collectors on one port is a coin toss at runtime rather than an error at build time. +Every hive's own collector reaches this one by its gateway name, +`swarm.otel.domain` (default `otel.`) — the same +by-domain-through-the-gateway shape every other swarm service uses, not a +loopback URL an operator has to redirect. There is nothing to set on a hive +that does not run the swarm's services; the name resolves through the +gateway either way. + | Option | When you'd touch it | |---|---| -| `swarm.otel.url` | **On every hive that does not run the swarm's services.** It defaults to this host, so a hive left at the default forwards into nothing and loses its telemetry silently. Point it at the services host: `"http://services-host.example:4319"`. | +| `swarm.otel.domain` | Only to rename it — the default already resolves correctly for every hive in the swarm. | | `swarm.otel.port` | Only if something else on the services host already claims `4319`. | With neither `otel.endpoint` nor the store enabled, this collector is diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 11ca025d..53d43329 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -14,11 +14,41 @@ # upstream options declared below describe that far end and are read one # tier up — they stay here because they mean what they have always meant. { + pkgs, lib, config, ... }: +let + # This tier now reaches the swarm's collector by name through the + # gateway (`swarm-otel.nix`'s `domain`) instead of a loopback URL, so it + # needs the same hive-CA trust every other host consumer of an `https://` + # swarm-service name needs — see `swarm-controller.nix` for the sibling + # wiring this copies. + # + # `hostUnit`: `opentelemetry-collector` is a host systemd service, not a + # container, so it reads the CA from the host path and the bundle oneshot + # waits on `hive-tls-ca.service` itself. `enable`: `imports` is + # unconditional at the host's top level, so without it a hive with this + # tier off would still get a bundle oneshot and a phantom + # `opentelemetry-collector` service holding an `SSL_CERT_FILE`. + caTrust = import ./lib/hive-ca-trust.nix { + inherit lib; + tlsCfg = config.services.hyperhive.tls; + gatewayCfg = config.services.hyperhive.gateway; + }; +in { + imports = [ + (caTrust.trustBundle { + inherit pkgs; + name = "hive-otel"; + consumers = [ "opentelemetry-collector" ]; + hostUnit = true; + enable = config.services.hyperhive.otel.enable; + }) + ]; + options.services.hyperhive.otel = { enable = lib.mkEnableOption '' hive-wide export of every agent's Claude Code stats (token usage, @@ -225,14 +255,15 @@ # store exporter needs `metrics_endpoint` while this one must # not have it. # - # Addressed by the option rather than by a loopback literal: - # the default already points at the co-located tier, and a - # hive whose swarm collector lives elsewhere then names it in - # config instead of needing this file changed. A loopback - # literal is correct only while listener and caller share a - # netns, an assumption that has cost this project two - # outages. - endpoint = config.services.hyperhive.swarm.otel.url; + # By name through the gateway, not a loopback literal: a + # loopback literal is correct only while listener and caller + # share a netns, an assumption that has cost this project two + # outages, and it is exactly the split-host case a swarm + # service name exists to make a config fact rather than a code + # change. `https://` because that name resolves through the + # gateway even on a co-located host — see `caTrust` above for + # the trust half that makes this verify. + endpoint = "https://${config.services.hyperhive.swarm.otel.domain}"; }; service.pipelines.metrics = { diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 7460c85e..0eacc0de 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -32,6 +32,14 @@ let swarmCfg = config.services.hyperhive.swarm; otelCfg = config.services.hyperhive.otel; vmCfg = config.services.hyperhive.swarm.victoriametrics; + hyperhiveCfg = config.services.hyperhive; + gatewayCfg = hyperhiveCfg.gateway; + swarmDomain = hyperhiveCfg.swarm.domain; + + # Total on a null swarm domain for the same reason every sibling module is: + # the required-domain assertion in hive-network.nix should be what an + # operator sees, not a coercion error from here. + domainBase = if swarmDomain == null then "invalid" else swarmDomain; in { options.services.hyperhive.swarm.otel = { @@ -47,8 +55,8 @@ in wherever the shared services live rather than on every hive. A hive that does not run it still runs its own hive-tier collector - (`services.hyperhive.otel.enable`) and points it here with - {option}`services.hyperhive.swarm.otel.url`. + (`services.hyperhive.otel.enable`) and reaches this one by name, at + {option}`services.hyperhive.swarm.otel.domain`. ''; }; @@ -99,29 +107,47 @@ in ''; }; - url = lib.mkOption { + domain = lib.mkOption { type = lib.types.str; - default = "http://127.0.0.1:${toString cfg.port}"; - defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString config.services.hyperhive.swarm.otel.port}"''; + default = "otel.${domainBase}"; + defaultText = lib.literalExpression ''"otel.''${services.hyperhive.swarm.domain}"''; description = '' - Where the **hive** tier sends what it receives — this collector's - OTLP/HTTP base URL. + Name the gateway serves this on. A sibling of the swarm's other + service names, so the swarm-services sub-CA can issue for it — see + `hive-tls.nix` for why a service name being a sibling rather than a + child decides which CA may sign it. - The default addresses it on loopback, which is correct while the - two tiers share a host: every swarm container runs in the host's - network namespace, so a swarm service is reachable there exactly - as the metrics store already is. - - ⚠️ That default is a *default*, not an assumption baked into the - exporter. A hive whose swarm collector runs elsewhere sets this to - that host's address, and nothing else changes — a loopback literal - written directly into the exporter would have made the split-host - case a code change instead of a config one. + This is what the **hive** tier's exporter reaches — the hive + collector is a plain producer against this name exactly like every + other client of a swarm service, resolved locally by dnsmasq on a + co-located host and over the real network otherwise. There is no + separate loopback-vs-remote knob to get wrong: `swarm-nats` is the + deliberate exception to this pattern (its cross-hive reach is the + wireguard mesh, not the gateway), everything else in this swarm + addresses its siblings by name. ''; }; }; config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) { + # The gateway name, inside `cfg.enable` — that guard is the load-bearing + # part. Every hive in a swarm may know this collector exists, but only + # the host that RUNS it may claim the name; a client hive declaring the + # vhost would answer for a service it does not have. + services.hyperhive.gateway.localNames = [ cfg.domain ]; + + # OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as + # every sibling swarm service: TLS terminates here, then plain http to + # the co-located container over loopback (shared netns, like the store + # this collector writes to). + services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { + listen = gatewayCfg.lib.listen; + extraConfig = gatewayCfg.lib.securityHeaders; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}"; + }; + }; + assertions = [ { # The tier exists to hold the upstream credential and to write the diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix index f12d84dd..80c3800a 100644 --- a/nix/host-modules/swarm-required-services.nix +++ b/nix/host-modules/swarm-required-services.nix @@ -74,7 +74,7 @@ in # The collector that feeds the pair above, and the only tier holding # the upstream credential. Same rule as the rest: once per swarm, # optional, and a hive that is not the service host is a *client* of - # it (`swarm.otel.url`) rather than a second one. + # it (by name, `swarm.otel.domain`) rather than a second one. otel.enable = lib.mkDefault swarmCfg.enableRequiredServices; };