From 094a54e7857794fa9ff7c06abf2422df49b9ad47 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 18 Aug 2026 13:09:49 +0200 Subject: [PATCH] feat(#3125): the hive tier forwards to the swarm tier and holds nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hive collector's only exporter becomes the swarm's collector, and the upstream credential, the metrics-store exporter and the choice of destination all move one tier up. Its assertion goes with them: 'endpoint or a local store' was the right rule while this tier picked the destination, and is the wrong one now. A hive that runs no swarm services has neither, forwards to a swarm collector elsewhere, and is correctly configured — the rule that replaces it lives in swarm-otel.nix, where the destinations are. The option descriptions here described a topology with one collector in it: endpoint and protocol are not what agents are handed (they get the derived first hop, see hive-c0re/environment.nix), and the credential is not read by this tier. --- nix/host-modules/otel.nix | 250 ++++++++++++++++---------------------- 1 file changed, 102 insertions(+), 148 deletions(-) diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 7d67d5aa..6a756941 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -1,11 +1,18 @@ -# Hive-wide OTEL stats export. Set ONCE here at host level; the -# meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the -# HYPERHIVE_OTEL_* env exported off hive-c0re's unit (see -# ./hive-c0re) and injects the matching `hyperhive.otel.*` build-time -# config into EVERY agent (mirroring the CA-cert injection), so each -# agent's harness exports its own Claude Code stats directly to the -# collector. There is no per-agent opt-in — this is the single switch -# for the whole hive. +# Hive-wide OTEL stats export, and the HIVE tier of the collector pair. +# Set ONCE here at host level; the meta-flake renderer +# (`hive-c0re/src/meta.rs::otel_config`) reads the HYPERHIVE_OTEL_* env +# exported off hive-c0re's unit (see ./hive-c0re) and injects the matching +# `hyperhive.otel.*` build-time config into EVERY agent (mirroring the +# CA-cert injection), so each agent's harness exports its own Claude Code +# stats directly to the collector. There is no per-agent opt-in — this is +# the single switch for the whole hive. +# +# This tier receives from this hive's agents and forwards to the swarm's +# collector (./swarm-otel.nix). It holds no credential and picks no +# destination: an agent's samples cross a hive boundary exactly once, and +# what happens after that is the swarm's decision, not a hive's. The +# upstream options declared below describe that far end and are read one +# tier up — they stay here because they mean what they have always meant. { lib, config, @@ -18,11 +25,15 @@ cost, tool calls) to an OTLP endpoint via Claude Code's built-in OpenTelemetry. One switch for all agents. - Enabling this also runs a collector on the host: there is exactly - one way telemetry leaves this hive, and it is through that + Enabling this also runs this hive's collector on the host: there is + exactly one way telemetry leaves this hive, and it is through that collector. Agents export unauthenticated to a bridge address only - their own containers can reach, and the collector is the only - holder of the upstream credential — an agent never sees it. + their own containers can reach. + + That collector forwards to the swarm's + ({option}`services.hyperhive.swarm.otel.enable`), which holds the + upstream credential and writes the swarm's store. So an agent never + sees the credential, and neither does this tier. ⚠️ The collector is therefore in the path of all telemetry. It runs on the same host as the agents and restarts on failure, and @@ -36,15 +47,19 @@ default = ""; example = "https://collector.example.com/otel"; description = '' - Upstream OTLP endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT` for - every agent. + Upstream OTLP endpoint: where telemetry ultimately goes, after it + has left the swarm. - Required when `enable` is true, **unless** this host runs the - swarm's metrics store - ({option}`services.hyperhive.swarm.victoriametrics.enable`) — that - store is a destination in its own right, and with both configured - telemetry goes to both. With neither, `enable` is refused rather - than silently exporting nowhere. + Read by the swarm's collector + ({option}`services.hyperhive.swarm.otel.enable`), which is the only + tier that holds the upstream credential. An agent is handed the + *first* hop instead — this hive's own collector — so this value is + never given to a container. + + Optional. Leave it empty and the swarm's own metrics store + ({option}`services.hyperhive.swarm.victoriametrics.enable`) is the + destination; that is a complete deployment, not a degraded one. + Set both and telemetry goes to both. ''; }; @@ -56,7 +71,12 @@ ]; default = "http/protobuf"; description = '' - OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`. + OTLP wire protocol for the **upstream** link, honoured by the + swarm collector's exporter. + + Not what agents speak: their first hop is this hive's collector, + whose OTLP/HTTP receiver takes protobuf whatever the upstream + wants (see `hive-c0re/environment.nix`). ''; }; @@ -73,12 +93,12 @@ upstream auth header as `NAME=value` (e.g. `Authorization=Bearer `). - **Only the host-side collector reads this.** It reaches the - collector as an `EnvironmentFile`, so the value is never read by - nix, never copied into the store or the generated config, and - never passed in argv — and it is never forwarded into an agent - container, which is the point of the collector existing. Must be - absolute. + **Only the swarm's collector reads this** — the one tier that + talks to the upstream. It arrives as an `EnvironmentFile`, so the + value is never read by nix, never copied into the store or the + generated config, and never passed in argv; and it reaches + neither an agent container nor this hive's own collector, which + is the point of the tiers existing. Must be absolute. Leave null if the upstream needs no auth header; the collector then sends none rather than an empty one. @@ -161,131 +181,65 @@ }; }; - config = lib.mkMerge [ - (lib.mkIf config.services.hyperhive.c0re.enable { - assertions = lib.optionals config.services.hyperhive.otel.enable [ - { - # Telemetry has to go SOMEWHERE, but "somewhere" stopped meaning - # "an upstream endpoint" once the swarm grew its own store: a hive - # running `swarm.victoriametrics` is a complete destination on its - # own, and requiring an external endpoint as well would make the - # all-local mode impossible to express. - # - # This only ever relaxes the old rule — every config that passed - # before still passes. - assertion = - config.services.hyperhive.otel.endpoint != "" - || config.services.hyperhive.swarm.victoriametrics.enable; - message = '' - services.hyperhive.otel.enable is true but telemetry has nowhere - to go: services.hyperhive.otel.endpoint is empty and - services.hyperhive.swarm.victoriametrics.enable is false. + config = lib.mkIf config.services.hyperhive.otel.enable ( + let + otel = config.services.hyperhive.otel; + listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}"; + swarmName = "otlphttp/swarm"; + in + { + # Reachable from agent containers and nowhere else: this opens + # the port on the bridge interface only. + services.hyperhive.network.exposeHostPorts = [ otel.collector.port ]; - Set the endpoint to export upstream, or enable the swarm's - metrics store to keep telemetry on this host. - ''; - } - ]; - }) + services.opentelemetry-collector = { + enable = true; + # `validateConfigFile` defaults to `isStorePath configFile`, + # and `configFile` is null on the `settings` path — so the + # upstream default is OFF for exactly the way this module + # configures it. Turning it on runs `otelcol validate` at + # build time, which is the collector checking its own config. + # ⚠️ It is a PARSER, not a wiring check, and the gap is wider + # than "no sample was sent": measured 2026-08-15, `validate` + # ACCEPTS a receiver naming an auth extension that is absent + # from the build, and the collector then dies at startup with + # `Failed to start component`. So a green build does not + # prove this config STARTS, never mind that a sample arrives. + validateConfigFile = true; + settings = { + receivers.otlp.protocols.http.endpoint = listen; - (lib.mkIf config.services.hyperhive.otel.enable ( - ( - let - otel = config.services.hyperhive.otel; - listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}"; - # `otel.protocol` describes the UPSTREAM link and always did. - # Inserting a collector splits the path in two, and the - # upstream half is the one that has to keep honouring it — so - # the exporter is chosen by it, rather than the option quietly - # becoming "how agents talk to the collector". The agent half - # is pinned to OTLP/HTTP by the receiver below (derived in - # hive-c0re/environment.nix). - vmCfg = config.services.hyperhive.swarm.victoriametrics; - # Two independent destinations, either of which may be absent: an - # upstream the operator named, and the swarm's own store when this - # host runs it. The assertion above guarantees at least one. - upstreamConfigured = otel.endpoint != ""; - localStore = vmCfg.enable; - - storeName = "otlphttp/victoriametrics"; - store = { - # ⚠️ `metrics_endpoint`, NOT `endpoint`, and the difference is - # invisible until you read the far end: `endpoint` is a BASE that - # otlphttp appends `/v1/metrics` to, while VictoriaMetrics serves - # OTLP at `/opentelemetry/api/v1/push`. With `endpoint` the - # collector still answers 200 to its own clients and the samples - # are silently posted to a path that does not exist. - # `metrics_endpoint` is used verbatim. + # One destination, and it is the swarm's collector. This tier + # holds no upstream credential and writes no store: it receives + # from this hive's agents and forwards, which is the whole of + # its job. Everything that decides where telemetry ultimately + # goes lives one tier up, in ./swarm-otel.nix. + exporters.${swarmName} = { + # ⚠️ Plain `endpoint`, and the exporter beside this one in + # ./swarm-otel.nix warns against exactly that spelling — read + # both before "fixing" either. The difference is the far end, + # not the exporter: `endpoint` is a BASE that otlphttp appends + # `/v1/metrics` to, which is precisely the path an OTLP/HTTP + # receiver serves. VictoriaMetrics is the odd one out, serving + # OTLP at `/opentelemetry/api/v1/push`, and that is why the + # store exporter needs `metrics_endpoint` while this one must + # not have it. # - # Measured end-to-end rather than read: a real sample crossed a - # real collector into a real store, and the same probe with - # `endpoint` never arrived — see `state/probe-3265-collector-to-vm.sh`. - metrics_endpoint = "http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push"; + # Addressed by the option rather than by a loopback literal: + # the default already points at the co-located tier, and a + # hive whose swarm collector lives elsewhere then names it in + # config instead of needing this file changed. A loopback + # literal is correct only while listener and caller share a + # netns — the assumption that cost #2860 and #3363. + endpoint = config.services.hyperhive.swarm.otel.url; }; - grpcUpstream = otel.protocol == "grpc"; - upstreamName = if grpcUpstream then "otlp" else "otlphttp"; - upstream = { - endpoint = otel.endpoint; - } - // lib.optionalAttrs (otel.headersCredential != null) { - # The value is interpolated by the collector at runtime from - # its environment, never by nix. `EnvironmentFile` below is - # what puts it there. No credential configured means no - # header at all — an upstream that needs no auth is a - # legitimate deployment, and rendering `${env:…}` for a - # variable nothing sets would send the literal. - headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}"; - } - // lib.optionalAttrs (otel.protocol == "http/json") { encoding = "json"; }; - in - { - # Reachable from agent containers and nowhere else: this opens - # the port on the bridge interface only. - services.hyperhive.network.exposeHostPorts = [ otel.collector.port ]; - - services.opentelemetry-collector = { - enable = true; - # `validateConfigFile` defaults to `isStorePath configFile`, - # and `configFile` is null on the `settings` path — so the - # upstream default is OFF for exactly the way this module - # configures it. Turning it on runs `otelcol validate` at - # build time, which is the collector checking its own config. - # ⚠️ It is a PARSER, not a wiring check, and the gap is wider - # than "no sample was sent": measured 2026-08-15, `validate` - # ACCEPTS a receiver naming an auth extension that is absent - # from the build, and the collector then dies at startup with - # `Failed to start component`. So a green build does not - # prove this config STARTS, never mind that a sample arrives. - validateConfigFile = true; - settings = { - receivers.otlp.protocols.http.endpoint = listen; - exporters = - lib.optionalAttrs upstreamConfigured { ${upstreamName} = upstream; } - // lib.optionalAttrs localStore { ${storeName} = store; }; - service.pipelines.metrics = { - receivers = [ "otlp" ]; - # Fan-out, not a choice: with both configured the same - # samples go upstream AND into the swarm's store. A local - # store is for looking at this swarm; an upstream is for - # whoever aggregates across swarms, and neither replaces - # the other. - exporters = lib.optional upstreamConfigured upstreamName ++ lib.optional localStore storeName; - }; - }; + service.pipelines.metrics = { + receivers = [ "otlp" ]; + exporters = [ swarmName ]; }; - - # The credential file is already `NAME=value`, which is - # systemd's EnvironmentFile format — so the secret reaches the - # process as an environment variable without ever being read by - # nix, written to the store, or passed in argv. - systemd.services.opentelemetry-collector.serviceConfig = - lib.optionalAttrs (otel.headersCredential != null) - { - EnvironmentFile = otel.headersCredential; - }; - } - ) - )) - ]; + }; + }; + } + ); }