From b8af1525ff6c576597517d97eb12f1d3d4c20acf Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 15 Aug 2026 00:34:31 +0200 Subject: [PATCH 1/8] feat(otel): options for a host-side collector, off by default Adds services.hyperhive.otel.collector.{enable,port}. No implementation yet - this commit only declares the surface, so the option docs and the default-off invariant can be reviewed on their own. The reason the collector exists is the credential: without it every agent needs headersCredential to reach the upstream, and the harness delivers that token into the agent's own settings.json where the agent can read it. The option text states the cost too - the collector becomes a dependency in an export path that is currently direct. --- nix/host-modules/otel.nix | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index d12da00b..79154cd0 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -88,6 +88,46 @@ ''; }; + collector.enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run an OpenTelemetry collector on this host and have agents + export to it instead of straight to `endpoint`. + + The point is the credential. Without this, every agent needs + `headersCredential` in order to talk to the upstream — and the + harness delivers it into the agent's own `settings.json`, where + the agent can read it. With a collector the token stops at the + host: the collector holds it, agents send unauthenticated to a + bridge address only their own containers can reach. + + ⚠️ It also makes the collector a dependency in the export path. + Today each harness exports directly, so telemetry survives + anything host-side being down. That property is traded for the + credential reduction; the collector is on the same host as the + agents, so the window is small, but it is not zero. + + Off by default, and off means *absent*: no unit, no port, and + `endpoint` keeps its current meaning for every agent. + ''; + }; + + collector.port = lib.mkOption { + type = lib.types.port; + default = 4318; + description = '' + Port the collector's OTLP/HTTP receiver listens on, at + `services.hyperhive.network.bridgeIp`. 4318 is the OTLP/HTTP + default. + + The port is contributed to + `services.hyperhive.network.exposeHostPorts`, which opens it on + the bridge interface only — so it is reachable from agent + containers and not from the outside world. + ''; + }; + metricIntervalMs = lib.mkOption { type = lib.types.nullOr lib.types.ints.positive; default = null; From 9ad4db842a8461a3b0244e94c346bbeec5787228 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 15 Aug 2026 01:19:30 +0200 Subject: [PATCH 2/8] feat(otel): the collector unit, off by default Renders services.opentelemetry-collector on the host: OTLP/HTTP receiver on the bridge address, otlphttp exporter to otel.endpoint, and the upstream credential delivered as EnvironmentFile so the collector interpolates it at runtime and nix never sees the value. Three things worth knowing, each measured rather than assumed: - network.exposeHostPorts already exists and is wired (it opens the port on the bridge interface only), so bridge reachability costs nothing. - validateConfigFile defaults to isStorePath configFile, which is null on the settings path - so upstream's default is OFF for exactly the way this module configures it. Set true: it runs otelcol validate at build time. It parses, it does not prove delivery. - headersCredential's file is already NAME=value, i.e. EnvironmentFile format, verified against a real settings.json rather than the doc. An assertion refuses collector.enable with no headersCredential: the collector exists to be the only holder of that token, and without one it is indirection that reads as security. --- nix/host-modules/otel.nix | 100 +++++++++++++++++++++++++++++++++++--- 1 file changed, 92 insertions(+), 8 deletions(-) diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 79154cd0..66d89855 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -113,6 +113,26 @@ ''; }; + collector.upstreamHeaderName = lib.mkOption { + type = lib.types.str; + default = "Authorization"; + description = '' + Name of the HTTP header the collector sends upstream, whose + *value* comes from `headersCredential`. + + The name is here and the value is not, and that split is forced + rather than chosen: the collector models exporter headers as a + static map, so rendering them means nix reading the value — the + one thing `headersCredential` being a path exists to prevent. + A name is public, a value is not. + + ⇒ exactly one header is expressible this way. A credential + carrying several (`a=1,b=2`) would be read as a single value, + which is why the shape is a named header rather than an opaque + blob: a second header has to be *declared*, not smuggled. + ''; + }; + collector.port = lib.mkOption { type = lib.types.port; default = 4318; @@ -146,12 +166,76 @@ }; }; - config = lib.mkIf config.services.hyperhive.c0re.enable { - assertions = lib.optionals config.services.hyperhive.otel.enable [ - { - assertion = config.services.hyperhive.otel.endpoint != ""; - message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty."; - } - ]; - }; + config = lib.mkMerge [ + (lib.mkIf config.services.hyperhive.c0re.enable { + assertions = lib.optionals config.services.hyperhive.otel.enable [ + { + assertion = config.services.hyperhive.otel.endpoint != ""; + message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty."; + } + ]; + }) + + (lib.mkIf (config.services.hyperhive.otel.enable && config.services.hyperhive.otel.collector.enable) + ( + let + otel = config.services.hyperhive.otel; + listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}"; + in + { + assertions = [ + { + # The collector's whole purpose is to hold the credential so + # agents do not have to. With none configured it is pure + # indirection, and the operator has almost certainly not got + # the deployment they think they have. + assertion = otel.headersCredential != null; + message = '' + services.hyperhive.otel.collector.enable is true but + otel.headersCredential is null. The collector exists to be + the only holder of the upstream credential; with no + credential it just forwards, and every agent's exporter + would be unauthenticated end to end. + ''; + } + ]; + + # Reachable from agent containers and nowhere else: this opens + # the port on the bridge interface only. + services.hyperhive.network.exposeHostPorts = [ otel.collector.port ]; + + services.opentelemetry-collector = { + enable = true; + # `validateConfigFile` defaults to `isStorePath configFile`, + # and `configFile` is null on the `settings` path — so the + # upstream default is OFF for exactly the way this module + # configures it. Turning it on runs `otelcol validate` at + # build time, which is the collector checking its own config. + # ⚠️ It parses; it does not prove a sample arrives. + validateConfigFile = true; + settings = { + receivers.otlp.protocols.http.endpoint = listen; + exporters.otlphttp = { + endpoint = otel.endpoint; + # The value is interpolated by the collector at runtime + # from its environment, never by nix. `EnvironmentFile` + # below is what puts it there. + headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}"; + }; + service.pipelines.metrics = { + receivers = [ "otlp" ]; + exporters = [ "otlphttp" ]; + }; + }; + }; + + # The credential file is already `NAME=value`, which is + # systemd's EnvironmentFile format — so the secret reaches the + # process as an environment variable without ever being read by + # nix, written to the store, or passed in argv. + systemd.services.opentelemetry-collector.serviceConfig.EnvironmentFile = otel.headersCredential; + } + ) + ) + ]; } From 2a2f5a754ebc2a5cb76f4bcca23f73879f69ed90 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 15 Aug 2026 09:31:42 +0200 Subject: [PATCH 3/8] feat(otel): point agents at the collector, and stop shipping them the token MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent-facing endpoint becomes derived rather than `otel.endpoint` re-defined: with the collector on, agents and hive-c0re's own exporter send to the bridge address; with it off the expression is the old value character for character. `otel.endpoint` keeps meaning "where telemetry ultimately goes" on every hive that already sets it. Suppressing HYPERHIVE_OTEL_HEADERS_CREDENTIAL is the half that actually closes the hole this issue is about. The credential path host -> container is sound (/run/credentials is dr-x------ root), but the hive-otel-header oneshot then writes the value into the agent's own settings.json, mode 0600 and owned by the agent — so today every agent can read the hive's upstream telemetry credential. With a collector there is nothing to forward. The protocol is derived for the same reason the endpoint is: the first hop is now the collector's OTLP/HTTP receiver, while otel.protocol describes the upstream link. --- nix/host-modules/hive-c0re/environment.nix | 30 +++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 3046f46c..ba345ca8 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -81,15 +81,39 @@ in # don't render no-op env lines. let otel = config.services.hyperhive.otel; + # `otel.endpoint` keeps meaning "where telemetry ultimately goes", + # on every hive, whether or not a collector runs. What changes with + # a collector is only where the *first* hop lands — so the + # agent-facing value is DERIVED here rather than by redefining the + # option. Redefining it would migrate the meaning of a value every + # existing hive already has set, silently, while still evaluating. + # With the collector off this expression is `otel.endpoint`, i.e. + # byte-identical to before the collector existed. + collectorOn = otel.collector.enable; in { - HYPERHIVE_OTEL_ENDPOINT = otel.endpoint; - HYPERHIVE_OTEL_PROTOCOL = otel.protocol; + HYPERHIVE_OTEL_ENDPOINT = + if collectorOn then + "http://${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}" + else + otel.endpoint; + # The first hop is the collector's OTLP/HTTP receiver, which speaks + # protobuf regardless of what the upstream wants — `otel.protocol` + # describes the *upstream* link, and the collector's own exporter is + # what has to honour it (see nix/host-modules/otel.nix). + HYPERHIVE_OTEL_PROTOCOL = if collectorOn then "http/protobuf" else otel.protocol; } // lib.optionalAttrs (otel.extraResourceAttributes != "") { HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES = otel.extraResourceAttributes; } - // lib.optionalAttrs (otel.headersCredential != null) { + // lib.optionalAttrs (otel.headersCredential != null && !collectorOn) { + # This is the variable that puts the upstream token in an agent's + # own `~/.claude/settings.json` (host_config.rs forwards it into the + # container as an nspawn credential; claude-settings.nix's + # `hive-otel-header` oneshot then writes it into a file the agent + # can read). Not emitting it is what actually removes the token from + # agent containers — the collector holding the credential is only + # half of it, and the half that is invisible in a diff. HYPERHIVE_OTEL_HEADERS_CREDENTIAL = otel.headersCredential; } // lib.optionalAttrs (otel.metricIntervalMs != null) { From 1fc267b880cfb2731610a3bc3e38f7bc06a0d10a Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 15 Aug 2026 09:33:11 +0200 Subject: [PATCH 4/8] feat(otel): the collector's upstream exporter honours otel.protocol The module hardcoded otlphttp, so a hive with protocol = "grpc" would have had its agents' protocol respected end to end before the collector and silently rewritten to HTTP after it. Splitting the path in two makes the upstream half the one that has to keep honouring the option; the agent half is pinned to the receiver's protocol and derived, not configured. --- nix/host-modules/otel.nix | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 66d89855..75e3ec49 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -181,6 +181,23 @@ let otel = config.services.hyperhive.otel; listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}"; + # `otel.protocol` describes the UPSTREAM link and always did. + # Inserting a collector splits the path in two, and the + # upstream half is the one that has to keep honouring it — so + # the exporter is chosen by it, rather than the option quietly + # becoming "how agents talk to the collector". The agent half + # is pinned to OTLP/HTTP by the receiver below (derived in + # hive-c0re/environment.nix). + grpcUpstream = otel.protocol == "grpc"; + upstreamName = if grpcUpstream then "otlp" else "otlphttp"; + upstream = { + endpoint = otel.endpoint; + # The value is interpolated by the collector at runtime from + # its environment, never by nix. `EnvironmentFile` below is + # what puts it there. + headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}"; + } + // lib.optionalAttrs (otel.protocol == "http/json") { encoding = "json"; }; in { assertions = [ @@ -215,16 +232,10 @@ validateConfigFile = true; settings = { receivers.otlp.protocols.http.endpoint = listen; - exporters.otlphttp = { - endpoint = otel.endpoint; - # The value is interpolated by the collector at runtime - # from its environment, never by nix. `EnvironmentFile` - # below is what puts it there. - headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}"; - }; + exporters.${upstreamName} = upstream; service.pipelines.metrics = { receivers = [ "otlp" ]; - exporters = [ "otlphttp" ]; + exporters = [ upstreamName ]; }; }; }; From f7fbad7655585d71d5f37c7e99ea0edbb60c78c4 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 15 Aug 2026 09:57:41 +0200 Subject: [PATCH 5/8] docs(otel): document the collector in the page the index points at argus's non-blocking note on #3278: docs/observability.md is what this repo's reading-paths index names as 'what OTEL options are available', and it did not mention collector.enable/port/upstreamHeaderName at all. The nix docstrings covered it, but not where a reader following the established path would look. Carries the two things a docstring is a poor home for: that endpoint keeps meaning 'where telemetry ultimately goes' (the agent-facing value is derived, so an existing deployment is unaffected), and the availability trade the collector makes against the direct-export property this page already promises. --- docs/observability.md | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/docs/observability.md b/docs/observability.md index 28350376..a8cbfbfd 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -93,6 +93,63 @@ metrics on process exit, so interval tuning is not required for metrics to be exported. A lower value gives more frequent intermediate flushes within long-running turns — cosmetic, not a correctness knob. +## Running a collector on the host (`otel.collector`) + +By default every agent exports **straight to `endpoint`**, which means every +agent needs `headersCredential` to authenticate — and the harness delivers that +token into the agent's own `~/.claude/settings.json`, a file the agent can read. +`0600` protects it from other containers, not from the agent itself. + +With `services.hyperhive.otel.collector.enable = true`, a collector runs on the +host and holds the credential instead. Agents export **unauthenticated** to a +bridge address only their own containers can reach; the collector adds the +upstream header and forwards to `endpoint`. + +```nix +services.hyperhive.otel = { + enable = true; + endpoint = "https://collector.example.com/otel"; # still the upstream + headersCredential = "/run/secrets/otel-headers"; # now only the host reads it + collector.enable = true; +}; +``` + +**`endpoint` keeps meaning "where telemetry ultimately goes."** Turning the +collector on does not redefine it — the agent-facing value is *derived* +(`http://:`), so an existing deployment's `endpoint` +keeps working unchanged. The bridge port is contributed to +`exposeHostPorts` automatically; there is nothing to open by hand. + +⚠️ **The trade:** without a collector each harness exports directly, so +telemetry survives anything host-side being down. A local collector is a new +dependency in that path. It runs on the same host as the agents, so the window +is small — but it is not zero. + +### `services.hyperhive.otel.collector.enable` — bool, default `false` + +Off means *absent*: no unit, no port, and `endpoint` keeps its current meaning +for every agent. Requires `headersCredential` to be set — a collector with no +credential is pure indirection, and an assertion says so rather than letting it +deploy. + +### `services.hyperhive.otel.collector.port` — port, default `4318` + +The OTLP/HTTP port the collector listens on, bound to the bridge IP only. + +### `services.hyperhive.otel.collector.upstreamHeaderName` — string, default `"Authorization"` + +Name of the header the collector sends upstream. The **value** comes from the +credential file at runtime (`EnvironmentFile` → `${env:}`), never from +nix — so header names are config and header values are secrets, which is the +only split the collector's static header map can express. + +⚠️ **`endpoint` must be valid for `protocol`.** The upstream exporter follows +`otel.protocol` (`grpc` → the gRPC exporter, otherwise OTLP/HTTP), and the gRPC +exporter takes an *address*: `https://host/path` is a legal +`OTEL_EXPORTER_OTLP_ENDPOINT` for HTTP but fails as gRPC with *"missing port in +address"*. The collector's config is validated at build time, so a mismatch is +a build error naming the reason rather than telemetry silently going nowhere. + ## Network access Agent containers can only reach the host on ports 80 and 443 by default. If @@ -112,6 +169,10 @@ services.hyperhive.otel.endpoint = "http://10.42.0.1:4318"; The bridge IP is the host's address on the `hvbr0` bridge, typically `10.42.0.1`. See `docs/network.md::Reaching host services` for details. +This is the manual form of what `otel.collector.enable` does for you — with the +collector on, the port is contributed and the endpoint derived, so neither line +above is needed. + ## Built-in resource labels Every agent's export includes these resource attributes automatically: From 9549cdf9bdf80014f3ad67a9088a8838eec2c4b2 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 15 Aug 2026 10:21:09 +0200 Subject: [PATCH 6/8] feat(otel): one path out of the hive, not two mara: 'there should only be the one via otel-collector'. Collapses collector.enable away - enabling otel runs the collector, and agents always export to it. The argument for it is the same one the collector exists for: as long as the direct path stays selectable, the credential-in-settings.json hole stays selectable. An option that can reintroduce a hole is a hole with extra steps. HYPERHIVE_OTEL_HEADERS_CREDENTIAL is now never forwarded to containers at all rather than conditionally. Two consequences, both deliberate: - the enable option's own promise changes. It said each harness exports directly so telemetry survives anything host-side being down; that is now false for every hive rather than for opted-in ones. Stated in the option text rather than left for a reader to discover. - the collector-requires-headersCredential assertion is gone. It was correct only under the shape it was written for: when the collector was opt-in FOR the credential, opting in without one was pure indirection. With one path it isn't - an upstream needing no auth header is legitimate, and the assertion would have rejected it. The upstream header block is now omitted entirely in that case, rather than rendering an ${env:...} reference nothing sets. --- nix/host-modules/hive-c0re/environment.nix | 40 +++++------- nix/host-modules/otel.nix | 75 ++++++++-------------- 2 files changed, 41 insertions(+), 74 deletions(-) diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index ba345ca8..46be1d8e 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -81,41 +81,31 @@ in # don't render no-op env lines. let otel = config.services.hyperhive.otel; - # `otel.endpoint` keeps meaning "where telemetry ultimately goes", - # on every hive, whether or not a collector runs. What changes with - # a collector is only where the *first* hop lands — so the - # agent-facing value is DERIVED here rather than by redefining the - # option. Redefining it would migrate the meaning of a value every - # existing hive already has set, silently, while still evaluating. - # With the collector off this expression is `otel.endpoint`, i.e. - # byte-identical to before the collector existed. - collectorOn = otel.collector.enable; in { - HYPERHIVE_OTEL_ENDPOINT = - if collectorOn then - "http://${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}" - else - otel.endpoint; + # `otel.endpoint` means "where telemetry ultimately goes" and keeps + # that meaning; what agents are handed is the *first hop*, which is + # always this hive's own collector. Deriving it rather than + # redefining `endpoint` is what lets every existing deployment keep + # its configured value untouched. + HYPERHIVE_OTEL_ENDPOINT = "http://${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}"; # The first hop is the collector's OTLP/HTTP receiver, which speaks # protobuf regardless of what the upstream wants — `otel.protocol` # describes the *upstream* link, and the collector's own exporter is # what has to honour it (see nix/host-modules/otel.nix). - HYPERHIVE_OTEL_PROTOCOL = if collectorOn then "http/protobuf" else otel.protocol; + HYPERHIVE_OTEL_PROTOCOL = "http/protobuf"; } // lib.optionalAttrs (otel.extraResourceAttributes != "") { HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES = otel.extraResourceAttributes; } - // lib.optionalAttrs (otel.headersCredential != null && !collectorOn) { - # This is the variable that puts the upstream token in an agent's - # own `~/.claude/settings.json` (host_config.rs forwards it into the - # container as an nspawn credential; claude-settings.nix's - # `hive-otel-header` oneshot then writes it into a file the agent - # can read). Not emitting it is what actually removes the token from - # agent containers — the collector holding the credential is only - # half of it, and the half that is invisible in a diff. - HYPERHIVE_OTEL_HEADERS_CREDENTIAL = otel.headersCredential; - } + # HYPERHIVE_OTEL_HEADERS_CREDENTIAL is deliberately NOT emitted, and + # its absence is the security half of this design. It is the variable + # that put the upstream token in an agent's own settings.json: + # host_config.rs forwards it into the container as an nspawn + # credential, and claude-settings.nix's `hive-otel-header` oneshot + # then writes the value into a file the agent can read. The collector + # holding the credential achieves nothing while the harness keeps + # handing out a copy — so there is exactly one holder, on the host. // lib.optionalAttrs (otel.metricIntervalMs != null) { HYPERHIVE_OTEL_METRIC_INTERVAL_MS = toString otel.metricIntervalMs; } diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 75e3ec49..5a516dc8 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -16,9 +16,19 @@ enable = lib.mkEnableOption '' hive-wide export of every agent's Claude Code stats (token usage, cost, tool calls) to an OTLP endpoint via Claude Code's built-in - OpenTelemetry. One switch for all agents; each harness exports - directly to the collector, so it keeps working even when hive-c0re - is down + OpenTelemetry. One switch for all agents. + + Enabling this also runs a collector on the host: there is exactly + one way telemetry leaves this hive, and it is through that + collector. Agents export unauthenticated to a bridge address only + their own containers can reach, and the collector is the only + holder of the upstream credential — an agent never sees it. + + ⚠️ The collector is therefore in the path of all telemetry. It runs + on the same host as the agents and restarts on failure, and + telemetry is not the control plane, so degraded telemetry is not + degraded operation — but the export no longer survives independently + of anything host-side ''; endpoint = lib.mkOption { @@ -88,31 +98,6 @@ ''; }; - collector.enable = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - Run an OpenTelemetry collector on this host and have agents - export to it instead of straight to `endpoint`. - - The point is the credential. Without this, every agent needs - `headersCredential` in order to talk to the upstream — and the - harness delivers it into the agent's own `settings.json`, where - the agent can read it. With a collector the token stops at the - host: the collector holds it, agents send unauthenticated to a - bridge address only their own containers can reach. - - ⚠️ It also makes the collector a dependency in the export path. - Today each harness exports directly, so telemetry survives - anything host-side being down. That property is traded for the - credential reduction; the collector is on the same host as the - agents, so the window is small, but it is not zero. - - Off by default, and off means *absent*: no unit, no port, and - `endpoint` keeps its current meaning for every agent. - ''; - }; - collector.upstreamHeaderName = lib.mkOption { type = lib.types.str; default = "Authorization"; @@ -176,7 +161,7 @@ ]; }) - (lib.mkIf (config.services.hyperhive.otel.enable && config.services.hyperhive.otel.collector.enable) + (lib.mkIf config.services.hyperhive.otel.enable ( ( let otel = config.services.hyperhive.otel; @@ -192,31 +177,19 @@ upstreamName = if grpcUpstream then "otlp" else "otlphttp"; upstream = { endpoint = otel.endpoint; + } + // lib.optionalAttrs (otel.headersCredential != null) { # The value is interpolated by the collector at runtime from # its environment, never by nix. `EnvironmentFile` below is - # what puts it there. + # what puts it there. No credential configured means no + # header at all — an upstream that needs no auth is a + # legitimate deployment, and rendering `${env:…}` for a + # variable nothing sets would send the literal. headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}"; } // lib.optionalAttrs (otel.protocol == "http/json") { encoding = "json"; }; in { - assertions = [ - { - # The collector's whole purpose is to hold the credential so - # agents do not have to. With none configured it is pure - # indirection, and the operator has almost certainly not got - # the deployment they think they have. - assertion = otel.headersCredential != null; - message = '' - services.hyperhive.otel.collector.enable is true but - otel.headersCredential is null. The collector exists to be - the only holder of the upstream credential; with no - credential it just forwards, and every agent's exporter - would be unauthenticated end to end. - ''; - } - ]; - # Reachable from agent containers and nowhere else: this opens # the port on the bridge interface only. services.hyperhive.network.exposeHostPorts = [ otel.collector.port ]; @@ -244,9 +217,13 @@ # systemd's EnvironmentFile format — so the secret reaches the # process as an environment variable without ever being read by # nix, written to the store, or passed in argv. - systemd.services.opentelemetry-collector.serviceConfig.EnvironmentFile = otel.headersCredential; + systemd.services.opentelemetry-collector.serviceConfig = + lib.optionalAttrs (otel.headersCredential != null) + { + EnvironmentFile = otel.headersCredential; + }; } ) - ) + )) ]; } From 513eb5729d1a30fd23688adaa3234478052602c1 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 15 Aug 2026 10:22:12 +0200 Subject: [PATCH 7/8] docs(otel): one path, and stop describing a forwarding that no longer happens The headersCredential section still said hive-c0re forwards the file into each agent container - that is exactly the behaviour this change removes, so the doc contradicted the code rather than merely lagging it. Same for the collector section, which presented the two paths the change collapses. Also drops the dangling collector.enable reference in the network section. --- docs/observability.md | 87 ++++++++++++++++++--------------------- nix/host-modules/otel.nix | 23 ++++++----- 2 files changed, 54 insertions(+), 56 deletions(-) diff --git a/docs/observability.md b/docs/observability.md index a8cbfbfd..13bf6864 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -16,12 +16,20 @@ services.hyperhive.otel = { }; ``` -`enable` is the single gate. `endpoint` (required when enabled) is the OTLP -HTTP endpoint; hive-c0re injects it as `OTEL_EXPORTER_OTLP_ENDPOINT` into -every agent's systemd service via the generated meta flake. +`enable` is the single gate. `endpoint` (required when enabled) is where +telemetry ultimately goes. -Each agent's harness (hive-ag3nt) exports directly to the collector — the -pipeline keeps working even when hive-c0re is down. +**There is exactly one way telemetry leaves a hive: through the collector that +`enable` starts on the host.** Agents never talk to `endpoint` themselves — +they export unauthenticated to a bridge address only their own containers can +reach, and the collector forwards upstream with the auth header. So the +upstream credential exists in one place, on the host, and no agent ever holds +a copy. + +⚠️ **The collector is therefore in the path of all telemetry.** It runs on the +same host as the agents and restarts on failure, and telemetry is not the +control plane — degraded telemetry is not degraded operation — but the export +no longer survives independently of anything host-side. ## Options reference @@ -43,17 +51,17 @@ OTLP wire protocol, passed as `OTEL_EXPORTER_OTLP_PROTOCOL`. Accepted values: ### `services.hyperhive.otel.headersCredential` — string or null, default `null` -Absolute path to a secret file on the host whose contents become -`OTEL_EXPORTER_OTLP_HEADERS` (e.g. `Authorization=Bearer `). +Absolute path to a secret file on the host holding the upstream auth header as +`NAME=value` (e.g. `Authorization=Bearer `). -hive-c0re forwards this host file into each agent container via -`systemd-nspawn --load-credential=otel-headers:`; the inner harness unit -inherits it by name. The token is never copied into the nix store, generated -config, a bind mount, or argv. +**Only the host collector reads it.** It arrives as an `EnvironmentFile` on the +collector's unit, so the value is never read by nix, never copied into the +store or the generated config, never passed in argv — and **never forwarded +into an agent container**. An agent cannot read the hive's upstream credential +because it is never given one. -Leave `null` if the endpoint needs no auth header. A configured-but-missing -file is skipped with a log warning — OTEL still exports, just without the auth -header. +Leave `null` if the upstream needs no auth header; the collector then sends +none rather than an empty one. ```nix services.hyperhive.otel = { @@ -93,44 +101,31 @@ metrics on process exit, so interval tuning is not required for metrics to be exported. A lower value gives more frequent intermediate flushes within long-running turns — cosmetic, not a correctness knob. -## Running a collector on the host (`otel.collector`) +## The host collector -By default every agent exports **straight to `endpoint`**, which means every -agent needs `headersCredential` to authenticate — and the harness delivers that -token into the agent's own `~/.claude/settings.json`, a file the agent can read. -`0600` protects it from other containers, not from the agent itself. - -With `services.hyperhive.otel.collector.enable = true`, a collector runs on the -host and holds the credential instead. Agents export **unauthenticated** to a -bridge address only their own containers can reach; the collector adds the -upstream header and forwards to `endpoint`. +`enable` starts an OpenTelemetry collector on the host. It is not optional and +there is no second path — that is the whole point: ```nix services.hyperhive.otel = { enable = true; - endpoint = "https://collector.example.com/otel"; # still the upstream - headersCredential = "/run/secrets/otel-headers"; # now only the host reads it - collector.enable = true; + endpoint = "https://collector.example.com/otel"; # the upstream + headersCredential = "/run/secrets/otel-headers"; # only the host reads it }; ``` -**`endpoint` keeps meaning "where telemetry ultimately goes."** Turning the -collector on does not redefine it — the agent-facing value is *derived* +**Why it isn't a knob.** Exporting straight to `endpoint` means every agent +needs the credential to authenticate — and the harness delivers that token into +the agent's own `~/.claude/settings.json`, a file the agent can read. `0600` +protects it from other containers, not from the agent itself. As long as the +direct path stays *selectable*, that hole stays selectable; an option that can +reintroduce it is a hole with extra steps. + +**`endpoint` keeps meaning "where telemetry ultimately goes."** The collector +does not redefine it — the agent-facing value is *derived* (`http://:`), so an existing deployment's `endpoint` -keeps working unchanged. The bridge port is contributed to -`exposeHostPorts` automatically; there is nothing to open by hand. - -⚠️ **The trade:** without a collector each harness exports directly, so -telemetry survives anything host-side being down. A local collector is a new -dependency in that path. It runs on the same host as the agents, so the window -is small — but it is not zero. - -### `services.hyperhive.otel.collector.enable` — bool, default `false` - -Off means *absent*: no unit, no port, and `endpoint` keeps its current meaning -for every agent. Requires `headersCredential` to be set — a collector with no -credential is pure indirection, and an assertion says so rather than letting it -deploy. +keeps working unchanged. The bridge port is contributed to `exposeHostPorts` +automatically; there is nothing to open by hand. ### `services.hyperhive.otel.collector.port` — port, default `4318` @@ -169,9 +164,9 @@ services.hyperhive.otel.endpoint = "http://10.42.0.1:4318"; The bridge IP is the host's address on the `hvbr0` bridge, typically `10.42.0.1`. See `docs/network.md::Reaching host services` for details. -This is the manual form of what `otel.collector.enable` does for you — with the -collector on, the port is contributed and the endpoint derived, so neither line -above is needed. +⚠️ **You do not need either line for hyperhive's own telemetry** — `otel.enable` +contributes the collector's port and derives the endpoint itself. The above is +for pointing something *else* at a host-local service. ## Built-in resource labels diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 5a516dc8..87a07713 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -62,16 +62,19 @@ default = null; example = "/run/secrets/otel-headers"; description = '' - Absolute path to an operator-provided secret file whose contents - become `OTEL_EXPORTER_OTLP_HEADERS` (e.g. - `Authorization=Bearer `). hive-c0re forwards this host - file into each agent container's credential store via - systemd-nspawn `--load-credential=otel-headers:`; the inner - harness unit inherits it by name (`LoadCredential`), so the token - is never copied into the nix store, the generated config, a bind - mount, or argv. Must be absolute. Leave null if the endpoint - needs no auth header. A configured-but-missing file is skipped - with a log warning (OTEL still exports, without the auth header). + Absolute path to an operator-provided secret file holding the + upstream auth header as `NAME=value` (e.g. + `Authorization=Bearer `). + + **Only the host-side collector reads this.** It reaches the + collector as an `EnvironmentFile`, so the value is never read by + nix, never copied into the store or the generated config, and + never passed in argv — and it is never forwarded into an agent + container, which is the point of the collector existing. Must be + absolute. + + Leave null if the upstream needs no auth header; the collector + then sends none rather than an empty one. ''; }; From 865a1cc9acf84402216dc1d049904293a22b20c5 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 15 Aug 2026 11:33:13 +0200 Subject: [PATCH 8/8] docs(otel): say what the agent->collector hop is and is not argus on #3280: the receiver has no auth extension - the nixpkgs module passes settings straight through and nothing wires one on - so 'unauthenticated to a bridge address' means reachable from an agent container, not presents a credential. The firewall is the whole access control. Consequence, stated because it is a choice rather than an oversight: any agent can push arbitrary OTLP and the collector forwards it under the operator's credential, including data smuggled out in resource attributes. That is a different risk from the one the collector fixes, and strictly smaller than what preceded it - before, every agent held the credential itself and could do all of that plus use the token anywhere else. The collector removes the token and keeps the pipe. Same principle this PR already applies to the availability trade: state it where the reader is, rather than let it be discovered. --- docs/observability.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/observability.md b/docs/observability.md index 13bf6864..5a3fbea0 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -31,6 +31,27 @@ same host as the agents and restarts on failure, and telemetry is not the control plane — degraded telemetry is not degraded operation — but the export no longer survives independently of anything host-side. +### what the agent→collector hop is and isn't + +**It has no application-level auth.** The receiver takes any OTLP that reaches +it; what bounds who can reach it is the firewall — `exposeHostPorts` opens the +port on the bridge interface only. So "unauthenticated to a bridge address" +means *reachable from an agent container*, not *presents a credential*. + +The consequence, stated because it is a choice rather than an oversight: **any +agent can push arbitrary OTLP, and the collector forwards it upstream under the +operator's credential.** It cannot tell a container's genuine Claude Code stats +from anything else shaped like OTLP arriving on that port — including data +smuggled out in resource attributes on an otherwise-legitimate export. + +That is a **different risk from the one the collector fixes**, and strictly +smaller than what preceded it: before, every agent held the upstream credential +itself, so it could do all of the above *and* use the token anywhere else. The +collector removes the token and keeps the pipe. Agents are inside the trust +boundary (`docs/security.md`: capability = accepted risk), so an agent being +able to *send* is an accepted extension of that boundary — but it is not +closed by this design, and nothing here should be read as closing it. + ## Options reference ### `services.hyperhive.otel.enable` — bool, default `false`