From 9549cdf9bdf80014f3ad67a9088a8838eec2c4b2 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 15 Aug 2026 10:21:09 +0200 Subject: [PATCH] feat(otel): one path out of the hive, not two mara: 'there should only be the one via otel-collector'. Collapses collector.enable away - enabling otel runs the collector, and agents always export to it. The argument for it is the same one the collector exists for: as long as the direct path stays selectable, the credential-in-settings.json hole stays selectable. An option that can reintroduce a hole is a hole with extra steps. HYPERHIVE_OTEL_HEADERS_CREDENTIAL is now never forwarded to containers at all rather than conditionally. Two consequences, both deliberate: - the enable option's own promise changes. It said each harness exports directly so telemetry survives anything host-side being down; that is now false for every hive rather than for opted-in ones. Stated in the option text rather than left for a reader to discover. - the collector-requires-headersCredential assertion is gone. It was correct only under the shape it was written for: when the collector was opt-in FOR the credential, opting in without one was pure indirection. With one path it isn't - an upstream needing no auth header is legitimate, and the assertion would have rejected it. The upstream header block is now omitted entirely in that case, rather than rendering an ${env:...} reference nothing sets. --- nix/host-modules/hive-c0re/environment.nix | 40 +++++------- nix/host-modules/otel.nix | 75 ++++++++-------------- 2 files changed, 41 insertions(+), 74 deletions(-) diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index ba345ca8..46be1d8e 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -81,41 +81,31 @@ in # don't render no-op env lines. let otel = config.services.hyperhive.otel; - # `otel.endpoint` keeps meaning "where telemetry ultimately goes", - # on every hive, whether or not a collector runs. What changes with - # a collector is only where the *first* hop lands — so the - # agent-facing value is DERIVED here rather than by redefining the - # option. Redefining it would migrate the meaning of a value every - # existing hive already has set, silently, while still evaluating. - # With the collector off this expression is `otel.endpoint`, i.e. - # byte-identical to before the collector existed. - collectorOn = otel.collector.enable; in { - HYPERHIVE_OTEL_ENDPOINT = - if collectorOn then - "http://${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}" - else - otel.endpoint; + # `otel.endpoint` means "where telemetry ultimately goes" and keeps + # that meaning; what agents are handed is the *first hop*, which is + # always this hive's own collector. Deriving it rather than + # redefining `endpoint` is what lets every existing deployment keep + # its configured value untouched. + HYPERHIVE_OTEL_ENDPOINT = "http://${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}"; # The first hop is the collector's OTLP/HTTP receiver, which speaks # protobuf regardless of what the upstream wants — `otel.protocol` # describes the *upstream* link, and the collector's own exporter is # what has to honour it (see nix/host-modules/otel.nix). - HYPERHIVE_OTEL_PROTOCOL = if collectorOn then "http/protobuf" else otel.protocol; + HYPERHIVE_OTEL_PROTOCOL = "http/protobuf"; } // lib.optionalAttrs (otel.extraResourceAttributes != "") { HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES = otel.extraResourceAttributes; } - // lib.optionalAttrs (otel.headersCredential != null && !collectorOn) { - # This is the variable that puts the upstream token in an agent's - # own `~/.claude/settings.json` (host_config.rs forwards it into the - # container as an nspawn credential; claude-settings.nix's - # `hive-otel-header` oneshot then writes it into a file the agent - # can read). Not emitting it is what actually removes the token from - # agent containers — the collector holding the credential is only - # half of it, and the half that is invisible in a diff. - HYPERHIVE_OTEL_HEADERS_CREDENTIAL = otel.headersCredential; - } + # HYPERHIVE_OTEL_HEADERS_CREDENTIAL is deliberately NOT emitted, and + # its absence is the security half of this design. It is the variable + # that put the upstream token in an agent's own settings.json: + # host_config.rs forwards it into the container as an nspawn + # credential, and claude-settings.nix's `hive-otel-header` oneshot + # then writes the value into a file the agent can read. The collector + # holding the credential achieves nothing while the harness keeps + # handing out a copy — so there is exactly one holder, on the host. // lib.optionalAttrs (otel.metricIntervalMs != null) { HYPERHIVE_OTEL_METRIC_INTERVAL_MS = toString otel.metricIntervalMs; } diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 75e3ec49..5a516dc8 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -16,9 +16,19 @@ enable = lib.mkEnableOption '' hive-wide export of every agent's Claude Code stats (token usage, cost, tool calls) to an OTLP endpoint via Claude Code's built-in - OpenTelemetry. One switch for all agents; each harness exports - directly to the collector, so it keeps working even when hive-c0re - is down + OpenTelemetry. One switch for all agents. + + Enabling this also runs a collector on the host: there is exactly + one way telemetry leaves this hive, and it is through that + collector. Agents export unauthenticated to a bridge address only + their own containers can reach, and the collector is the only + holder of the upstream credential — an agent never sees it. + + ⚠️ The collector is therefore in the path of all telemetry. It runs + on the same host as the agents and restarts on failure, and + telemetry is not the control plane, so degraded telemetry is not + degraded operation — but the export no longer survives independently + of anything host-side ''; endpoint = lib.mkOption { @@ -88,31 +98,6 @@ ''; }; - collector.enable = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - Run an OpenTelemetry collector on this host and have agents - export to it instead of straight to `endpoint`. - - The point is the credential. Without this, every agent needs - `headersCredential` in order to talk to the upstream — and the - harness delivers it into the agent's own `settings.json`, where - the agent can read it. With a collector the token stops at the - host: the collector holds it, agents send unauthenticated to a - bridge address only their own containers can reach. - - ⚠️ It also makes the collector a dependency in the export path. - Today each harness exports directly, so telemetry survives - anything host-side being down. That property is traded for the - credential reduction; the collector is on the same host as the - agents, so the window is small, but it is not zero. - - Off by default, and off means *absent*: no unit, no port, and - `endpoint` keeps its current meaning for every agent. - ''; - }; - collector.upstreamHeaderName = lib.mkOption { type = lib.types.str; default = "Authorization"; @@ -176,7 +161,7 @@ ]; }) - (lib.mkIf (config.services.hyperhive.otel.enable && config.services.hyperhive.otel.collector.enable) + (lib.mkIf config.services.hyperhive.otel.enable ( ( let otel = config.services.hyperhive.otel; @@ -192,31 +177,19 @@ upstreamName = if grpcUpstream then "otlp" else "otlphttp"; upstream = { endpoint = otel.endpoint; + } + // lib.optionalAttrs (otel.headersCredential != null) { # The value is interpolated by the collector at runtime from # its environment, never by nix. `EnvironmentFile` below is - # what puts it there. + # what puts it there. No credential configured means no + # header at all — an upstream that needs no auth is a + # legitimate deployment, and rendering `${env:…}` for a + # variable nothing sets would send the literal. headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}"; } // lib.optionalAttrs (otel.protocol == "http/json") { encoding = "json"; }; in { - assertions = [ - { - # The collector's whole purpose is to hold the credential so - # agents do not have to. With none configured it is pure - # indirection, and the operator has almost certainly not got - # the deployment they think they have. - assertion = otel.headersCredential != null; - message = '' - services.hyperhive.otel.collector.enable is true but - otel.headersCredential is null. The collector exists to be - the only holder of the upstream credential; with no - credential it just forwards, and every agent's exporter - would be unauthenticated end to end. - ''; - } - ]; - # Reachable from agent containers and nowhere else: this opens # the port on the bridge interface only. services.hyperhive.network.exposeHostPorts = [ otel.collector.port ]; @@ -244,9 +217,13 @@ # systemd's EnvironmentFile format — so the secret reaches the # process as an environment variable without ever being read by # nix, written to the store, or passed in argv. - systemd.services.opentelemetry-collector.serviceConfig.EnvironmentFile = otel.headersCredential; + systemd.services.opentelemetry-collector.serviceConfig = + lib.optionalAttrs (otel.headersCredential != null) + { + EnvironmentFile = otel.headersCredential; + }; } ) - ) + )) ]; }