From 8b14d959d62e4b6e29dacec408cbdae9e8e36668 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 18 Aug 2026 22:38:13 +0200 Subject: [PATCH] otel: move the generic OTLP environment out of claude's settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The endpoint, protocol, temporality preference and resource labels were shipped only inside claude's managed settings json, so they applied to claude's own process. hive-bash-daemon, hive-mcp-http, hive-matrix-daemon and hive-forge-notify are systemd *siblings* of claude rather than its children, so nothing shipped there could ever reach them: `hive-metric` invoked from a tool call exited with "OTEL_EXPORTER_OTLP_ENDPOINT not set", which is the honest failure of a value it structurally could not see. Declare those variables container-wide in a new agent module instead — systemd.globalEnvironment for every unit PID 1 starts, environment.variables for login shells. Both are needed and neither implies the other; NIX_REMOTE is set both ways for the same reason. Claude keeps only what is genuinely its own: the telemetry master flag, the feedback-survey flag, the version label, and which signals it exports. A different producer in the same container may legitimately emit only metrics. The hyperhive.otel.* options move across with them. They have more than one consumer now, so their home is the OTEL module rather than the claude one. --- docs/observability.md | 10 +- nix/agent-modules/claude-settings.nix | 141 ++++----------------- nix/agent-modules/default.nix | 1 + nix/agent-modules/otel.nix | 168 ++++++++++++++++++++++++++ 4 files changed, 204 insertions(+), 116 deletions(-) create mode 100644 nix/agent-modules/otel.nix diff --git a/docs/observability.md b/docs/observability.md index 19daedc6..5565f19a 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -231,7 +231,15 @@ for pointing something *else* at a host-local service. ## Built-in resource labels -Every agent's export includes these resource attributes automatically: +The OTLP variables (`OTEL_EXPORTER_OTLP_ENDPOINT`, `_PROTOCOL`, +`OTEL_RESOURCE_ATTRIBUTES`, the temporality preference) are set **container +wide** — in systemd's `DefaultEnvironment` and in `/etc/profile` — so every +process in an agent container exports to the hive's collector without any +per-tool wiring. That covers Claude Code, `hive-metric`, and anything you run +yourself from a tool call or `hivectl shell`. + +Every agent's export therefore includes these resource attributes +automatically: | Attribute | Value | |-----------|-------| diff --git a/nix/agent-modules/claude-settings.nix b/nix/agent-modules/claude-settings.nix index 93b3aa43..45941b5b 100644 --- a/nix/agent-modules/claude-settings.nix +++ b/nix/agent-modules/claude-settings.nix @@ -1,7 +1,11 @@ # Everything that shapes claude-code's own configuration inside the -# container: the managed settings json (base env + OTEL), the -# onboarding/trust seed, the runtime OTEL auth-header injection, and -# the plugin/marketplace install lists the harness reads at boot. +# container: the managed settings json (base env + claude's own +# telemetry switches), the onboarding/trust seed, and the +# plugin/marketplace install lists the harness reads at boot. +# +# The generic OTLP environment — endpoint, protocol, resource labels — +# is NOT here: it belongs to every producer in the container, not to +# claude, and lives in `otel.nix`. { pkgs, lib, @@ -12,18 +16,17 @@ let userName = config.hyperhive.user.name; homeDir = "/home/${userName}"; # Hive-wide OpenTelemetry config (host-driven; baked in per-agent by - # meta.rs `otel_config`). + # meta.rs `otel_config`). Options declared in `otel.nix`, which also + # exports the generic OTLP environment this container's producers read. otelCfg = config.hyperhive.otel; - # Hive/swarm display names, read from the per-agent options meta.rs - # renders (NOT from `environment.variables` — those carry the same names - # at *runtime* only, so reading them here silently yielded "unknown" on - # every agent while the process env held the right answer). `null` means - # the hive did not name itself; "unknown" is then an honest label rather - # than a guess. + # Hive display name, read from the per-agent option meta.rs renders + # (NOT from `environment.variables` — that carries the same name at + # *runtime* only, so reading it here silently yielded "unknown" on + # every agent while the process env held the right answer). `null` + # means the hive did not name itself; "unknown" is then an honest label + # rather than a guess. hiveDisplayName = if config.hyperhive.hiveName == null then "unknown" else config.hyperhive.hiveName; - swarmDisplayName = - if config.hyperhive.swarmName == null then "unknown" else config.hyperhive.swarmName; # Effective per-agent MemoryMax=, in bytes, injected by meta.rs's # per-agent flake render (`hyperhive.claudeMemoryMaxBytes`). `null` # when the effective cap is unbounded ("infinity") or a RAM @@ -68,121 +71,29 @@ let // lib.optionalAttrs (memoryMaxBytes != null) { BUN_JSC_forceRAMSize = toString (memoryMaxBytes * 75 / 100); }; - # OTEL environment Claude Code reads to export metrics/logs/traces. - # Shipped via the managed claude settings json (below), which claude - # auto-discovers for BOTH the harness turn-loop and `hivectl choom` — - # so telemetry parity is declarative, with no launch wrapper. - # - # There is no auth header here, and no mechanism to add one. An agent - # exports to the hive's own collector, which is the only thing holding - # a credential for anything upstream; nothing an agent can read is a - # secret to the swarm. An earlier revision forwarded the operator's - # upstream token into this container and merged it into the agent's own - # `~/.claude/settings.json` — which handed every agent the hive's - # credential, and was removed with the direct-export path it served. + # Claude Code's own telemetry switches — what it emits, and whether it + # emits at all. Everything an OTEL SDK reads generically (endpoint, + # protocol, temporality, resource labels) is deliberately NOT here: it + # lives in `otel.nix` as container environment, because claude is one + # producer in this container and not the owner of the pipe. Shipping + # those in claude's managed settings put them on claude's process only, + # and `hive-metric` — a sibling of claude, not a child — could not see + # the endpoint at all. otelSettingsEnv = { CLAUDE_CODE_ENABLE_TELEMETRY = "1"; # Attach feedback-survey data to the OTEL pipeline. CLAUDE_CODE_ENABLE_FEEDBACK_SURVEY_FOR_OTEL = "1"; + # Which signals claude exports. A different producer in this same + # container may legitimately emit only metrics, so this stays a + # claude decision rather than container-wide config. OTEL_METRICS_EXPORTER = "otlp"; OTEL_LOGS_EXPORTER = "otlp"; OTEL_TRACES_EXPORTER = "otlp"; - OTEL_EXPORTER_OTLP_PROTOCOL = otelCfg.protocol; - OTEL_EXPORTER_OTLP_ENDPOINT = otelCfg.endpoint; - # Force CUMULATIVE temporality — Claude Code defaults to DELTA, - # which Prometheus/Mimir-family backends (incl. grafana-lgtm) - # silently drop without a deltatocumulative processor. - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; - OTEL_RESOURCE_ATTRIBUTES = - "service.name=hyperhive-agent,agent=${userName},hive=${hiveDisplayName},swarm=${swarmDisplayName}" - + lib.optionalString (otelCfg.extraResourceAttributes != "") ",${otelCfg.extraResourceAttributes}"; # Include the Claude Code version label in emitted metrics. OTEL_METRICS_INCLUDE_VERSION = "1"; - } - // lib.optionalAttrs (otelCfg.metricIntervalMs != null) { - OTEL_METRIC_EXPORT_INTERVAL = toString otelCfg.metricIntervalMs; }; in { - # OTEL stats export is configured ONCE at host level via - # `services.hyperhive.otel.*` (see nix/host-modules/hive-c0re.nix) and - # injected into every agent's build by the meta-flake renderer - # (`hive-c0re/src/meta.rs::otel_config`). These per-agent options are - # the build-time implementation surface that injection writes into; - # they are not meant to be set directly in an agent.nix. Marked - # `internal` so the host option is the only documented operator knob. - options.hyperhive.otel = { - enable = lib.mkOption { - type = lib.types.bool; - default = false; - internal = true; - description = '' - Export this agent's Claude Code stats (token usage, cost, tool - calls) to an OTLP endpoint via Claude Code's built-in - OpenTelemetry. Each agent's harness exports directly to the - collector, so it keeps working even when hive-c0re is down. - Host-driven: set `services.hyperhive.otel.enable` instead. - ''; - }; - - endpoint = lib.mkOption { - type = lib.types.str; - default = ""; - internal = true; - description = '' - OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`. - Host-driven via `services.hyperhive.otel.endpoint`. - ''; - }; - - protocol = lib.mkOption { - type = lib.types.enum [ - "http/protobuf" - "http/json" - "grpc" - ]; - default = "http/protobuf"; - internal = true; - description = '' - OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`. - Host-driven via `services.hyperhive.otel.protocol`. - ''; - }; - - extraResourceAttributes = lib.mkOption { - type = lib.types.str; - default = ""; - internal = true; - description = '' - Extra comma-separated entries appended to - `OTEL_RESOURCE_ATTRIBUTES` after the built-in - `service.name` / `agent` / `hive` / `swarm` labels. - Host-driven via `services.hyperhive.otel.extraResourceAttributes`. - ''; - }; - - metricIntervalMs = lib.mkOption { - type = lib.types.nullOr lib.types.ints.positive; - default = null; - internal = true; - description = '' - Metric export interval in milliseconds, set as - `OTEL_METRIC_EXPORT_INTERVAL`. Null leaves Claude Code's 60s - default. Host-driven via `services.hyperhive.otel.metricIntervalMs`. - ''; - }; - - debug = lib.mkOption { - type = lib.types.bool; - default = false; - internal = true; - description = '' - Emit OTEL SDK diagnostics to stderr (`CLAUDE_CODE_OTEL_DIAG_STDERR=1`). - Host-driven via `services.hyperhive.otel.debug`. - ''; - }; - }; - # Build-time implementation surface for the JSC-heap-ceiling fix: # meta.rs's per-agent flake render injects this from the effective # `MemoryMax=` (per-agent `resource-limits.json` override, else the diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index fb49e100..42c9ce3c 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -31,6 +31,7 @@ ./matrix.nix ./mcp.nix ./network.nix + ./otel.nix ./packages.nix ./user.nix ./screen.nix diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix new file mode 100644 index 00000000..24867e6e --- /dev/null +++ b/nix/agent-modules/otel.nix @@ -0,0 +1,168 @@ +# OpenTelemetry wiring for an agent container: the per-agent options the +# meta flake injects (host-driven from `services.hyperhive.otel.*`), and +# the environment every OTLP producer inside the container reads. +# +# Claude Code is one producer here, not the owner. `hive-metric` — and +# any future in-container exporter — reads the same endpoint, protocol +# and resource labels, so they are container environment rather than +# claude settings. Claude's own switches (its telemetry master flag, +# which signals it emits) stay in `claude-settings.nix`, which reads the +# options declared below. +{ + lib, + config, + ... +}: +let + cfg = config.hyperhive.otel; + userName = config.hyperhive.user.name; + # Hive/swarm display names, read from the per-agent options meta.rs + # renders (NOT from `environment.variables` — those carry the same + # names at *runtime* only, so reading them here silently yielded + # "unknown" on every agent while the process env held the right + # answer). `null` means the hive did not name itself; "unknown" is then + # an honest label rather than a guess. + hiveDisplayName = + if config.hyperhive.hiveName == null then "unknown" else config.hyperhive.hiveName; + swarmDisplayName = + if config.hyperhive.swarmName == null then "unknown" else config.hyperhive.swarmName; + + # Resource labels every producer in this container stamps on what it + # emits. `service.name` names the container's role, not one binary + # inside it: claude's samples and `hive-metric`'s both come from this + # agent, and their metric names already tell them apart. + resourceAttributes = + "service.name=hyperhive-agent,agent=${userName},hive=${hiveDisplayName},swarm=${swarmDisplayName}" + + lib.optionalString (cfg.extraResourceAttributes != "") ",${cfg.extraResourceAttributes}"; + + # The variables an OTEL SDK reads on its own, in any language, in any + # process here. + # + # There is no auth header among them, and no mechanism to add one. An + # agent exports to the hive's own collector, which is the only thing + # holding a credential for anything upstream; nothing an agent can read + # is a secret to the swarm. An earlier revision forwarded the + # operator's upstream token into this container and merged it into the + # agent's own `~/.claude/settings.json` — which handed every agent the + # hive's credential, and was removed with the direct-export path it + # served. + producerEnv = { + OTEL_EXPORTER_OTLP_ENDPOINT = cfg.endpoint; + OTEL_EXPORTER_OTLP_PROTOCOL = cfg.protocol; + # Force CUMULATIVE temporality — Claude Code defaults to DELTA, + # which Prometheus/Mimir-family backends (incl. grafana-lgtm) + # silently drop without a deltatocumulative processor. + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; + OTEL_RESOURCE_ATTRIBUTES = resourceAttributes; + } + // lib.optionalAttrs (cfg.metricIntervalMs != null) { + OTEL_METRIC_EXPORT_INTERVAL = toString cfg.metricIntervalMs; + }; +in +{ + # OTEL stats export is configured ONCE at host level via + # `services.hyperhive.otel.*` (see nix/host-modules/otel.nix) and + # injected into every agent's build by the meta-flake renderer + # (`hive-c0re/src/meta.rs::otel_config`). These per-agent options are + # the build-time implementation surface that injection writes into; + # they are not meant to be set directly in an agent.nix. Marked + # `internal` so the host option is the only documented operator knob. + options.hyperhive.otel = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + internal = true; + description = '' + Export this agent's telemetry — Claude Code's stats (token + usage, cost, tool calls) and anything pushed with + `hive-metric` — to an OTLP endpoint. Each agent exports + directly to the hive's collector, so it keeps working even when + hive-c0re is down. Host-driven: set + `services.hyperhive.otel.enable` instead. + ''; + }; + + endpoint = lib.mkOption { + type = lib.types.str; + default = ""; + internal = true; + description = '' + OTLP collector endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT`. + Host-driven via `services.hyperhive.otel.endpoint`. + ''; + }; + + protocol = lib.mkOption { + type = lib.types.enum [ + "http/protobuf" + "http/json" + "grpc" + ]; + default = "http/protobuf"; + internal = true; + description = '' + OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`. + Host-driven via `services.hyperhive.otel.protocol`. + ''; + }; + + extraResourceAttributes = lib.mkOption { + type = lib.types.str; + default = ""; + internal = true; + description = '' + Extra comma-separated entries appended to + `OTEL_RESOURCE_ATTRIBUTES` after the built-in + `service.name` / `agent` / `hive` / `swarm` labels. + Host-driven via `services.hyperhive.otel.extraResourceAttributes`. + ''; + }; + + metricIntervalMs = lib.mkOption { + type = lib.types.nullOr lib.types.ints.positive; + default = null; + internal = true; + description = '' + Metric export interval in milliseconds, set as + `OTEL_METRIC_EXPORT_INTERVAL`. Null leaves the SDK default (60s + for Claude Code). Host-driven via + `services.hyperhive.otel.metricIntervalMs`. + ''; + }; + + debug = lib.mkOption { + type = lib.types.bool; + default = false; + internal = true; + description = '' + Emit OTEL SDK diagnostics to stderr (`CLAUDE_CODE_OTEL_DIAG_STDERR=1`). + Host-driven via `services.hyperhive.otel.debug`. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + # Two assignments, because they cover disjoint sets of processes and + # neither implies the other: + # + # - `systemd.globalEnvironment` is merged into the `Environment=` + # lines of every generated unit, so the harness, the + # bash/matrix/MCP daemons and everything the bash tool spawns + # beneath them all get it. This is the assignment that matters: + # those daemons are claude's *siblings*, not its children, so + # nothing shipped inside claude's own settings could ever reach + # them, and `hive-metric` invoked from a tool call exited with + # "OTEL_EXPORTER_OTLP_ENDPOINT not set". + # - `environment.variables` lands in `/etc/set-environment`, sourced + # by `/etc/profile` — login shells, i.e. `hivectl shell` and + # `hivectl choom`, which systemd does not start. + # + # `NIX_REMOTE` in `default.nix` is set both ways for this same + # reason. The difference is visible on any running agent: + # `HIVE_ASSETS_DIR` is an `environment.variables` entry and is absent + # from `systemctl show hive-bash-daemon.service -p Environment`, + # while `NIX_REMOTE` — the same value, also set globally — is there. + systemd.globalEnvironment = producerEnv; + environment.variables = producerEnv; + }; +}