From 141764c6eb5178ffd7b93d312f26b8d46ab99c15 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 27 Jun 2026 15:05:54 +0200 Subject: [PATCH] feat(#2023): ship OTEL via managed claude settings json, drop the wrapper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per mara: configure OTEL in the generated claude settings json (what the Claude Code docs suggest), not a launch wrapper or /etc shell file. claude-code auto-discovers /etc/claude-code/managed-settings.json in every context — the harness turn loop AND hivectl choom — so putting the OTEL env there gives telemetry parity declaratively, with no wrapper and no --settings plumbing. - managed-settings.json: was a static shared .source; now, when OTEL is enabled, a per-agent build-time jq merge of the base asset + an env block (jq at build, not eval-time readFile, to avoid IFD). OTEL off = the static asset verbatim. - otelSettingsEnv carries the static OTEL knobs + OTEL_RESOURCE_ATTRIBUTES with the agent name (build-time) and the hive/swarm names forwarded by meta.rs into environment.variables (mara: forward host config into agent config where needed). - removed the hive-serve-otel ExecStart wrapper, the per-unit otelEnv, and the otel-headers LoadCredential from the harness service — the harness binary emits no OTEL itself; only claude does, and it now reads the settings json directly. Known follow-ups (noted in code): the auth header (otel.headersCredential, opt-in/default-null) is a secret and can't live in the world-readable settings file — authenticated collectors need a runtime mechanism; this PR covers the unauthenticated default. nix fmt clean. --- nix/templates/harness-base.nix | 121 ++++++++++++++++----------------- 1 file changed, 60 insertions(+), 61 deletions(-) diff --git a/nix/templates/harness-base.nix b/nix/templates/harness-base.nix index c081d777..13a2f415 100644 --- a/nix/templates/harness-base.nix +++ b/nix/templates/harness-base.nix @@ -17,6 +17,40 @@ let # from `userName` to keep them coupled. userName = config.hyperhive.user.name; homeDir = "/home/${userName}"; + # Hive-wide OpenTelemetry config (host-driven; baked in per-agent by + # meta.rs `otel_config`). + otelCfg = config.hyperhive.otel; + # Hive/swarm display names are forwarded into each agent's build by + # meta.rs as `environment.variables` (per-agent, build-time strings), + # so they can be baked into the resource attributes below without a + # runtime shell. Absent (option unset) → "unknown". + hiveDisplayName = config.environment.variables.HYPERHIVE_HIVE_NAME or "unknown"; + swarmDisplayName = config.environment.variables.HYPERHIVE_SWARM_NAME or "unknown"; + # OTEL environment Claude Code reads to export metrics/logs/traces. + # Shipped via the managed claude settings json (below), which claude + # auto-discovers for BOTH the harness turn-loop and `hivectl choom` — + # so telemetry parity is declarative, with no launch wrapper. The + # auth header (`otel.headersCredential`) is deliberately NOT included: + # it's a secret and the settings file is world-readable; authenticated + # collectors need a runtime mechanism (tracked as a follow-up). + otelSettingsEnv = { + CLAUDE_CODE_ENABLE_TELEMETRY = "1"; + OTEL_METRICS_EXPORTER = "otlp"; + OTEL_LOGS_EXPORTER = "otlp"; + OTEL_TRACES_EXPORTER = "otlp"; + OTEL_EXPORTER_OTLP_PROTOCOL = otelCfg.protocol; + OTEL_EXPORTER_OTLP_ENDPOINT = otelCfg.endpoint; + # Force CUMULATIVE temporality — Claude Code defaults to DELTA, + # which Prometheus/Mimir-family backends (incl. grafana-lgtm) + # silently drop without a deltatocumulative processor. + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; + OTEL_RESOURCE_ATTRIBUTES = + "service.name=hyperhive-agent,agent=${userName},hive=${hiveDisplayName},swarm=${swarmDisplayName}" + + lib.optionalString (otelCfg.extraResourceAttributes != "") ",${otelCfg.extraResourceAttributes}"; + } + // lib.optionalAttrs (otelCfg.metricIntervalMs != null) { + OTEL_METRIC_EXPORT_INTERVAL = toString otelCfg.metricIntervalMs; + }; # Single source of truth for the default matrix homeserver URL, shared # by the `hyperhive.matrix.url` option default and the daemon-unit guard # that decides whether to set a unit-level HIVE_MATRIX_URL (so the two @@ -1073,8 +1107,25 @@ in # deliberately NOT shipped here: effort is controlled live via the # `--effort` CLI flag (HIVE_DEFAULT_EFFORT / the per-agent UI slider), # which managed scope would otherwise override and lock. + # Base hive-enforced settings, plus — when OTEL is enabled — an `env` + # block so Claude Code exports telemetry natively from the settings + # file it reads in every context (harness turn-loop AND `hivectl + # choom`), with no launch wrapper. With OTEL off it's the shared + # static asset verbatim; with OTEL on it's a per-agent merge (the + # resource attributes carry the agent name) done at BUILD time via + # `jq` — not eval-time `readFile`, which would be import-from- + # derivation. environment.etc."claude-code/managed-settings.json".source = - "${pkgs.hyperhive-assets}/share/hyperhive/prompts/claude-settings.json"; + let + baseSettings = "${pkgs.hyperhive-assets}/share/hyperhive/prompts/claude-settings.json"; + in + if !otelCfg.enable then + baseSettings + else + pkgs.runCommand "managed-settings.json" { nativeBuildInputs = [ pkgs.jq ]; } '' + jq --argjson env ${lib.escapeShellArg (builtins.toJSON otelSettingsEnv)} \ + '. + { env: $env }' ${baseSettings} > "$out" + ''; # Merged frontend static tree. Base = `${frontend.dist}/agent/`, # then each `extraFiles` entry is laid on top at its `target` @@ -1707,49 +1758,11 @@ in systemd.services.hive-ag3nt = let binary = "hive"; - otel = config.hyperhive.otel; - # Claude Code's native OpenTelemetry is env-driven; the harness - # spawns `claude` as a child which inherits this unit's env, so - # setting these here is all it takes to export per-agent stats. - otelEnv = lib.optionalAttrs otel.enable ( - { - CLAUDE_CODE_ENABLE_TELEMETRY = "1"; - OTEL_METRICS_EXPORTER = "otlp"; - OTEL_LOGS_EXPORTER = "otlp"; - # Route traces to OTLP too so any spans Claude Code emits land - # at the configured collector rather than a default exporter. - OTEL_TRACES_EXPORTER = "otlp"; - OTEL_EXPORTER_OTLP_PROTOCOL = otel.protocol; - OTEL_EXPORTER_OTLP_ENDPOINT = otel.endpoint; - # Force CUMULATIVE metric temporality. Claude Code defaults to - # DELTA, which Prometheus/Mimir-family backends (the common case, - # incl. grafana-lgtm) silently drop unless a deltatocumulative - # processor is wired — so delta = "logs arrive, metrics vanish". - # Cumulative is what those backends ingest natively. Verified: - # a delta export never registers the metric name; cumulative does. - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative"; - } - // lib.optionalAttrs (otel.metricIntervalMs != null) { - OTEL_METRIC_EXPORT_INTERVAL = toString otel.metricIntervalMs; - } - ); - # When OTEL is on, wrap the harness launch so (1) the bearer-token - # header is read from the systemd credential at start (never in the - # nix store or argv) and (2) the resource attributes are assembled - # from this agent's name (known at build time) plus the hive/swarm - # names (inherited HYPERHIVE_HIVE_NAME / HYPERHIVE_SWARM_NAME env, - # the same vars `identity::hive_name`/`swarm_name` read at runtime). - otelExecStart = pkgs.writeShellScript "hive-serve-otel" '' - set -eu - if [ -n "''${CREDENTIALS_DIRECTORY:-}" ] && [ -r "$CREDENTIALS_DIRECTORY/otel-headers" ]; then - OTEL_EXPORTER_OTLP_HEADERS="$(cat "$CREDENTIALS_DIRECTORY/otel-headers")" - export OTEL_EXPORTER_OTLP_HEADERS - fi - export OTEL_RESOURCE_ATTRIBUTES="service.name=hyperhive-agent,agent=${userName},hive=''${HYPERHIVE_HIVE_NAME:-unknown},swarm=''${HYPERHIVE_SWARM_NAME:-unknown}${ - lib.optionalString (otel.extraResourceAttributes != "") ",${otel.extraResourceAttributes}" - }" - exec ${pkgs.hyperhive}/bin/${binary} serve - ''; + # OTEL is shipped declaratively via the managed claude settings + # json (`environment.etc."claude-code/managed-settings.json"`, + # `otelSettingsEnv` in the top-level let) — claude reads it for + # both the harness turn-loop and `hivectl choom`, so there's no + # launch wrapper or per-unit OTEL env here anymore. in { description = "${binary} harness"; @@ -1773,7 +1786,6 @@ in # bind-mounts and gateway upstream config stay in sync. HIVE_WEB_SOCKET = "/run/hive-agent/${userName}/web.sock"; } - // otelEnv // lib.optionalAttrs config.hyperhive.gui.enable { # Tells the harness which fixed VNC port weston bound, and (by # its presence) that gui is enabled — the harness `/screen/ws` @@ -1784,13 +1796,9 @@ in HIVE_GUI_VNC_PORT = toString config.hyperhive.gui.vncPort; }; serviceConfig = { - ExecStart = if otel.enable then "${otelExecStart}" else "${pkgs.hyperhive}/bin/${binary} serve"; - # Pin the journal identity to the binary name. Without this, - # systemd derives SyslogIdentifier from the ExecStart basename — - # which under OTEL is the wrapper script's store path - # (`-hive-serve-otel`), so every agent's harness logs showed - # that opaque name instead of `hive`. Set explicitly so the - # identity is stable across the otel / non-otel ExecStart branches. + ExecStart = "${pkgs.hyperhive}/bin/${binary} serve"; + # Pin the journal identity to the binary name (otherwise systemd + # derives SyslogIdentifier from the ExecStart basename). SyslogIdentifier = binary; Restart = "on-failure"; RestartSec = 2; @@ -1801,15 +1809,6 @@ in RuntimeDirectory = "hive-config"; User = userName; Group = userName; - } - // lib.optionalAttrs (otel.enable && otel.headersCredential != null) { - # Inherit form (no `:path`): hive-c0re forwards the host file at - # `headersCredential` into this container's credential store via - # nspawn `--load-credential=otel-headers:` (see - # lifecycle.rs::hive_load_credentials). The path isn't reachable - # from inside the container, so we inherit the already-loaded - # credential by name rather than re-reading the host path here. - LoadCredential = [ "otel-headers" ]; }; };