From 5ca5433e0b2a8ff670ac2b7e63f391bf49b2f4eb Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 18 Aug 2026 23:06:14 +0200 Subject: [PATCH] otel: stop handing hive-c0re the upstream auth credential hive-c0re's container-resource exporter already targets this hive's own collector (environment.nix derives the bridge address), so the upstream header it was loaded with has nowhere to be presented: that hop is unauthenticated for every producer on the host, and the credential belongs to the swarm tier, which is the one that leaves the swarm. Drop the LoadCredential entry and the auth_headers() reader with it. The option itself stays -- swarm-otel.nix is its real consumer, via EnvironmentFile on the collector unit. Also corrects three descriptions that this makes false, or that were already false: the module doc claimed to reuse the config "Claude Code's in-container SDK export uses", which stopped being true when agents moved off that path; the nix comment claimed the secret is "the same one the agent containers get, forwarded via nspawn --load-credential", which lost its last producer earlier; and docs/observability.md described an Authorization header on a hop that will no longer send one. The headersCredential option's own docs already said it reaches "neither an agent container nor a hive's own collector" -- this makes that true rather than aspirational. --- docs/observability.md | 13 ++++--- hive-c0re/src/stats/otel_metrics.rs | 47 ++++++++------------------ nix/host-modules/hive-c0re/default.nix | 17 ++++------ 3 files changed, 27 insertions(+), 50 deletions(-) diff --git a/docs/observability.md b/docs/observability.md index 5565f19a..46f87bbd 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -256,10 +256,9 @@ reference above); custom per-data-point labels can be passed with When OTEL is enabled, **hive-c0re itself** also exports each agent container's resource load — the same cgroup gauges shown on the dashboard -LOAD tab — to the configured `endpoint`, reusing the same -`services.hyperhive.otel` config (no separate toggle). These come from the -host, not the in-container Claude SDK, so they cover containers even when -their agent is idle. +LOAD tab — to this hive's own collector, exactly like an agent does and with +no separate toggle. These come from the host, not the in-container Claude SDK, +so they cover containers even when their agent is idle. Emitted via the OpenTelemetry Rust SDK, using the [semconv `container.*`](https://opentelemetry.io/docs/specs/semconv/system/container-metrics/) @@ -284,9 +283,9 @@ labels (`hive`, `swarm`, …) ride on the resource via `extraResourceAttributes`. Cadence follows `metricIntervalMs` (default 60s). Transport is OTLP/HTTP -(JSON) to ``; the auth header is loaded onto hive-c0re's own unit -via systemd `LoadCredential` (from the same `headersCredential` file) and -sent as `Authorization`. +(JSON) to the hive collector's bridge address, with no auth header — that +first hop is unauthenticated for every producer on this host, and the upstream +credential stays on the swarm tier. ## Agent-emitted custom metrics (`hive-metric`) diff --git a/hive-c0re/src/stats/otel_metrics.rs b/hive-c0re/src/stats/otel_metrics.rs index e9d3291d..0883f04e 100644 --- a/hive-c0re/src/stats/otel_metrics.rs +++ b/hive-c0re/src/stats/otel_metrics.rs @@ -1,11 +1,11 @@ //! Per-agent container-resource OTEL export. hive-c0re already samples each //! agent container's cgroup load for the dashboard //! ([`super::container_stats`]); this rides those same gauges out to the -//! hive-wide OTLP endpoint, reusing the SAME `services.hyperhive.otel` config -//! (endpoint + auth header) that Claude Code's in-container SDK export uses — -//! no new toggle. The auth header is loaded onto hive-c0re's own unit via -//! systemd `LoadCredential` (see `nix/host-modules/hive-c0re`) and read from -//! `$CREDENTIALS_DIRECTORY/otel-headers`. +//! hive's own collector — the same first hop the agents use, arriving as +//! `HYPERHIVE_OTEL_ENDPOINT` (see `nix/host-modules/hive-c0re`). No toggle of +//! its own, and no credential: a hive's collector takes unauthenticated OTLP +//! on the bridge, and the only hop that presents anything is the swarm tier's, +//! which is the one that leaves the swarm. //! //! Emits the OTEL **semconv `container.*`** metrics with the standard //! `container.name` attribute (so off-the-shelf OTEL/Grafana container @@ -21,14 +21,13 @@ //! observable-instrument callbacks are sync. So an async task refreshes a //! shared snapshot on an interval, and the (sync) callbacks read it. -use std::collections::HashMap; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; use anyhow::{Context, Result}; use opentelemetry::KeyValue; use opentelemetry::metrics::MeterProvider as _; -use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig, WithHttpConfig}; +use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig}; use opentelemetry_sdk::Resource; use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; @@ -90,15 +89,16 @@ fn build_provider( // http/json — the only OTLP transport this crate enables (matching // hive-metric). The Claude SDK path honours `HYPERHIVE_OTEL_PROTOCOL` for // its own export; this exporter is always http/json. - let mut builder = MetricExporter::builder() + // + // No auth headers: the destination is this hive's own collector, which + // takes unauthenticated OTLP on the bridge. Adding one here would put the + // upstream credential on a hop that never uses it. + let exporter = MetricExporter::builder() .with_http() .with_endpoint(endpoint) - .with_protocol(Protocol::HttpJson); - let headers = auth_headers(); - if !headers.is_empty() { - builder = builder.with_headers(headers); - } - let exporter = builder.build().context("build OTLP metric exporter")?; + .with_protocol(Protocol::HttpJson) + .build() + .context("build OTLP metric exporter")?; // Drive the reader's export at `interval` so `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` // is the real export cadence (not just the snapshot-refresh cadence). The // refresher runs at the same interval, gather-first, so the snapshot is @@ -294,25 +294,6 @@ fn resource_attributes() -> Vec<(String, String)> { .unwrap_or_default() } -/// OTLP auth headers from the systemd credential at -/// `$CREDENTIALS_DIRECTORY/otel-headers` (an `OTEL_EXPORTER_OTLP_HEADERS`-style -/// `Key=Value` line). Empty when the credential is absent — the collector is -/// then assumed unauthenticated. -fn auth_headers() -> HashMap { - let Some(dir) = std::env::var("CREDENTIALS_DIRECTORY").ok() else { - return HashMap::new(); - }; - let path = std::path::Path::new(&dir).join("otel-headers"); - let Ok(raw) = std::fs::read_to_string(&path) else { - tracing::warn!( - "otel container-metrics: no auth header at $CREDENTIALS_DIRECTORY/otel-headers; \ - exporting without Authorization" - ); - return HashMap::new(); - }; - parse_kv(&raw).into_iter().collect() -} - /// Parse `key=value` pairs separated by commas and/or newlines. The value /// keeps any `=` after the first (so `Authorization=Bearer x=y` → `Bearer x=y`). fn parse_kv(s: &str) -> Vec<(String, String)> { diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 056b28b3..129063d7 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -273,16 +273,13 @@ in RuntimeDirectoryPreserve = "yes"; StateDirectory = "hyperhive"; StateDirectoryMode = "0750"; - # OTEL auth-header secret, loaded onto hive-c0re's own unit so its - # per-agent container-resource metrics exporter can read it - # at $CREDENTIALS_DIRECTORY/otel-headers — via systemd, not a world - # path. Same secret the agent containers get (forwarded there via - # nspawn --load-credential); this just also hands it to c0re itself. - # Empty list (no credential) when otel is off or no header is set. + # No OTEL credential here. hive-c0re's container-resource exporter + # targets this hive's own collector, which takes unauthenticated OTLP + # on the bridge; the upstream header belongs to the swarm tier + # (`swarm-otel.nix`), the only hop that leaves the swarm. Handing it to + # the daemon as well would put a secret on a process that has nowhere + # to present it. LoadCredential = - lib.optional ( - config.services.hyperhive.otel.enable && config.services.hyperhive.otel.headersCredential != null - ) "otel-headers:${config.services.hyperhive.otel.headersCredential}" # The swarm-queue client secret this hive authenticates with to # publish its own status. `LoadCredential` and not a copy: root # reads the plaintext at unit start and hive-core sees it 0400 @@ -290,7 +287,7 @@ in # and the daemon never needs read access to wherever it lives. # (The callout responder copies instead only because it # delivers into a container, across a filesystem boundary.) - ++ lib.optional ( + lib.optional ( config.services.hyperhive.swarm.statusPublish.clientSecretFile != null ) "swarm-status-client.secret:${config.services.hyperhive.swarm.statusPublish.clientSecretFile}"; # Sandboxing. hive-c0re is unprivileged (runs as hive-core, never