diff --git a/docs/observability.md b/docs/observability.md index 5565f19a..46f87bbd 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -256,10 +256,9 @@ reference above); custom per-data-point labels can be passed with When OTEL is enabled, **hive-c0re itself** also exports each agent container's resource load — the same cgroup gauges shown on the dashboard -LOAD tab — to the configured `endpoint`, reusing the same -`services.hyperhive.otel` config (no separate toggle). These come from the -host, not the in-container Claude SDK, so they cover containers even when -their agent is idle. +LOAD tab — to this hive's own collector, exactly like an agent does and with +no separate toggle. These come from the host, not the in-container Claude SDK, +so they cover containers even when their agent is idle. Emitted via the OpenTelemetry Rust SDK, using the [semconv `container.*`](https://opentelemetry.io/docs/specs/semconv/system/container-metrics/) @@ -284,9 +283,9 @@ labels (`hive`, `swarm`, …) ride on the resource via `extraResourceAttributes`. Cadence follows `metricIntervalMs` (default 60s). Transport is OTLP/HTTP -(JSON) to ``; the auth header is loaded onto hive-c0re's own unit -via systemd `LoadCredential` (from the same `headersCredential` file) and -sent as `Authorization`. +(JSON) to the hive collector's bridge address, with no auth header — that +first hop is unauthenticated for every producer on this host, and the upstream +credential stays on the swarm tier. ## Agent-emitted custom metrics (`hive-metric`) diff --git a/hive-c0re/src/stats/otel_metrics.rs b/hive-c0re/src/stats/otel_metrics.rs index e9d3291d..0883f04e 100644 --- a/hive-c0re/src/stats/otel_metrics.rs +++ b/hive-c0re/src/stats/otel_metrics.rs @@ -1,11 +1,11 @@ //! Per-agent container-resource OTEL export. hive-c0re already samples each //! agent container's cgroup load for the dashboard //! ([`super::container_stats`]); this rides those same gauges out to the -//! hive-wide OTLP endpoint, reusing the SAME `services.hyperhive.otel` config -//! (endpoint + auth header) that Claude Code's in-container SDK export uses — -//! no new toggle. The auth header is loaded onto hive-c0re's own unit via -//! systemd `LoadCredential` (see `nix/host-modules/hive-c0re`) and read from -//! `$CREDENTIALS_DIRECTORY/otel-headers`. +//! hive's own collector — the same first hop the agents use, arriving as +//! `HYPERHIVE_OTEL_ENDPOINT` (see `nix/host-modules/hive-c0re`). No toggle of +//! its own, and no credential: a hive's collector takes unauthenticated OTLP +//! on the bridge, and the only hop that presents anything is the swarm tier's, +//! which is the one that leaves the swarm. //! //! Emits the OTEL **semconv `container.*`** metrics with the standard //! `container.name` attribute (so off-the-shelf OTEL/Grafana container @@ -21,14 +21,13 @@ //! observable-instrument callbacks are sync. So an async task refreshes a //! shared snapshot on an interval, and the (sync) callbacks read it. -use std::collections::HashMap; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; use anyhow::{Context, Result}; use opentelemetry::KeyValue; use opentelemetry::metrics::MeterProvider as _; -use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig, WithHttpConfig}; +use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig}; use opentelemetry_sdk::Resource; use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; @@ -90,15 +89,16 @@ fn build_provider( // http/json — the only OTLP transport this crate enables (matching // hive-metric). The Claude SDK path honours `HYPERHIVE_OTEL_PROTOCOL` for // its own export; this exporter is always http/json. - let mut builder = MetricExporter::builder() + // + // No auth headers: the destination is this hive's own collector, which + // takes unauthenticated OTLP on the bridge. Adding one here would put the + // upstream credential on a hop that never uses it. + let exporter = MetricExporter::builder() .with_http() .with_endpoint(endpoint) - .with_protocol(Protocol::HttpJson); - let headers = auth_headers(); - if !headers.is_empty() { - builder = builder.with_headers(headers); - } - let exporter = builder.build().context("build OTLP metric exporter")?; + .with_protocol(Protocol::HttpJson) + .build() + .context("build OTLP metric exporter")?; // Drive the reader's export at `interval` so `HYPERHIVE_OTEL_METRIC_INTERVAL_MS` // is the real export cadence (not just the snapshot-refresh cadence). The // refresher runs at the same interval, gather-first, so the snapshot is @@ -294,25 +294,6 @@ fn resource_attributes() -> Vec<(String, String)> { .unwrap_or_default() } -/// OTLP auth headers from the systemd credential at -/// `$CREDENTIALS_DIRECTORY/otel-headers` (an `OTEL_EXPORTER_OTLP_HEADERS`-style -/// `Key=Value` line). Empty when the credential is absent — the collector is -/// then assumed unauthenticated. -fn auth_headers() -> HashMap { - let Some(dir) = std::env::var("CREDENTIALS_DIRECTORY").ok() else { - return HashMap::new(); - }; - let path = std::path::Path::new(&dir).join("otel-headers"); - let Ok(raw) = std::fs::read_to_string(&path) else { - tracing::warn!( - "otel container-metrics: no auth header at $CREDENTIALS_DIRECTORY/otel-headers; \ - exporting without Authorization" - ); - return HashMap::new(); - }; - parse_kv(&raw).into_iter().collect() -} - /// Parse `key=value` pairs separated by commas and/or newlines. The value /// keeps any `=` after the first (so `Authorization=Bearer x=y` → `Bearer x=y`). fn parse_kv(s: &str) -> Vec<(String, String)> { diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 056b28b3..129063d7 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -273,16 +273,13 @@ in RuntimeDirectoryPreserve = "yes"; StateDirectory = "hyperhive"; StateDirectoryMode = "0750"; - # OTEL auth-header secret, loaded onto hive-c0re's own unit so its - # per-agent container-resource metrics exporter can read it - # at $CREDENTIALS_DIRECTORY/otel-headers — via systemd, not a world - # path. Same secret the agent containers get (forwarded there via - # nspawn --load-credential); this just also hands it to c0re itself. - # Empty list (no credential) when otel is off or no header is set. + # No OTEL credential here. hive-c0re's container-resource exporter + # targets this hive's own collector, which takes unauthenticated OTLP + # on the bridge; the upstream header belongs to the swarm tier + # (`swarm-otel.nix`), the only hop that leaves the swarm. Handing it to + # the daemon as well would put a secret on a process that has nowhere + # to present it. LoadCredential = - lib.optional ( - config.services.hyperhive.otel.enable && config.services.hyperhive.otel.headersCredential != null - ) "otel-headers:${config.services.hyperhive.otel.headersCredential}" # The swarm-queue client secret this hive authenticates with to # publish its own status. `LoadCredential` and not a copy: root # reads the plaintext at unit start and hive-core sees it 0400 @@ -290,7 +287,7 @@ in # and the daemon never needs read access to wherever it lives. # (The callout responder copies instead only because it # delivers into a container, across a filesystem boundary.) - ++ lib.optional ( + lib.optional ( config.services.hyperhive.swarm.statusPublish.clientSecretFile != null ) "swarm-status-client.secret:${config.services.hyperhive.swarm.statusPublish.clientSecretFile}"; # Sandboxing. hive-c0re is unprivileged (runs as hive-core, never