feat(#2899): add host.arch + service.version to container metrics

The container-resource exporter identified the samples it sent by
container and by hive, but not by machine or by build — so a sample
could not be attributed to the host it came from or the deploy that
produced it.

Both go on the OTEL resource rather than on each data point, for the
same reason `hive` / `swarm` already do: they are constant across one
hive-c0re. Only per-container facts stay per-data-point.

- `host.arch` — mapped to the semconv spelling, not forwarded from
  rust's. The two disagree on exactly the architectures this runs on
  (`x86_64` / `aarch64` vs `amd64` / `arm64`), and the failure mode is
  silent: a dashboard filtering the standard value matches nothing.
  A test pins this, since nothing else would catch it.
- `service.version` — the running flake rev, via
  `auto_update::current_flake_rev`, the same source the dashboard
  snapshot and `get_agent_meta` already use. NOT the crate version:
  that's a workspace constant that never moves between deploys, so it
  could not answer "which build produced this sample?". Omitted rather
  than guessed when the flake ref carries no rev.

`spawn_exporter` takes the flake ref to reach the rev — the string it
needs, not the whole `Coordinator`, so the module's coupling doesn't
widen for one attribute.

The issue's third item, `container`, needs no change: `attrs()` has
emitted `container.name` per data point since this exporter landed.

Checked with clippy (`--all-targets -D warnings`), `cargo test -p
hive-c0re` (321 passed) and `nix fmt`. No option surface is touched, so
no nix-eval gate.
This commit is contained in:
atlas 2026-08-01 13:38:32 +02:00 committed by mara
commit d09ec31431
2 changed files with 63 additions and 8 deletions

View file

@ -519,7 +519,7 @@ async fn cmd_serve(
// cgroup gauges out to the configured OTLP endpoint, reusing the hive
// `services.hyperhive.otel` config (endpoint + LoadCredential auth).
// No-op when OTEL isn't configured.
crate::otel_metrics::spawn_exporter();
crate::otel_metrics::spawn_exporter(&coord.hyperhive_flake);
// build_logs.sqlite vacuum: c0re-side (single db). Failures kept
// 30d, successes 24h — see `build_logs::vacuum` for the rule.
crate::build_logs::spawn_vacuum(&coord);

View file

@ -10,7 +10,10 @@
//! Emits the OTEL **semconv `container.*`** metrics with the standard
//! `container.name` attribute (so off-the-shelf OTEL/Grafana container
//! dashboards work), plus the hive-specific `agent` / `hive` / `swarm` labels
//! for our own dashboards. Uses the OpenTelemetry Rust SDK (same crates as
//! for our own dashboards. The resource additionally carries semconv
//! `host.arch` and `service.version` (the running hyperhive flake rev), so a
//! sample can be attributed to a machine and a deploy.
//! Uses the OpenTelemetry Rust SDK (same crates as
//! `hive-metric`); the blocking OTLP client is deliberate — the metrics SDK's
//! `PeriodicReader` runs on a background thread with no Tokio reactor.
//!
@ -45,7 +48,7 @@ static PROVIDER: OnceLock<SdkMeterProvider> = OnceLock::new();
/// Spawn the container-resource OTEL exporter if OTEL is configured
/// (`HYPERHIVE_OTEL_ENDPOINT` non-empty — the same enable signal
/// [`crate::meta::otel_config`] uses). No-op otherwise. Call once at startup.
pub fn spawn_exporter() {
pub fn spawn_exporter(hyperhive_flake: &str) {
let Some(endpoint) = endpoint() else {
tracing::debug!("otel container-metrics: no endpoint configured, exporter disabled");
return;
@ -67,7 +70,7 @@ pub fn spawn_exporter() {
}
});
match build_provider(&endpoint, interval, snapshot) {
match build_provider(&endpoint, interval, snapshot, hyperhive_flake) {
Ok(provider) => {
let _ = PROVIDER.set(provider);
tracing::info!(%endpoint, ?interval, "otel container-metrics: exporter enabled");
@ -82,6 +85,7 @@ fn build_provider(
endpoint: &str,
interval: Duration,
snapshot: Arc<Mutex<Vec<ContainerResource>>>,
hyperhive_flake: &str,
) -> Result<SdkMeterProvider> {
// http/json — the only OTLP transport this crate enables (matching
// hive-metric). The Claude SDK path honours `HYPERHIVE_OTEL_PROTOCOL` for
@ -104,7 +108,7 @@ fn build_provider(
.build();
let provider = SdkMeterProvider::builder()
.with_reader(reader)
.with_resource(resource())
.with_resource(resource(hyperhive_flake))
.build();
register_instruments(&provider, snapshot);
Ok(provider)
@ -221,17 +225,49 @@ fn attrs(c: &ContainerResource) -> Vec<KeyValue> {
]
}
/// Resource: `service.name = hyperhive-c0re` plus whatever the operator set in
/// Resource: `service.name = hyperhive-c0re`, the host architecture, the
/// running hyperhive revision, plus whatever the operator set in
/// `extraResourceAttributes` (where the hive `hive` / `swarm` labels live, same
/// channel the agent-side export uses).
fn resource() -> Resource {
let mut builder = Resource::builder().with_service_name("hyperhive-c0re");
///
/// These belong on the resource rather than on each data point for the same
/// reason `hive` / `swarm` do: they are constant across one `hive-c0re`. Only
/// facts that vary per container (see [`attrs`]) are per-data-point.
///
/// `service.version` is the **flake revision**, not the crate version — the
/// crate version is a workspace constant that never moves between deploys, so
/// it could not answer "which build produced this sample?". `None` when the
/// flake ref carries no rev (a local path ref), in which case the attribute is
/// omitted rather than reported as a guess.
fn resource(hyperhive_flake: &str) -> Resource {
let mut builder = Resource::builder()
.with_service_name("hyperhive-c0re")
.with_attribute(KeyValue::new("host.arch", host_arch()));
if let Some(rev) = crate::auto_update::current_flake_rev(hyperhive_flake) {
builder = builder.with_attribute(KeyValue::new("service.version", rev));
}
for (k, v) in resource_attributes() {
builder = builder.with_attribute(KeyValue::new(k, v));
}
builder.build()
}
/// The host architecture as OTEL semconv spells it.
///
/// Rust's [`std::env::consts::ARCH`] and semconv `host.arch` disagree on the
/// two architectures this actually runs on: rust says `x86_64` / `aarch64`,
/// semconv says `amd64` / `arm64`. Forwarding rust's spelling would emit a
/// label that off-the-shelf dashboards silently fail to match — the same class
/// of problem as guessing a metric name. Anything else is passed through: a
/// non-standard value is more useful than a wrong standard one.
fn host_arch() -> &'static str {
match std::env::consts::ARCH {
"x86_64" => "amd64",
"aarch64" => "arm64",
other => other,
}
}
/// `HYPERHIVE_OTEL_ENDPOINT`, non-empty. The enable signal.
fn endpoint() -> Option<String> {
std::env::var("HYPERHIVE_OTEL_ENDPOINT")
@ -315,4 +351,23 @@ mod tests {
]
);
}
/// `host.arch` must be the semconv spelling, not rust's. The two disagree
/// on exactly the architectures hyperhive runs on, and the failure is
/// silent — a dashboard filtering `host.arch="amd64"` simply matches
/// nothing if we emit `x86_64`.
#[test]
fn host_arch_is_semconv_not_rust_spelling() {
assert!(
!matches!(host_arch(), "x86_64" | "aarch64"),
"host_arch() forwarded rust's ARCH spelling ({}) — semconv wants amd64 / arm64",
host_arch()
);
// On the architectures we actually ship, pin the exact expected value
// rather than only asserting the negative above.
#[cfg(target_arch = "x86_64")]
assert_eq!(host_arch(), "amd64");
#[cfg(target_arch = "aarch64")]
assert_eq!(host_arch(), "arm64");
}
}