feat(#2007): export per-agent container cpu/mem/disk via otel
hive-c0re already samples each agent container's cgroup load for the dashboard (stats/container_stats.rs); this rides those gauges out to the configured OTLP endpoint, reusing the existing services.hyperhive.otel config (endpoint + auth header) — no new toggle. - New stats/otel_metrics.rs: exports via the OpenTelemetry Rust SDK (same crates as hive-metric) with the semconv container.* metric names + container.name attribute so off-the-shelf OTel/Grafana dashboards work, plus the hive agent label. container.cpu.time (counter, s, from cumulative cpu.stat usage_usec), container.memory.usage, container.memory.usage.limit; memory peak / on-disk storage / instantaneous cpu percent stay hyperhive.* custom (no semconv equivalent). Observable instruments read a shared snapshot an async task refreshes (gather() is async; SDK callbacks sync). - container_stats: expose cpu_time_usec (cumulative) on ContainerResource. - The OTLP auth header is loaded onto hive-c0re's own unit via systemd LoadCredential and read from $CREDENTIALS_DIRECTORY/otel-headers. - docs/observability.md documents the host-emitted semconv metrics. Host-side export, so it covers containers even when their agent is idle.
This commit is contained in:
parent
c238ffe1ff
commit
419c9659a3
9 changed files with 396 additions and 2 deletions
|
|
@ -43,6 +43,11 @@ pub struct ContainerResource {
|
|||
/// Host-normalised CPU usage over the sample interval, as a
|
||||
/// percentage of total host CPU (0..100 across all cores).
|
||||
pub cpu_pct: f64,
|
||||
/// Cumulative CPU time (`cpu.stat` `usage_usec`, microseconds,
|
||||
/// monotonic). `None` if the read failed. Feeds the OTEL semconv
|
||||
/// `container.cpu.time` counter (which wants absolute cumulative time,
|
||||
/// converted to seconds), distinct from the sampled `cpu_pct`.
|
||||
pub cpu_time_usec: Option<u64>,
|
||||
/// Current memory usage (`memory.current`), bytes.
|
||||
pub mem_current_bytes: u64,
|
||||
/// High-water memory usage since container start (`memory.peak`),
|
||||
|
|
@ -262,7 +267,11 @@ pub async fn gather() -> Vec<ContainerResource> {
|
|||
|
||||
let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
|
||||
for (i, (name, dir)) in candidates.iter().enumerate() {
|
||||
let cpu_pct = match (t0[i], read_usage_usec(dir)) {
|
||||
// The second cumulative read — also exposed raw as `cpu_time_usec`
|
||||
// for the OTEL `container.cpu.time` counter (which wants absolute
|
||||
// cumulative CPU time, not the sampled percent).
|
||||
let usage_now = read_usage_usec(dir);
|
||||
let cpu_pct = match (t0[i], usage_now) {
|
||||
(Some(a), Some(b)) => {
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
|
|
@ -276,6 +285,7 @@ pub async fn gather() -> Vec<ContainerResource> {
|
|||
out.push(ContainerResource {
|
||||
name: name.clone(),
|
||||
cpu_pct,
|
||||
cpu_time_usec: usage_now,
|
||||
mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
|
||||
mem_peak_bytes: read_u64(&dir.join("memory.peak")),
|
||||
mem_max_bytes: read_mem_max(&dir.join("memory.max")),
|
||||
|
|
|
|||
Loading…
Reference in a new issue