hyperhive/hive-c0re/src/container_stats.rs

//! Live per-agent-container resource load (CPU + memory) for the
//! dashboard. Reads cgroup v2 stat files for each running agent
//! machine directly — same privsep-clean posture as the turn-stats
//! sqlite reads (`cpu.stat` / `memory.*` are world-readable; no
//! `hive-priv` needed). Runs host-side in hive-c0re, where
//! `/sys/fs/cgroup/machine.slice/` holds the nspawn machine scopes.
//!
//! No network: agents share the host network namespace
//! (`privateNetwork = false`), so there is no per-container net
//! counter to read. Per-agent network only becomes meaningful with the
//! netns-isolation roadmap (`docs/network.md`).
//!
//! CPU is cumulative (`usage_usec` is monotonic), so a single read is
//! meaningless — we sample every machine's counter, sleep one short
//! interval, sample again, and divide the delta by `interval × nproc`
//! to get a host-normalised percentage (0..100 across all cores).

use std::path::PathBuf;
use std::time::Duration;

use serde::Serialize;
use tokio::time::sleep;

use crate::coordinator::Coordinator;

/// Sampling interval for the two CPU reads. Short enough to keep the
/// endpoint snappy, long enough that the delta isn't dominated by read
/// jitter.
const CPU_SAMPLE: Duration = Duration::from_millis(200);

const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice";

#[derive(Debug, Serialize)]
pub struct ContainerResource {
    /// Agent name (without the `h-` machine prefix).
    pub name: String,
    /// Host-normalised CPU usage over the sample interval, as a
    /// percentage of total host CPU (0..100 across all cores).
    pub cpu_pct: f64,
    /// Current memory usage (`memory.current`), bytes.
    pub mem_current_bytes: u64,
    /// High-water memory usage since container start (`memory.peak`),
    /// bytes. `None` if the kernel doesn't expose `memory.peak`.
    pub mem_peak_bytes: Option<u64>,
    /// Memory ceiling (`memory.max`), bytes. `None` when unlimited
    /// (the file reads `max`).
    pub mem_max_bytes: Option<u64>,
}

/// systemd escapes the machine name in the cgroup scope dir
/// (`machine-<escaped>.scope`). For our machine names — `h-<agent>`,
/// agent ∈ `[a-z0-9_-]` — the only character systemd escapes is `-`,
/// which becomes `\x2d` (verified against `systemd-escape`).
fn escape_machine(machine: &str) -> String {
    machine.replace('-', "\\x2d")
}

fn scope_dir(machine: &str) -> PathBuf {
    PathBuf::from(MACHINE_SLICE).join(format!("machine-{}.scope", escape_machine(machine)))
}

/// Read a single unsigned integer from a one-line cgroup file.
fn read_u64(path: &std::path::Path) -> Option<u64> {
    std::fs::read_to_string(path)
        .ok()?
        .trim()
        .parse::<u64>()
        .ok()
}

/// `memory.max` reads `max` when there's no limit — map that to `None`.
fn read_mem_max(path: &std::path::Path) -> Option<u64> {
    let s = std::fs::read_to_string(path).ok()?;
    let s = s.trim();
    if s == "max" {
        None
    } else {
        s.parse::<u64>().ok()
    }
}

/// Parse `usage_usec` (cumulative CPU time, microseconds) from a
/// scope's `cpu.stat`.
fn read_usage_usec(dir: &std::path::Path) -> Option<u64> {
    let text = std::fs::read_to_string(dir.join("cpu.stat")).ok()?;
    for line in text.lines() {
        if let Some(rest) = line.strip_prefix("usage_usec ") {
            return rest.trim().parse::<u64>().ok();
        }
    }
    None
}

/// Total online host CPUs, for normalising CPU% to "% of total host".
/// Reads `/sys/devices/system/cpu/online` (a comma-separated list of
/// ranges, e.g. `0-19`) rather than `available_parallelism()`, which
/// reflects the hive-core process's CPU affinity — if hive-core is ever
/// affinity-pinned to a subset, that would under-count the denominator
/// and inflate the percentage. Falls back to the process count, then 1.
fn host_nproc() -> usize {
    fn from_sysfs() -> Option<usize> {
        let s = std::fs::read_to_string("/sys/devices/system/cpu/online").ok()?;
        let mut count = 0usize;
        for part in s.trim().split(',') {
            let part = part.trim();
            if part.is_empty() {
                continue;
            }
            if let Some((a, b)) = part.split_once('-') {
                let a: usize = a.trim().parse().ok()?;
                let b: usize = b.trim().parse().ok()?;
                count += b.saturating_sub(a) + 1;
            } else {
                part.parse::<usize>().ok()?;
                count += 1;
            }
        }
        (count > 0).then_some(count)
    }
    from_sysfs()
        .or_else(|| std::thread::available_parallelism().ok().map(usize::from))
        .unwrap_or(1)
}

/// Sample CPU + memory for every running agent container. Best-effort:
/// agents whose scope dir is absent (not running) or unreadable are
/// skipped, never fatal. Sorted by name for a stable display order.
pub async fn gather() -> Vec<ContainerResource> {
    // Machines that currently have a cgroup scope (= running). Agent
    // machine name is `h-<state-dir name>`.
    let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names()
        .into_iter()
        .filter_map(|name| {
            let dir = scope_dir(&format!("h-{name}"));
            dir.join("cpu.stat").exists().then_some((name, dir))
        })
        .collect();

    // First CPU sample for all, then one shared sleep, then the second
    // — so N agents cost one interval, not N.
    let t0: Vec<Option<u64>> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect();
    sleep(CPU_SAMPLE).await;

    #[allow(clippy::cast_precision_loss)]
    let nproc = host_nproc() as f64;
    #[allow(clippy::cast_precision_loss)]
    let interval_usec = CPU_SAMPLE.as_micros() as f64;

    let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
    for (i, (name, dir)) in candidates.iter().enumerate() {
        let cpu_pct = match (t0[i], read_usage_usec(dir)) {
            (Some(a), Some(b)) => {
                #[allow(clippy::cast_precision_loss)]
                let delta = b.saturating_sub(a) as f64;
                (delta / (interval_usec * nproc)) * 100.0
            }
            _ => 0.0,
        };
        out.push(ContainerResource {
            name: name.clone(),
            cpu_pct,
            mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
            mem_peak_bytes: read_u64(&dir.join("memory.peak")),
            mem_max_bytes: read_mem_max(&dir.join("memory.max")),
        });
    }
    out
}