//! Live per-agent-container resource load (CPU + memory) for the //! dashboard. Reads cgroup v2 stat files for each running agent //! machine directly — same privsep-clean posture as the turn-stats //! sqlite reads (`cpu.stat` / `memory.*` are world-readable; no //! `hive-priv` needed). Runs host-side in hive-c0re, where //! `/sys/fs/cgroup/machine.slice/` holds the nspawn machine scopes. //! //! No network: agents share the host network namespace //! (`privateNetwork = false`), so there is no per-container net //! counter to read. Per-agent network only becomes meaningful with the //! netns-isolation roadmap (`docs/network.md`). //! //! CPU is cumulative (`usage_usec` is monotonic), so a single read is //! meaningless — we sample every machine's counter, sleep one short //! interval, sample again, and divide the delta by `interval × nproc` //! to get a host-normalised percentage (0..100 across all cores). use std::path::PathBuf; use std::time::Duration; use serde::Serialize; use tokio::time::sleep; use crate::coordinator::Coordinator; /// Sampling interval for the two CPU reads. Short enough to keep the /// endpoint snappy, long enough that the delta isn't dominated by read /// jitter. const CPU_SAMPLE: Duration = Duration::from_millis(200); const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice"; #[derive(Debug, Serialize)] pub struct ContainerResource { /// Agent name (without the `h-` machine prefix). pub name: String, /// Host-normalised CPU usage over the sample interval, as a /// percentage of total host CPU (0..100 across all cores). pub cpu_pct: f64, /// Current memory usage (`memory.current`), bytes. pub mem_current_bytes: u64, /// High-water memory usage since container start (`memory.peak`), /// bytes. `None` if the kernel doesn't expose `memory.peak`. pub mem_peak_bytes: Option, /// Memory ceiling (`memory.max`), bytes. `None` when unlimited /// (the file reads `max`). pub mem_max_bytes: Option, } /// systemd escapes the machine name in the cgroup scope dir /// (`machine-.scope`). For our machine names — `h-`, /// agent ∈ `[a-z0-9_-]` — the only character systemd escapes is `-`, /// which becomes `\x2d` (verified against `systemd-escape`). fn escape_machine(machine: &str) -> String { machine.replace('-', "\\x2d") } fn scope_dir(machine: &str) -> PathBuf { PathBuf::from(MACHINE_SLICE).join(format!("machine-{}.scope", escape_machine(machine))) } /// Read a single unsigned integer from a one-line cgroup file. fn read_u64(path: &std::path::Path) -> Option { std::fs::read_to_string(path) .ok()? .trim() .parse::() .ok() } /// `memory.max` reads `max` when there's no limit — map that to `None`. fn read_mem_max(path: &std::path::Path) -> Option { let s = std::fs::read_to_string(path).ok()?; let s = s.trim(); if s == "max" { None } else { s.parse::().ok() } } /// Parse `usage_usec` (cumulative CPU time, microseconds) from a /// scope's `cpu.stat`. fn read_usage_usec(dir: &std::path::Path) -> Option { let text = std::fs::read_to_string(dir.join("cpu.stat")).ok()?; for line in text.lines() { if let Some(rest) = line.strip_prefix("usage_usec ") { return rest.trim().parse::().ok(); } } None } /// Total online host CPUs, for normalising CPU% to "% of total host". /// Reads `/sys/devices/system/cpu/online` (a comma-separated list of /// ranges, e.g. `0-19`) rather than `available_parallelism()`, which /// reflects the hive-core process's CPU affinity — if hive-core is ever /// affinity-pinned to a subset, that would under-count the denominator /// and inflate the percentage. Falls back to the process count, then 1. fn host_nproc() -> usize { fn from_sysfs() -> Option { let s = std::fs::read_to_string("/sys/devices/system/cpu/online").ok()?; let mut count = 0usize; for part in s.trim().split(',') { let part = part.trim(); if part.is_empty() { continue; } if let Some((a, b)) = part.split_once('-') { let a: usize = a.trim().parse().ok()?; let b: usize = b.trim().parse().ok()?; count += b.saturating_sub(a) + 1; } else { part.parse::().ok()?; count += 1; } } (count > 0).then_some(count) } from_sysfs() .or_else(|| std::thread::available_parallelism().ok().map(usize::from)) .unwrap_or(1) } /// Sample CPU + memory for every running agent container. Best-effort: /// agents whose scope dir is absent (not running) or unreadable are /// skipped, never fatal. Sorted by name for a stable display order. pub async fn gather() -> Vec { // Machines that currently have a cgroup scope (= running). Agent // machine name is `h-`. let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names() .into_iter() .filter_map(|name| { let dir = scope_dir(&format!("h-{name}")); dir.join("cpu.stat").exists().then_some((name, dir)) }) .collect(); // First CPU sample for all, then one shared sleep, then the second // — so N agents cost one interval, not N. let t0: Vec> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect(); sleep(CPU_SAMPLE).await; #[allow(clippy::cast_precision_loss)] let nproc = host_nproc() as f64; #[allow(clippy::cast_precision_loss)] let interval_usec = CPU_SAMPLE.as_micros() as f64; let mut out: Vec = Vec::with_capacity(candidates.len()); for (i, (name, dir)) in candidates.iter().enumerate() { let cpu_pct = match (t0[i], read_usage_usec(dir)) { (Some(a), Some(b)) => { #[allow(clippy::cast_precision_loss)] let delta = b.saturating_sub(a) as f64; (delta / (interval_usec * nproc)) * 100.0 } _ => 0.0, }; out.push(ContainerResource { name: name.clone(), cpu_pct, mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0), mem_peak_bytes: read_u64(&dir.join("memory.peak")), mem_max_bytes: read_mem_max(&dir.join("memory.max")), }); } out }