feat(stats): per-container cpu/mem load (#1424 p3)

C0NT41N3R L04D on the SYST3M tab + GET /api/container-resources.

Backend (hive-c0re/src/container_stats.rs): reads cgroup v2 cpu.stat +
memory.{current,peak,max} for each running agent machine
(machine-h\x2d<name>.scope under machine.slice), read-only/world-
readable so no hive-priv. CPU is a two-sample (~200ms) host-normalised
percentage; one shared sleep covers all agents. Skips agents whose
scope dir is absent (= not running). Network omitted: agents share the
host netns, no per-container counter.

Frontend: a polled C0NT41N3R L04D table on SYST3M (agent / cpu / mem /
peak / limit with meter bars), reusing the ST4TS table style. Polls
/api/container-resources every 5s only while the tab is active.

Backend reviewed-in-principle by damocles (path escaping + cpu delta
math); ping for the on-host sign-off.

hive-c0re/src/container_stats.rs

@@ -0,0 +1,137 @@
+//! Live per-agent-container resource load (CPU + memory) for the
+//! dashboard. Reads cgroup v2 stat files for each running agent
+//! machine directly — same privsep-clean posture as the turn-stats
+//! sqlite reads (`cpu.stat` / `memory.*` are world-readable; no
+//! `hive-priv` needed). Runs host-side in hive-c0re, where
+//! `/sys/fs/cgroup/machine.slice/` holds the nspawn machine scopes.
+//!
+//! No network: agents share the host network namespace
+//! (`privateNetwork = false`), so there is no per-container net
+//! counter to read. Per-agent network only becomes meaningful with the
+//! netns-isolation roadmap (`docs/network.md`).
+//!
+//! CPU is cumulative (`usage_usec` is monotonic), so a single read is
+//! meaningless — we sample every machine's counter, sleep one short
+//! interval, sample again, and divide the delta by `interval × nproc`
+//! to get a host-normalised percentage (0..100 across all cores).
+
+use std::path::PathBuf;
+use std::time::Duration;
+
+use serde::Serialize;
+use tokio::time::sleep;
+
+use crate::coordinator::Coordinator;
+
+/// Sampling interval for the two CPU reads. Short enough to keep the
+/// endpoint snappy, long enough that the delta isn't dominated by read
+/// jitter.
+const CPU_SAMPLE: Duration = Duration::from_millis(200);
+
+const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice";
+
+#[derive(Debug, Serialize)]
+pub struct ContainerResource {
+    /// Agent name (without the `h-` machine prefix).
+    pub name: String,
+    /// Host-normalised CPU usage over the sample interval, as a
+    /// percentage of total host CPU (0..100 across all cores).
+    pub cpu_pct: f64,
+    /// Current memory usage (`memory.current`), bytes.
+    pub mem_current_bytes: u64,
+    /// High-water memory usage since container start (`memory.peak`),
+    /// bytes. `None` if the kernel doesn't expose `memory.peak`.
+    pub mem_peak_bytes: Option<u64>,
+    /// Memory ceiling (`memory.max`), bytes. `None` when unlimited
+    /// (the file reads `max`).
+    pub mem_max_bytes: Option<u64>,
+}
+
+/// systemd escapes the machine name in the cgroup scope dir
+/// (`machine-<escaped>.scope`). For our machine names — `h-<agent>`,
+/// agent ∈ `[a-z0-9_-]` — the only character systemd escapes is `-`,
+/// which becomes `\x2d` (verified against `systemd-escape`).
+fn escape_machine(machine: &str) -> String {
+    machine.replace('-', "\\x2d")
+}
+
+fn scope_dir(machine: &str) -> PathBuf {
+    PathBuf::from(MACHINE_SLICE).join(format!("machine-{}.scope", escape_machine(machine)))
+}
+
+/// Read a single unsigned integer from a one-line cgroup file.
+fn read_u64(path: &std::path::Path) -> Option<u64> {
+    std::fs::read_to_string(path)
+        .ok()?
+        .trim()
+        .parse::<u64>()
+        .ok()
+}
+
+/// `memory.max` reads `max` when there's no limit — map that to `None`.
+fn read_mem_max(path: &std::path::Path) -> Option<u64> {
+    let s = std::fs::read_to_string(path).ok()?;
+    let s = s.trim();
+    if s == "max" {
+        None
+    } else {
+        s.parse::<u64>().ok()
+    }
+}
+
+/// Parse `usage_usec` (cumulative CPU time, microseconds) from a
+/// scope's `cpu.stat`.
+fn read_usage_usec(dir: &std::path::Path) -> Option<u64> {
+    let text = std::fs::read_to_string(dir.join("cpu.stat")).ok()?;
+    for line in text.lines() {
+        if let Some(rest) = line.strip_prefix("usage_usec ") {
+            return rest.trim().parse::<u64>().ok();
+        }
+    }
+    None
+}
+
+/// Sample CPU + memory for every running agent container. Best-effort:
+/// agents whose scope dir is absent (not running) or unreadable are
+/// skipped, never fatal. Sorted by name for a stable display order.
+pub async fn gather() -> Vec<ContainerResource> {
+    // Machines that currently have a cgroup scope (= running). Agent
+    // machine name is `h-<state-dir name>`.
+    let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names()
+        .into_iter()
+        .filter_map(|name| {
+            let dir = scope_dir(&format!("h-{name}"));
+            dir.join("cpu.stat").exists().then_some((name, dir))
+        })
+        .collect();
+
+    // First CPU sample for all, then one shared sleep, then the second
+    // — so N agents cost one interval, not N.
+    let t0: Vec<Option<u64>> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect();
+    sleep(CPU_SAMPLE).await;
+
+    #[allow(clippy::cast_precision_loss)]
+    let nproc = std::thread::available_parallelism().map_or(1.0, |n| n.get() as f64);
+    #[allow(clippy::cast_precision_loss)]
+    let interval_usec = CPU_SAMPLE.as_micros() as f64;
+
+    let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
+    for (i, (name, dir)) in candidates.iter().enumerate() {
+        let cpu_pct = match (t0[i], read_usage_usec(dir)) {
+            (Some(a), Some(b)) => {
+                #[allow(clippy::cast_precision_loss)]
+                let delta = b.saturating_sub(a) as f64;
+                (delta / (interval_usec * nproc)) * 100.0
+            }
+            _ => 0.0,
+        };
+        out.push(ContainerResource {
+            name: name.clone(),
+            cpu_pct,
+            mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
+            mem_peak_bytes: read_u64(&dir.join("memory.peak")),
+            mem_max_bytes: read_mem_max(&dir.join("memory.max")),
+        });
+    }
+    out
+}

hive-c0re/src/dashboard.rs

@@ -68,6 +68,7 @@ pub async fn serve(port: u16, coord: Arc<Coordinator>) -> Result<()> {
         .route("/api/state-file", get(get_state_file))
         .route("/api/reminders", get(api_reminders))
         .route("/api/stats-hive", get(api_stats_hive))
+        .route("/api/container-resources", get(api_container_resources))
         .route("/api/build-logs", get(get_build_logs_all))
         .route("/api/build-logs/{agent}", get(get_build_logs_agent))
         .route("/api/build-logs/id/{id}", get(get_build_log_full))
@@ -1748,6 +1749,12 @@ async fn api_stats_hive(axum::extract::Query(q): axum::extract::Query<StatsHiveQ
     axum::Json(crate::hive_stats::hive_snapshot(window)).into_response()
 }
 
+/// Live per-agent-container CPU + memory load from cgroup v2. Samples
+/// CPU over a short interval (~200 ms), so this call briefly awaits.
+async fn api_container_resources() -> Response {
+    axum::Json(crate::container_stats::gather().await).into_response()
+}
+
 #[derive(Deserialize)]
 struct BuildLogsQuery {
     /// Maximum number of rows to return. Capped server-side at 50

hive-c0re/src/lib.rs

@@ -23,6 +23,7 @@ pub mod broker;
 pub mod build_logs;
 pub mod capabilities;
 pub mod client;
+pub mod container_stats;
 pub mod container_view;
 pub mod coordinator;
 pub mod crash_watch;