From 03ea6d1bdadd0c96be6c28aec919a02441edf856 Mon Sep 17 00:00:00 2001
From: iris <iris@hyperhive.local>
Date: Fri, 5 Jun 2026 23:00:27 +0200
Subject: [PATCH] feat(stats): per-container cpu/mem load (#1424 p3)

C0NT41N3R L04D on the SYST3M tab + GET /api/container-resources.

Backend (hive-c0re/src/container_stats.rs): reads cgroup v2 cpu.stat +
memory.{current,peak,max} for each running agent machine
(machine-h\x2d<name>.scope under machine.slice), read-only/world-
readable so no hive-priv. CPU is a two-sample (~200ms) host-normalised
percentage; one shared sleep covers all agents. Skips agents whose
scope dir is absent (= not running). Network omitted: agents share the
host netns, no per-container counter.

Frontend: a polled C0NT41N3R L04D table on SYST3M (agent / cpu / mem /
peak / limit with meter bars), reusing the ST4TS table style. Polls
/api/container-resources every 5s only while the tab is active.

Backend reviewed-in-principle by damocles (path escaping + cpu delta
math); ping for the on-host sign-off.
---
 docs/web-ui/dashboard.md                      |  22 +++
 frontend/packages/dashboard/src/dashboard.css |  26 ++++
 frontend/packages/dashboard/src/index.html    |  10 ++
 frontend/packages/dashboard/src/tabs.js       |  97 +++++++++++++
 hive-c0re/src/container_stats.rs              | 137 ++++++++++++++++++
 hive-c0re/src/dashboard.rs                    |   7 +
 hive-c0re/src/lib.rs                          |   1 +
 7 files changed, 300 insertions(+)
 create mode 100644 hive-c0re/src/container_stats.rs

diff --git a/docs/web-ui/dashboard.md b/docs/web-ui/dashboard.md
index fef65252..3baa5a59 100644
--- a/docs/web-ui/dashboard.md
+++ b/docs/web-ui/dashboard.md
@@ -153,6 +153,22 @@ age + claude-creds badge). Two actions: `⊕ R3V1V3` (queues a
 Spawn approval; existing state is reused), `PURG3` (wipes
 state + applied dirs; `POST /purge-tombstone/{name}`).
 
+**C0NT41N3R L04D** — live CPU + memory per agent container, read
+straight from cgroup v2 on the host (`cpu.stat`, `memory.current`,
+`memory.peak`, `memory.max` under
+`/sys/fs/cgroup/machine.slice/machine-h\x2d<name>.scope/`). CPU is a
+host-normalised percentage (0..100 across all cores) sampled over a
+short (~200 ms) two-read interval; memory shows current + peak with a
+bar against the `memory.max` quota. Backed by
+`GET /api/container-resources` (`container_stats.rs`), which reads the
+files read-only (world-readable; no `hive-priv`) and skips agents whose
+scope dir is absent (= not running). Pull-only: `tabs.js` polls every
+5 s **only while the SYST3M tab is active** (CPU needs a fresh sample
+each refresh), and stops on tab change. Network is intentionally
+omitted — agents share the host netns, so there is no per-container net
+counter (per-agent network needs the netns-isolation roadmap in
+`docs/network.md`).
+
 ## P3RM1SS10NS tab
 
 Per-agent permission configuration. Two sections, each rendered as a
@@ -838,6 +854,12 @@ that's a browser-level decision, not ours.
   zero-turn dbs); returns swarm totals, a busiest-first per-agent
   rollup, swarm model mix, and a labelled `est_cost_usd`. Window
   defaults to `24h`.
+- `GET /api/container-resources` — live per-agent-container CPU +
+  memory from cgroup v2 (SYST3M › C0NT41N3R L04D panel). Returns one
+  row per running agent (`name`, `cpu_pct`, `mem_current_bytes`,
+  `mem_peak_bytes`, `mem_max_bytes`); samples CPU over ~200 ms so the
+  call briefly awaits. Skips non-running agents (no scope dir). No
+  network field — agents share the host netns.
 - `POST /cancel-reminder/{id}` — hard-delete a pending reminder.
 - `POST /retry-reminder/{id}` — re-arm a reminder whose delivery
   failed (clears the failure state so the scheduler retries).
diff --git a/frontend/packages/dashboard/src/dashboard.css b/frontend/packages/dashboard/src/dashboard.css
index 4c707dbd..5d1f3294 100644
--- a/frontend/packages/dashboard/src/dashboard.css
+++ b/frontend/packages/dashboard/src/dashboard.css
@@ -1609,3 +1609,29 @@ body.dashboard-shell.has-selection { padding-bottom: 4.5em; }
   color: var(--muted);
   font-variant-numeric: tabular-nums;
 }
+
+/* ─── SYST3M › C0NT41N3R L04D: live cgroup cpu/mem ─────────────────────────────
+   Reuses the `.hive-stats-table` styling from the ST4TS tab; only the
+   inline meter bar is new. */
+.cload-meter {
+  display: inline-block;
+  width: 6em;
+  height: 10px;
+  vertical-align: middle;
+  margin-left: 6px;
+  background: var(--bg-elev);
+  border: 1px solid var(--border);
+  border-radius: 3px;
+  overflow: hidden;
+}
+.cload-meter .fill {
+  display: block;
+  height: 100%;
+  background: var(--green);
+}
+.cload-meter .fill.warn {
+  background: var(--amber);
+}
+.cload-meter .fill.hot {
+  background: var(--red);
+}
diff --git a/frontend/packages/dashboard/src/index.html b/frontend/packages/dashboard/src/index.html
index e81b52c0..2b3f8bc1 100644
--- a/frontend/packages/dashboard/src/index.html
+++ b/frontend/packages/dashboard/src/index.html
@@ -191,6 +191,16 @@
         <p class="meta">loading…</p>
       </div>
 
+      <!-- C0NT41N3R L04D: live cpu + memory per agent container, read
+           from cgroup v2 on the host. Polled only while this tab is
+           active (cpu needs a short two-sample read each refresh). -->
+      <h2>◆ C0NT41N3R L04D ◆</h2>
+      <div class="divider">══════════════════════════════════════════════════════════════</div>
+      <p class="meta">live cpu + memory per agent container, from cgroup v2 on the host. cpu is % of total host capacity (all cores), sampled over ~200ms each refresh; polled every 5s while this tab is open. network is omitted on purpose — agents share the host netns, so there is no per-container counter (see <code>docs/network.md</code>).</p>
+      <div id="container-load-section">
+        <p class="meta">loading…</p>
+      </div>
+
     </section>
 
     <!-- P3RM1SS10NS: per-agent capability grants + tool-group
diff --git a/frontend/packages/dashboard/src/tabs.js b/frontend/packages/dashboard/src/tabs.js
index 3e22ee8e..b37bc7cb 100644
--- a/frontend/packages/dashboard/src/tabs.js
+++ b/frontend/packages/dashboard/src/tabs.js
@@ -4076,6 +4076,9 @@ window.marked = marked;
     }
     // ST4TS: hive-wide rollup is a pull (no SSE) — fetch on activation.
     if (target === 'stats') { refreshHiveStats(); }
+    // SYST3M › C0NT41N3R L04D: live cgroup poll only while the tab is
+    // open (cpu needs a short two-sample read each refresh).
+    if (target === 'system') { startContainerLoadPolling(); } else { stopContainerLoadPolling(); }
   }
   // ─── tabbar overflow menu ────────────────────────────────────────────────
   // Tabs with `data-overflow="default"` (LOGS, SETTINGS) always live in
@@ -4230,6 +4233,100 @@ window.marked = marked;
   }
   bindHiveStatsWindows();
 
+  // ─── SYST3M › C0NT41N3R L04D: live per-container cgroup cpu/mem ────────────
+  // Pull-only, polled at 5s ONLY while the SYST3M tab is active (cpu is a
+  // short two-sample read on the server). Data from /api/container-resources.
+  let containerLoadTimer = null;
+
+  function cloadFmtBytes(n) {
+    if (!Number.isFinite(n) || n <= 0) return '0';
+    const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
+    let i = 0; let v = n;
+    while (v >= 1024 && i < u.length - 1) { v /= 1024; i += 1; }
+    return v.toFixed(v < 10 && i > 0 ? 1 : 0) + ' ' + u[i];
+  }
+  function cloadMeter(pct) {
+    const p = Math.max(0, Math.min(100, pct));
+    const cls = p >= 90 ? 'hot' : (p >= 70 ? 'warn' : '');
+    const m = document.createElement('span');
+    m.className = 'cload-meter';
+    m.title = p.toFixed(0) + '%';
+    const f = document.createElement('span');
+    f.className = 'fill' + (cls ? ' ' + cls : '');
+    f.style.width = p + '%';
+    m.append(f);
+    return m;
+  }
+
+  function renderContainerLoad(rows) {
+    const root = $('container-load-section');
+    if (!root) return;
+    if (!Array.isArray(rows) || rows.length === 0) {
+      root.replaceChildren();
+      const p = document.createElement('p'); p.className = 'meta';
+      p.textContent = 'no running agent containers'; root.append(p);
+      return;
+    }
+    const table = document.createElement('table');
+    table.className = 'hive-stats-table';
+    table.innerHTML = '<thead><tr><th>agent</th><th>cpu</th><th>memory</th>'
+      + '<th>peak</th><th>limit</th></tr></thead>';
+    const tb = document.createElement('tbody');
+    for (const r of rows) {
+      const tr = document.createElement('tr');
+
+      const name = document.createElement('td'); name.textContent = r.name; tr.append(name);
+
+      const cpu = document.createElement('td'); cpu.className = 'num';
+      cpu.append((Number(r.cpu_pct) || 0).toFixed(1) + '%', cloadMeter(Number(r.cpu_pct) || 0));
+      tr.append(cpu);
+
+      const mem = document.createElement('td'); mem.className = 'num';
+      const memCur = Number(r.mem_current_bytes) || 0;
+      if (r.mem_max_bytes) {
+        mem.append(cloadFmtBytes(memCur), cloadMeter(100 * memCur / r.mem_max_bytes));
+      } else {
+        mem.textContent = cloadFmtBytes(memCur);
+      }
+      tr.append(mem);
+
+      const peak = document.createElement('td'); peak.className = 'num';
+      peak.textContent = r.mem_peak_bytes ? cloadFmtBytes(Number(r.mem_peak_bytes)) : '—';
+      tr.append(peak);
+
+      const lim = document.createElement('td'); lim.className = 'num';
+      lim.textContent = r.mem_max_bytes ? cloadFmtBytes(Number(r.mem_max_bytes)) : '∞';
+      tr.append(lim);
+
+      tb.append(tr);
+    }
+    table.append(tb);
+    root.replaceChildren(table);
+  }
+
+  async function refreshContainerLoad() {
+    try {
+      const resp = await fetch('/api/container-resources');
+      if (!resp.ok) throw new Error('http ' + resp.status);
+      renderContainerLoad(await resp.json());
+    } catch (e) {
+      const root = $('container-load-section');
+      if (root) {
+        root.replaceChildren();
+        const p = document.createElement('p'); p.className = 'meta';
+        p.textContent = 'container load fetch failed: ' + e; root.append(p);
+      }
+    }
+  }
+  function startContainerLoadPolling() {
+    refreshContainerLoad();
+    if (containerLoadTimer) return;
+    containerLoadTimer = setInterval(refreshContainerLoad, 5000);
+  }
+  function stopContainerLoadPolling() {
+    if (containerLoadTimer) { clearInterval(containerLoadTimer); containerLoadTimer = null; }
+  }
+
   function syncTabFromHash() {
     const h = (window.location.hash || '#swarm').replace(/^#/, '');
     activateTab(h);
diff --git a/hive-c0re/src/container_stats.rs b/hive-c0re/src/container_stats.rs
new file mode 100644
index 00000000..20b25138
--- /dev/null
+++ b/hive-c0re/src/container_stats.rs
@@ -0,0 +1,137 @@
+//! Live per-agent-container resource load (CPU + memory) for the
+//! dashboard. Reads cgroup v2 stat files for each running agent
+//! machine directly — same privsep-clean posture as the turn-stats
+//! sqlite reads (`cpu.stat` / `memory.*` are world-readable; no
+//! `hive-priv` needed). Runs host-side in hive-c0re, where
+//! `/sys/fs/cgroup/machine.slice/` holds the nspawn machine scopes.
+//!
+//! No network: agents share the host network namespace
+//! (`privateNetwork = false`), so there is no per-container net
+//! counter to read. Per-agent network only becomes meaningful with the
+//! netns-isolation roadmap (`docs/network.md`).
+//!
+//! CPU is cumulative (`usage_usec` is monotonic), so a single read is
+//! meaningless — we sample every machine's counter, sleep one short
+//! interval, sample again, and divide the delta by `interval × nproc`
+//! to get a host-normalised percentage (0..100 across all cores).
+
+use std::path::PathBuf;
+use std::time::Duration;
+
+use serde::Serialize;
+use tokio::time::sleep;
+
+use crate::coordinator::Coordinator;
+
+/// Sampling interval for the two CPU reads. Short enough to keep the
+/// endpoint snappy, long enough that the delta isn't dominated by read
+/// jitter.
+const CPU_SAMPLE: Duration = Duration::from_millis(200);
+
+const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice";
+
+#[derive(Debug, Serialize)]
+pub struct ContainerResource {
+    /// Agent name (without the `h-` machine prefix).
+    pub name: String,
+    /// Host-normalised CPU usage over the sample interval, as a
+    /// percentage of total host CPU (0..100 across all cores).
+    pub cpu_pct: f64,
+    /// Current memory usage (`memory.current`), bytes.
+    pub mem_current_bytes: u64,
+    /// High-water memory usage since container start (`memory.peak`),
+    /// bytes. `None` if the kernel doesn't expose `memory.peak`.
+    pub mem_peak_bytes: Option<u64>,
+    /// Memory ceiling (`memory.max`), bytes. `None` when unlimited
+    /// (the file reads `max`).
+    pub mem_max_bytes: Option<u64>,
+}
+
+/// systemd escapes the machine name in the cgroup scope dir
+/// (`machine-<escaped>.scope`). For our machine names — `h-<agent>`,
+/// agent ∈ `[a-z0-9_-]` — the only character systemd escapes is `-`,
+/// which becomes `\x2d` (verified against `systemd-escape`).
+fn escape_machine(machine: &str) -> String {
+    machine.replace('-', "\\x2d")
+}
+
+fn scope_dir(machine: &str) -> PathBuf {
+    PathBuf::from(MACHINE_SLICE).join(format!("machine-{}.scope", escape_machine(machine)))
+}
+
+/// Read a single unsigned integer from a one-line cgroup file.
+fn read_u64(path: &std::path::Path) -> Option<u64> {
+    std::fs::read_to_string(path)
+        .ok()?
+        .trim()
+        .parse::<u64>()
+        .ok()
+}
+
+/// `memory.max` reads `max` when there's no limit — map that to `None`.
+fn read_mem_max(path: &std::path::Path) -> Option<u64> {
+    let s = std::fs::read_to_string(path).ok()?;
+    let s = s.trim();
+    if s == "max" {
+        None
+    } else {
+        s.parse::<u64>().ok()
+    }
+}
+
+/// Parse `usage_usec` (cumulative CPU time, microseconds) from a
+/// scope's `cpu.stat`.
+fn read_usage_usec(dir: &std::path::Path) -> Option<u64> {
+    let text = std::fs::read_to_string(dir.join("cpu.stat")).ok()?;
+    for line in text.lines() {
+        if let Some(rest) = line.strip_prefix("usage_usec ") {
+            return rest.trim().parse::<u64>().ok();
+        }
+    }
+    None
+}
+
+/// Sample CPU + memory for every running agent container. Best-effort:
+/// agents whose scope dir is absent (not running) or unreadable are
+/// skipped, never fatal. Sorted by name for a stable display order.
+pub async fn gather() -> Vec<ContainerResource> {
+    // Machines that currently have a cgroup scope (= running). Agent
+    // machine name is `h-<state-dir name>`.
+    let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names()
+        .into_iter()
+        .filter_map(|name| {
+            let dir = scope_dir(&format!("h-{name}"));
+            dir.join("cpu.stat").exists().then_some((name, dir))
+        })
+        .collect();
+
+    // First CPU sample for all, then one shared sleep, then the second
+    // — so N agents cost one interval, not N.
+    let t0: Vec<Option<u64>> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect();
+    sleep(CPU_SAMPLE).await;
+
+    #[allow(clippy::cast_precision_loss)]
+    let nproc = std::thread::available_parallelism().map_or(1.0, |n| n.get() as f64);
+    #[allow(clippy::cast_precision_loss)]
+    let interval_usec = CPU_SAMPLE.as_micros() as f64;
+
+    let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
+    for (i, (name, dir)) in candidates.iter().enumerate() {
+        let cpu_pct = match (t0[i], read_usage_usec(dir)) {
+            (Some(a), Some(b)) => {
+                #[allow(clippy::cast_precision_loss)]
+                let delta = b.saturating_sub(a) as f64;
+                (delta / (interval_usec * nproc)) * 100.0
+            }
+            _ => 0.0,
+        };
+        out.push(ContainerResource {
+            name: name.clone(),
+            cpu_pct,
+            mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
+            mem_peak_bytes: read_u64(&dir.join("memory.peak")),
+            mem_max_bytes: read_mem_max(&dir.join("memory.max")),
+        });
+    }
+    out
+}
diff --git a/hive-c0re/src/dashboard.rs b/hive-c0re/src/dashboard.rs
index 3739027f..1ca26b7b 100644
--- a/hive-c0re/src/dashboard.rs
+++ b/hive-c0re/src/dashboard.rs
@@ -68,6 +68,7 @@ pub async fn serve(port: u16, coord: Arc<Coordinator>) -> Result<()> {
         .route("/api/state-file", get(get_state_file))
         .route("/api/reminders", get(api_reminders))
         .route("/api/stats-hive", get(api_stats_hive))
+        .route("/api/container-resources", get(api_container_resources))
         .route("/api/build-logs", get(get_build_logs_all))
         .route("/api/build-logs/{agent}", get(get_build_logs_agent))
         .route("/api/build-logs/id/{id}", get(get_build_log_full))
@@ -1748,6 +1749,12 @@ async fn api_stats_hive(axum::extract::Query(q): axum::extract::Query<StatsHiveQ
     axum::Json(crate::hive_stats::hive_snapshot(window)).into_response()
 }
 
+/// Live per-agent-container CPU + memory load from cgroup v2. Samples
+/// CPU over a short interval (~200 ms), so this call briefly awaits.
+async fn api_container_resources() -> Response {
+    axum::Json(crate::container_stats::gather().await).into_response()
+}
+
 #[derive(Deserialize)]
 struct BuildLogsQuery {
     /// Maximum number of rows to return. Capped server-side at 50
diff --git a/hive-c0re/src/lib.rs b/hive-c0re/src/lib.rs
index dce299a7..1bba3cad 100644
--- a/hive-c0re/src/lib.rs
+++ b/hive-c0re/src/lib.rs
@@ -23,6 +23,7 @@ pub mod broker;
 pub mod build_logs;
 pub mod capabilities;
 pub mod client;
+pub mod container_stats;
 pub mod container_view;
 pub mod coordinator;
 pub mod crash_watch;