feat(stats): per-container cpu/mem load (#1424 p3)

C0NT41N3R L04D on the SYST3M tab + GET /api/container-resources.

Backend (hive-c0re/src/container_stats.rs): reads cgroup v2 cpu.stat +
memory.{current,peak,max} for each running agent machine
(machine-h\x2d<name>.scope under machine.slice), read-only/world-
readable so no hive-priv. CPU is a two-sample (~200ms) host-normalised
percentage; one shared sleep covers all agents. Skips agents whose
scope dir is absent (= not running). Network omitted: agents share the
host netns, no per-container counter.

Frontend: a polled C0NT41N3R L04D table on SYST3M (agent / cpu / mem /
peak / limit with meter bars), reusing the ST4TS table style. Polls
/api/container-resources every 5s only while the tab is active.

Backend reviewed-in-principle by damocles (path escaping + cpu delta
math); ping for the on-host sign-off.
This commit is contained in:
iris 2026-06-05 23:00:27 +02:00 committed by mara
commit 03ea6d1bda
7 changed files with 300 additions and 0 deletions

View file

@ -153,6 +153,22 @@ age + claude-creds badge). Two actions: `⊕ R3V1V3` (queues a
Spawn approval; existing state is reused), `PURG3` (wipes
state + applied dirs; `POST /purge-tombstone/{name}`).
**C0NT41N3R L04D** — live CPU + memory per agent container, read
straight from cgroup v2 on the host (`cpu.stat`, `memory.current`,
`memory.peak`, `memory.max` under
`/sys/fs/cgroup/machine.slice/machine-h\x2d<name>.scope/`). CPU is a
host-normalised percentage (0..100 across all cores) sampled over a
short (~200 ms) two-read interval; memory shows current + peak with a
bar against the `memory.max` quota. Backed by
`GET /api/container-resources` (`container_stats.rs`), which reads the
files read-only (world-readable; no `hive-priv`) and skips agents whose
scope dir is absent (= not running). Pull-only: `tabs.js` polls every
5 s **only while the SYST3M tab is active** (CPU needs a fresh sample
each refresh), and stops on tab change. Network is intentionally
omitted — agents share the host netns, so there is no per-container net
counter (per-agent network needs the netns-isolation roadmap in
`docs/network.md`).
## P3RM1SS10NS tab
Per-agent permission configuration. Two sections, each rendered as a
@ -838,6 +854,12 @@ that's a browser-level decision, not ours.
zero-turn dbs); returns swarm totals, a busiest-first per-agent
rollup, swarm model mix, and a labelled `est_cost_usd`. Window
defaults to `24h`.
- `GET /api/container-resources` — live per-agent-container CPU +
memory from cgroup v2 (SYST3M C0NT41N3R L04D panel). Returns one
row per running agent (`name`, `cpu_pct`, `mem_current_bytes`,
`mem_peak_bytes`, `mem_max_bytes`); samples CPU over ~200 ms so the
call briefly awaits. Skips non-running agents (no scope dir). No
network field — agents share the host netns.
- `POST /cancel-reminder/{id}` — hard-delete a pending reminder.
- `POST /retry-reminder/{id}` — re-arm a reminder whose delivery
failed (clears the failure state so the scheduler retries).

View file

@ -1609,3 +1609,29 @@ body.dashboard-shell.has-selection { padding-bottom: 4.5em; }
color: var(--muted);
font-variant-numeric: tabular-nums;
}
/* SYST3M C0NT41N3R L04D: live cgroup cpu/mem
Reuses the `.hive-stats-table` styling from the ST4TS tab; only the
inline meter bar is new. */
.cload-meter {
display: inline-block;
width: 6em;
height: 10px;
vertical-align: middle;
margin-left: 6px;
background: var(--bg-elev);
border: 1px solid var(--border);
border-radius: 3px;
overflow: hidden;
}
.cload-meter .fill {
display: block;
height: 100%;
background: var(--green);
}
.cload-meter .fill.warn {
background: var(--amber);
}
.cload-meter .fill.hot {
background: var(--red);
}

View file

@ -191,6 +191,16 @@
<p class="meta">loading…</p>
</div>
<!-- C0NT41N3R L04D: live cpu + memory per agent container, read
from cgroup v2 on the host. Polled only while this tab is
active (cpu needs a short two-sample read each refresh). -->
<h2>◆ C0NT41N3R L04D ◆</h2>
<div class="divider">══════════════════════════════════════════════════════════════</div>
<p class="meta">live cpu + memory per agent container, from cgroup v2 on the host. cpu is % of total host capacity (all cores), sampled over ~200ms each refresh; polled every 5s while this tab is open. network is omitted on purpose — agents share the host netns, so there is no per-container counter (see <code>docs/network.md</code>).</p>
<div id="container-load-section">
<p class="meta">loading…</p>
</div>
</section>
<!-- P3RM1SS10NS: per-agent capability grants + tool-group

View file

@ -4076,6 +4076,9 @@ window.marked = marked;
}
// ST4TS: hive-wide rollup is a pull (no SSE) — fetch on activation.
if (target === 'stats') { refreshHiveStats(); }
// SYST3M C0NT41N3R L04D: live cgroup poll only while the tab is
// open (cpu needs a short two-sample read each refresh).
if (target === 'system') { startContainerLoadPolling(); } else { stopContainerLoadPolling(); }
}
// ─── tabbar overflow menu ────────────────────────────────────────────────
// Tabs with `data-overflow="default"` (LOGS, SETTINGS) always live in
@ -4230,6 +4233,100 @@ window.marked = marked;
}
bindHiveStatsWindows();
// ─── SYST3M C0NT41N3R L04D: live per-container cgroup cpu/mem ────────────
// Pull-only, polled at 5s ONLY while the SYST3M tab is active (cpu is a
// short two-sample read on the server). Data from /api/container-resources.
let containerLoadTimer = null;
function cloadFmtBytes(n) {
if (!Number.isFinite(n) || n <= 0) return '0';
const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
let i = 0; let v = n;
while (v >= 1024 && i < u.length - 1) { v /= 1024; i += 1; }
return v.toFixed(v < 10 && i > 0 ? 1 : 0) + ' ' + u[i];
}
function cloadMeter(pct) {
const p = Math.max(0, Math.min(100, pct));
const cls = p >= 90 ? 'hot' : (p >= 70 ? 'warn' : '');
const m = document.createElement('span');
m.className = 'cload-meter';
m.title = p.toFixed(0) + '%';
const f = document.createElement('span');
f.className = 'fill' + (cls ? ' ' + cls : '');
f.style.width = p + '%';
m.append(f);
return m;
}
function renderContainerLoad(rows) {
const root = $('container-load-section');
if (!root) return;
if (!Array.isArray(rows) || rows.length === 0) {
root.replaceChildren();
const p = document.createElement('p'); p.className = 'meta';
p.textContent = 'no running agent containers'; root.append(p);
return;
}
const table = document.createElement('table');
table.className = 'hive-stats-table';
table.innerHTML = '<thead><tr><th>agent</th><th>cpu</th><th>memory</th>'
+ '<th>peak</th><th>limit</th></tr></thead>';
const tb = document.createElement('tbody');
for (const r of rows) {
const tr = document.createElement('tr');
const name = document.createElement('td'); name.textContent = r.name; tr.append(name);
const cpu = document.createElement('td'); cpu.className = 'num';
cpu.append((Number(r.cpu_pct) || 0).toFixed(1) + '%', cloadMeter(Number(r.cpu_pct) || 0));
tr.append(cpu);
const mem = document.createElement('td'); mem.className = 'num';
const memCur = Number(r.mem_current_bytes) || 0;
if (r.mem_max_bytes) {
mem.append(cloadFmtBytes(memCur), cloadMeter(100 * memCur / r.mem_max_bytes));
} else {
mem.textContent = cloadFmtBytes(memCur);
}
tr.append(mem);
const peak = document.createElement('td'); peak.className = 'num';
peak.textContent = r.mem_peak_bytes ? cloadFmtBytes(Number(r.mem_peak_bytes)) : '—';
tr.append(peak);
const lim = document.createElement('td'); lim.className = 'num';
lim.textContent = r.mem_max_bytes ? cloadFmtBytes(Number(r.mem_max_bytes)) : '∞';
tr.append(lim);
tb.append(tr);
}
table.append(tb);
root.replaceChildren(table);
}
async function refreshContainerLoad() {
try {
const resp = await fetch('/api/container-resources');
if (!resp.ok) throw new Error('http ' + resp.status);
renderContainerLoad(await resp.json());
} catch (e) {
const root = $('container-load-section');
if (root) {
root.replaceChildren();
const p = document.createElement('p'); p.className = 'meta';
p.textContent = 'container load fetch failed: ' + e; root.append(p);
}
}
}
function startContainerLoadPolling() {
refreshContainerLoad();
if (containerLoadTimer) return;
containerLoadTimer = setInterval(refreshContainerLoad, 5000);
}
function stopContainerLoadPolling() {
if (containerLoadTimer) { clearInterval(containerLoadTimer); containerLoadTimer = null; }
}
function syncTabFromHash() {
const h = (window.location.hash || '#swarm').replace(/^#/, '');
activateTab(h);

View file

@ -0,0 +1,137 @@
//! Live per-agent-container resource load (CPU + memory) for the
//! dashboard. Reads cgroup v2 stat files for each running agent
//! machine directly — same privsep-clean posture as the turn-stats
//! sqlite reads (`cpu.stat` / `memory.*` are world-readable; no
//! `hive-priv` needed). Runs host-side in hive-c0re, where
//! `/sys/fs/cgroup/machine.slice/` holds the nspawn machine scopes.
//!
//! No network: agents share the host network namespace
//! (`privateNetwork = false`), so there is no per-container net
//! counter to read. Per-agent network only becomes meaningful with the
//! netns-isolation roadmap (`docs/network.md`).
//!
//! CPU is cumulative (`usage_usec` is monotonic), so a single read is
//! meaningless — we sample every machine's counter, sleep one short
//! interval, sample again, and divide the delta by `interval × nproc`
//! to get a host-normalised percentage (0..100 across all cores).
use std::path::PathBuf;
use std::time::Duration;
use serde::Serialize;
use tokio::time::sleep;
use crate::coordinator::Coordinator;
/// Sampling interval for the two CPU reads. Short enough to keep the
/// endpoint snappy, long enough that the delta isn't dominated by read
/// jitter.
const CPU_SAMPLE: Duration = Duration::from_millis(200);
const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice";
#[derive(Debug, Serialize)]
pub struct ContainerResource {
/// Agent name (without the `h-` machine prefix).
pub name: String,
/// Host-normalised CPU usage over the sample interval, as a
/// percentage of total host CPU (0..100 across all cores).
pub cpu_pct: f64,
/// Current memory usage (`memory.current`), bytes.
pub mem_current_bytes: u64,
/// High-water memory usage since container start (`memory.peak`),
/// bytes. `None` if the kernel doesn't expose `memory.peak`.
pub mem_peak_bytes: Option<u64>,
/// Memory ceiling (`memory.max`), bytes. `None` when unlimited
/// (the file reads `max`).
pub mem_max_bytes: Option<u64>,
}
/// systemd escapes the machine name in the cgroup scope dir
/// (`machine-<escaped>.scope`). For our machine names — `h-<agent>`,
/// agent ∈ `[a-z0-9_-]` — the only character systemd escapes is `-`,
/// which becomes `\x2d` (verified against `systemd-escape`).
fn escape_machine(machine: &str) -> String {
machine.replace('-', "\\x2d")
}
fn scope_dir(machine: &str) -> PathBuf {
PathBuf::from(MACHINE_SLICE).join(format!("machine-{}.scope", escape_machine(machine)))
}
/// Read a single unsigned integer from a one-line cgroup file.
fn read_u64(path: &std::path::Path) -> Option<u64> {
std::fs::read_to_string(path)
.ok()?
.trim()
.parse::<u64>()
.ok()
}
/// `memory.max` reads `max` when there's no limit — map that to `None`.
fn read_mem_max(path: &std::path::Path) -> Option<u64> {
let s = std::fs::read_to_string(path).ok()?;
let s = s.trim();
if s == "max" {
None
} else {
s.parse::<u64>().ok()
}
}
/// Parse `usage_usec` (cumulative CPU time, microseconds) from a
/// scope's `cpu.stat`.
fn read_usage_usec(dir: &std::path::Path) -> Option<u64> {
let text = std::fs::read_to_string(dir.join("cpu.stat")).ok()?;
for line in text.lines() {
if let Some(rest) = line.strip_prefix("usage_usec ") {
return rest.trim().parse::<u64>().ok();
}
}
None
}
/// Sample CPU + memory for every running agent container. Best-effort:
/// agents whose scope dir is absent (not running) or unreadable are
/// skipped, never fatal. Sorted by name for a stable display order.
pub async fn gather() -> Vec<ContainerResource> {
// Machines that currently have a cgroup scope (= running). Agent
// machine name is `h-<state-dir name>`.
let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names()
.into_iter()
.filter_map(|name| {
let dir = scope_dir(&format!("h-{name}"));
dir.join("cpu.stat").exists().then_some((name, dir))
})
.collect();
// First CPU sample for all, then one shared sleep, then the second
// — so N agents cost one interval, not N.
let t0: Vec<Option<u64>> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect();
sleep(CPU_SAMPLE).await;
#[allow(clippy::cast_precision_loss)]
let nproc = std::thread::available_parallelism().map_or(1.0, |n| n.get() as f64);
#[allow(clippy::cast_precision_loss)]
let interval_usec = CPU_SAMPLE.as_micros() as f64;
let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
for (i, (name, dir)) in candidates.iter().enumerate() {
let cpu_pct = match (t0[i], read_usage_usec(dir)) {
(Some(a), Some(b)) => {
#[allow(clippy::cast_precision_loss)]
let delta = b.saturating_sub(a) as f64;
(delta / (interval_usec * nproc)) * 100.0
}
_ => 0.0,
};
out.push(ContainerResource {
name: name.clone(),
cpu_pct,
mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
mem_peak_bytes: read_u64(&dir.join("memory.peak")),
mem_max_bytes: read_mem_max(&dir.join("memory.max")),
});
}
out
}

View file

@ -68,6 +68,7 @@ pub async fn serve(port: u16, coord: Arc<Coordinator>) -> Result<()> {
.route("/api/state-file", get(get_state_file))
.route("/api/reminders", get(api_reminders))
.route("/api/stats-hive", get(api_stats_hive))
.route("/api/container-resources", get(api_container_resources))
.route("/api/build-logs", get(get_build_logs_all))
.route("/api/build-logs/{agent}", get(get_build_logs_agent))
.route("/api/build-logs/id/{id}", get(get_build_log_full))
@ -1748,6 +1749,12 @@ async fn api_stats_hive(axum::extract::Query(q): axum::extract::Query<StatsHiveQ
axum::Json(crate::hive_stats::hive_snapshot(window)).into_response()
}
/// Live per-agent-container CPU + memory load from cgroup v2. Samples
/// CPU over a short interval (~200 ms), so this call briefly awaits.
async fn api_container_resources() -> Response {
axum::Json(crate::container_stats::gather().await).into_response()
}
#[derive(Deserialize)]
struct BuildLogsQuery {
/// Maximum number of rows to return. Capped server-side at 50

View file

@ -23,6 +23,7 @@ pub mod broker;
pub mod build_logs;
pub mod capabilities;
pub mod client;
pub mod container_stats;
pub mod container_view;
pub mod coordinator;
pub mod crash_watch;