feat(stats): per-container cpu/mem load (#1424 p3)
C0NT41N3R L04D on the SYST3M tab + GET /api/container-resources.
Backend (hive-c0re/src/container_stats.rs): reads cgroup v2 cpu.stat +
memory.{current,peak,max} for each running agent machine
(machine-h\x2d<name>.scope under machine.slice), read-only/world-
readable so no hive-priv. CPU is a two-sample (~200ms) host-normalised
percentage; one shared sleep covers all agents. Skips agents whose
scope dir is absent (= not running). Network omitted: agents share the
host netns, no per-container counter.
Frontend: a polled C0NT41N3R L04D table on SYST3M (agent / cpu / mem /
peak / limit with meter bars), reusing the ST4TS table style. Polls
/api/container-resources every 5s only while the tab is active.
Backend reviewed-in-principle by damocles (path escaping + cpu delta
math); ping for the on-host sign-off.
This commit is contained in:
parent
14c7b0d406
commit
03ea6d1bda
7 changed files with 300 additions and 0 deletions
|
|
@ -153,6 +153,22 @@ age + claude-creds badge). Two actions: `⊕ R3V1V3` (queues a
|
|||
Spawn approval; existing state is reused), `PURG3` (wipes
|
||||
state + applied dirs; `POST /purge-tombstone/{name}`).
|
||||
|
||||
**C0NT41N3R L04D** — live CPU + memory per agent container, read
|
||||
straight from cgroup v2 on the host (`cpu.stat`, `memory.current`,
|
||||
`memory.peak`, `memory.max` under
|
||||
`/sys/fs/cgroup/machine.slice/machine-h\x2d<name>.scope/`). CPU is a
|
||||
host-normalised percentage (0..100 across all cores) sampled over a
|
||||
short (~200 ms) two-read interval; memory shows current + peak with a
|
||||
bar against the `memory.max` quota. Backed by
|
||||
`GET /api/container-resources` (`container_stats.rs`), which reads the
|
||||
files read-only (world-readable; no `hive-priv`) and skips agents whose
|
||||
scope dir is absent (= not running). Pull-only: `tabs.js` polls every
|
||||
5 s **only while the SYST3M tab is active** (CPU needs a fresh sample
|
||||
each refresh), and stops on tab change. Network is intentionally
|
||||
omitted — agents share the host netns, so there is no per-container net
|
||||
counter (per-agent network needs the netns-isolation roadmap in
|
||||
`docs/network.md`).
|
||||
|
||||
## P3RM1SS10NS tab
|
||||
|
||||
Per-agent permission configuration. Two sections, each rendered as a
|
||||
|
|
@ -838,6 +854,12 @@ that's a browser-level decision, not ours.
|
|||
zero-turn dbs); returns swarm totals, a busiest-first per-agent
|
||||
rollup, swarm model mix, and a labelled `est_cost_usd`. Window
|
||||
defaults to `24h`.
|
||||
- `GET /api/container-resources` — live per-agent-container CPU +
|
||||
memory from cgroup v2 (SYST3M › C0NT41N3R L04D panel). Returns one
|
||||
row per running agent (`name`, `cpu_pct`, `mem_current_bytes`,
|
||||
`mem_peak_bytes`, `mem_max_bytes`); samples CPU over ~200 ms so the
|
||||
call briefly awaits. Skips non-running agents (no scope dir). No
|
||||
network field — agents share the host netns.
|
||||
- `POST /cancel-reminder/{id}` — hard-delete a pending reminder.
|
||||
- `POST /retry-reminder/{id}` — re-arm a reminder whose delivery
|
||||
failed (clears the failure state so the scheduler retries).
|
||||
|
|
|
|||
|
|
@ -1609,3 +1609,29 @@ body.dashboard-shell.has-selection { padding-bottom: 4.5em; }
|
|||
color: var(--muted);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
/* ─── SYST3M › C0NT41N3R L04D: live cgroup cpu/mem ─────────────────────────────
|
||||
Reuses the `.hive-stats-table` styling from the ST4TS tab; only the
|
||||
inline meter bar is new. */
|
||||
.cload-meter {
|
||||
display: inline-block;
|
||||
width: 6em;
|
||||
height: 10px;
|
||||
vertical-align: middle;
|
||||
margin-left: 6px;
|
||||
background: var(--bg-elev);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 3px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.cload-meter .fill {
|
||||
display: block;
|
||||
height: 100%;
|
||||
background: var(--green);
|
||||
}
|
||||
.cload-meter .fill.warn {
|
||||
background: var(--amber);
|
||||
}
|
||||
.cload-meter .fill.hot {
|
||||
background: var(--red);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -191,6 +191,16 @@
|
|||
<p class="meta">loading…</p>
|
||||
</div>
|
||||
|
||||
<!-- C0NT41N3R L04D: live cpu + memory per agent container, read
|
||||
from cgroup v2 on the host. Polled only while this tab is
|
||||
active (cpu needs a short two-sample read each refresh). -->
|
||||
<h2>◆ C0NT41N3R L04D ◆</h2>
|
||||
<div class="divider">══════════════════════════════════════════════════════════════</div>
|
||||
<p class="meta">live cpu + memory per agent container, from cgroup v2 on the host. cpu is % of total host capacity (all cores), sampled over ~200ms each refresh; polled every 5s while this tab is open. network is omitted on purpose — agents share the host netns, so there is no per-container counter (see <code>docs/network.md</code>).</p>
|
||||
<div id="container-load-section">
|
||||
<p class="meta">loading…</p>
|
||||
</div>
|
||||
|
||||
</section>
|
||||
|
||||
<!-- P3RM1SS10NS: per-agent capability grants + tool-group
|
||||
|
|
|
|||
|
|
@ -4076,6 +4076,9 @@ window.marked = marked;
|
|||
}
|
||||
// ST4TS: hive-wide rollup is a pull (no SSE) — fetch on activation.
|
||||
if (target === 'stats') { refreshHiveStats(); }
|
||||
// SYST3M › C0NT41N3R L04D: live cgroup poll only while the tab is
|
||||
// open (cpu needs a short two-sample read each refresh).
|
||||
if (target === 'system') { startContainerLoadPolling(); } else { stopContainerLoadPolling(); }
|
||||
}
|
||||
// ─── tabbar overflow menu ────────────────────────────────────────────────
|
||||
// Tabs with `data-overflow="default"` (LOGS, SETTINGS) always live in
|
||||
|
|
@ -4230,6 +4233,100 @@ window.marked = marked;
|
|||
}
|
||||
bindHiveStatsWindows();
|
||||
|
||||
// ─── SYST3M › C0NT41N3R L04D: live per-container cgroup cpu/mem ────────────
|
||||
// Pull-only, polled at 5s ONLY while the SYST3M tab is active (cpu is a
|
||||
// short two-sample read on the server). Data from /api/container-resources.
|
||||
let containerLoadTimer = null;
|
||||
|
||||
function cloadFmtBytes(n) {
|
||||
if (!Number.isFinite(n) || n <= 0) return '0';
|
||||
const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||
let i = 0; let v = n;
|
||||
while (v >= 1024 && i < u.length - 1) { v /= 1024; i += 1; }
|
||||
return v.toFixed(v < 10 && i > 0 ? 1 : 0) + ' ' + u[i];
|
||||
}
|
||||
function cloadMeter(pct) {
|
||||
const p = Math.max(0, Math.min(100, pct));
|
||||
const cls = p >= 90 ? 'hot' : (p >= 70 ? 'warn' : '');
|
||||
const m = document.createElement('span');
|
||||
m.className = 'cload-meter';
|
||||
m.title = p.toFixed(0) + '%';
|
||||
const f = document.createElement('span');
|
||||
f.className = 'fill' + (cls ? ' ' + cls : '');
|
||||
f.style.width = p + '%';
|
||||
m.append(f);
|
||||
return m;
|
||||
}
|
||||
|
||||
function renderContainerLoad(rows) {
|
||||
const root = $('container-load-section');
|
||||
if (!root) return;
|
||||
if (!Array.isArray(rows) || rows.length === 0) {
|
||||
root.replaceChildren();
|
||||
const p = document.createElement('p'); p.className = 'meta';
|
||||
p.textContent = 'no running agent containers'; root.append(p);
|
||||
return;
|
||||
}
|
||||
const table = document.createElement('table');
|
||||
table.className = 'hive-stats-table';
|
||||
table.innerHTML = '<thead><tr><th>agent</th><th>cpu</th><th>memory</th>'
|
||||
+ '<th>peak</th><th>limit</th></tr></thead>';
|
||||
const tb = document.createElement('tbody');
|
||||
for (const r of rows) {
|
||||
const tr = document.createElement('tr');
|
||||
|
||||
const name = document.createElement('td'); name.textContent = r.name; tr.append(name);
|
||||
|
||||
const cpu = document.createElement('td'); cpu.className = 'num';
|
||||
cpu.append((Number(r.cpu_pct) || 0).toFixed(1) + '%', cloadMeter(Number(r.cpu_pct) || 0));
|
||||
tr.append(cpu);
|
||||
|
||||
const mem = document.createElement('td'); mem.className = 'num';
|
||||
const memCur = Number(r.mem_current_bytes) || 0;
|
||||
if (r.mem_max_bytes) {
|
||||
mem.append(cloadFmtBytes(memCur), cloadMeter(100 * memCur / r.mem_max_bytes));
|
||||
} else {
|
||||
mem.textContent = cloadFmtBytes(memCur);
|
||||
}
|
||||
tr.append(mem);
|
||||
|
||||
const peak = document.createElement('td'); peak.className = 'num';
|
||||
peak.textContent = r.mem_peak_bytes ? cloadFmtBytes(Number(r.mem_peak_bytes)) : '—';
|
||||
tr.append(peak);
|
||||
|
||||
const lim = document.createElement('td'); lim.className = 'num';
|
||||
lim.textContent = r.mem_max_bytes ? cloadFmtBytes(Number(r.mem_max_bytes)) : '∞';
|
||||
tr.append(lim);
|
||||
|
||||
tb.append(tr);
|
||||
}
|
||||
table.append(tb);
|
||||
root.replaceChildren(table);
|
||||
}
|
||||
|
||||
async function refreshContainerLoad() {
|
||||
try {
|
||||
const resp = await fetch('/api/container-resources');
|
||||
if (!resp.ok) throw new Error('http ' + resp.status);
|
||||
renderContainerLoad(await resp.json());
|
||||
} catch (e) {
|
||||
const root = $('container-load-section');
|
||||
if (root) {
|
||||
root.replaceChildren();
|
||||
const p = document.createElement('p'); p.className = 'meta';
|
||||
p.textContent = 'container load fetch failed: ' + e; root.append(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
function startContainerLoadPolling() {
|
||||
refreshContainerLoad();
|
||||
if (containerLoadTimer) return;
|
||||
containerLoadTimer = setInterval(refreshContainerLoad, 5000);
|
||||
}
|
||||
function stopContainerLoadPolling() {
|
||||
if (containerLoadTimer) { clearInterval(containerLoadTimer); containerLoadTimer = null; }
|
||||
}
|
||||
|
||||
function syncTabFromHash() {
|
||||
const h = (window.location.hash || '#swarm').replace(/^#/, '');
|
||||
activateTab(h);
|
||||
|
|
|
|||
137
hive-c0re/src/container_stats.rs
Normal file
137
hive-c0re/src/container_stats.rs
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
//! Live per-agent-container resource load (CPU + memory) for the
|
||||
//! dashboard. Reads cgroup v2 stat files for each running agent
|
||||
//! machine directly — same privsep-clean posture as the turn-stats
|
||||
//! sqlite reads (`cpu.stat` / `memory.*` are world-readable; no
|
||||
//! `hive-priv` needed). Runs host-side in hive-c0re, where
|
||||
//! `/sys/fs/cgroup/machine.slice/` holds the nspawn machine scopes.
|
||||
//!
|
||||
//! No network: agents share the host network namespace
|
||||
//! (`privateNetwork = false`), so there is no per-container net
|
||||
//! counter to read. Per-agent network only becomes meaningful with the
|
||||
//! netns-isolation roadmap (`docs/network.md`).
|
||||
//!
|
||||
//! CPU is cumulative (`usage_usec` is monotonic), so a single read is
|
||||
//! meaningless — we sample every machine's counter, sleep one short
|
||||
//! interval, sample again, and divide the delta by `interval × nproc`
|
||||
//! to get a host-normalised percentage (0..100 across all cores).
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::Serialize;
|
||||
use tokio::time::sleep;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
/// Sampling interval for the two CPU reads. Short enough to keep the
|
||||
/// endpoint snappy, long enough that the delta isn't dominated by read
|
||||
/// jitter.
|
||||
const CPU_SAMPLE: Duration = Duration::from_millis(200);
|
||||
|
||||
const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice";
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ContainerResource {
|
||||
/// Agent name (without the `h-` machine prefix).
|
||||
pub name: String,
|
||||
/// Host-normalised CPU usage over the sample interval, as a
|
||||
/// percentage of total host CPU (0..100 across all cores).
|
||||
pub cpu_pct: f64,
|
||||
/// Current memory usage (`memory.current`), bytes.
|
||||
pub mem_current_bytes: u64,
|
||||
/// High-water memory usage since container start (`memory.peak`),
|
||||
/// bytes. `None` if the kernel doesn't expose `memory.peak`.
|
||||
pub mem_peak_bytes: Option<u64>,
|
||||
/// Memory ceiling (`memory.max`), bytes. `None` when unlimited
|
||||
/// (the file reads `max`).
|
||||
pub mem_max_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
/// systemd escapes the machine name in the cgroup scope dir
|
||||
/// (`machine-<escaped>.scope`). For our machine names — `h-<agent>`,
|
||||
/// agent ∈ `[a-z0-9_-]` — the only character systemd escapes is `-`,
|
||||
/// which becomes `\x2d` (verified against `systemd-escape`).
|
||||
fn escape_machine(machine: &str) -> String {
|
||||
machine.replace('-', "\\x2d")
|
||||
}
|
||||
|
||||
fn scope_dir(machine: &str) -> PathBuf {
|
||||
PathBuf::from(MACHINE_SLICE).join(format!("machine-{}.scope", escape_machine(machine)))
|
||||
}
|
||||
|
||||
/// Read a single unsigned integer from a one-line cgroup file.
|
||||
fn read_u64(path: &std::path::Path) -> Option<u64> {
|
||||
std::fs::read_to_string(path)
|
||||
.ok()?
|
||||
.trim()
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// `memory.max` reads `max` when there's no limit — map that to `None`.
|
||||
fn read_mem_max(path: &std::path::Path) -> Option<u64> {
|
||||
let s = std::fs::read_to_string(path).ok()?;
|
||||
let s = s.trim();
|
||||
if s == "max" {
|
||||
None
|
||||
} else {
|
||||
s.parse::<u64>().ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse `usage_usec` (cumulative CPU time, microseconds) from a
|
||||
/// scope's `cpu.stat`.
|
||||
fn read_usage_usec(dir: &std::path::Path) -> Option<u64> {
|
||||
let text = std::fs::read_to_string(dir.join("cpu.stat")).ok()?;
|
||||
for line in text.lines() {
|
||||
if let Some(rest) = line.strip_prefix("usage_usec ") {
|
||||
return rest.trim().parse::<u64>().ok();
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Sample CPU + memory for every running agent container. Best-effort:
|
||||
/// agents whose scope dir is absent (not running) or unreadable are
|
||||
/// skipped, never fatal. Sorted by name for a stable display order.
|
||||
pub async fn gather() -> Vec<ContainerResource> {
|
||||
// Machines that currently have a cgroup scope (= running). Agent
|
||||
// machine name is `h-<state-dir name>`.
|
||||
let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names()
|
||||
.into_iter()
|
||||
.filter_map(|name| {
|
||||
let dir = scope_dir(&format!("h-{name}"));
|
||||
dir.join("cpu.stat").exists().then_some((name, dir))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// First CPU sample for all, then one shared sleep, then the second
|
||||
// — so N agents cost one interval, not N.
|
||||
let t0: Vec<Option<u64>> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect();
|
||||
sleep(CPU_SAMPLE).await;
|
||||
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
let nproc = std::thread::available_parallelism().map_or(1.0, |n| n.get() as f64);
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
let interval_usec = CPU_SAMPLE.as_micros() as f64;
|
||||
|
||||
let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
|
||||
for (i, (name, dir)) in candidates.iter().enumerate() {
|
||||
let cpu_pct = match (t0[i], read_usage_usec(dir)) {
|
||||
(Some(a), Some(b)) => {
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
let delta = b.saturating_sub(a) as f64;
|
||||
(delta / (interval_usec * nproc)) * 100.0
|
||||
}
|
||||
_ => 0.0,
|
||||
};
|
||||
out.push(ContainerResource {
|
||||
name: name.clone(),
|
||||
cpu_pct,
|
||||
mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
|
||||
mem_peak_bytes: read_u64(&dir.join("memory.peak")),
|
||||
mem_max_bytes: read_mem_max(&dir.join("memory.max")),
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
|
@ -68,6 +68,7 @@ pub async fn serve(port: u16, coord: Arc<Coordinator>) -> Result<()> {
|
|||
.route("/api/state-file", get(get_state_file))
|
||||
.route("/api/reminders", get(api_reminders))
|
||||
.route("/api/stats-hive", get(api_stats_hive))
|
||||
.route("/api/container-resources", get(api_container_resources))
|
||||
.route("/api/build-logs", get(get_build_logs_all))
|
||||
.route("/api/build-logs/{agent}", get(get_build_logs_agent))
|
||||
.route("/api/build-logs/id/{id}", get(get_build_log_full))
|
||||
|
|
@ -1748,6 +1749,12 @@ async fn api_stats_hive(axum::extract::Query(q): axum::extract::Query<StatsHiveQ
|
|||
axum::Json(crate::hive_stats::hive_snapshot(window)).into_response()
|
||||
}
|
||||
|
||||
/// Live per-agent-container CPU + memory load from cgroup v2. Samples
|
||||
/// CPU over a short interval (~200 ms), so this call briefly awaits.
|
||||
async fn api_container_resources() -> Response {
|
||||
axum::Json(crate::container_stats::gather().await).into_response()
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct BuildLogsQuery {
|
||||
/// Maximum number of rows to return. Capped server-side at 50
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ pub mod broker;
|
|||
pub mod build_logs;
|
||||
pub mod capabilities;
|
||||
pub mod client;
|
||||
pub mod container_stats;
|
||||
pub mod container_view;
|
||||
pub mod coordinator;
|
||||
pub mod crash_watch;
|
||||
|
|
|
|||
Loading…
Reference in a new issue