diff --git a/docs/web-ui/dashboard.md b/docs/web-ui/dashboard.md index e43f8a92..cb5660c3 100644 --- a/docs/web-ui/dashboard.md +++ b/docs/web-ui/dashboard.md @@ -195,7 +195,11 @@ bar against the `memory.max` quota. Backed by files read-only (world-readable; no `hive-priv`) and skips agents whose scope dir is absent (= not running). Pull-only: `core.js` polls every 5 s **only while the C0NT41N3R L04D sub-tab is active** (CPU needs a fresh -sample each refresh), and stops on sub-tab change. Network is intentionally +sample each refresh), and stops on sub-tab change. Disk size (`disk_bytes`) +rides the same row but is fed by a separate ~5 min background `du` sampler +(state dir + container writable rootfs, shared nix store excluded via +`du -x`), so the 5 s poll stays cheap cgroup-only reads; the row carries the +last-sampled value (`null` until the first sample). Network is intentionally omitted — agents share the host netns, so there is no per-container net counter (per-agent network needs the netns-isolation roadmap in `docs/network.md`). @@ -994,9 +998,13 @@ that's a browser-level decision, not ours. - `GET /api/container-resources` — live per-agent-container CPU + memory from cgroup v2 (C0R3 › C0NT41N3R L04D panel). Returns one row per running agent (`name`, `cpu_pct`, `mem_current_bytes`, - `mem_peak_bytes`, `mem_max_bytes`); samples CPU over ~200 ms so the - call briefly awaits. Skips non-running agents (no scope dir). No - network field — agents share the host netns. + `mem_peak_bytes`, `mem_max_bytes`, `disk_bytes`); samples CPU over + ~200 ms so the call briefly awaits. Skips non-running agents (no scope + dir). No network field — agents share the host netns. `disk_bytes` is + the last value from a **separate slow sampler** (not this hot path): + a background `du -sxb` of the agent's state dir + container writable + rootfs every ~5 min, `-x` excluding the shared read-only nix store. + `null` until the first sample lands. - `POST /cancel-reminder/{id}` — hard-delete a pending reminder. - `POST /retry-reminder/{id}` — re-arm a reminder whose delivery failed (clears the failure state so the scheduler retries). diff --git a/hive-c0re/src/container_stats.rs b/hive-c0re/src/container_stats.rs index 0e95bd9d..898747d2 100644 --- a/hive-c0re/src/container_stats.rs +++ b/hive-c0re/src/container_stats.rs @@ -19,7 +19,9 @@ //! interval, sample again, and divide the delta by `interval × nproc` //! to get a host-normalised percentage (0..100 across all cores). +use std::collections::HashMap; use std::path::PathBuf; +use std::sync::{OnceLock, RwLock}; use std::time::Duration; use serde::Serialize; @@ -49,6 +51,88 @@ pub struct ContainerResource { /// Memory ceiling (`memory.max`), bytes. `None` when unlimited /// (the file reads `max`). pub mem_max_bytes: Option, + /// Last-sampled total disk usage — the agent's state dir plus the + /// container's writable rootfs, excluding the shared read-only nix + /// store — in bytes. `None` until the slow background sampler + /// ([`disk_sampler_loop`]) has populated the cache at least once. + /// Decoupled from this hot per-poll path because a `du` tree-walk is + /// far too expensive at the 5s `/api/container-load` cadence. + pub disk_bytes: Option, +} + +/// Cadence for the slow disk sampler. Disk size changes slowly relative +/// to CPU/mem, and a `du` walk is expensive, so this runs far less often +/// than the per-poll cgroup reads — the row just carries the last value. +const DISK_SAMPLE_INTERVAL: Duration = Duration::from_mins(5); + +/// Root of the per-container writable rootfs trees that nixos-containers +/// manages (`/var/lib/nixos-containers/`). +const NIXOS_CONTAINERS_ROOT: &str = "/var/lib/nixos-containers"; + +/// Last-sampled per-agent disk usage (bytes), keyed by agent name. +/// Written by [`disk_sampler_loop`] (~every 5 min), read by [`gather`] +/// so the hot poll never runs `du`. +fn disk_cache() -> &'static RwLock> { + static CACHE: OnceLock>> = OnceLock::new(); + CACHE.get_or_init(|| RwLock::new(HashMap::new())) +} + +/// `du -sxb ` → apparent total bytes. `-s` summary, `-b` apparent +/// size, `-x` stay on one filesystem so the walk never descends into the +/// bind-mounted read-only shared nix store (or any other bind mount) — +/// exactly the per-container-only measure we want. Best-effort: a +/// missing/unreadable path (e.g. a stopped container with no rootfs) +/// yields `None`, which the caller treats as 0. +async fn du_bytes(path: &str) -> Option { + let out = tokio::process::Command::new("du") + .args(["-sxb", path]) + .output() + .await + .ok()?; + if !out.status.success() { + return None; + } + let stdout = String::from_utf8_lossy(&out.stdout); + stdout.split_whitespace().next()?.parse::().ok() +} + +/// Total per-agent disk: state dir (`/agents//state`, the host +/// bind-mount) plus the container's writable rootfs +/// (`/var/lib/nixos-containers/h-`). Each measured with `du -sxb` +/// so the shared nix store and other bind mounts are excluded. A path +/// that doesn't exist contributes 0. +async fn measure_agent_disk(name: &str) -> u64 { + let state_dir = format!("/agents/{name}/state"); + let rootfs = format!("{NIXOS_CONTAINERS_ROOT}/h-{name}"); + let mut total = 0u64; + for path in [state_dir, rootfs] { + if let Some(bytes) = du_bytes(&path).await { + total = total.saturating_add(bytes); + } + } + total +} + +/// Background loop: every [`DISK_SAMPLE_INTERVAL`], recompute disk usage +/// for every kept-state agent and refresh [`disk_cache`]. The first pass +/// runs immediately so the dashboard shows disk shortly after boot. +/// Runs forever; spawn once at hive-c0re startup via [`spawn_disk_sampler`]. +pub async fn disk_sampler_loop() { + loop { + for name in Coordinator::kept_state_names() { + let bytes = measure_agent_disk(&name).await; + if let Ok(mut cache) = disk_cache().write() { + cache.insert(name, bytes); + } + } + sleep(DISK_SAMPLE_INTERVAL).await; + } +} + +/// Spawn the slow disk sampler as a detached background task. Called once +/// from `main` alongside the other host-side loops. +pub fn spawn_disk_sampler() { + tokio::spawn(disk_sampler_loop()); } /// Cgroup directory for a container's machine. @@ -162,6 +246,11 @@ pub async fn gather() -> Vec { )] let interval_usec = CPU_SAMPLE.as_micros() as f64; + // Snapshot the slow disk cache once (cheap clone of a small map) so + // the per-agent loop below is a plain map lookup — no `du` on this + // hot path. Agents not yet sampled simply carry `disk_bytes = None`. + let disk = disk_cache().read().map(|c| c.clone()).unwrap_or_default(); + let mut out: Vec = Vec::with_capacity(candidates.len()); for (i, (name, dir)) in candidates.iter().enumerate() { let cpu_pct = match (t0[i], read_usage_usec(dir)) { @@ -181,6 +270,7 @@ pub async fn gather() -> Vec { mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0), mem_peak_bytes: read_u64(&dir.join("memory.peak")), mem_max_bytes: read_mem_max(&dir.join("memory.max")), + disk_bytes: disk.get(name).copied(), }); } out diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index 8d9efacd..3cf6496c 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -394,6 +394,11 @@ async fn cmd_serve( // Per-agent bash-tasks file vacuum: host-side so the harness // cannot disable it. Deletes terminal task trios older than 48h. bash_tasks_vacuum::spawn(&coord); + // Slow per-container disk sampler: a `du` of each agent's state dir + // + writable rootfs every ~5 min, cached so the 5s container-load + // poll stays cheap cgroup-only reads. Feeds `disk_bytes` on the LOAD + // tab. See container_stats::disk_sampler_loop. + hive_c0re::container_stats::spawn_disk_sampler(); // build_logs.sqlite vacuum: c0re-side (single db). Failures kept // 30d, successes 24h — see `build_logs::vacuum` for the rule. hive_c0re::build_logs::spawn_vacuum(&coord);