feat(c0re): per-container disk-size sampler on container-load (#1659)

docs/web-ui/dashboard.md

@@ -195,7 +195,11 @@ bar against the `memory.max` quota. Backed by
 files read-only (world-readable; no `hive-priv`) and skips agents whose
 scope dir is absent (= not running). Pull-only: `core.js` polls every
 5 s **only while the C0NT41N3R L04D sub-tab is active** (CPU needs a fresh
-sample each refresh), and stops on sub-tab change. Network is intentionally
+sample each refresh), and stops on sub-tab change. Disk size (`disk_bytes`)
+rides the same row but is fed by a separate ~5 min background `du` sampler
+(state dir + container writable rootfs, shared nix store excluded via
+`du -x`), so the 5 s poll stays cheap cgroup-only reads; the row carries the
+last-sampled value (`null` until the first sample). Network is intentionally
 omitted — agents share the host netns, so there is no per-container net
 counter (per-agent network needs the netns-isolation roadmap in
 `docs/network.md`).
@@ -994,9 +998,13 @@ that's a browser-level decision, not ours.
 - `GET /api/container-resources` — live per-agent-container CPU +
   memory from cgroup v2 (C0R3 › C0NT41N3R L04D panel). Returns one
   row per running agent (`name`, `cpu_pct`, `mem_current_bytes`,
-  `mem_peak_bytes`, `mem_max_bytes`); samples CPU over ~200 ms so the
-  call briefly awaits. Skips non-running agents (no scope dir). No
-  network field — agents share the host netns.
+  `mem_peak_bytes`, `mem_max_bytes`, `disk_bytes`); samples CPU over
+  ~200 ms so the call briefly awaits. Skips non-running agents (no scope
+  dir). No network field — agents share the host netns. `disk_bytes` is
+  the last value from a **separate slow sampler** (not this hot path):
+  a background `du -sxb` of the agent's state dir + container writable
+  rootfs every ~5 min, `-x` excluding the shared read-only nix store.
+  `null` until the first sample lands.
 - `POST /cancel-reminder/{id}` — hard-delete a pending reminder.
 - `POST /retry-reminder/{id}` — re-arm a reminder whose delivery
   failed (clears the failure state so the scheduler retries).

hive-c0re/src/container_stats.rs

@@ -19,7 +19,9 @@
 //! interval, sample again, and divide the delta by `interval × nproc`
 //! to get a host-normalised percentage (0..100 across all cores).
 
+use std::collections::HashMap;
 use std::path::PathBuf;
+use std::sync::{OnceLock, RwLock};
 use std::time::Duration;
 
 use serde::Serialize;
@@ -49,6 +51,88 @@ pub struct ContainerResource {
     /// Memory ceiling (`memory.max`), bytes. `None` when unlimited
     /// (the file reads `max`).
     pub mem_max_bytes: Option<u64>,
+    /// Last-sampled total disk usage — the agent's state dir plus the
+    /// container's writable rootfs, excluding the shared read-only nix
+    /// store — in bytes. `None` until the slow background sampler
+    /// ([`disk_sampler_loop`]) has populated the cache at least once.
+    /// Decoupled from this hot per-poll path because a `du` tree-walk is
+    /// far too expensive at the 5s `/api/container-load` cadence.
+    pub disk_bytes: Option<u64>,
+}
+
+/// Cadence for the slow disk sampler. Disk size changes slowly relative
+/// to CPU/mem, and a `du` walk is expensive, so this runs far less often
+/// than the per-poll cgroup reads — the row just carries the last value.
+const DISK_SAMPLE_INTERVAL: Duration = Duration::from_mins(5);
+
+/// Root of the per-container writable rootfs trees that nixos-containers
+/// manages (`/var/lib/nixos-containers/<machine>`).
+const NIXOS_CONTAINERS_ROOT: &str = "/var/lib/nixos-containers";
+
+/// Last-sampled per-agent disk usage (bytes), keyed by agent name.
+/// Written by [`disk_sampler_loop`] (~every 5 min), read by [`gather`]
+/// so the hot poll never runs `du`.
+fn disk_cache() -> &'static RwLock<HashMap<String, u64>> {
+    static CACHE: OnceLock<RwLock<HashMap<String, u64>>> = OnceLock::new();
+    CACHE.get_or_init(|| RwLock::new(HashMap::new()))
+}
+
+/// `du -sxb <path>` → apparent total bytes. `-s` summary, `-b` apparent
+/// size, `-x` stay on one filesystem so the walk never descends into the
+/// bind-mounted read-only shared nix store (or any other bind mount) —
+/// exactly the per-container-only measure we want. Best-effort: a
+/// missing/unreadable path (e.g. a stopped container with no rootfs)
+/// yields `None`, which the caller treats as 0.
+async fn du_bytes(path: &str) -> Option<u64> {
+    let out = tokio::process::Command::new("du")
+        .args(["-sxb", path])
+        .output()
+        .await
+        .ok()?;
+    if !out.status.success() {
+        return None;
+    }
+    let stdout = String::from_utf8_lossy(&out.stdout);
+    stdout.split_whitespace().next()?.parse::<u64>().ok()
+}
+
+/// Total per-agent disk: state dir (`/agents/<name>/state`, the host
+/// bind-mount) plus the container's writable rootfs
+/// (`/var/lib/nixos-containers/h-<name>`). Each measured with `du -sxb`
+/// so the shared nix store and other bind mounts are excluded. A path
+/// that doesn't exist contributes 0.
+async fn measure_agent_disk(name: &str) -> u64 {
+    let state_dir = format!("/agents/{name}/state");
+    let rootfs = format!("{NIXOS_CONTAINERS_ROOT}/h-{name}");
+    let mut total = 0u64;
+    for path in [state_dir, rootfs] {
+        if let Some(bytes) = du_bytes(&path).await {
+            total = total.saturating_add(bytes);
+        }
+    }
+    total
+}
+
+/// Background loop: every [`DISK_SAMPLE_INTERVAL`], recompute disk usage
+/// for every kept-state agent and refresh [`disk_cache`]. The first pass
+/// runs immediately so the dashboard shows disk shortly after boot.
+/// Runs forever; spawn once at hive-c0re startup via [`spawn_disk_sampler`].
+pub async fn disk_sampler_loop() {
+    loop {
+        for name in Coordinator::kept_state_names() {
+            let bytes = measure_agent_disk(&name).await;
+            if let Ok(mut cache) = disk_cache().write() {
+                cache.insert(name, bytes);
+            }
+        }
+        sleep(DISK_SAMPLE_INTERVAL).await;
+    }
+}
+
+/// Spawn the slow disk sampler as a detached background task. Called once
+/// from `main` alongside the other host-side loops.
+pub fn spawn_disk_sampler() {
+    tokio::spawn(disk_sampler_loop());
 }
 
 /// Cgroup directory for a container's machine.
@@ -162,6 +246,11 @@ pub async fn gather() -> Vec<ContainerResource> {
     )]
     let interval_usec = CPU_SAMPLE.as_micros() as f64;
 
+    // Snapshot the slow disk cache once (cheap clone of a small map) so
+    // the per-agent loop below is a plain map lookup — no `du` on this
+    // hot path. Agents not yet sampled simply carry `disk_bytes = None`.
+    let disk = disk_cache().read().map(|c| c.clone()).unwrap_or_default();
+
     let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
     for (i, (name, dir)) in candidates.iter().enumerate() {
         let cpu_pct = match (t0[i], read_usage_usec(dir)) {
@@ -181,6 +270,7 @@ pub async fn gather() -> Vec<ContainerResource> {
             mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
             mem_peak_bytes: read_u64(&dir.join("memory.peak")),
             mem_max_bytes: read_mem_max(&dir.join("memory.max")),
+            disk_bytes: disk.get(name).copied(),
         });
     }
     out

hive-c0re/src/main.rs

@@ -394,6 +394,11 @@ async fn cmd_serve(
     // Per-agent bash-tasks file vacuum: host-side so the harness
     // cannot disable it. Deletes terminal task trios older than 48h.
     bash_tasks_vacuum::spawn(&coord);
+    // Slow per-container disk sampler: a `du` of each agent's state dir
+    // + writable rootfs every ~5 min, cached so the 5s container-load
+    // poll stays cheap cgroup-only reads. Feeds `disk_bytes` on the LOAD
+    // tab. See container_stats::disk_sampler_loop.
+    hive_c0re::container_stats::spawn_disk_sampler();
     // build_logs.sqlite vacuum: c0re-side (single db). Failures kept
     // 30d, successes 24h — see `build_logs::vacuum` for the rule.
     hive_c0re::build_logs::spawn_vacuum(&coord);