refactor(hive-c0re): group src-root files into submodules

stores/ (sqlite-backed host stores + db helper), stats/, agent_config/,
workers/ — pure git-mv moves; crate-root re-exports keep every
crate::<module> path compiling. flake_check stays at root (synchronous
approval-flow validation, not a background worker)

hive-c0re/src/stats/container_stats.rs

@@ -0,0 +1,307 @@
+//! Live per-agent-container resource load (CPU + memory) for the
+//! dashboard. Reads cgroup v2 stat files for each running agent
+//! machine directly — same privsep-clean posture as the turn-stats
+//! sqlite reads (`cpu.stat` / `memory.*` are world-readable; no
+//! `hive-priv` needed). Runs host-side in hive-c0re, where
+//! `/sys/fs/cgroup/machine.slice/` holds the per-container cgroups. Because
+//! nixos-container runs `systemd-nspawn --keep-unit` (with
+//! `Slice = "machine.slice"`), each container's cgroup is its launching
+//! service unit `container@<machine>.service` — not a machined
+//! `machine-<name>.scope`. See [`scope_dir`].
+//!
+//! No network: agents share the host network namespace
+//! (`privateNetwork = false`), so there is no per-container net
+//! counter to read. Per-agent network only becomes meaningful with the
+//! netns-isolation roadmap (`docs/network.md`).
+//!
+//! CPU is cumulative (`usage_usec` is monotonic), so a single read is
+//! meaningless — we sample every machine's counter, sleep one short
+//! interval, sample again, and divide the delta by `interval × nproc`
+//! to get a host-normalised percentage (0..100 across all cores).
+
+use std::collections::HashMap;
+use std::path::PathBuf;
+use std::sync::{OnceLock, RwLock};
+use std::time::Duration;
+
+use serde::Serialize;
+use tokio::time::sleep;
+
+use crate::coordinator::Coordinator;
+
+/// Sampling interval for the two CPU reads. Short enough to keep the
+/// endpoint snappy, long enough that the delta isn't dominated by read
+/// jitter.
+const CPU_SAMPLE: Duration = Duration::from_millis(200);
+
+const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice";
+
+#[derive(Debug, Serialize)]
+pub struct ContainerResource {
+    /// Agent name (without the `h-` machine prefix).
+    pub name: String,
+    /// Host-normalised CPU usage over the sample interval, as a
+    /// percentage of total host CPU (0..100 across all cores).
+    pub cpu_pct: f64,
+    /// Current memory usage (`memory.current`), bytes.
+    pub mem_current_bytes: u64,
+    /// High-water memory usage since container start (`memory.peak`),
+    /// bytes. `None` if the kernel doesn't expose `memory.peak`.
+    pub mem_peak_bytes: Option<u64>,
+    /// Memory ceiling (`memory.max`), bytes. `None` when unlimited
+    /// (the file reads `max`).
+    pub mem_max_bytes: Option<u64>,
+    /// Last-sampled total disk usage — the agent's state dir plus the
+    /// container's writable rootfs, excluding the shared read-only nix
+    /// store — in bytes. `None` until the slow background sampler
+    /// ([`disk_sampler_loop`]) has populated the cache at least once.
+    /// Decoupled from this hot per-poll path because a `du` tree-walk is
+    /// far too expensive at the 5s `/api/container-load` cadence.
+    pub disk_bytes: Option<u64>,
+}
+
+/// Cadence for the slow disk sampler. Disk size changes slowly relative
+/// to CPU/mem, and a `du` walk is expensive, so this runs far less often
+/// than the per-poll cgroup reads — the row just carries the last value.
+const DISK_SAMPLE_INTERVAL: Duration = Duration::from_mins(5);
+
+/// Root of the per-container writable rootfs trees that nixos-containers
+/// manages (`/var/lib/nixos-containers/<machine>`).
+const NIXOS_CONTAINERS_ROOT: &str = "/var/lib/nixos-containers";
+
+/// Last-sampled per-agent disk usage (bytes), keyed by agent name.
+/// Written by [`disk_sampler_loop`] (~every 5 min), read by [`gather`]
+/// so the hot poll never runs `du`.
+fn disk_cache() -> &'static RwLock<HashMap<String, u64>> {
+    static CACHE: OnceLock<RwLock<HashMap<String, u64>>> = OnceLock::new();
+    CACHE.get_or_init(|| RwLock::new(HashMap::new()))
+}
+
+/// `du -sxb <path>` → apparent total bytes. `-s` summary, `-b` apparent
+/// size, `-x` stay on one filesystem so the walk never descends into the
+/// bind-mounted read-only shared nix store (or any other bind mount) —
+/// exactly the per-container-only measure we want. Best-effort: a
+/// missing/unreadable path (e.g. a stopped container with no rootfs)
+/// yields `None`, which the caller treats as 0.
+async fn du_bytes(path: &std::path::Path) -> Option<u64> {
+    let out = tokio::process::Command::new("du")
+        .arg("-sxb")
+        .arg(path)
+        .output()
+        .await
+        .ok()?;
+    if !out.status.success() {
+        return None;
+    }
+    let stdout = String::from_utf8_lossy(&out.stdout);
+    stdout.split_whitespace().next()?.parse::<u64>().ok()
+}
+
+/// Total per-agent disk: the agent's host-side state dir
+/// (`/var/lib/hyperhive/agents/<name>/state`) plus the container's writable
+/// rootfs (`/var/lib/nixos-containers/h-<name>`). Each measured with `du -sxb`
+/// so the shared nix store and other bind mounts are excluded. A path that
+/// doesn't exist contributes 0.
+///
+/// The state dir is resolved via [`Coordinator::agent_notes_dir`] — the same
+/// host path the dashboard reads agent state from. Hardcoding the
+/// container-internal bind-mount path (`/agents/<name>/state`) instead made
+/// `du` fail host-side (that path only exists inside the container), so every
+/// agent's state-dir contribution was 0; with the writable rootfs nearly empty
+/// (almost everything is bind-mounted), that surfaced as all agents reporting
+/// 0 disk.
+async fn measure_agent_disk(name: &str) -> u64 {
+    let state_dir = Coordinator::agent_notes_dir(name);
+    let rootfs = PathBuf::from(format!("{NIXOS_CONTAINERS_ROOT}/h-{name}"));
+    let mut total = 0u64;
+    for path in [state_dir, rootfs] {
+        if let Some(bytes) = du_bytes(&path).await {
+            total = total.saturating_add(bytes);
+        }
+    }
+    total
+}
+
+/// Background loop: every [`DISK_SAMPLE_INTERVAL`], recompute disk usage
+/// for every kept-state agent and refresh [`disk_cache`]. The first pass
+/// runs immediately so the dashboard shows disk shortly after boot.
+/// Runs forever; spawn once at hive-c0re startup via [`spawn_disk_sampler`].
+pub async fn disk_sampler_loop() {
+    loop {
+        for name in Coordinator::kept_state_names() {
+            let bytes = measure_agent_disk(&name).await;
+            if let Ok(mut cache) = disk_cache().write() {
+                cache.insert(name, bytes);
+            }
+        }
+        sleep(DISK_SAMPLE_INTERVAL).await;
+    }
+}
+
+/// Spawn the slow disk sampler as a detached background task. Called once
+/// from `main` alongside the other host-side loops.
+pub fn spawn_disk_sampler() {
+    tokio::spawn(disk_sampler_loop());
+}
+
+/// Cgroup directory for a container's machine.
+///
+/// nixos-container runs `systemd-nspawn --keep-unit` with
+/// `Slice = "machine.slice"` (see nixpkgs
+/// `virtualisation/nixos-containers.nix`). `--keep-unit` means nspawn does
+/// **not** create a separate machined `machine-<name>.scope` — the
+/// container's cgroup *is* the launching service unit,
+/// `container@<machine>.service`, placed under `machine.slice`.
+/// systemd-machined still logs "New machine <name>" (registration), but the
+/// cgroup stays on the service unit. So the path is
+/// `machine.slice/container@<machine>.service`, and the service unit name is
+/// used verbatim — no `\x2d` escaping (that only applies when a string is
+/// converted *into* a scope/slice unit name, not to an already-formed
+/// instance unit; the journal shows the literal `container@h-<agent>.service`).
+fn scope_dir(machine: &str) -> PathBuf {
+    PathBuf::from(MACHINE_SLICE).join(format!("container@{machine}.service"))
+}
+
+/// Read a single unsigned integer from a one-line cgroup file.
+fn read_u64(path: &std::path::Path) -> Option<u64> {
+    std::fs::read_to_string(path)
+        .ok()?
+        .trim()
+        .parse::<u64>()
+        .ok()
+}
+
+/// `memory.max` reads `max` when there's no limit — map that to `None`.
+fn read_mem_max(path: &std::path::Path) -> Option<u64> {
+    let s = std::fs::read_to_string(path).ok()?;
+    let s = s.trim();
+    if s == "max" {
+        None
+    } else {
+        s.parse::<u64>().ok()
+    }
+}
+
+/// Parse `usage_usec` (cumulative CPU time, microseconds) from a
+/// scope's `cpu.stat`.
+fn read_usage_usec(dir: &std::path::Path) -> Option<u64> {
+    let text = std::fs::read_to_string(dir.join("cpu.stat")).ok()?;
+    for line in text.lines() {
+        if let Some(rest) = line.strip_prefix("usage_usec ") {
+            return rest.trim().parse::<u64>().ok();
+        }
+    }
+    None
+}
+
+/// Total online host CPUs, for normalising CPU% to "% of total host".
+/// Reads `/sys/devices/system/cpu/online` (a comma-separated list of
+/// ranges, e.g. `0-19`) rather than `available_parallelism()`, which
+/// reflects the hive-core process's CPU affinity — if hive-core is ever
+/// affinity-pinned to a subset, that would under-count the denominator
+/// and inflate the percentage. Falls back to the process count, then 1.
+fn host_nproc() -> usize {
+    fn from_sysfs() -> Option<usize> {
+        let s = std::fs::read_to_string("/sys/devices/system/cpu/online").ok()?;
+        let mut count = 0usize;
+        for part in s.trim().split(',') {
+            let part = part.trim();
+            if part.is_empty() {
+                continue;
+            }
+            if let Some((a, b)) = part.split_once('-') {
+                let a: usize = a.trim().parse().ok()?;
+                let b: usize = b.trim().parse().ok()?;
+                count += b.saturating_sub(a) + 1;
+            } else {
+                part.parse::<usize>().ok()?;
+                count += 1;
+            }
+        }
+        (count > 0).then_some(count)
+    }
+    from_sysfs()
+        .or_else(|| std::thread::available_parallelism().ok().map(usize::from))
+        .unwrap_or(1)
+}
+
+/// Sample CPU + memory for every running agent container. Best-effort:
+/// agents whose scope dir is absent (not running) or unreadable are
+/// skipped, never fatal. Sorted by name for a stable display order.
+pub async fn gather() -> Vec<ContainerResource> {
+    // Machines that currently have a cgroup scope (= running). Agent
+    // machine name is `h-<state-dir name>`.
+    let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names()
+        .into_iter()
+        .filter_map(|name| {
+            let dir = scope_dir(&format!("h-{name}"));
+            dir.join("cpu.stat").exists().then_some((name, dir))
+        })
+        .collect();
+
+    // First CPU sample for all, then one shared sleep, then the second
+    // — so N agents cost one interval, not N.
+    let t0: Vec<Option<u64>> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect();
+    sleep(CPU_SAMPLE).await;
+
+    #[allow(
+        clippy::cast_precision_loss,
+        reason = "CPU-utilisation math; the operands (core count, microsecond sample interval, cgroup time delta) stay well under f64's 2^53 exact-integer range, so no precision is lost"
+    )]
+    let nproc = host_nproc() as f64;
+    #[allow(
+        clippy::cast_precision_loss,
+        reason = "CPU-utilisation math; the operands (core count, microsecond sample interval, cgroup time delta) stay well under f64's 2^53 exact-integer range, so no precision is lost"
+    )]
+    let interval_usec = CPU_SAMPLE.as_micros() as f64;
+
+    // Snapshot the slow disk cache once (cheap clone of a small map) so
+    // the per-agent loop below is a plain map lookup — no `du` on this
+    // hot path. Agents not yet sampled simply carry `disk_bytes = None`.
+    let disk = disk_cache().read().map(|c| c.clone()).unwrap_or_default();
+
+    let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
+    for (i, (name, dir)) in candidates.iter().enumerate() {
+        let cpu_pct = match (t0[i], read_usage_usec(dir)) {
+            (Some(a), Some(b)) => {
+                #[allow(
+                    clippy::cast_precision_loss,
+                    reason = "CPU-utilisation math; the operands (core count, microsecond sample interval, cgroup time delta) stay well under f64's 2^53 exact-integer range, so no precision is lost"
+                )]
+                let delta = b.saturating_sub(a) as f64;
+                (delta / (interval_usec * nproc)) * 100.0
+            }
+            _ => 0.0,
+        };
+        out.push(ContainerResource {
+            name: name.clone(),
+            cpu_pct,
+            mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
+            mem_peak_bytes: read_u64(&dir.join("memory.peak")),
+            mem_max_bytes: read_mem_max(&dir.join("memory.max")),
+            disk_bytes: disk.get(name).copied(),
+        });
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn scope_dir_is_the_keep_unit_service_under_machine_slice() {
+        // nixos-container `--keep-unit` keeps the cgroup on the service
+        // unit `container@<machine>.service` (literal name, no `\x2d`),
+        // under machine.slice — NOT a machined `machine-<name>.scope`.
+        assert_eq!(
+            scope_dir("h-atlas"),
+            PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-atlas.service")
+        );
+        // Underscores in agent names are likewise verbatim.
+        assert_eq!(
+            scope_dir("h-foo_bar"),
+            PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-foo_bar.service")
+        );
+    }
+}

hive-c0re/src/stats/hive_stats.rs

@@ -0,0 +1,450 @@
+//! Hive-wide turn-stats aggregation for the dashboard's swarm stats
+//! view. Reads every agent's per-agent
+//! `hyperhive-turn-stats.sqlite` read-only and rolls the rows up into
+//! swarm totals + a per-agent rollup + model mix + a *labelled*
+//! cost estimate.
+//!
+//! Why re-read the rows here instead of reusing `hive-ag3nt`'s
+//! `stats.rs`: that module lives in a different crate (the agent
+//! harness) which hive-c0re can't import. The stable contract is the
+//! turn-stats *schema*, so we run a focused query against it. If we
+//! ever want a single source of truth, the row-read + aggregation can
+//! be lifted into a shared crate — overkill for now.
+//!
+//! Privsep: the sqlite files are mode 0644 owned by the agent user;
+//! `hive-core` can read them fine, but cannot write/delete (which is why
+//! retention sweeps run agent-side in the harness, not here). We open
+//! read-only so an in-flight harness writer never blocks us.
+
+use std::collections::HashMap;
+use std::path::Path;
+use std::time::Duration;
+
+use rusqlite::{Connection, OpenFlags};
+use serde::{Deserialize, Serialize};
+
+use crate::coordinator::Coordinator;
+use hive_sh4re::wire_time::now_unix;
+
+/// Window accepted by `/api/stats-hive?window=`. Maps to a lookback
+/// span; the hive view is a flat rollup (no per-bucket trend — the
+/// per-agent `/stats` page owns the trend charts).
+#[derive(Debug, Clone, Copy)]
+pub enum Window {
+    Hour,
+    FourHour,
+    Day,
+    ThreeDay,
+    Week,
+    Month,
+    /// All available data: aggregate over every recorded turn since the
+    /// earliest (`from == 0`), with no fixed lookback.
+    All,
+}
+
+impl Window {
+    #[must_use]
+    pub fn parse(s: &str) -> Self {
+        match s {
+            "1h" => Self::Hour,
+            "4h" => Self::FourHour,
+            "3d" => Self::ThreeDay,
+            "7d" => Self::Week,
+            "30d" => Self::Month,
+            "all" => Self::All,
+            _ => Self::Day,
+        }
+    }
+
+    fn label(self) -> &'static str {
+        match self {
+            Self::Hour => "1h",
+            Self::FourHour => "4h",
+            Self::Day => "24h",
+            Self::ThreeDay => "3d",
+            Self::Week => "7d",
+            Self::Month => "30d",
+            Self::All => "all",
+        }
+    }
+
+    fn span_secs(self) -> i64 {
+        match self {
+            Self::Hour => 3600,
+            Self::FourHour => 4 * 3600,
+            Self::Day => 24 * 3600,
+            Self::ThreeDay => 3 * 24 * 3600,
+            Self::Week => 7 * 24 * 3600,
+            Self::Month => 30 * 24 * 3600,
+            // `All` has no fixed lookback; `hive_snapshot` uses `from == 0`
+            // (every recorded turn). 0 here is a consistent safe fallback.
+            Self::All => 0,
+        }
+    }
+}
+
+/// Approximate USD price per **million** tokens for one model class.
+/// The four fields map to the four token kinds the turn-stats schema
+/// records. This is a deliberately rough estimate — Anthropic list
+/// pricing drifts, so the dashboard labels the figure as an estimate.
+///
+/// Operators keep the table current without a code change via the
+/// `services.hyperhive.modelPrices` nix option (passed to
+/// `hive-c0re serve --model-prices <json>`, held on the `Coordinator`
+/// as a [`PriceTable`]). Any model not covered by the operator table
+/// falls back to [`builtin_prices`].
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct Prices {
+    pub input: f64,
+    pub output: f64,
+    pub cache_read: f64,
+    pub cache_write: f64,
+}
+
+/// Operator-tunable model→price map. Keys are matched
+/// case-insensitively as a substring of the model id (e.g. `"sonnet"`
+/// matches `"claude-sonnet-4-5"`); the longest matching key wins so a
+/// specific entry beats a generic family name. An empty map means
+/// every model uses [`builtin_prices`].
+pub type PriceTable = HashMap<String, Prices>;
+
+/// Built-in fallback pricing, per **million** tokens. Used when the
+/// operator's [`PriceTable`] has no key matching the model. Figures are
+/// current Anthropic list prices for the Claude 4.x family (Opus 4.x,
+/// Sonnet 4.x, Haiku 4.5); `cache_write` uses the 1-hour cache-TTL price,
+/// which is the default through the Claude subscription the agents run
+/// on. Keep in sync with the `services.hyperhive.modelPrices` nix default
+/// (`nix/modules/hive-c0re.nix`).
+fn builtin_prices(model: &str) -> Prices {
+    let m = model.to_ascii_lowercase();
+    if m.contains("opus") {
+        Prices {
+            input: 5.0,
+            output: 25.0,
+            cache_read: 0.5,
+            cache_write: 10.0,
+        }
+    } else if m.contains("haiku") {
+        Prices {
+            input: 1.0,
+            output: 5.0,
+            cache_read: 0.1,
+            cache_write: 2.0,
+        }
+    } else {
+        // sonnet + unknown fallback
+        Prices {
+            input: 3.0,
+            output: 15.0,
+            cache_read: 0.3,
+            cache_write: 6.0,
+        }
+    }
+}
+
+/// Resolve the price for `model`: prefer the operator-provided
+/// `table` (longest matching key wins, so a specific
+/// `claude-3-5-sonnet` entry beats a generic `sonnet`), else fall back
+/// to [`builtin_prices`].
+fn resolve_prices(model: &str, table: &PriceTable) -> Prices {
+    let m = model.to_ascii_lowercase();
+    let mut best: Option<(usize, Prices)> = None;
+    for (k, v) in table {
+        let key = k.to_ascii_lowercase();
+        if key.is_empty() || !m.contains(&key) {
+            continue;
+        }
+        let better = match best {
+            Some((blen, _)) => key.len() > blen,
+            None => true,
+        };
+        if better {
+            best = Some((key.len(), *v));
+        }
+    }
+    match best {
+        Some((_, p)) => p,
+        None => builtin_prices(&m),
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct KeyCount {
+    pub key: String,
+    pub count: u64,
+}
+
+#[derive(Debug, Serialize)]
+pub struct AgentRollup {
+    pub name: String,
+    pub turns: u64,
+    pub input_tokens: u64,
+    pub output_tokens: u64,
+    pub cache_read_tokens: u64,
+    pub cache_creation_tokens: u64,
+    /// Labelled estimate — see [`Prices`].
+    pub est_cost_usd: f64,
+}
+
+#[derive(Debug, Serialize)]
+pub struct HiveStats {
+    pub window: &'static str,
+    pub from: i64,
+    pub now: i64,
+    /// Number of agents that had at least one turn in the window.
+    pub active_agents: u64,
+    pub total_turns: u64,
+    pub total_input_tokens: u64,
+    pub total_output_tokens: u64,
+    pub total_cache_read_tokens: u64,
+    pub total_cache_creation_tokens: u64,
+    /// Labelled estimate — see [`Prices`].
+    pub est_cost_usd: f64,
+    /// Per-agent rollup, busiest (most turns) first.
+    pub agents: Vec<AgentRollup>,
+    /// Turns per model across the whole swarm, busiest first.
+    pub model_mix: Vec<KeyCount>,
+    /// Most-run normalised bash-command heads ("favorite tools") across
+    /// the whole swarm, busiest first, capped to 10. Empty until the
+    /// hive-bash-mcp capture has recorded `bash_commands` rows on at
+    /// least one active agent.
+    pub bash_mix: Vec<KeyCount>,
+}
+
+#[derive(Default)]
+struct AgentAgg {
+    turns: u64,
+    input: u64,
+    output: u64,
+    cache_read: u64,
+    cache_creation: u64,
+    cost: f64,
+    models: HashMap<String, u64>,
+    /// Normalised bash-command head → invocation count, from the agent's
+    /// `bash_commands` table (written by hive-bash-mcp). Empty when that
+    /// capture hasn't run for this agent (table absent).
+    bash: HashMap<String, u64>,
+}
+
+#[allow(
+    clippy::cast_sign_loss,
+    clippy::cast_possible_truncation,
+    reason = "v.max(0) clamps to a non-negative value first, so the i64->u64 cast is exact: no sign loss, and no truncation since u64 covers all non-negative i64"
+)]
+fn u64_from_i64(v: i64) -> u64 {
+    v.max(0) as u64
+}
+
+/// Aggregate one agent's turn-stats over `[from, now]`. Errors bubble
+/// up so the caller can skip a single bad/locked db without failing
+/// the whole endpoint.
+fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<AgentAgg> {
+    let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
+    // turn_stats is rollback-journal (not WAL): a read landing while the
+    // harness is mid-INSERT would get SQLITE_BUSY and drop that active
+    // agent from the rollup. Wait out the brief write instead.
+    conn.busy_timeout(Duration::from_millis(500))?;
+    let mut stmt = conn.prepare(
+        "SELECT model, input_tokens, output_tokens,
+                cache_read_input_tokens, cache_creation_input_tokens
+         FROM turn_stats
+         WHERE started_at >= ?1",
+    )?;
+    let mut agg = AgentAgg::default();
+    let rows = stmt.query_map([from], |row| {
+        Ok((
+            row.get::<_, String>(0)?,
+            u64_from_i64(row.get::<_, i64>(1)?),
+            u64_from_i64(row.get::<_, i64>(2)?),
+            u64_from_i64(row.get::<_, i64>(3)?),
+            u64_from_i64(row.get::<_, i64>(4)?),
+        ))
+    })?;
+    for r in rows {
+        let (model, input, output, cache_read, cache_creation) = r?;
+        agg.turns += 1;
+        agg.input = agg.input.saturating_add(input);
+        agg.output = agg.output.saturating_add(output);
+        agg.cache_read = agg.cache_read.saturating_add(cache_read);
+        agg.cache_creation = agg.cache_creation.saturating_add(cache_creation);
+        let p = resolve_prices(&model, prices);
+        #[allow(
+            clippy::cast_precision_loss,
+            reason = "token counts stay well under f64's 2^53 exact-integer range, so this cost computation loses no precision"
+        )]
+        {
+            agg.cost += (input as f64 * p.input
+                + output as f64 * p.output
+                + cache_read as f64 * p.cache_read
+                + cache_creation as f64 * p.cache_write)
+                / 1_000_000.0;
+        }
+        *agg.models.entry(model).or_insert(0) += 1;
+    }
+    agg.bash = read_bash_heads(&conn, from);
+    Ok(agg)
+}
+
+/// Tally normalised bash-command heads from the agent's `bash_commands`
+/// table (`ts INTEGER, head TEXT`, written by hive-bash-mcp) over
+/// `[from, now]`. Best-effort + isolated from `read_agent`'s error path:
+/// a missing table (capture hasn't run for this agent) or any read error
+/// yields an empty map rather than propagating, so the favorite-tools
+/// rollup simply omits that agent and never fails the whole endpoint.
+/// `ts` is unix seconds, matching the `from` cutoff.
+fn read_bash_heads(conn: &Connection, from: i64) -> HashMap<String, u64> {
+    let mut out: HashMap<String, u64> = HashMap::new();
+    let Ok(mut stmt) =
+        conn.prepare("SELECT head, COUNT(*) FROM bash_commands WHERE ts >= ?1 GROUP BY head")
+    else {
+        return out;
+    };
+    let Ok(rows) = stmt.query_map([from], |row| {
+        Ok((
+            row.get::<_, String>(0)?,
+            u64_from_i64(row.get::<_, i64>(1)?),
+        ))
+    }) else {
+        return out;
+    };
+    for (head, count) in rows.flatten() {
+        *out.entry(head).or_insert(0) += count;
+    }
+    out
+}
+
+/// Build the swarm-wide rollup. Best-effort: a missing or unreadable
+/// per-agent db is skipped (logged), never fatal.
+#[must_use]
+pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
+    let now = now_unix();
+    // Fixed windows look back a constant span; `all` aggregates every
+    // recorded turn (`from == 0`). The hive rollup isn't time-bucketed, so
+    // unlike the per-agent snapshot it needs no adaptive bucket sizing.
+    let from = match window {
+        Window::All => 0,
+        _ => now - window.span_secs(),
+    };
+
+    let mut agents: Vec<AgentRollup> = Vec::new();
+    let mut model_mix: HashMap<String, u64> = HashMap::new();
+    let mut bash_mix: HashMap<String, u64> = HashMap::new();
+    let mut total_turns = 0u64;
+    let mut total_input = 0u64;
+    let mut total_output = 0u64;
+    let mut total_cache_read = 0u64;
+    let mut total_cache_creation = 0u64;
+    let mut total_cost = 0.0f64;
+    let mut active_agents = 0u64;
+
+    for name in Coordinator::kept_state_names() {
+        let path = Coordinator::agent_harness_dir(&name).join("hyperhive-turn-stats.sqlite");
+        if !path.exists() {
+            continue;
+        }
+        let agg = match read_agent(&path, from, prices) {
+            Ok(a) => a,
+            Err(e) => {
+                tracing::warn!(agent = %name, error = ?e, "hive-stats: read failed; skipping");
+                continue;
+            }
+        };
+        if agg.turns == 0 {
+            continue;
+        }
+        active_agents += 1;
+        total_turns += agg.turns;
+        total_input = total_input.saturating_add(agg.input);
+        total_output = total_output.saturating_add(agg.output);
+        total_cache_read = total_cache_read.saturating_add(agg.cache_read);
+        total_cache_creation = total_cache_creation.saturating_add(agg.cache_creation);
+        total_cost += agg.cost;
+        for (m, c) in &agg.models {
+            *model_mix.entry(m.clone()).or_insert(0) += c;
+        }
+        for (h, c) in &agg.bash {
+            *bash_mix.entry(h.clone()).or_insert(0) += c;
+        }
+        agents.push(AgentRollup {
+            name,
+            turns: agg.turns,
+            input_tokens: agg.input,
+            output_tokens: agg.output,
+            cache_read_tokens: agg.cache_read,
+            cache_creation_tokens: agg.cache_creation,
+            est_cost_usd: agg.cost,
+        });
+    }
+
+    // Busiest agents first.
+    agents.sort_by(|a, b| b.turns.cmp(&a.turns).then_with(|| a.name.cmp(&b.name)));
+
+    let mut model_mix: Vec<KeyCount> = model_mix
+        .into_iter()
+        .map(|(key, count)| KeyCount { key, count })
+        .collect();
+    model_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
+
+    // Busiest commands first, capped to a top-10 "favorite tools" list.
+    let mut bash_mix: Vec<KeyCount> = bash_mix
+        .into_iter()
+        .map(|(key, count)| KeyCount { key, count })
+        .collect();
+    bash_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
+    bash_mix.truncate(10);
+
+    HiveStats {
+        window: window.label(),
+        from,
+        now,
+        active_agents,
+        total_turns,
+        total_input_tokens: total_input,
+        total_output_tokens: total_output,
+        total_cache_read_tokens: total_cache_read,
+        total_cache_creation_tokens: total_cache_creation,
+        est_cost_usd: total_cost,
+        agents,
+        model_mix,
+        bash_mix,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// `read_bash_heads` tallies per-head counts over the window and is
+    /// isolated from a missing table (returns empty, never errors).
+    #[test]
+    fn bash_heads_tally_and_window() {
+        let conn = Connection::open_in_memory().unwrap();
+        conn.execute_batch("CREATE TABLE bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL);")
+            .unwrap();
+        let now = now_unix();
+        for (ts, head) in [
+            (now - 100, "cargo"),
+            (now - 200, "cargo"),
+            (now - 300, "git"),
+            (now - 10_000, "rg"), // outside a 1h-ish cutoff below
+        ] {
+            conn.execute(
+                "INSERT INTO bash_commands (ts, head) VALUES (?1, ?2)",
+                rusqlite::params![ts, head],
+            )
+            .unwrap();
+        }
+        let heads = read_bash_heads(&conn, now - 3600);
+        assert_eq!(heads.get("cargo").copied(), Some(2));
+        assert_eq!(heads.get("git").copied(), Some(1));
+        assert_eq!(heads.get("rg").copied(), None); // window cutoff excludes it
+    }
+
+    /// A missing `bash_commands` table degrades to an empty tally rather
+    /// than erroring — the pre-capture window on a fresh agent.
+    #[test]
+    fn bash_heads_missing_table_is_empty() {
+        let conn = Connection::open_in_memory().unwrap();
+        assert!(read_bash_heads(&conn, 0).is_empty());
+    }
+}

hive-c0re/src/stats/host_stats.rs

@@ -0,0 +1,260 @@
+//! Host-system probes and the derived **server-warning** list that the
+//! dashboard renders as a top-of-page banner.
+//!
+//! The first (and currently only) producer is a host disk-usage check:
+//! the nix store filling up is what ENOSPC-failed CI before the GC
+//! guardrails were documented. hive-c0re runs on the host (not inside a
+//! container), so it can `statvfs` the store path directly and warn the
+//! operator *before* an ENOSPC, not after.
+//!
+//! [`server_warnings`] is the public surface: it returns a flat list of
+//! [`ServerWarning`]s for `/api/state`. The dashboard renders whatever it
+//! returns, coloured by `level`, so adding a new system warning (memory
+//! pressure, a failed unit, …) is a backend-only change — no frontend
+//! edit. Keep producers cheap; this runs on every `/api/state` assembly.
+
+use std::collections::HashMap;
+
+use serde::Serialize;
+
+use crate::container_view::ContainerView;
+
+/// One server-level warning for the dashboard's top-of-page banner.
+#[derive(Debug, Clone, Serialize)]
+pub struct ServerWarning {
+    /// Stable kind id (e.g. `"disk_pressure"`) — lets the frontend dedupe
+    /// or special-case without parsing the message.
+    pub kind: &'static str,
+    /// Severity: `"warn"` (amber) or `"crit"` (red). The banner picks its
+    /// colour from this; everything else is just the message text.
+    pub level: &'static str,
+    /// Human-readable, already-formatted message shown in the banner.
+    pub message: String,
+}
+
+/// Percent-used past which the host nix store earns a disk-pressure
+/// warning; above [`DISK_CRIT_PCT`] it escalates to `crit`.
+const DISK_WARN_PCT: f64 = 85.0;
+const DISK_CRIT_PCT: f64 = 95.0;
+
+/// Collect the current server-level warnings for the dashboard banner.
+/// Each producer pushes zero or more [`ServerWarning`]s; the frontend
+/// renders whatever this returns. Cheap to call on every `/api/state`
+/// assembly (currently a single `statvfs`).
+#[must_use]
+pub fn server_warnings() -> Vec<ServerWarning> {
+    let mut out = Vec::new();
+    if let Some(d) = nix_disk_usage()
+        && d.used_pct >= DISK_WARN_PCT
+    {
+        #[allow(
+            clippy::cast_precision_loss,
+            reason = "byte counts stay well under f64's 2^53 exact-integer range, so this GiB conversion loses no precision"
+        )]
+        let free_gib = d.free_bytes as f64 / (1024.0 * 1024.0 * 1024.0);
+        out.push(ServerWarning {
+            kind: "disk_pressure",
+            level: if d.used_pct >= DISK_CRIT_PCT {
+                "crit"
+            } else {
+                "warn"
+            },
+            message: format!(
+                "host nix store {:.0}% full ({free_gib:.1} GiB free) \
+                 — garbage-collect the store before it runs out of space",
+                d.used_pct
+            ),
+        });
+    }
+    out
+}
+
+/// Agent-state warnings derived from the live container snapshot the
+/// dashboard already holds: agents that need a claude login, and agents
+/// that are crashing. Kept separate from [`server_warnings`] (host
+/// probes) because the caller owns the container list + crash counts;
+/// the dashboard concatenates both into one banner list.
+///
+/// `crash_counts` is `Coordinator::recent_crash_counts(window)` — agent →
+/// number of crashes inside that window — so a crash-looping agent shows
+/// its repeat count rather than a single point-in-time flap.
+#[must_use]
+pub fn agent_state_warnings<S: std::hash::BuildHasher>(
+    containers: &[ContainerView],
+    crash_counts: &HashMap<String, usize, S>,
+) -> Vec<ServerWarning> {
+    let mut out = Vec::new();
+
+    // `needs_login` is already running-gated in `container_view::build_all`,
+    // so a stopped container never lights this.
+    let mut pending: Vec<&str> = containers
+        .iter()
+        .filter(|c| c.needs_login)
+        .map(|c| c.name.as_str())
+        .collect();
+    if !pending.is_empty() {
+        pending.sort_unstable();
+        out.push(ServerWarning {
+            kind: "pending_logins",
+            level: "warn",
+            message: format!(
+                "{n} agent{plural} {verb} claude login: {list} \
+                 — open the agent's page in the dashboard to complete the login",
+                n = pending.len(),
+                plural = if pending.len() == 1 { "" } else { "s" },
+                verb = if pending.len() == 1 { "needs" } else { "need" },
+                list = pending.join(", "),
+            ),
+        });
+    }
+
+    if !crash_counts.is_empty() {
+        let mut crashing: Vec<(&String, usize)> =
+            crash_counts.iter().map(|(a, n)| (a, *n)).collect();
+        crashing.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0)));
+        let list = crashing
+            .iter()
+            .map(|(a, n)| format!("{a} (×{n})"))
+            .collect::<Vec<_>>()
+            .join(", ");
+        out.push(ServerWarning {
+            kind: "agents_crashing",
+            level: "crit",
+            message: format!(
+                "{n} agent{plural} crashing: {list} \
+                 — check the container journal (`hivectl logs <agent>`)",
+                n = crashing.len(),
+                plural = if crashing.len() == 1 { "" } else { "s" },
+            ),
+        });
+    }
+
+    out
+}
+
+/// Disk usage for the filesystem backing the host nix store — internal to
+/// the disk-pressure producer above.
+struct DiskUsage {
+    /// Space available to unprivileged writers, in bytes.
+    free_bytes: u64,
+    /// Percentage used, 0–100. Mirrors `df`'s use% — `used / (used +
+    /// available)` — so the threshold means what the operator sees in `df`.
+    used_pct: f64,
+}
+
+/// Probe disk usage for the filesystem containing the nix store (`/nix`),
+/// falling back to `/` when `/nix` isn't its own mount. Returns `None` if
+/// the `statvfs` syscall fails (path missing, permission, etc.).
+fn nix_disk_usage() -> Option<DiskUsage> {
+    disk_usage("/nix").or_else(|| disk_usage("/"))
+}
+
+fn disk_usage(path: &str) -> Option<DiskUsage> {
+    let c_path = std::ffi::CString::new(path).ok()?;
+    // SAFETY: `statvfs` reads only through the valid NUL-terminated
+    // `c_path` pointer and writes into the zeroed `stat` we own. We check
+    // the return code before reading any field.
+    let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
+    let rc = unsafe { libc::statvfs(c_path.as_ptr(), &raw mut stat) };
+    if rc != 0 {
+        return None;
+    }
+    // statvfs fields are `c_ulong` (== u64 on the x86_64-linux host this
+    // runs on); the arithmetic below stays in that native width.
+    let frsize = stat.f_frsize;
+    let total_blocks = stat.f_blocks;
+    let free_blocks = stat.f_bfree;
+    let avail_blocks = stat.f_bavail;
+    if total_blocks == 0 || frsize == 0 {
+        return None;
+    }
+    let free_bytes = avail_blocks.saturating_mul(frsize);
+    // df's use%: used / (used + available). `used` counts root-reserved
+    // blocks (total - bfree); `available` is the unprivileged free
+    // (bavail), so the percentage matches what `df` reports.
+    let used_blocks = total_blocks.saturating_sub(free_blocks);
+    let capacity = used_blocks.saturating_add(avail_blocks);
+    let used_pct = if capacity == 0 {
+        0.0
+    } else {
+        #[allow(
+            clippy::cast_precision_loss,
+            reason = "block counts stay well under f64's 2^53 exact-integer range, so this percentage computation loses no precision"
+        )]
+        let raw = used_blocks as f64 / capacity as f64 * 100.0;
+        raw
+    };
+    Some(DiskUsage {
+        free_bytes,
+        used_pct,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn cv(name: &str, needs_login: bool) -> ContainerView {
+        ContainerView {
+            name: name.to_owned(),
+            container: format!("h-{name}"),
+            port: 0,
+            running: true,
+            needs_update: false,
+            needs_login,
+            deployed_sha: None,
+            pending_reminders: 0,
+            parent: None,
+            active_model: None,
+        }
+    }
+
+    #[test]
+    fn no_agent_warnings_when_all_healthy() {
+        let containers = [cv("alice", false), cv("bob", false)];
+        assert!(agent_state_warnings(&containers, &HashMap::new()).is_empty());
+    }
+
+    #[test]
+    fn pending_logins_lists_sorted_agents() {
+        let containers = [cv("zoe", true), cv("amy", false), cv("bob", true)];
+        let w = agent_state_warnings(&containers, &HashMap::new());
+        assert_eq!(w.len(), 1);
+        assert_eq!(w[0].kind, "pending_logins");
+        assert_eq!(w[0].level, "warn");
+        // sorted, login-needing only, count reflected
+        assert!(
+            w[0].message
+                .starts_with("2 agents need claude login: bob, zoe")
+        );
+    }
+
+    #[test]
+    fn singular_grammar_for_one_agent() {
+        let containers = [cv("solo", true)];
+        let w = agent_state_warnings(&containers, &HashMap::new());
+        assert!(w[0].message.starts_with("1 agent needs claude login: solo"));
+    }
+
+    #[test]
+    fn crashing_warning_is_crit_and_count_ordered() {
+        let crashes = HashMap::from([("flap".to_owned(), 5), ("blip".to_owned(), 1)]);
+        let w = agent_state_warnings(&[], &crashes);
+        assert_eq!(w.len(), 1);
+        assert_eq!(w[0].kind, "agents_crashing");
+        assert_eq!(w[0].level, "crit");
+        // higher crash count first
+        assert!(w[0].message.contains("flap (×5), blip (×1)"));
+    }
+
+    #[test]
+    fn both_warnings_coexist() {
+        let containers = [cv("a", true)];
+        let crashes = HashMap::from([("b".to_owned(), 2)]);
+        let kinds: Vec<&str> = agent_state_warnings(&containers, &crashes)
+            .iter()
+            .map(|w| w.kind)
+            .collect();
+        assert_eq!(kinds, ["pending_logins", "agents_crashing"]);
+    }
+}

hive-c0re/src/stats/mod.rs

@@ -0,0 +1,8 @@
+//! Metrics aggregation for the dashboard: hive-wide turn-stats
+//! rollups, host-system probes / server warnings, and live
+//! per-container cgroup load. Each submodule is re-exported at the
+//! crate root, so `crate::hive_stats::…` etc. keep working unchanged.
+
+pub mod container_stats;
+pub mod hive_stats;
+pub mod host_stats;