//! Host-system probes and the derived **server-warning** list that the //! dashboard renders as a top-of-page banner. //! //! The first (and currently only) producer is a host disk-usage check: //! the nix store filling up is what ENOSPC-failed CI before the GC //! guardrails were documented. hive-c0re runs on the host (not inside a //! container), so it can `statvfs` the store path directly and warn the //! operator *before* an ENOSPC, not after. //! //! [`server_warnings`] is the public surface: it returns a flat list of //! [`ServerWarning`]s for `/api/state`. The dashboard renders whatever it //! returns, coloured by `level`, so adding a new system warning (memory //! pressure, a failed unit, …) is a backend-only change — no frontend //! edit. Keep producers cheap; this runs on every `/api/state` assembly. use std::collections::HashMap; use serde::Serialize; use crate::container_view::ContainerView; /// One server-level warning for the dashboard's top-of-page banner. #[derive(Debug, Clone, Serialize)] pub struct ServerWarning { /// Stable kind id (e.g. `"disk_pressure"`) — lets the frontend dedupe /// or special-case without parsing the message. pub kind: &'static str, /// Severity: `"warn"` (amber) or `"crit"` (red). The banner picks its /// colour from this; everything else is just the message text. pub level: &'static str, /// Human-readable, already-formatted message shown in the banner. pub message: String, } /// Percent-used past which the host nix store earns a disk-pressure /// warning; above [`DISK_CRIT_PCT`] it escalates to `crit`. const DISK_WARN_PCT: f64 = 85.0; const DISK_CRIT_PCT: f64 = 95.0; /// Collect the current server-level warnings for the dashboard banner. /// Each producer pushes zero or more [`ServerWarning`]s; the frontend /// renders whatever this returns. Cheap to call on every `/api/state` /// assembly (currently a single `statvfs`). #[must_use] pub fn server_warnings() -> Vec { let mut out = Vec::new(); if let Some(d) = nix_disk_usage() && d.used_pct >= DISK_WARN_PCT { #[allow( clippy::cast_precision_loss, reason = "byte counts stay well under f64's 2^53 exact-integer range, so this GiB conversion loses no precision" )] let free_gib = d.free_bytes as f64 / (1024.0 * 1024.0 * 1024.0); out.push(ServerWarning { kind: "disk_pressure", level: if d.used_pct >= DISK_CRIT_PCT { "crit" } else { "warn" }, message: format!( "host nix store {:.0}% full ({free_gib:.1} GiB free) \ — garbage-collect the store before it runs out of space", d.used_pct ), }); } out } /// Agent-state warnings derived from the live container snapshot the /// dashboard already holds: agents that need a claude login, and agents /// that are crashing. Kept separate from [`server_warnings`] (host /// probes) because the caller owns the container list + crash counts; /// the dashboard concatenates both into one banner list. /// /// `crash_counts` is `Coordinator::recent_crash_counts(window)` — agent → /// number of crashes inside that window — so a crash-looping agent shows /// its repeat count rather than a single point-in-time flap. #[must_use] pub fn agent_state_warnings( containers: &[ContainerView], crash_counts: &HashMap, ) -> Vec { let mut out = Vec::new(); // `needs_login` is already running-gated in `container_view::build_all`, // so a stopped container never lights this. let mut pending: Vec<&str> = containers .iter() .filter(|c| c.needs_login) .map(|c| c.name.as_str()) .collect(); if !pending.is_empty() { pending.sort_unstable(); out.push(ServerWarning { kind: "pending_logins", level: "warn", message: format!( "{n} agent{plural} {verb} claude login: {list} \ — run `hivectl login ` to authenticate", n = pending.len(), plural = if pending.len() == 1 { "" } else { "s" }, verb = if pending.len() == 1 { "needs" } else { "need" }, list = pending.join(", "), ), }); } if !crash_counts.is_empty() { let mut crashing: Vec<(&String, usize)> = crash_counts.iter().map(|(a, n)| (a, *n)).collect(); crashing.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0))); let list = crashing .iter() .map(|(a, n)| format!("{a} (×{n})")) .collect::>() .join(", "); out.push(ServerWarning { kind: "agents_crashing", level: "crit", message: format!( "{n} agent{plural} crashing: {list} \ — check the container journal (`hivectl logs `)", n = crashing.len(), plural = if crashing.len() == 1 { "" } else { "s" }, ), }); } out } /// Disk usage for the filesystem backing the host nix store — internal to /// the disk-pressure producer above. struct DiskUsage { /// Space available to unprivileged writers, in bytes. free_bytes: u64, /// Percentage used, 0–100. Mirrors `df`'s use% — `used / (used + /// available)` — so the threshold means what the operator sees in `df`. used_pct: f64, } /// Probe disk usage for the filesystem containing the nix store (`/nix`), /// falling back to `/` when `/nix` isn't its own mount. Returns `None` if /// the `statvfs` syscall fails (path missing, permission, etc.). fn nix_disk_usage() -> Option { disk_usage("/nix").or_else(|| disk_usage("/")) } fn disk_usage(path: &str) -> Option { let c_path = std::ffi::CString::new(path).ok()?; // SAFETY: `statvfs` reads only through the valid NUL-terminated // `c_path` pointer and writes into the zeroed `stat` we own. We check // the return code before reading any field. let mut stat: libc::statvfs = unsafe { std::mem::zeroed() }; let rc = unsafe { libc::statvfs(c_path.as_ptr(), &raw mut stat) }; if rc != 0 { return None; } // statvfs fields are `c_ulong` (== u64 on the x86_64-linux host this // runs on); the arithmetic below stays in that native width. let frsize = stat.f_frsize; let total_blocks = stat.f_blocks; let free_blocks = stat.f_bfree; let avail_blocks = stat.f_bavail; if total_blocks == 0 || frsize == 0 { return None; } let free_bytes = avail_blocks.saturating_mul(frsize); // df's use%: used / (used + available). `used` counts root-reserved // blocks (total - bfree); `available` is the unprivileged free // (bavail), so the percentage matches what `df` reports. let used_blocks = total_blocks.saturating_sub(free_blocks); let capacity = used_blocks.saturating_add(avail_blocks); let used_pct = if capacity == 0 { 0.0 } else { #[allow( clippy::cast_precision_loss, reason = "block counts stay well under f64's 2^53 exact-integer range, so this percentage computation loses no precision" )] let raw = used_blocks as f64 / capacity as f64 * 100.0; raw }; Some(DiskUsage { free_bytes, used_pct, }) } #[cfg(test)] mod tests { use super::*; fn cv(name: &str, needs_login: bool) -> ContainerView { ContainerView { name: name.to_owned(), container: format!("h-{name}"), port: 0, running: true, needs_update: false, needs_login, deployed_sha: None, pending_reminders: 0, parent: None, } } #[test] fn no_agent_warnings_when_all_healthy() { let containers = [cv("alice", false), cv("bob", false)]; assert!(agent_state_warnings(&containers, &HashMap::new()).is_empty()); } #[test] fn pending_logins_lists_sorted_agents() { let containers = [cv("zoe", true), cv("amy", false), cv("bob", true)]; let w = agent_state_warnings(&containers, &HashMap::new()); assert_eq!(w.len(), 1); assert_eq!(w[0].kind, "pending_logins"); assert_eq!(w[0].level, "warn"); // sorted, login-needing only, count reflected assert!( w[0].message .starts_with("2 agents need claude login: bob, zoe") ); } #[test] fn singular_grammar_for_one_agent() { let containers = [cv("solo", true)]; let w = agent_state_warnings(&containers, &HashMap::new()); assert!(w[0].message.starts_with("1 agent needs claude login: solo")); } #[test] fn crashing_warning_is_crit_and_count_ordered() { let crashes = HashMap::from([("flap".to_owned(), 5), ("blip".to_owned(), 1)]); let w = agent_state_warnings(&[], &crashes); assert_eq!(w.len(), 1); assert_eq!(w[0].kind, "agents_crashing"); assert_eq!(w[0].level, "crit"); // higher crash count first assert!(w[0].message.contains("flap (×5), blip (×1)")); } #[test] fn both_warnings_coexist() { let containers = [cv("a", true)]; let crashes = HashMap::from([("b".to_owned(), 2)]); let kinds: Vec<&str> = agent_state_warnings(&containers, &crashes) .iter() .map(|w| w.kind) .collect(); assert_eq!(kinds, ["pending_logins", "agents_crashing"]); } }