//! Host-system probes that raise hive-level warnings into the push-based //! [`crate::warnings`] registry. //! //! The first producer is a host disk-usage check: the nix store filling up //! is what ENOSPC-failed CI before the GC guardrails were documented. //! hive-c0re runs on the host (not inside a container), so it can `statvfs` //! the store path directly and warn the operator *before* an ENOSPC, not //! after. [`refresh_disk_warning`] drives a held guard from a periodic task //! (see `spawn` in `main.rs`). The same probe backs the //! `hyperhive.hive.nix_store.utilization` gauge ([`disk_utilization`]), which is //! what the early-warning Grafana alerts read. //! //! The registry has no dashboard banner any more. A raised warning reaches //! the operator as the `hyperhive.hive.warning` gauge and a Grafana alert //! on it (see `docs/scheduler/observability.md`). use serde::Serialize; use utoipa::ToSchema; use crate::container_view::ContainerView; /// One hive-level warning in the [`crate::warnings`] registry. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct ServerWarning { /// Stable kind id (e.g. `"disk_pressure"`), the `kind` label of the /// `hyperhive.hive.warning` gauge. pub kind: &'static str, /// Severity: `"warn"` or `"crit"`. `crit` degrades readiness. pub level: &'static str, /// Human-readable, already-formatted message. pub message: String, } /// Percent-used past which the host nix store earns a disk-pressure /// warning; above [`DISK_CRIT_PCT`] it escalates to `crit`. const DISK_WARN_PCT: f64 = 85.0; const DISK_CRIT_PCT: f64 = 95.0; /// Pure disk-pressure decision: given usage, return the warning /// `(level, message)` or `None` when below the warn threshold. Split from /// the probe + guard plumbing so the threshold + formatting is unit-tested /// without a real `statvfs`. fn disk_pressure_warning(used_pct: f64, free_bytes: u64) -> Option<(&'static str, String)> { if used_pct < DISK_WARN_PCT { return None; } #[allow( clippy::cast_precision_loss, reason = "byte counts stay well under f64's 2^53 exact-integer range, so this GiB conversion loses no precision" )] let free_gib = free_bytes as f64 / (1024.0 * 1024.0 * 1024.0); let level = if used_pct >= DISK_CRIT_PCT { "crit" } else { "warn" }; let message = format!( "host nix store {used_pct:.0}% full ({free_gib:.1} GiB free) \ — garbage-collect the store before it runs out of space" ); Some((level, message)) } /// One tick of the disk-pressure watch: probe the store and reconcile the /// held warning `guard` — raise or refresh it while usage is over /// threshold, drop it (clearing the warning) once back under. Driven by the /// periodic disk-watch task spawned in `main.rs`. pub fn refresh_disk_warning(guard: &mut Option) { let decision = nix_disk_usage().and_then(|d| disk_pressure_warning(d.used_pct, d.free_bytes)); match decision { Some((level, message)) => match guard { Some(g) => g.update(level, message), None => { *guard = Some(crate::warnings::set_warning( "disk_pressure", level, message, )); } }, // Back under threshold (or probe failed) — drop the guard to clear. None => *guard = None, } } /// Fraction (0–1) of the filesystem holding the nix store that is in use, /// the value behind the `hyperhive.hive.nix_store.utilization` gauge. `None` when /// the probe fails, so a failing probe reports no sample rather than zero. #[must_use] pub fn disk_utilization() -> Option { nix_disk_usage().map(|d| d.used_pct / 100.0) } /// The held `pending_logins` warning; see [`refresh_pending_logins`]. static PENDING_LOGINS: std::sync::Mutex> = std::sync::Mutex::new(None); /// Reconcile the `pending_logins` warning with the current container /// snapshot: raise or refresh it while any agent needs a claude login, /// drop it once none does. Driven by the container rescan in the /// coordinator, which already builds the snapshot every poll. pub fn refresh_pending_logins(containers: &[ContainerView]) { let warning = agent_state_warnings(containers).into_iter().next(); let mut guard = PENDING_LOGINS .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); match (warning, guard.as_ref()) { (Some(w), Some(g)) => g.update(w.level, w.message), (Some(w), None) => { *guard = Some(crate::warnings::set_warning(w.kind, w.level, w.message)); } (None, _) => *guard = None, } } /// Agent-state warnings derived from the live container snapshot: agents /// that need a claude login. Takes the container list from its caller /// because that is where the snapshot lives. /// /// Container crashes are not a warning here: the crash-watcher logs /// `container crash detected` and a swarm-level grafana alert /// (`hh-container-crashed` in `nix/host-modules/swarm-grafana.nix`) matches /// that line, with no agent in the loop. #[must_use] pub fn agent_state_warnings(containers: &[ContainerView]) -> Vec { let mut out = Vec::new(); // `needs_login` is already running-gated in `container_view::build_all`, // so a stopped container never lights this. let mut pending: Vec<&str> = containers .iter() .filter(|c| c.needs_login) .map(|c| c.name.as_str()) .collect(); if !pending.is_empty() { pending.sort_unstable(); out.push(ServerWarning { kind: "pending_logins", level: "warn", message: format!( "{n} agent{plural} {verb} claude login: {list} \ — open the agent's page in the dashboard to complete the login", n = pending.len(), plural = if pending.len() == 1 { "" } else { "s" }, verb = if pending.len() == 1 { "needs" } else { "need" }, list = pending.join(", "), ), }); } out } /// Disk usage for the filesystem backing the host nix store — internal to /// the disk-pressure producer above. struct DiskUsage { /// Space available to unprivileged writers, in bytes. free_bytes: u64, /// Percentage used, 0–100. Mirrors `df`'s use% — `used / (used + /// available)` — so the threshold means what the operator sees in `df`. used_pct: f64, } /// Probe disk usage for the filesystem containing the nix store (`/nix`), /// falling back to `/` when `/nix` isn't its own mount. Returns `None` if /// the `statvfs` syscall fails (path missing, permission, etc.). fn nix_disk_usage() -> Option { disk_usage("/nix").or_else(|| disk_usage("/")) } fn disk_usage(path: &str) -> Option { let c_path = std::ffi::CString::new(path).ok()?; // SAFETY: `statvfs` reads only through the valid NUL-terminated // `c_path` pointer and writes into the zeroed `stat` we own. We check // the return code before reading any field. let mut stat: libc::statvfs = unsafe { std::mem::zeroed() }; let rc = unsafe { libc::statvfs(c_path.as_ptr(), &raw mut stat) }; if rc != 0 { return None; } // statvfs fields are `c_ulong` (== u64 on the x86_64-linux host this // runs on); the arithmetic below stays in that native width. let frsize = stat.f_frsize; let total_blocks = stat.f_blocks; let free_blocks = stat.f_bfree; let avail_blocks = stat.f_bavail; if total_blocks == 0 || frsize == 0 { return None; } let free_bytes = avail_blocks.saturating_mul(frsize); // df's use%: used / (used + available). `used` counts root-reserved // blocks (total - bfree); `available` is the unprivileged free // (bavail), so the percentage matches what `df` reports. let used_blocks = total_blocks.saturating_sub(free_blocks); let capacity = used_blocks.saturating_add(avail_blocks); let used_pct = if capacity == 0 { 0.0 } else { #[allow( clippy::cast_precision_loss, reason = "block counts stay well under f64's 2^53 exact-integer range, so this percentage computation loses no precision" )] let raw = used_blocks as f64 / capacity as f64 * 100.0; raw }; Some(DiskUsage { free_bytes, used_pct, }) } #[cfg(test)] mod tests { use super::*; #[test] fn disk_below_threshold_is_none() { assert!(disk_pressure_warning(84.9, 100 << 30).is_none()); } #[test] fn disk_warn_band_is_amber() { let (level, msg) = disk_pressure_warning(90.0, 20 << 30).expect("warn"); assert_eq!(level, "warn"); assert!(msg.contains("90% full")); assert!(msg.contains("20.0 GiB free")); } #[test] fn disk_crit_band_is_red() { let (level, _) = disk_pressure_warning(95.0, 1 << 30).expect("crit"); assert_eq!(level, "crit"); } fn cv(name: &str, needs_login: bool) -> ContainerView { ContainerView { name: name.to_owned(), container: format!("h-{name}"), port: 0, running: true, failed: false, needs_update: false, needs_login, deployed_sha: None, active_model: None, status_text: None, status_set_at: None, paused: false, } } #[test] fn no_agent_warnings_when_all_healthy() { let containers = [cv("alice", false), cv("bob", false)]; assert!(agent_state_warnings(&containers).is_empty()); } #[test] fn pending_logins_lists_sorted_agents() { let containers = [cv("zoe", true), cv("amy", false), cv("bob", true)]; let w = agent_state_warnings(&containers); assert_eq!(w.len(), 1); assert_eq!(w[0].kind, "pending_logins"); assert_eq!(w[0].level, "warn"); // sorted, login-needing only, count reflected assert!( w[0].message .starts_with("2 agents need claude login: bob, zoe") ); } #[test] fn singular_grammar_for_one_agent() { let containers = [cv("solo", true)]; let w = agent_state_warnings(&containers); assert!(w[0].message.starts_with("1 agent needs claude login: solo")); } /// The registry entry tracks the snapshot it is refreshed from: raised /// when an agent needs a login, rewritten when the set changes, and gone /// when none does. This is the only test that touches `pending_logins`, /// so reading the process-global registry by that kind stays sound under /// the parallel test runner. #[test] fn pending_logins_warning_follows_the_container_snapshot() { let current = || { crate::warnings::snapshot() .into_iter() .find(|w| w.kind == "pending_logins") }; assert!( current().is_none(), "nothing raised before the first refresh" ); refresh_pending_logins(&[cv("amy", true)]); assert!( current().is_some_and(|w| w.message.starts_with("1 agent needs claude login: amy")), "an agent needing a login raises the warning" ); refresh_pending_logins(&[cv("amy", true), cv("bob", true)]); assert!( current().is_some_and(|w| w.message.starts_with("2 agents need claude login")), "a second agent rewrites the same entry" ); refresh_pending_logins(&[cv("amy", false), cv("bob", false)]); assert!(current().is_none(), "no agent needing a login clears it"); } /// The gauge value is a fraction, so a percent leaking through would put /// every alert threshold 100x off. #[test] fn disk_utilization_is_a_fraction() { if let Some(used) = disk_utilization() { assert!((0.0..=1.0).contains(&used), "got {used}"); } } }