hyperhive/hive-c0re/src/stats/host_stats.rs

//! Host-system probes that raise hive-level warnings into the push-based
//! [`crate::warnings`] registry.
//!
//! The first producer is a host disk-usage check: the nix store filling up
//! is what ENOSPC-failed CI before the GC guardrails were documented.
//! hive-c0re runs on the host (not inside a container), so it can `statvfs`
//! the store path directly and warn the operator *before* an ENOSPC, not
//! after. [`refresh_disk_warning`] drives a held guard from a periodic task
//! (see `spawn` in `main.rs`). The same probe backs the
//! `hyperhive.hive.nix_store.utilization` gauge ([`disk_utilization`]), which is
//! what the early-warning Grafana alerts read.
//!
//! The registry has no dashboard banner any more. A raised warning reaches
//! the operator as the `hyperhive.hive.warning` gauge and a Grafana alert
//! on it (see `docs/scheduler/observability.md`).

use serde::Serialize;
use utoipa::ToSchema;

use crate::container_view::ContainerView;

/// One hive-level warning in the [`crate::warnings`] registry.
#[derive(Debug, Clone, Serialize, ToSchema)]
pub struct ServerWarning {
    /// Stable kind id (e.g. `"disk_pressure"`), the `kind` label of the
    /// `hyperhive.hive.warning` gauge.
    pub kind: &'static str,
    /// Severity: `"warn"` or `"crit"`. `crit` degrades readiness.
    pub level: &'static str,
    /// Human-readable, already-formatted message.
    pub message: String,
}

/// Percent-used past which the host nix store earns a disk-pressure
/// warning; above [`DISK_CRIT_PCT`] it escalates to `crit`.
const DISK_WARN_PCT: f64 = 85.0;
const DISK_CRIT_PCT: f64 = 95.0;

/// Pure disk-pressure decision: given usage, return the warning
/// `(level, message)` or `None` when below the warn threshold. Split from
/// the probe + guard plumbing so the threshold + formatting is unit-tested
/// without a real `statvfs`.
fn disk_pressure_warning(used_pct: f64, free_bytes: u64) -> Option<(&'static str, String)> {
    if used_pct < DISK_WARN_PCT {
        return None;
    }
    #[allow(
        clippy::cast_precision_loss,
        reason = "byte counts stay well under f64's 2^53 exact-integer range, so this GiB conversion loses no precision"
    )]
    let free_gib = free_bytes as f64 / (1024.0 * 1024.0 * 1024.0);
    let level = if used_pct >= DISK_CRIT_PCT {
        "crit"
    } else {
        "warn"
    };
    let message = format!(
        "host nix store {used_pct:.0}% full ({free_gib:.1} GiB free) \
         — garbage-collect the store before it runs out of space"
    );
    Some((level, message))
}

/// One tick of the disk-pressure watch: probe the store and reconcile the
/// held warning `guard` — raise or refresh it while usage is over
/// threshold, drop it (clearing the warning) once back under. Driven by the
/// periodic disk-watch task spawned in `main.rs`.
pub fn refresh_disk_warning(guard: &mut Option<crate::warnings::WarningGuard>) {
    let decision = nix_disk_usage().and_then(|d| disk_pressure_warning(d.used_pct, d.free_bytes));
    match decision {
        Some((level, message)) => match guard {
            Some(g) => g.update(level, message),
            None => {
                *guard = Some(crate::warnings::set_warning(
                    "disk_pressure",
                    level,
                    message,
                ));
            }
        },
        // Back under threshold (or probe failed) — drop the guard to clear.
        None => *guard = None,
    }
}

/// Fraction (0–1) of the filesystem holding the nix store that is in use,
/// the value behind the `hyperhive.hive.nix_store.utilization` gauge. `None` when
/// the probe fails, so a failing probe reports no sample rather than zero.
#[must_use]
pub fn disk_utilization() -> Option<f64> {
    nix_disk_usage().map(|d| d.used_pct / 100.0)
}

/// The held `pending_logins` warning; see [`refresh_pending_logins`].
static PENDING_LOGINS: std::sync::Mutex<Option<crate::warnings::WarningGuard>> =
    std::sync::Mutex::new(None);

/// Reconcile the `pending_logins` warning with the current container
/// snapshot: raise or refresh it while any agent needs a claude login,
/// drop it once none does. Driven by the container rescan in the
/// coordinator, which already builds the snapshot every poll.
pub fn refresh_pending_logins(containers: &[ContainerView]) {
    let warning = agent_state_warnings(containers).into_iter().next();
    let mut guard = PENDING_LOGINS
        .lock()
        .unwrap_or_else(std::sync::PoisonError::into_inner);
    match (warning, guard.as_ref()) {
        (Some(w), Some(g)) => g.update(w.level, w.message),
        (Some(w), None) => {
            *guard = Some(crate::warnings::set_warning(w.kind, w.level, w.message));
        }
        (None, _) => *guard = None,
    }
}

/// Agent-state warnings derived from the live container snapshot: agents
/// that need a claude login. Takes the container list from its caller
/// because that is where the snapshot lives.
///
/// Container crashes are not a warning here: the crash-watcher logs
/// `container crash detected` and a swarm-level grafana alert
/// (`hh-container-crashed` in `nix/host-modules/swarm-grafana.nix`) matches
/// that line, with no agent in the loop.
#[must_use]
pub fn agent_state_warnings(containers: &[ContainerView]) -> Vec<ServerWarning> {
    let mut out = Vec::new();

    // `needs_login` is already running-gated in `container_view::build_all`,
    // so a stopped container never lights this.
    let mut pending: Vec<&str> = containers
        .iter()
        .filter(|c| c.needs_login)
        .map(|c| c.name.as_str())
        .collect();
    if !pending.is_empty() {
        pending.sort_unstable();
        out.push(ServerWarning {
            kind: "pending_logins",
            level: "warn",
            message: format!(
                "{n} agent{plural} {verb} claude login: {list} \
                 — open the agent's page in the dashboard to complete the login",
                n = pending.len(),
                plural = if pending.len() == 1 { "" } else { "s" },
                verb = if pending.len() == 1 { "needs" } else { "need" },
                list = pending.join(", "),
            ),
        });
    }

    out
}

/// Disk usage for the filesystem backing the host nix store — internal to
/// the disk-pressure producer above.
struct DiskUsage {
    /// Space available to unprivileged writers, in bytes.
    free_bytes: u64,
    /// Percentage used, 0–100. Mirrors `df`'s use% — `used / (used +
    /// available)` — so the threshold means what the operator sees in `df`.
    used_pct: f64,
}

/// Probe disk usage for the filesystem containing the nix store (`/nix`),
/// falling back to `/` when `/nix` isn't its own mount. Returns `None` if
/// the `statvfs` syscall fails (path missing, permission, etc.).
fn nix_disk_usage() -> Option<DiskUsage> {
    disk_usage("/nix").or_else(|| disk_usage("/"))
}

fn disk_usage(path: &str) -> Option<DiskUsage> {
    let c_path = std::ffi::CString::new(path).ok()?;
    // SAFETY: `statvfs` reads only through the valid NUL-terminated
    // `c_path` pointer and writes into the zeroed `stat` we own. We check
    // the return code before reading any field.
    let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
    let rc = unsafe { libc::statvfs(c_path.as_ptr(), &raw mut stat) };
    if rc != 0 {
        return None;
    }
    // statvfs fields are `c_ulong` (== u64 on the x86_64-linux host this
    // runs on); the arithmetic below stays in that native width.
    let frsize = stat.f_frsize;
    let total_blocks = stat.f_blocks;
    let free_blocks = stat.f_bfree;
    let avail_blocks = stat.f_bavail;
    if total_blocks == 0 || frsize == 0 {
        return None;
    }
    let free_bytes = avail_blocks.saturating_mul(frsize);
    // df's use%: used / (used + available). `used` counts root-reserved
    // blocks (total - bfree); `available` is the unprivileged free
    // (bavail), so the percentage matches what `df` reports.
    let used_blocks = total_blocks.saturating_sub(free_blocks);
    let capacity = used_blocks.saturating_add(avail_blocks);
    let used_pct = if capacity == 0 {
        0.0
    } else {
        #[allow(
            clippy::cast_precision_loss,
            reason = "block counts stay well under f64's 2^53 exact-integer range, so this percentage computation loses no precision"
        )]
        let raw = used_blocks as f64 / capacity as f64 * 100.0;
        raw
    };
    Some(DiskUsage {
        free_bytes,
        used_pct,
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn disk_below_threshold_is_none() {
        assert!(disk_pressure_warning(84.9, 100 << 30).is_none());
    }

    #[test]
    fn disk_warn_band_is_amber() {
        let (level, msg) = disk_pressure_warning(90.0, 20 << 30).expect("warn");
        assert_eq!(level, "warn");
        assert!(msg.contains("90% full"));
        assert!(msg.contains("20.0 GiB free"));
    }

    #[test]
    fn disk_crit_band_is_red() {
        let (level, _) = disk_pressure_warning(95.0, 1 << 30).expect("crit");
        assert_eq!(level, "crit");
    }

    fn cv(name: &str, needs_login: bool) -> ContainerView {
        ContainerView {
            name: name.to_owned(),
            container: format!("h-{name}"),
            port: 0,
            running: true,
            failed: false,
            needs_update: false,
            needs_login,
            deployed_sha: None,
            active_model: None,
            status_text: None,
            status_set_at: None,
            paused: false,
        }
    }

    #[test]
    fn no_agent_warnings_when_all_healthy() {
        let containers = [cv("alice", false), cv("bob", false)];
        assert!(agent_state_warnings(&containers).is_empty());
    }

    #[test]
    fn pending_logins_lists_sorted_agents() {
        let containers = [cv("zoe", true), cv("amy", false), cv("bob", true)];
        let w = agent_state_warnings(&containers);
        assert_eq!(w.len(), 1);
        assert_eq!(w[0].kind, "pending_logins");
        assert_eq!(w[0].level, "warn");
        // sorted, login-needing only, count reflected
        assert!(
            w[0].message
                .starts_with("2 agents need claude login: bob, zoe")
        );
    }

    #[test]
    fn singular_grammar_for_one_agent() {
        let containers = [cv("solo", true)];
        let w = agent_state_warnings(&containers);
        assert!(w[0].message.starts_with("1 agent needs claude login: solo"));
    }

    /// The registry entry tracks the snapshot it is refreshed from: raised
    /// when an agent needs a login, rewritten when the set changes, and gone
    /// when none does. This is the only test that touches `pending_logins`,
    /// so reading the process-global registry by that kind stays sound under
    /// the parallel test runner.
    #[test]
    fn pending_logins_warning_follows_the_container_snapshot() {
        let current = || {
            crate::warnings::snapshot()
                .into_iter()
                .find(|w| w.kind == "pending_logins")
        };
        assert!(
            current().is_none(),
            "nothing raised before the first refresh"
        );

        refresh_pending_logins(&[cv("amy", true)]);
        assert!(
            current().is_some_and(|w| w.message.starts_with("1 agent needs claude login: amy")),
            "an agent needing a login raises the warning"
        );

        refresh_pending_logins(&[cv("amy", true), cv("bob", true)]);
        assert!(
            current().is_some_and(|w| w.message.starts_with("2 agents need claude login")),
            "a second agent rewrites the same entry"
        );

        refresh_pending_logins(&[cv("amy", false), cv("bob", false)]);
        assert!(current().is_none(), "no agent needing a login clears it");
    }

    /// The gauge value is a fraction, so a percent leaking through would put
    /// every alert threshold 100x off.
    #[test]
    fn disk_utilization_is_a_fraction() {
        if let Some(used) = disk_utilization() {
            assert!((0.0..=1.0).contains(&used), "got {used}");
        }
    }
}