diff --git a/Cargo.lock b/Cargo.lock index 309bca59..df808e26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1530,6 +1530,7 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", + "libc", "reqwest 0.13.1", "rmcp", "rusqlite", diff --git a/docs/turn-loop.md b/docs/turn-loop.md index 2c2321c6..a8e7ded7 100644 --- a/docs/turn-loop.md +++ b/docs/turn-loop.md @@ -101,6 +101,15 @@ installs claude plugins, spawns `forge_notify::run` + `web_ui::serve`, and either drops into `serve_loop` directly (`Online`) or parks on the login flow first (`NeedsLogin`). +`spawn_todo_socket` opens the todos store and, alongside +`todo_server::run` (the socket the out-of-process producers dial), +spawns `disk_watch::run` — an *in-process* todo producer, so it shares +the store + wake `Notify` directly rather than dialling its own socket. +It raises a keyed `disk` todo when the filesystem backing the agent's +state gets tight, naming the agent's own biggest directories; the +summary is bucketed and carries no raw byte counts, so an unchanged +situation re-upserts as `changed == false` and never re-wakes. + Plugin install failures are not fatal: each entry comes back as a human-readable failure string that gets routed via `Surface::send_to_parent` to the agent's topology parent (the diff --git a/hive-agent/Cargo.toml b/hive-agent/Cargo.toml index caeadbf5..db9222f8 100644 --- a/hive-agent/Cargo.toml +++ b/hive-agent/Cargo.toml @@ -23,6 +23,7 @@ hive-claude.workspace = true hive-agent-sock.workspace = true hive-core-agent-sock.workspace = true hive-sh4re.workspace = true +libc.workspace = true rmcp.workspace = true rusqlite.workspace = true schemars.workspace = true diff --git a/hive-agent/src/disk_watch.rs b/hive-agent/src/disk_watch.rs new file mode 100644 index 00000000..7866fcd0 --- /dev/null +++ b/hive-agent/src/disk_watch.rs @@ -0,0 +1,369 @@ +//! Disk-pressure watch: raises a loose-ends-v2 todo when the filesystem +//! backing this agent's state gets tight, pointing at the agent's own +//! biggest directories so the todo says where the bytes actually went. +//! +//! Harness-local by design. hive-c0re cannot push a todo — the todo store +//! and its wake `Notify` live inside the container, reachable only over +//! `HIVE_AGENT_SOCKET` — and the operator explicitly ruled out core +//! wiring for this. Running in-process also means we skip the socket +//! entirely and call [`crate::todos::Todos`] directly. +//! +//! Anti-nag: the todo is keyed (`disk`/[`TODO_KEY`]), and the summary is +//! deliberately *stable* — the percentage is bucketed and no raw byte +//! counts appear in it. An unchanged summary makes `upsert` report +//! `changed == false`, so a steadily-full disk sits quiet in the +//! loose-ends list instead of waking the agent every tick. Only crossing +//! into a new bucket (or a change in which directories are big enough to +//! list) speaks up again. Dropping back under the threshold clears the +//! todo. + +use std::ffi::CString; +use std::fmt::Write as _; +use std::os::unix::ffi::OsStrExt as _; +use std::os::unix::fs::MetadataExt as _; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use tokio::sync::Notify; + +use crate::todos::Todos; + +/// How often the filesystem is probed. The `statvfs` is free; the +/// directory scan behind it only runs once we're already over threshold. +const CHECK_INTERVAL: Duration = Duration::from_mins(15); +/// Percent-used past which the agent gets a todo. Matches the operator's +/// ask ("more than 80%"). +const WARN_PCT: u64 = 80; +/// Percentages are reported in buckets this wide. This is the anti-nag +/// knob: a disk drifting 89% → 91% keeps the same summary and stays +/// silent; a jump to 95% earns a fresh wake. +const BUCKET_PCT: u64 = 5; +/// Producing subsystem marker for the todo row. +const SUBSYSTEM: &str = "disk"; +/// Fixed dedup key — there is only ever one disk todo per agent. +const TODO_KEY: &str = "usage"; +/// Only directories at least this large are worth naming. Coarse on +/// purpose: the listed set shouldn't churn while a build runs. +const BIG_DIR_BYTES: u64 = 5 << 30; +/// At most this many directories are named in the todo. +const MAX_LISTED: usize = 5; +/// Directories deeper than this (relative to a scan root) are summed into +/// their parent but never listed by name — a todo pointing at +/// `…/target/debug/build/foo-1a2b/out` helps nobody. +const REPORT_DEPTH: usize = 4; +/// Hard recursion cap, independent of [`REPORT_DEPTH`]: bounds stack +/// depth on a pathological tree. +const MAX_DEPTH: usize = 24; +/// Directory entries the scan will look at before giving up. Bounds the +/// worst case on a huge tree; a truncated scan just names fewer dirs. +const SCAN_BUDGET: u64 = 400_000; + +/// Background loop: periodically reconcile the disk todo. Detached task — +/// runs for the harness's lifetime; errors are logged, never fatal. +pub async fn run(todos: Arc, wake: Arc) { + loop { + // Sleep first: boot is the busiest the container gets, and a disk + // that's been full for a week can wait one interval. + tokio::time::sleep(CHECK_INTERVAL).await; + // The scan walks the agent's tree — blocking work, off the runtime. + let summary = match tokio::task::spawn_blocking(probe).await { + Ok(s) => s, + Err(e) => { + tracing::warn!(error = %e, "disk watch probe panicked"); + continue; + } + }; + reconcile(&todos, &wake, summary.as_deref()); + } +} + +/// Apply one probe result to the todo store: raise/refresh while over +/// threshold, clear once back under. Split from [`run`] so the store +/// interaction is testable without a timer. +fn reconcile(todos: &Todos, wake: &Notify, summary: Option<&str>) { + match summary { + Some(summary) => match todos.upsert(SUBSYSTEM, Some(TODO_KEY), summary, None) { + // Only a genuine change wakes the agent — an identical summary + // means the situation is unchanged and already in its list. + Ok((_, true)) => wake.notify_one(), + Ok((_, false)) => {} + Err(e) => tracing::warn!(error = ?e, "disk todo upsert failed"), + }, + None => { + if let Err(e) = todos.clear(SUBSYSTEM, Some(TODO_KEY)) { + tracing::warn!(error = ?e, "disk todo clear failed"); + } + } + } +} + +/// One probe: measure the filesystem backing the state dir and, when it's +/// over threshold, scan the agent's own tree for the big directories worth +/// naming. `None` means "nothing to report" — under threshold, or the +/// syscall failed. +fn probe() -> Option { + let state = crate::paths::state_dir(); + let pct = used_pct(&state)?; + if pct < WARN_PCT { + return None; + } + summary_for(pct, &big_dirs()) +} + +/// Pure rendering half of [`probe`]: the todo text for a given usage +/// percentage + set of oversized directories. Separated so the threshold +/// and — more importantly — the *stability* of the summary are unit-tested +/// without a real filesystem. +fn summary_for(pct: u64, dirs: &[PathBuf]) -> Option { + if pct < WARN_PCT { + return None; + } + let bucket = pct / BUCKET_PCT * BUCKET_PCT; + let mut out = format!( + "disk over {bucket}% full on the filesystem holding your state — free up space if you safely can.\n\ + Only delete things that are actually big (tens of GiB); a few MB of notes won't move the needle.\n\ + First candidates: regenerable build output in your workspace (`target/`, `node_modules/`, `dist/`).\n\ + Never delete anything still needed. If nothing is safe to drop, tell the operator you need more space \ + rather than forcing it — be cautious with deletions in general." + ); + if !dirs.is_empty() { + out.push_str( + "\nBiggest directories under your own tree (`du -sh` them before removing anything):", + ); + for dir in dirs { + // Infallible: writing into a String. + let _ = write!(out, "\n - {}", dir.display()); + } + } + Some(out) +} + +/// Percent of the filesystem containing `path` that is in use, as `df` +/// reports it (`used / (used + available)`, so root-reserved blocks count +/// as used). `None` if the `statvfs` syscall fails. +fn used_pct(path: &Path) -> Option { + let c_path = CString::new(path.as_os_str().as_bytes()).ok()?; + // SAFETY: `statvfs` reads only through the valid NUL-terminated + // `c_path` pointer and writes into the zeroed `stat` we own. The + // return code is checked before any field is read. + let mut stat: libc::statvfs = unsafe { std::mem::zeroed() }; + let rc = unsafe { libc::statvfs(c_path.as_ptr(), &raw mut stat) }; + if rc != 0 { + return None; + } + let used = stat.f_blocks.checked_sub(stat.f_bfree)?; + let capacity = used.checked_add(stat.f_bavail)?; + if capacity == 0 { + return None; + } + // Integer math on block counts — no float, so no rounding to explain. + Some(used.saturating_mul(100) / capacity) +} + +/// Scan roots: everything this agent owns and could plausibly free. The +/// `/agents/