Watch
0
0
Fork
You've already forked hyperhive
0

disk_watch: silence the todo when the agent's total footprint is tiny

gate the big-dir todo on agent_footprint_bytes() >= 3*BIG_DIR_BYTES (15gib)
in addition to the existing pct/empty-dirs checks, so a small agent whose
disk-usage% crosses the warn line because the *filesystem* is full (not
because this agent owns much of it) stays silent even when big_dirs()
happens to name one oversized subdir inside an otherwise small tree.

closes #4635
This commit is contained in:
damocles 2026-10-04 15:03:16 +02:00 • committed by mara
commit a8f47b3f69

View file

@ -57,6 +57,13 @@ const TODO_KEY: &str = "usage";
/// Only directories at least this large are worth naming. Coarse on
/// purpose: the listed set shouldn't churn while a build runs.
const BIG_DIR_BYTES: u64 = 5 << 30;
/// Floor on the agent's total footprint (sum across every `roots()`
/// tree) below which the todo never fires, regardless of any single
/// subdir crossing `BIG_DIR_BYTES` on its own — an agent whose whole
/// tree is this small cannot meaningfully move a large filesystem's
/// usage even by deleting everything it owns, so naming one oversized
/// subdir inside a small total isn't worth a turn to act on.
const MIN_AGENT_FOOTPRINT_BYTES: u64 = 3 * BIG_DIR_BYTES;
/// At most this many directories are named in the todo.
const MAX_LISTED: usize = 5;
/// Directories deeper than this (relative to a scan root) are summed into
@ -119,21 +126,23 @@ fn probe() -> Option<String> {
if pct < WARN_PCT {
return None;
}
summary_for(pct, &big_dirs())
summary_for(pct, agent_footprint_bytes(), &big_dirs())
}
/// Pure rendering half of [`probe`]: the todo text for a given usage
/// percentage + set of oversized directories. Separated so the threshold
/// and — more importantly — the *stability* of the summary are unit-tested
/// without a real filesystem.
/// percentage, total agent footprint, and set of oversized directories.
/// Separated so the thresholds and — more importantly — the *stability*
/// of the summary are unit-tested without a real filesystem.
///
/// Silent unless the agent owns something worth deleting. A full disk the
/// agent did not fill is not its problem to solve: it cannot free host
/// bytes, so the todo would only ever cost a turn to conclude "not
/// actionable". The shared store filling up is the host's signal, not an
/// agent's.
fn summary_for(pct: u64, dirs: &[PathBuf]) -> Option<String> {
if pct < WARN_PCT || dirs.is_empty() {
/// agent's. Same reasoning extends to [`MIN_AGENT_FOOTPRINT_BYTES`]: a
/// small total footprint means there is nothing here worth the bother
/// even when one subdir happens to cross [`BIG_DIR_BYTES`] on its own.
fn summary_for(pct: u64, footprint: u64, dirs: &[PathBuf]) -> Option<String> {
if pct < WARN_PCT || dirs.is_empty() || footprint < MIN_AGENT_FOOTPRINT_BYTES {
return None;
}
let bucket = pct / BUCKET_PCT * BUCKET_PCT;
@ -201,6 +210,26 @@ fn roots() -> Vec<PathBuf> {
kept
}
/// Sum of bytes under every `roots()` tree combined — the agent's total
/// footprint on the filesystem [`probe`] measured. A second full walk
/// alongside [`big_dirs`]'s own: both are already gated behind
/// `pct >= WARN_PCT`, so neither runs on a healthy disk, and the extra
/// walk trades a little redundant `read_dir` work for keeping this and
/// [`big_dirs`] as two small, independently obvious functions instead of
/// one that returns a tuple nobody asked for.
fn agent_footprint_bytes() -> u64 {
let mut budget = SCAN_BUDGET;
let mut discard = Vec::new();
let mut total: u64 = 0;
for root in roots() {
let Ok(md) = std::fs::metadata(&root) else {
continue;
};
total = total.saturating_add(dir_bytes(&root, md.dev(), 0, &mut budget, &mut discard));
}
total
}
/// Walk the agent's own tree and return the directories worth naming:
/// at least [`BIG_DIR_BYTES`], deepest-first (a `target/` dir, not the
/// `/agents/<label>` that contains it), capped at [`MAX_LISTED`].
@ -284,14 +313,26 @@ mod tests {
PathBuf::from(s)
}
/// A footprint comfortably over the floor, for tests that aren't
/// exercising the floor itself.
const BIG_FOOTPRINT: u64 = MIN_AGENT_FOOTPRINT_BYTES * 2;
#[test]
fn under_threshold_is_silent() {
assert!(summary_for(WARN_PCT - 1, &[p("/agents/a/state/hh/target")]).is_none());
assert!(
summary_for(
WARN_PCT - 1,
BIG_FOOTPRINT,
&[p("/agents/a/state/hh/target")]
)
.is_none()
);
}
#[test]
fn at_threshold_speaks_up() {
let s = summary_for(WARN_PCT, &[p("/agents/a/state/hh/target")]).expect("todo");
let s =
summary_for(WARN_PCT, BIG_FOOTPRINT, &[p("/agents/a/state/hh/target")]).expect("todo");
assert!(s.contains("over 80% full"));
assert!(s.contains("tens of GiB"));
assert!(s.contains("tell the operator"));
@ -302,7 +343,35 @@ mod tests {
/// free host bytes, so naming no directories means saying nothing.
#[test]
fn full_disk_with_nothing_big_of_ours_is_silent() {
assert!(summary_for(99, &[]).is_none());
assert!(summary_for(99, BIG_FOOTPRINT, &[]).is_none());
}
/// The bug this exists for (hive-agent#4635's report): one subdir can
/// cross `BIG_DIR_BYTES` on its own while the agent's whole footprint
/// stays small enough that deleting it wouldn't move a large
/// filesystem's usage at all — that must stay silent too.
#[test]
fn a_small_total_footprint_is_silent_even_with_a_named_dir() {
assert!(
summary_for(
90,
MIN_AGENT_FOOTPRINT_BYTES - 1,
&[p("/agents/a/state/hh/target")]
)
.is_none()
);
}
#[test]
fn a_footprint_at_the_floor_speaks_up() {
assert!(
summary_for(
90,
MIN_AGENT_FOOTPRINT_BYTES,
&[p("/agents/a/state/hh/target")]
)
.is_some()
);
}
/// The anti-nag property: usage drifting inside one bucket must render
@ -311,23 +380,23 @@ mod tests {
#[test]
fn drift_inside_a_bucket_is_identical() {
let dirs = [p("/agents/a/state/hh/target")];
let a = summary_for(85, &dirs).expect("todo");
let b = summary_for(89, &dirs).expect("todo");
let a = summary_for(85, BIG_FOOTPRINT, &dirs).expect("todo");
let b = summary_for(89, BIG_FOOTPRINT, &dirs).expect("todo");
assert_eq!(a, b);
}
#[test]
fn crossing_a_bucket_changes_the_summary() {
let dirs = [p("/agents/a/state/hh/target")];
let a = summary_for(89, &dirs).expect("todo");
let b = summary_for(90, &dirs).expect("todo");
let a = summary_for(89, BIG_FOOTPRINT, &dirs).expect("todo");
let b = summary_for(90, BIG_FOOTPRINT, &dirs).expect("todo");
assert_ne!(a, b);
assert!(b.contains("over 90% full"));
}
#[test]
fn big_dirs_are_named() {
let s = summary_for(90, &[p("/agents/x/state/wt/target")]).expect("todo");
let s = summary_for(90, BIG_FOOTPRINT, &[p("/agents/x/state/wt/target")]).expect("todo");
assert!(s.contains("/agents/x/state/wt/target"));
assert!(s.contains("du -sh"));
}
@ -376,7 +445,8 @@ mod tests {
let dir = tempfile::tempdir().expect("tempdir");
let todos = Todos::open(&dir.path().join("state.sqlite")).expect("open");
let wake = Notify::new();
let summary = summary_for(90, &[p("/agents/a/state/hh/target")]).expect("todo");
let summary =
summary_for(90, BIG_FOOTPRINT, &[p("/agents/a/state/hh/target")]).expect("todo");
reconcile(&todos, &wake, Some(&summary));
assert_eq!(todos.list(Some(SUBSYSTEM)).expect("list").len(), 1);