vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

container stats: bound hive-c0re's wait for the disk measurement, no total for an agent with no paths

Browse source 
hive-c0re waited on hive-priv's answer with no timeout, so a hung
hive-priv would stall the disk sampler indefinitely. The wait is now
hive-priv's own 10 min bound plus a minute (the bound is shared as
MEASURE_AGENT_DISK_TIMEOUT); a timeout is an error, so the agent shows
unknown.

An agent with none of state/, work/ or the rootfs has no measurable
size. hive-priv now answers with an error for it instead of a total of
0, which read as a real measurement.
This commit is contained in:
atlas 2026-10-05 16:33:11 +02:00 • committed by mara
commit 960cf9ce30
3 changed files with 131 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

100
hive-priv/src/main.rs
View file

@ -23,9 +23,9 @@ use std::path::{Path, PathBuf};
use anyhow::{Context as _, Result, anyhow, bail};
use hive_priv_sock::{
AGENT_PREFIX, AGENT_RUNTIME_ROOT, AGENT_STATE_ROOT, BindMount, CredentialMount, InfraAction,
InfraContainer, JournalQuery, META_DIR, MIGRATE_STAGING_ROOT, NetworkIsolation,
PAUSED_MARKER_FILE, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream,
PrivStreamLine, SIBLING_CONTAINERS,
InfraContainer, JournalQuery, MEASURE_AGENT_DISK_TIMEOUT, META_DIR, MIGRATE_STAGING_ROOT,
NetworkIsolation, PAUSED_MARKER_FILE, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse,
PrivStream, PrivStreamLine, SIBLING_CONTAINERS,
};
use tokio::io::{AsyncWriteExt, BufReader};
use tokio::net::unix::OwnedWriteHalf;
@ -1684,10 +1684,6 @@ async fn read_subvolume_usage(agent_name: &str) -> Result<(String, String)> {
))
}
/// Wall-clock bound on one [`measure_agent_disk`] walk. It only stops a walk
/// wedged on I/O from holding the caller's sampler forever.
const DISK_USAGE_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(10);
/// The host paths `MeasureAgentDisk` sums for `agent_name`: its `state/`
/// and `work/` dirs under `AGENT_STATE_ROOT` and its container rootfs.
/// These are the only paths that op ever walks.
@ -1703,10 +1699,8 @@ fn agent_disk_paths(agent_name: &str) -> [PathBuf; 3] {
}
/// `MeasureAgentDisk` — one `du` over the [`agent_disk_paths`] that exist,
/// returning the grand total in bytes. `-x` keeps each walk on its
/// operand's filesystem (so the bind-mounted nix store is never counted),
/// `-P` never follows a symlink, `-b` counts apparent size, `-c` adds the
/// `total` line [`parse_du_total`] reads.
/// returning the grand total in bytes. An agent with none of them has no
/// measurable size, which is an error rather than a total of 0.
async fn measure_agent_disk(agent_name: &str) -> Result<(String, String)> {
let mut paths = Vec::new();
for path in agent_disk_paths(agent_name) {
@ -1716,18 +1710,33 @@ async fn measure_agent_disk(agent_name: &str) -> Result<(String, String)> {
Err(e) => return Err(e).with_context(|| format!("stat {}", path.display())),
}
}
// `du` with no operand would measure its working directory instead.
let total = du_total(agent_name, du_command(&paths)?, MEASURE_AGENT_DISK_TIMEOUT).await?;
Ok((total.to_string(), String::new()))
}
/// `du -s -x -b -c -P -- <paths>`. `-x` keeps each walk on its operand's
/// filesystem (so the bind-mounted nix store is never counted), `-P` never
/// follows a symlink, `-b` counts apparent size, `-c` adds the `total` line
/// [`parse_du_total`] reads. Errors on no paths: `du` with no operand would
/// measure its working directory instead.
fn du_command(paths: &[PathBuf]) -> Result<Command> {
if paths.is_empty() {
return Ok(("0".to_owned(), String::new()));
bail!("none of the agent's state, work or rootfs paths exist");
}
let mut cmd = Command::new("du");
cmd.args(["-s", "-x", "-b", "-c", "-P", "--"])
.args(&paths)
.kill_on_drop(true);
let Ok(out) = tokio::time::timeout(DISK_USAGE_TIMEOUT, cmd.output()).await else {
cmd.args(["-s", "-x", "-b", "-c", "-P", "--"]).args(paths);
Ok(cmd)
}
/// Run `cmd` (a [`du_command`]) for at most `limit` and parse its total.
/// The child is killed on timeout. A timeout, a spawn failure, a non-zero
/// exit or output without a total is an error, never a partial total.
async fn du_total(agent_name: &str, mut cmd: Command, limit: std::time::Duration) -> Result<u64> {
cmd.kill_on_drop(true);
let Ok(out) = tokio::time::timeout(limit, cmd.output()).await else {
bail!(
"du for {agent_name} timed out after {}s; killed it",
DISK_USAGE_TIMEOUT.as_secs()
limit.as_secs()
);
};
let out = out.with_context(|| format!("spawn du for {agent_name}"))?;
@ -1743,9 +1752,8 @@ async fn measure_agent_disk(agent_name: &str) -> Result<(String, String)> {
);
}
let stdout = String::from_utf8_lossy(&out.stdout);
let total = parse_du_total(&stdout)
.ok_or_else(|| anyhow!("no `total` line in du output for {agent_name}: {stdout:?}"))?;
Ok((total.to_string(), String::new()))
parse_du_total(&stdout)
.ok_or_else(|| anyhow!("no `total` line in du output for {agent_name}: {stdout:?}"))
}
/// The byte count on `du -c`'s final `<bytes>\ttotal` line. `None` when the
@ -3178,16 +3186,17 @@ fn ensure_socket_dir_in(root: &Path, name: &str) -> Result<()> {
#[cfg(test)]
mod tests {
use super::{
BindMount, BoundedRun, OwnedFd, PAUSED_MARKER_FILE, PrivRequest, StagedFile,
BindMount, BoundedRun, Command, OwnedFd, PAUSED_MARKER_FILE, PrivRequest, StagedFile,
agent_disk_paths, check_fd_agreement, contains_secret_shaped_run, describe_forge_admin,
ensure_plain_filename, ensure_socket_dir_in, exec_measure_agent_disk, git_overlay_flags,
limits_dropin_body, open_dir, open_export_dest, parse_du_total, partial_name, publish_file,
redact_secret_line, remove_marker_in, run_bounded, single_output_path, toplevel_attr,
validate_credential_name, validate_snapshot_name, write_agent_dir_file,
write_bridge_dns_marker_in,
du_command, du_total, ensure_plain_filename, ensure_socket_dir_in, exec_measure_agent_disk,
git_overlay_flags, limits_dropin_body, open_dir, open_export_dest, parse_du_total,
partial_name, publish_file, redact_secret_line, remove_marker_in, run_bounded,
single_output_path, toplevel_attr, validate_credential_name, validate_snapshot_name,
write_agent_dir_file, write_bridge_dns_marker_in,
};
use std::path::PathBuf;
use std::sync::atomic::{AtomicU32, Ordering};
use std::time::Duration;
fn bind(container_path: &str) -> BindMount {
BindMount {
@ -4240,4 +4249,41 @@ mod tests {
assert_eq!(parse_du_total("-1\ttotal\n"), None);
assert_eq!(parse_du_total("100 total\n"), None);
}
#[test]
fn no_existing_paths_is_an_error_not_zero() {
let err = du_command(&[]).expect_err("no operands must not run du");
assert!(err.to_string().contains("none of"), "{err}");
}
#[tokio::test]
async fn non_zero_du_exit_is_an_error() {
let err = du_total("iris", Command::new("false"), Duration::from_secs(5))
.await
.expect_err("a failed du must not yield a total");
assert!(err.to_string().contains("failed"), "{err}");
}
#[tokio::test]
async fn a_walk_past_the_limit_is_killed_and_errors() {
let mut cmd = Command::new("sleep");
cmd.arg("30");
let started = std::time::Instant::now();
let err = du_total("iris", cmd, Duration::from_millis(50))
.await
.expect_err("a walk past the limit must fail");
assert!(err.to_string().contains("timed out"), "{err}");
assert!(started.elapsed() < Duration::from_secs(10));
}
#[tokio::test]
async fn du_total_sums_a_real_walk() {
let dir = std::env::temp_dir().join(format!("hive-priv-du-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("f"), b"hello").unwrap();
let cmd = du_command(std::slice::from_ref(&dir)).unwrap();
let total = du_total("iris", cmd, Duration::from_secs(30)).await;
std::fs::remove_dir_all(&dir).unwrap();
assert!(total.unwrap() >= 5);
}
}