meta nix/git calls under META_LOCK time out instead of wedging every deploy
A hung forge fetch inside `nix flake update hyperhive` (or any other meta git/nix child) now fails after its deadline with an error naming the command, the child is killed, and META_LOCK, BuildSlot and MetaWindow are released. The boot sweep already treats that failure as a warning and fans out the rebuilds. A wait on META_LOCK past 60s logs a warning naming the waiter and the holder.
This commit is contained in:
parent
8d70afd93e
commit
7cde4e7b15
6 changed files with 270 additions and 58 deletions
|
|
@ -473,3 +473,48 @@ async fn a_systemctl_answer_inside_the_bound_passes_through() {
|
|||
.await;
|
||||
assert_eq!(got, Some(7));
|
||||
}
|
||||
|
||||
/// A child that outlives the deadline yields an error naming the command and
|
||||
/// the deadline, and is killed rather than orphaned.
|
||||
#[tokio::test]
|
||||
async fn run_with_deadline_times_out_and_kills_the_child() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let pidfile = dir.path().join("pid");
|
||||
let mut cmd = tokio::process::Command::new("sh");
|
||||
cmd.arg("-c")
|
||||
.arg(format!("echo $$ > {}; exec sleep 30", pidfile.display()));
|
||||
let err = run_with_deadline(
|
||||
&mut cmd,
|
||||
std::time::Duration::from_millis(500),
|
||||
"fake fetch",
|
||||
)
|
||||
.await
|
||||
.expect_err("a sleeping child must hit the deadline");
|
||||
let msg = format!("{err:#}");
|
||||
assert!(msg.contains("fake fetch timed out after 0.5s"), "{msg}");
|
||||
|
||||
let pid = std::fs::read_to_string(&pidfile)
|
||||
.expect("child wrote its pid before the deadline")
|
||||
.trim()
|
||||
.to_owned();
|
||||
let proc = std::path::PathBuf::from("/proc").join(&pid);
|
||||
for _ in 0..100 {
|
||||
if !proc.exists() {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
panic!("child {pid} still running after the deadline dropped its future");
|
||||
}
|
||||
|
||||
/// Presence control for the timeout test: a child that finishes in time
|
||||
/// returns its output.
|
||||
#[tokio::test]
|
||||
async fn run_with_deadline_returns_output_of_a_fast_child() {
|
||||
let mut cmd = tokio::process::Command::new("echo");
|
||||
cmd.arg("hi");
|
||||
let out = run_with_deadline(&mut cmd, std::time::Duration::from_secs(30), "echo")
|
||||
.await
|
||||
.expect("echo finishes");
|
||||
assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "hi");
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue