meta nix/git calls under META_LOCK time out instead of wedging every deploy

A hung forge fetch inside `nix flake update hyperhive` (or any other meta
git/nix child) now fails after its deadline with an error naming the command,
the child is killed, and META_LOCK, BuildSlot and MetaWindow are released. The
boot sweep already treats that failure as a warning and fans out the rebuilds.
A wait on META_LOCK past 60s logs a warning naming the waiter and the holder.

hive-c0re/src/lifecycle/git.rs

@@ -3,10 +3,36 @@
 //! fetch proposal commits into the applied repo.
 
 use std::path::Path;
+use std::process::Output;
+use std::time::Duration;
 
 use anyhow::{Context, Result, bail};
 use tokio::process::Command;
 
+/// Deadline for a child that talks to a remote (forge fetch/push, `nix flake
+/// update`/`lock`): a healthy one takes seconds, so ten minutes only ever
+/// fires on a hang.
+pub const FETCH_DEADLINE: Duration = Duration::from_mins(10);
+
+/// Deadline for a git child that touches only the local repo.
+pub const GIT_LOCAL_DEADLINE: Duration = Duration::from_mins(2);
+
+/// Run `cmd` to completion and capture its output, failing once `deadline`
+/// passes. `what` names the command in the error. The child is killed when the
+/// deadline drops its future, so a hung remote cannot leave an orphan behind.
+pub async fn run_with_deadline(
+    cmd: &mut Command,
+    deadline: Duration,
+    what: &str,
+) -> Result<Output> {
+    // `kill_on_drop` is what turns the timeout's drop into a kill.
+    cmd.kill_on_drop(true);
+    tokio::time::timeout(deadline, cmd.output())
+        .await
+        .map_err(|_| anyhow::anyhow!("{what} timed out after {}s", deadline.as_secs_f64()))?
+        .with_context(|| what.to_owned())
+}
+
 const GIT_NAME: &str = "c0re";
 const GIT_EMAIL: &str = "c0re@hyperhive.local";
 
@@ -80,14 +106,11 @@ pub fn git_command_authed(auth_header: &str) -> Command {
     cmd
 }
 
-/// Run `cmd` as `git <args>` in `dir`, erroring on a non-zero exit.
-async fn run(mut cmd: Command, dir: &Path, args: &[&str]) -> Result<()> {
-    let out = cmd
-        .current_dir(dir)
-        .args(args)
-        .output()
-        .await
-        .with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?;
+/// Run `cmd` as `git <args>` in `dir` within `deadline`, erroring on a
+/// non-zero exit.
+async fn run(mut cmd: Command, dir: &Path, args: &[&str], deadline: Duration) -> Result<()> {
+    let what = format!("git {} in {}", args.join(" "), dir.display());
+    let out = run_with_deadline(cmd.current_dir(dir).args(args), deadline, &what).await?;
     if !out.status.success() {
         bail!(
             "git {} failed ({}): {}",
@@ -100,14 +123,14 @@ async fn run(mut cmd: Command, dir: &Path, args: &[&str]) -> Result<()> {
 }
 
 pub async fn git(dir: &Path, args: &[&str]) -> Result<()> {
-    run(git_command(), dir, args).await
+    run(git_command(), dir, args, GIT_LOCAL_DEADLINE).await
 }
 
 /// [`git`] against an authenticated remote — for the paths that talk to the
 /// forge. The credential rides the environment, so `args` (and therefore the
 /// error above) stay free of it.
 pub async fn git_authed(dir: &Path, args: &[&str], auth_header: &str) -> Result<()> {
-    run(git_command_authed(auth_header), dir, args).await
+    run(git_command_authed(auth_header), dir, args, FETCH_DEADLINE).await
 }
 
 /// Resolve `refname` (a tag, branch, or sha) in `dir` to its full sha.

hive-c0re/src/lifecycle/mod.rs

@@ -8,8 +8,9 @@ mod setup;
 mod tests;
 
 pub use git::{
-    git, git_authed, git_command, git_command_authed, git_is_ancestor, git_read_tree_reset,
-    git_rev_parse, git_show_file, git_tag, git_update_ref, git_update_ref_cas,
+    FETCH_DEADLINE, GIT_LOCAL_DEADLINE, git, git_authed, git_command, git_command_authed,
+    git_is_ancestor, git_read_tree_reset, git_rev_parse, git_show_file, git_tag, git_update_ref,
+    git_update_ref_cas, run_with_deadline,
 };
 pub use host_config::{hand_permissions, write_dropins};
 pub use setup::{

hive-c0re/src/lifecycle/tests.rs

@@ -473,3 +473,48 @@ async fn a_systemctl_answer_inside_the_bound_passes_through() {
     .await;
     assert_eq!(got, Some(7));
 }
+
+/// A child that outlives the deadline yields an error naming the command and
+/// the deadline, and is killed rather than orphaned.
+#[tokio::test]
+async fn run_with_deadline_times_out_and_kills_the_child() {
+    let dir = tempfile::tempdir().expect("tempdir");
+    let pidfile = dir.path().join("pid");
+    let mut cmd = tokio::process::Command::new("sh");
+    cmd.arg("-c")
+        .arg(format!("echo $$ > {}; exec sleep 30", pidfile.display()));
+    let err = run_with_deadline(
+        &mut cmd,
+        std::time::Duration::from_millis(500),
+        "fake fetch",
+    )
+    .await
+    .expect_err("a sleeping child must hit the deadline");
+    let msg = format!("{err:#}");
+    assert!(msg.contains("fake fetch timed out after 0.5s"), "{msg}");
+
+    let pid = std::fs::read_to_string(&pidfile)
+        .expect("child wrote its pid before the deadline")
+        .trim()
+        .to_owned();
+    let proc = std::path::PathBuf::from("/proc").join(&pid);
+    for _ in 0..100 {
+        if !proc.exists() {
+            return;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+    }
+    panic!("child {pid} still running after the deadline dropped its future");
+}
+
+/// Presence control for the timeout test: a child that finishes in time
+/// returns its output.
+#[tokio::test]
+async fn run_with_deadline_returns_output_of_a_fast_child() {
+    let mut cmd = tokio::process::Command::new("echo");
+    cmd.arg("hi");
+    let out = run_with_deadline(&mut cmd, std::time::Duration::from_secs(30), "echo")
+        .await
+        .expect("echo finishes");
+    assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "hi");
+}