fix(#2560): bound migration shellouts with timeouts to prevent boot hang

hive-c0re/src/migrate.rs

@@ -6,6 +6,7 @@
 
 use std::path::Path;
 use std::sync::Arc;
+use std::time::Duration;
 
 use anyhow::{Context, Result};
 use tokio::process::Command;
@@ -17,6 +18,18 @@ use crate::tool_groups;
 
 const KILL_SWITCH: &str = "HIVE_SKIP_META_MIGRATION";
 
+/// Per-shellout timeouts for the blocking startup migration. `run` is
+/// awaited *before* the daemon starts serving (main.rs), so any child
+/// process that wedges here freezes the whole daemon — admin socket +
+/// dashboard included — with no diagnostics: a git/container shellout was
+/// observed blocked for 86min under a concurrent `nixos-rebuild`. Every
+/// shellout now runs under a timeout that kills the child on elapse, so a
+/// stuck migration degrades to a logged warning instead of a hung boot.
+/// Git ops are quick; `nixos-container update` can legitimately trigger a
+/// nix build, so it gets a much longer budget.
+const GIT_TIMEOUT: Duration = Duration::from_secs(120);
+const CONTAINER_TIMEOUT: Duration = Duration::from_secs(600);
+
 /// Substring that identifies the *current* agent flake boilerplate.
 /// Bumped whenever the template changes so the startup migration
 /// re-renders existing agents onto the new shape. Today the marker
@@ -46,28 +59,40 @@ pub async fn run(coord: &Arc<Coordinator>) -> Result<()> {
     // Phase 0: move harness-owned files out of state/ into harness/.
     // Idempotent — rename is a no-op if the source doesn't exist and
     // the destination already does.
+    tracing::debug!("migration: phase 0 (harness files)");
     for name in &names {
         migrate_harness_files(name);
     }
 
     // Phase 1 + 2: per-agent applied + proposed.
+    tracing::debug!("migration: phase 1+2 (applied + proposed repos)");
     for name in &names {
+        tracing::debug!(%name, "migration: applied+proposed");
         if let Err(e) = migrate_applied_repo(name).await {
             tracing::warn!(%name, error = ?e, "migration: applied repo rewrite failed");
         }
-        if let Err(e) =
-            lifecycle::setup_proposed(&Coordinator::agent_proposed_dir(name), name).await
-        {
-            tracing::warn!(%name, error = ?e, "migration: setup_proposed failed");
+        let proposed_dir = Coordinator::agent_proposed_dir(name);
+        let proposed = lifecycle::setup_proposed(&proposed_dir, name);
+        match tokio::time::timeout(GIT_TIMEOUT, proposed).await {
+            Ok(Err(e)) => tracing::warn!(%name, error = ?e, "migration: setup_proposed failed"),
+            Err(_) => {
+                tracing::warn!(%name, timeout = ?GIT_TIMEOUT, "migration: setup_proposed timed out — skipping");
+            }
+            Ok(Ok(())) => {}
         }
     }
 
     // Phase 3: meta repo.
+    tracing::debug!("migration: phase 3 (meta sync_agents)");
     let agents = lifecycle::agents_for_meta_listing()
         .await
         .unwrap_or_default();
-    if let Err(e) = meta::sync_agents(&coord.hive_env(), &agents).await {
-        tracing::warn!(error = ?e, "migration: meta sync_agents failed");
+    match tokio::time::timeout(GIT_TIMEOUT, meta::sync_agents(&coord.hive_env(), &agents)).await {
+        Ok(Err(e)) => tracing::warn!(error = ?e, "migration: meta sync_agents failed"),
+        Err(_) => {
+            tracing::warn!(timeout = ?GIT_TIMEOUT, "migration: meta sync_agents timed out — skipping");
+        }
+        Ok(Ok(())) => {}
     }
 
     // Phase 4: container repoint, guarded by marker.
@@ -75,6 +100,7 @@ pub async fn run(coord: &Arc<Coordinator>) -> Result<()> {
         tracing::debug!("migration: phase 4 marker present, skipping repoint");
         return Ok(());
     }
+    tracing::debug!("migration: phase 4 (container repoint)");
     let mut all_ok = true;
     for name in &names {
         // Mark Rebuilding so the crash watcher skips this container
@@ -301,11 +327,14 @@ async fn migrate_applied_repo(name: &str) -> Result<()> {
 async fn repoint_container(name: &str) -> Result<()> {
     let container = lifecycle::container_name(name);
     let flake_ref = format!("{}#{name}", crate::paths::meta_root().display());
-    let out = Command::new("nixos-container")
-        .args(["update", &container, "--flake", &flake_ref])
-        .output()
-        .await
-        .with_context(|| format!("nixos-container update {container}"))?;
+    let mut cmd = Command::new("nixos-container");
+    cmd.args(["update", &container, "--flake", &flake_ref]);
+    let out = output_with_timeout(
+        cmd,
+        CONTAINER_TIMEOUT,
+        &format!("nixos-container update {container}"),
+    )
+    .await?;
     if !out.status.success() {
         anyhow::bail!(
             "nixos-container update {container} exited {}: {}",
@@ -346,13 +375,27 @@ fn backfill_manager_tool_groups(names: &[String]) {
     }
 }
 
+/// Run a command to completion under a timeout, capturing its output. On
+/// timeout the child is killed (`kill_on_drop`) and an error is returned,
+/// so a wedged shellout can never freeze startup migration. `what` is a
+/// human label surfaced in the timeout error + `with_context`.
+async fn output_with_timeout(
+    mut cmd: Command,
+    timeout: Duration,
+    what: &str,
+) -> Result<std::process::Output> {
+    cmd.kill_on_drop(true);
+    match tokio::time::timeout(timeout, cmd.output()).await {
+        Ok(r) => r.with_context(|| format!("run {what}")),
+        Err(_) => anyhow::bail!("{what} timed out after {timeout:?} (child killed)"),
+    }
+}
+
 async fn raw_git(dir: &Path, args: &[&str]) -> Result<()> {
-    let out = lifecycle::git_command()
-        .current_dir(dir)
-        .args(args)
-        .output()
-        .await
-        .with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?;
+    let mut cmd = lifecycle::git_command();
+    cmd.current_dir(dir).args(args);
+    let label = format!("git {} in {}", args.join(" "), dir.display());
+    let out = output_with_timeout(cmd, GIT_TIMEOUT, &label).await?;
     if !out.status.success() {
         anyhow::bail!(
             "git {} failed: {}",