From d00b34910246508a187848e40530b23f0a78a7de Mon Sep 17 00:00:00 2001 From: damocles Date: Fri, 17 Jul 2026 14:54:48 +0200 Subject: [PATCH] fix(#2560): bound migration shellouts with timeouts to prevent boot hang --- hive-c0re/src/migrate.rs | 77 +++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/hive-c0re/src/migrate.rs b/hive-c0re/src/migrate.rs index 5ed34f51..7a1eaaec 100644 --- a/hive-c0re/src/migrate.rs +++ b/hive-c0re/src/migrate.rs @@ -6,6 +6,7 @@ use std::path::Path; use std::sync::Arc; +use std::time::Duration; use anyhow::{Context, Result}; use tokio::process::Command; @@ -17,6 +18,18 @@ use crate::tool_groups; const KILL_SWITCH: &str = "HIVE_SKIP_META_MIGRATION"; +/// Per-shellout timeouts for the blocking startup migration. `run` is +/// awaited *before* the daemon starts serving (main.rs), so any child +/// process that wedges here freezes the whole daemon — admin socket + +/// dashboard included — with no diagnostics: a git/container shellout was +/// observed blocked for 86min under a concurrent `nixos-rebuild`. Every +/// shellout now runs under a timeout that kills the child on elapse, so a +/// stuck migration degrades to a logged warning instead of a hung boot. +/// Git ops are quick; `nixos-container update` can legitimately trigger a +/// nix build, so it gets a much longer budget. +const GIT_TIMEOUT: Duration = Duration::from_secs(120); +const CONTAINER_TIMEOUT: Duration = Duration::from_secs(600); + /// Substring that identifies the *current* agent flake boilerplate. /// Bumped whenever the template changes so the startup migration /// re-renders existing agents onto the new shape. Today the marker @@ -46,28 +59,40 @@ pub async fn run(coord: &Arc) -> Result<()> { // Phase 0: move harness-owned files out of state/ into harness/. // Idempotent — rename is a no-op if the source doesn't exist and // the destination already does. + tracing::debug!("migration: phase 0 (harness files)"); for name in &names { migrate_harness_files(name); } // Phase 1 + 2: per-agent applied + proposed. + tracing::debug!("migration: phase 1+2 (applied + proposed repos)"); for name in &names { + tracing::debug!(%name, "migration: applied+proposed"); if let Err(e) = migrate_applied_repo(name).await { tracing::warn!(%name, error = ?e, "migration: applied repo rewrite failed"); } - if let Err(e) = - lifecycle::setup_proposed(&Coordinator::agent_proposed_dir(name), name).await - { - tracing::warn!(%name, error = ?e, "migration: setup_proposed failed"); + let proposed_dir = Coordinator::agent_proposed_dir(name); + let proposed = lifecycle::setup_proposed(&proposed_dir, name); + match tokio::time::timeout(GIT_TIMEOUT, proposed).await { + Ok(Err(e)) => tracing::warn!(%name, error = ?e, "migration: setup_proposed failed"), + Err(_) => { + tracing::warn!(%name, timeout = ?GIT_TIMEOUT, "migration: setup_proposed timed out — skipping"); + } + Ok(Ok(())) => {} } } // Phase 3: meta repo. + tracing::debug!("migration: phase 3 (meta sync_agents)"); let agents = lifecycle::agents_for_meta_listing() .await .unwrap_or_default(); - if let Err(e) = meta::sync_agents(&coord.hive_env(), &agents).await { - tracing::warn!(error = ?e, "migration: meta sync_agents failed"); + match tokio::time::timeout(GIT_TIMEOUT, meta::sync_agents(&coord.hive_env(), &agents)).await { + Ok(Err(e)) => tracing::warn!(error = ?e, "migration: meta sync_agents failed"), + Err(_) => { + tracing::warn!(timeout = ?GIT_TIMEOUT, "migration: meta sync_agents timed out — skipping"); + } + Ok(Ok(())) => {} } // Phase 4: container repoint, guarded by marker. @@ -75,6 +100,7 @@ pub async fn run(coord: &Arc) -> Result<()> { tracing::debug!("migration: phase 4 marker present, skipping repoint"); return Ok(()); } + tracing::debug!("migration: phase 4 (container repoint)"); let mut all_ok = true; for name in &names { // Mark Rebuilding so the crash watcher skips this container @@ -301,11 +327,14 @@ async fn migrate_applied_repo(name: &str) -> Result<()> { async fn repoint_container(name: &str) -> Result<()> { let container = lifecycle::container_name(name); let flake_ref = format!("{}#{name}", crate::paths::meta_root().display()); - let out = Command::new("nixos-container") - .args(["update", &container, "--flake", &flake_ref]) - .output() - .await - .with_context(|| format!("nixos-container update {container}"))?; + let mut cmd = Command::new("nixos-container"); + cmd.args(["update", &container, "--flake", &flake_ref]); + let out = output_with_timeout( + cmd, + CONTAINER_TIMEOUT, + &format!("nixos-container update {container}"), + ) + .await?; if !out.status.success() { anyhow::bail!( "nixos-container update {container} exited {}: {}", @@ -346,13 +375,27 @@ fn backfill_manager_tool_groups(names: &[String]) { } } +/// Run a command to completion under a timeout, capturing its output. On +/// timeout the child is killed (`kill_on_drop`) and an error is returned, +/// so a wedged shellout can never freeze startup migration. `what` is a +/// human label surfaced in the timeout error + `with_context`. +async fn output_with_timeout( + mut cmd: Command, + timeout: Duration, + what: &str, +) -> Result { + cmd.kill_on_drop(true); + match tokio::time::timeout(timeout, cmd.output()).await { + Ok(r) => r.with_context(|| format!("run {what}")), + Err(_) => anyhow::bail!("{what} timed out after {timeout:?} (child killed)"), + } +} + async fn raw_git(dir: &Path, args: &[&str]) -> Result<()> { - let out = lifecycle::git_command() - .current_dir(dir) - .args(args) - .output() - .await - .with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?; + let mut cmd = lifecycle::git_command(); + cmd.current_dir(dir).args(args); + let label = format!("git {} in {}", args.join(" "), dir.display()); + let out = output_with_timeout(cmd, GIT_TIMEOUT, &label).await?; if !out.status.success() { anyhow::bail!( "git {} failed: {}",