diff --git a/hive-c0re/src/lifecycle/git.rs b/hive-c0re/src/lifecycle/git.rs index fa35696c..a54299d9 100644 --- a/hive-c0re/src/lifecycle/git.rs +++ b/hive-c0re/src/lifecycle/git.rs @@ -47,10 +47,17 @@ pub(super) async fn git_commit(dir: &Path, message: &str) -> Result<()> { /// Spawn `git` honoring the `HYPERHIVE_GIT` env var (absolute path baked in /// by the NixOS module), falling back to bare `git` (PATH lookup) otherwise. +/// +/// `kill_on_drop(true)`: if the caller's future is dropped before the child +/// exits — e.g. a `tokio::time::timeout` around startup migration fires — the +/// git child is killed instead of orphaned (left retrying an unreachable +/// forge). No-op on normal completion, where the child has already exited. #[must_use] pub fn git_command() -> Command { let exe = std::env::var("HYPERHIVE_GIT").unwrap_or_else(|_| "git".into()); - Command::new(exe) + let mut cmd = Command::new(exe); + cmd.kill_on_drop(true); + cmd } pub async fn git(dir: &Path, args: &[&str]) -> Result<()> { diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 7d824d38..1d111c83 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -1391,6 +1391,10 @@ async fn nix_output(dir: &Path, args: &[&str]) -> Result { Command::new("nix") .current_dir(dir) .args(nix_argv(args)) + // Kill the nix child if the caller's future is dropped (e.g. the + // startup-migration timeout around `sync_agents` fires) rather than + // orphaning it against an unreachable forge. No-op on normal exit. + .kill_on_drop(true) .output() .await .with_context(|| format!("nix {} in {}", args.join(" "), dir.display())) diff --git a/hive-c0re/src/migrate.rs b/hive-c0re/src/migrate.rs index 5ed34f51..f15dc8f4 100644 --- a/hive-c0re/src/migrate.rs +++ b/hive-c0re/src/migrate.rs @@ -6,6 +6,7 @@ use std::path::Path; use std::sync::Arc; +use std::time::Duration; use anyhow::{Context, Result}; use tokio::process::Command; @@ -17,6 +18,18 @@ use crate::tool_groups; const KILL_SWITCH: &str = "HIVE_SKIP_META_MIGRATION"; +/// Per-shellout timeouts for the blocking startup migration. `run` is +/// awaited *before* the daemon starts serving (main.rs), so any child +/// process that wedges here freezes the whole daemon — admin socket + +/// dashboard included — with no diagnostics: a git/container shellout was +/// observed blocked for 86min under a concurrent `nixos-rebuild`. Every +/// shellout now runs under a timeout that kills the child on elapse, so a +/// stuck migration degrades to a logged warning instead of a hung boot. +/// Git ops are quick; `nixos-container update` can legitimately trigger a +/// nix build, so it gets a much longer budget. +const GIT_TIMEOUT: Duration = Duration::from_mins(2); +const CONTAINER_TIMEOUT: Duration = Duration::from_mins(10); + /// Substring that identifies the *current* agent flake boilerplate. /// Bumped whenever the template changes so the startup migration /// re-renders existing agents onto the new shape. Today the marker @@ -46,28 +59,40 @@ pub async fn run(coord: &Arc) -> Result<()> { // Phase 0: move harness-owned files out of state/ into harness/. // Idempotent — rename is a no-op if the source doesn't exist and // the destination already does. + tracing::debug!("migration: phase 0 (harness files)"); for name in &names { migrate_harness_files(name); } // Phase 1 + 2: per-agent applied + proposed. + tracing::debug!("migration: phase 1+2 (applied + proposed repos)"); for name in &names { + tracing::debug!(%name, "migration: applied+proposed"); if let Err(e) = migrate_applied_repo(name).await { tracing::warn!(%name, error = ?e, "migration: applied repo rewrite failed"); } - if let Err(e) = - lifecycle::setup_proposed(&Coordinator::agent_proposed_dir(name), name).await - { - tracing::warn!(%name, error = ?e, "migration: setup_proposed failed"); + let proposed_dir = Coordinator::agent_proposed_dir(name); + let proposed = lifecycle::setup_proposed(&proposed_dir, name); + match tokio::time::timeout(GIT_TIMEOUT, proposed).await { + Ok(Err(e)) => tracing::warn!(%name, error = ?e, "migration: setup_proposed failed"), + Err(_) => { + tracing::warn!(%name, timeout = ?GIT_TIMEOUT, "migration: setup_proposed timed out — skipping"); + } + Ok(Ok(())) => {} } } // Phase 3: meta repo. + tracing::debug!("migration: phase 3 (meta sync_agents)"); let agents = lifecycle::agents_for_meta_listing() .await .unwrap_or_default(); - if let Err(e) = meta::sync_agents(&coord.hive_env(), &agents).await { - tracing::warn!(error = ?e, "migration: meta sync_agents failed"); + match tokio::time::timeout(GIT_TIMEOUT, meta::sync_agents(&coord.hive_env(), &agents)).await { + Ok(Err(e)) => tracing::warn!(error = ?e, "migration: meta sync_agents failed"), + Err(_) => { + tracing::warn!(timeout = ?GIT_TIMEOUT, "migration: meta sync_agents timed out — skipping"); + } + Ok(Ok(())) => {} } // Phase 4: container repoint, guarded by marker. @@ -75,6 +100,7 @@ pub async fn run(coord: &Arc) -> Result<()> { tracing::debug!("migration: phase 4 marker present, skipping repoint"); return Ok(()); } + tracing::debug!("migration: phase 4 (container repoint)"); let mut all_ok = true; for name in &names { // Mark Rebuilding so the crash watcher skips this container @@ -301,11 +327,14 @@ async fn migrate_applied_repo(name: &str) -> Result<()> { async fn repoint_container(name: &str) -> Result<()> { let container = lifecycle::container_name(name); let flake_ref = format!("{}#{name}", crate::paths::meta_root().display()); - let out = Command::new("nixos-container") - .args(["update", &container, "--flake", &flake_ref]) - .output() - .await - .with_context(|| format!("nixos-container update {container}"))?; + let mut cmd = Command::new("nixos-container"); + cmd.args(["update", &container, "--flake", &flake_ref]); + let out = output_with_timeout( + cmd, + CONTAINER_TIMEOUT, + &format!("nixos-container update {container}"), + ) + .await?; if !out.status.success() { anyhow::bail!( "nixos-container update {container} exited {}: {}", @@ -346,13 +375,27 @@ fn backfill_manager_tool_groups(names: &[String]) { } } +/// Run a command to completion under a timeout, capturing its output. On +/// timeout the child is killed (`kill_on_drop`) and an error is returned, +/// so a wedged shellout can never freeze startup migration. `what` is a +/// human label surfaced in the timeout error + `with_context`. +async fn output_with_timeout( + mut cmd: Command, + timeout: Duration, + what: &str, +) -> Result { + cmd.kill_on_drop(true); + match tokio::time::timeout(timeout, cmd.output()).await { + Ok(r) => r.with_context(|| format!("run {what}")), + Err(_) => anyhow::bail!("{what} timed out after {timeout:?} (child killed)"), + } +} + async fn raw_git(dir: &Path, args: &[&str]) -> Result<()> { - let out = lifecycle::git_command() - .current_dir(dir) - .args(args) - .output() - .await - .with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?; + let mut cmd = lifecycle::git_command(); + cmd.current_dir(dir).args(args); + let label = format!("git {} in {}", args.join(" "), dir.display()); + let out = output_with_timeout(cmd, GIT_TIMEOUT, &label).await?; if !out.status.success() { anyhow::bail!( "git {} failed: {}", diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index ce89b2fe..d2cc0b74 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -34,6 +34,15 @@ let [credential "http://${config.services.hyperhive.forge.domain}"] helper = hive-forge username = core + [http] + # Abort a stalled fetch instead of blocking startup indefinitely. If a + # git+http transfer (e.g. `nix flake lock` of the forge-hosted config + # inputs) drops below 1 KiB/s for 30s -- forge unreachable / not ready + # on cold boot -- git fails fast rather than hanging the whole daemon. + # Pairs with GIT_TERMINAL_PROMPT=0 (no credential-prompt hang) and the + # migration shellout timeout in migrate.rs. + lowSpeedLimit = 1024 + lowSpeedTime = 30 ''; # git credential helper for hive-core's authenticated fetches of the diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 7b65ee02..285c75c2 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -18,6 +18,15 @@ in # the writable StateDirectory. HOME = "/var/lib/hyperhive"; HYPERHIVE_GIT = "${pkgs.git}/bin/git"; + # Never let git block on an interactive credential prompt. hive-core is a + # TTY-less system user, so a prompt (e.g. the forge credential helper + # returns nothing because the forge isn't reachable yet on cold boot) + # would hang forever — this is what froze the whole daemon during startup + # migration's `nix flake lock` of the forge-hosted config inputs. With + # this set, git fails fast instead of prompting. Paired with the git http + # low-speed abort in `safeDirGitconfig` (bounds a stalled transfer) and + # the 120s migration shellout timeout in migrate.rs. + GIT_TERMINAL_PROMPT = "0"; # No HIVE_STATIC_DIR: the gateway static-serves the dashboard dist # (see the hive-gateway module); this router is API-only. # Path to the base agent frontend dist. hive-c0re's