diff --git a/hive-c0re/src/lifecycle/git.rs b/hive-c0re/src/lifecycle/git.rs index a54299d9..fa35696c 100644 --- a/hive-c0re/src/lifecycle/git.rs +++ b/hive-c0re/src/lifecycle/git.rs @@ -47,17 +47,10 @@ pub(super) async fn git_commit(dir: &Path, message: &str) -> Result<()> { /// Spawn `git` honoring the `HYPERHIVE_GIT` env var (absolute path baked in /// by the NixOS module), falling back to bare `git` (PATH lookup) otherwise. -/// -/// `kill_on_drop(true)`: if the caller's future is dropped before the child -/// exits — e.g. a `tokio::time::timeout` around startup migration fires — the -/// git child is killed instead of orphaned (left retrying an unreachable -/// forge). No-op on normal completion, where the child has already exited. #[must_use] pub fn git_command() -> Command { let exe = std::env::var("HYPERHIVE_GIT").unwrap_or_else(|_| "git".into()); - let mut cmd = Command::new(exe); - cmd.kill_on_drop(true); - cmd + Command::new(exe) } pub async fn git(dir: &Path, args: &[&str]) -> Result<()> { diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 1d111c83..7d824d38 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -1391,10 +1391,6 @@ async fn nix_output(dir: &Path, args: &[&str]) -> Result { Command::new("nix") .current_dir(dir) .args(nix_argv(args)) - // Kill the nix child if the caller's future is dropped (e.g. the - // startup-migration timeout around `sync_agents` fires) rather than - // orphaning it against an unreachable forge. No-op on normal exit. - .kill_on_drop(true) .output() .await .with_context(|| format!("nix {} in {}", args.join(" "), dir.display())) diff --git a/hive-c0re/src/migrate.rs b/hive-c0re/src/migrate.rs index f15dc8f4..5ed34f51 100644 --- a/hive-c0re/src/migrate.rs +++ b/hive-c0re/src/migrate.rs @@ -6,7 +6,6 @@ use std::path::Path; use std::sync::Arc; -use std::time::Duration; use anyhow::{Context, Result}; use tokio::process::Command; @@ -18,18 +17,6 @@ use crate::tool_groups; const KILL_SWITCH: &str = "HIVE_SKIP_META_MIGRATION"; -/// Per-shellout timeouts for the blocking startup migration. `run` is -/// awaited *before* the daemon starts serving (main.rs), so any child -/// process that wedges here freezes the whole daemon — admin socket + -/// dashboard included — with no diagnostics: a git/container shellout was -/// observed blocked for 86min under a concurrent `nixos-rebuild`. Every -/// shellout now runs under a timeout that kills the child on elapse, so a -/// stuck migration degrades to a logged warning instead of a hung boot. -/// Git ops are quick; `nixos-container update` can legitimately trigger a -/// nix build, so it gets a much longer budget. -const GIT_TIMEOUT: Duration = Duration::from_mins(2); -const CONTAINER_TIMEOUT: Duration = Duration::from_mins(10); - /// Substring that identifies the *current* agent flake boilerplate. /// Bumped whenever the template changes so the startup migration /// re-renders existing agents onto the new shape. Today the marker @@ -59,40 +46,28 @@ pub async fn run(coord: &Arc) -> Result<()> { // Phase 0: move harness-owned files out of state/ into harness/. // Idempotent — rename is a no-op if the source doesn't exist and // the destination already does. - tracing::debug!("migration: phase 0 (harness files)"); for name in &names { migrate_harness_files(name); } // Phase 1 + 2: per-agent applied + proposed. - tracing::debug!("migration: phase 1+2 (applied + proposed repos)"); for name in &names { - tracing::debug!(%name, "migration: applied+proposed"); if let Err(e) = migrate_applied_repo(name).await { tracing::warn!(%name, error = ?e, "migration: applied repo rewrite failed"); } - let proposed_dir = Coordinator::agent_proposed_dir(name); - let proposed = lifecycle::setup_proposed(&proposed_dir, name); - match tokio::time::timeout(GIT_TIMEOUT, proposed).await { - Ok(Err(e)) => tracing::warn!(%name, error = ?e, "migration: setup_proposed failed"), - Err(_) => { - tracing::warn!(%name, timeout = ?GIT_TIMEOUT, "migration: setup_proposed timed out — skipping"); - } - Ok(Ok(())) => {} + if let Err(e) = + lifecycle::setup_proposed(&Coordinator::agent_proposed_dir(name), name).await + { + tracing::warn!(%name, error = ?e, "migration: setup_proposed failed"); } } // Phase 3: meta repo. - tracing::debug!("migration: phase 3 (meta sync_agents)"); let agents = lifecycle::agents_for_meta_listing() .await .unwrap_or_default(); - match tokio::time::timeout(GIT_TIMEOUT, meta::sync_agents(&coord.hive_env(), &agents)).await { - Ok(Err(e)) => tracing::warn!(error = ?e, "migration: meta sync_agents failed"), - Err(_) => { - tracing::warn!(timeout = ?GIT_TIMEOUT, "migration: meta sync_agents timed out — skipping"); - } - Ok(Ok(())) => {} + if let Err(e) = meta::sync_agents(&coord.hive_env(), &agents).await { + tracing::warn!(error = ?e, "migration: meta sync_agents failed"); } // Phase 4: container repoint, guarded by marker. @@ -100,7 +75,6 @@ pub async fn run(coord: &Arc) -> Result<()> { tracing::debug!("migration: phase 4 marker present, skipping repoint"); return Ok(()); } - tracing::debug!("migration: phase 4 (container repoint)"); let mut all_ok = true; for name in &names { // Mark Rebuilding so the crash watcher skips this container @@ -327,14 +301,11 @@ async fn migrate_applied_repo(name: &str) -> Result<()> { async fn repoint_container(name: &str) -> Result<()> { let container = lifecycle::container_name(name); let flake_ref = format!("{}#{name}", crate::paths::meta_root().display()); - let mut cmd = Command::new("nixos-container"); - cmd.args(["update", &container, "--flake", &flake_ref]); - let out = output_with_timeout( - cmd, - CONTAINER_TIMEOUT, - &format!("nixos-container update {container}"), - ) - .await?; + let out = Command::new("nixos-container") + .args(["update", &container, "--flake", &flake_ref]) + .output() + .await + .with_context(|| format!("nixos-container update {container}"))?; if !out.status.success() { anyhow::bail!( "nixos-container update {container} exited {}: {}", @@ -375,27 +346,13 @@ fn backfill_manager_tool_groups(names: &[String]) { } } -/// Run a command to completion under a timeout, capturing its output. On -/// timeout the child is killed (`kill_on_drop`) and an error is returned, -/// so a wedged shellout can never freeze startup migration. `what` is a -/// human label surfaced in the timeout error + `with_context`. -async fn output_with_timeout( - mut cmd: Command, - timeout: Duration, - what: &str, -) -> Result { - cmd.kill_on_drop(true); - match tokio::time::timeout(timeout, cmd.output()).await { - Ok(r) => r.with_context(|| format!("run {what}")), - Err(_) => anyhow::bail!("{what} timed out after {timeout:?} (child killed)"), - } -} - async fn raw_git(dir: &Path, args: &[&str]) -> Result<()> { - let mut cmd = lifecycle::git_command(); - cmd.current_dir(dir).args(args); - let label = format!("git {} in {}", args.join(" "), dir.display()); - let out = output_with_timeout(cmd, GIT_TIMEOUT, &label).await?; + let out = lifecycle::git_command() + .current_dir(dir) + .args(args) + .output() + .await + .with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?; if !out.status.success() { anyhow::bail!( "git {} failed: {}", diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index d2cc0b74..ce89b2fe 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -34,15 +34,6 @@ let [credential "http://${config.services.hyperhive.forge.domain}"] helper = hive-forge username = core - [http] - # Abort a stalled fetch instead of blocking startup indefinitely. If a - # git+http transfer (e.g. `nix flake lock` of the forge-hosted config - # inputs) drops below 1 KiB/s for 30s -- forge unreachable / not ready - # on cold boot -- git fails fast rather than hanging the whole daemon. - # Pairs with GIT_TERMINAL_PROMPT=0 (no credential-prompt hang) and the - # migration shellout timeout in migrate.rs. - lowSpeedLimit = 1024 - lowSpeedTime = 30 ''; # git credential helper for hive-core's authenticated fetches of the diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 285c75c2..7b65ee02 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -18,15 +18,6 @@ in # the writable StateDirectory. HOME = "/var/lib/hyperhive"; HYPERHIVE_GIT = "${pkgs.git}/bin/git"; - # Never let git block on an interactive credential prompt. hive-core is a - # TTY-less system user, so a prompt (e.g. the forge credential helper - # returns nothing because the forge isn't reachable yet on cold boot) - # would hang forever — this is what froze the whole daemon during startup - # migration's `nix flake lock` of the forge-hosted config inputs. With - # this set, git fails fast instead of prompting. Paired with the git http - # low-speed abort in `safeDirGitconfig` (bounds a stalled transfer) and - # the 120s migration shellout timeout in migrate.rs. - GIT_TERMINAL_PROMPT = "0"; # No HIVE_STATIC_DIR: the gateway static-serves the dashboard dist # (see the hive-gateway module); this router is API-only. # Path to the base agent frontend dist. hive-c0re's