Compare commits

..
5 changed files with 18 additions and 90 deletions

View file

@ -47,17 +47,10 @@ pub(super) async fn git_commit(dir: &Path, message: &str) -> Result<()> {
/// Spawn `git` honoring the `HYPERHIVE_GIT` env var (absolute path baked in /// Spawn `git` honoring the `HYPERHIVE_GIT` env var (absolute path baked in
/// by the NixOS module), falling back to bare `git` (PATH lookup) otherwise. /// by the NixOS module), falling back to bare `git` (PATH lookup) otherwise.
///
/// `kill_on_drop(true)`: if the caller's future is dropped before the child
/// exits — e.g. a `tokio::time::timeout` around startup migration fires — the
/// git child is killed instead of orphaned (left retrying an unreachable
/// forge). No-op on normal completion, where the child has already exited.
#[must_use] #[must_use]
pub fn git_command() -> Command { pub fn git_command() -> Command {
let exe = std::env::var("HYPERHIVE_GIT").unwrap_or_else(|_| "git".into()); let exe = std::env::var("HYPERHIVE_GIT").unwrap_or_else(|_| "git".into());
let mut cmd = Command::new(exe); Command::new(exe)
cmd.kill_on_drop(true);
cmd
} }
pub async fn git(dir: &Path, args: &[&str]) -> Result<()> { pub async fn git(dir: &Path, args: &[&str]) -> Result<()> {

View file

@ -1391,10 +1391,6 @@ async fn nix_output(dir: &Path, args: &[&str]) -> Result<std::process::Output> {
Command::new("nix") Command::new("nix")
.current_dir(dir) .current_dir(dir)
.args(nix_argv(args)) .args(nix_argv(args))
// Kill the nix child if the caller's future is dropped (e.g. the
// startup-migration timeout around `sync_agents` fires) rather than
// orphaning it against an unreachable forge. No-op on normal exit.
.kill_on_drop(true)
.output() .output()
.await .await
.with_context(|| format!("nix {} in {}", args.join(" "), dir.display())) .with_context(|| format!("nix {} in {}", args.join(" "), dir.display()))

View file

@ -6,7 +6,6 @@
use std::path::Path; use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use tokio::process::Command; use tokio::process::Command;
@ -18,18 +17,6 @@ use crate::tool_groups;
const KILL_SWITCH: &str = "HIVE_SKIP_META_MIGRATION"; const KILL_SWITCH: &str = "HIVE_SKIP_META_MIGRATION";
/// Per-shellout timeouts for the blocking startup migration. `run` is
/// awaited *before* the daemon starts serving (main.rs), so any child
/// process that wedges here freezes the whole daemon — admin socket +
/// dashboard included — with no diagnostics: a git/container shellout was
/// observed blocked for 86min under a concurrent `nixos-rebuild`. Every
/// shellout now runs under a timeout that kills the child on elapse, so a
/// stuck migration degrades to a logged warning instead of a hung boot.
/// Git ops are quick; `nixos-container update` can legitimately trigger a
/// nix build, so it gets a much longer budget.
const GIT_TIMEOUT: Duration = Duration::from_mins(2);
const CONTAINER_TIMEOUT: Duration = Duration::from_mins(10);
/// Substring that identifies the *current* agent flake boilerplate. /// Substring that identifies the *current* agent flake boilerplate.
/// Bumped whenever the template changes so the startup migration /// Bumped whenever the template changes so the startup migration
/// re-renders existing agents onto the new shape. Today the marker /// re-renders existing agents onto the new shape. Today the marker
@ -59,40 +46,28 @@ pub async fn run(coord: &Arc<Coordinator>) -> Result<()> {
// Phase 0: move harness-owned files out of state/ into harness/. // Phase 0: move harness-owned files out of state/ into harness/.
// Idempotent — rename is a no-op if the source doesn't exist and // Idempotent — rename is a no-op if the source doesn't exist and
// the destination already does. // the destination already does.
tracing::debug!("migration: phase 0 (harness files)");
for name in &names { for name in &names {
migrate_harness_files(name); migrate_harness_files(name);
} }
// Phase 1 + 2: per-agent applied + proposed. // Phase 1 + 2: per-agent applied + proposed.
tracing::debug!("migration: phase 1+2 (applied + proposed repos)");
for name in &names { for name in &names {
tracing::debug!(%name, "migration: applied+proposed");
if let Err(e) = migrate_applied_repo(name).await { if let Err(e) = migrate_applied_repo(name).await {
tracing::warn!(%name, error = ?e, "migration: applied repo rewrite failed"); tracing::warn!(%name, error = ?e, "migration: applied repo rewrite failed");
} }
let proposed_dir = Coordinator::agent_proposed_dir(name); if let Err(e) =
let proposed = lifecycle::setup_proposed(&proposed_dir, name); lifecycle::setup_proposed(&Coordinator::agent_proposed_dir(name), name).await
match tokio::time::timeout(GIT_TIMEOUT, proposed).await { {
Ok(Err(e)) => tracing::warn!(%name, error = ?e, "migration: setup_proposed failed"), tracing::warn!(%name, error = ?e, "migration: setup_proposed failed");
Err(_) => {
tracing::warn!(%name, timeout = ?GIT_TIMEOUT, "migration: setup_proposed timed out — skipping");
}
Ok(Ok(())) => {}
} }
} }
// Phase 3: meta repo. // Phase 3: meta repo.
tracing::debug!("migration: phase 3 (meta sync_agents)");
let agents = lifecycle::agents_for_meta_listing() let agents = lifecycle::agents_for_meta_listing()
.await .await
.unwrap_or_default(); .unwrap_or_default();
match tokio::time::timeout(GIT_TIMEOUT, meta::sync_agents(&coord.hive_env(), &agents)).await { if let Err(e) = meta::sync_agents(&coord.hive_env(), &agents).await {
Ok(Err(e)) => tracing::warn!(error = ?e, "migration: meta sync_agents failed"), tracing::warn!(error = ?e, "migration: meta sync_agents failed");
Err(_) => {
tracing::warn!(timeout = ?GIT_TIMEOUT, "migration: meta sync_agents timed out — skipping");
}
Ok(Ok(())) => {}
} }
// Phase 4: container repoint, guarded by marker. // Phase 4: container repoint, guarded by marker.
@ -100,7 +75,6 @@ pub async fn run(coord: &Arc<Coordinator>) -> Result<()> {
tracing::debug!("migration: phase 4 marker present, skipping repoint"); tracing::debug!("migration: phase 4 marker present, skipping repoint");
return Ok(()); return Ok(());
} }
tracing::debug!("migration: phase 4 (container repoint)");
let mut all_ok = true; let mut all_ok = true;
for name in &names { for name in &names {
// Mark Rebuilding so the crash watcher skips this container // Mark Rebuilding so the crash watcher skips this container
@ -327,14 +301,11 @@ async fn migrate_applied_repo(name: &str) -> Result<()> {
async fn repoint_container(name: &str) -> Result<()> { async fn repoint_container(name: &str) -> Result<()> {
let container = lifecycle::container_name(name); let container = lifecycle::container_name(name);
let flake_ref = format!("{}#{name}", crate::paths::meta_root().display()); let flake_ref = format!("{}#{name}", crate::paths::meta_root().display());
let mut cmd = Command::new("nixos-container"); let out = Command::new("nixos-container")
cmd.args(["update", &container, "--flake", &flake_ref]); .args(["update", &container, "--flake", &flake_ref])
let out = output_with_timeout( .output()
cmd, .await
CONTAINER_TIMEOUT, .with_context(|| format!("nixos-container update {container}"))?;
&format!("nixos-container update {container}"),
)
.await?;
if !out.status.success() { if !out.status.success() {
anyhow::bail!( anyhow::bail!(
"nixos-container update {container} exited {}: {}", "nixos-container update {container} exited {}: {}",
@ -375,27 +346,13 @@ fn backfill_manager_tool_groups(names: &[String]) {
} }
} }
/// Run a command to completion under a timeout, capturing its output. On
/// timeout the child is killed (`kill_on_drop`) and an error is returned,
/// so a wedged shellout can never freeze startup migration. `what` is a
/// human label surfaced in the timeout error + `with_context`.
async fn output_with_timeout(
mut cmd: Command,
timeout: Duration,
what: &str,
) -> Result<std::process::Output> {
cmd.kill_on_drop(true);
match tokio::time::timeout(timeout, cmd.output()).await {
Ok(r) => r.with_context(|| format!("run {what}")),
Err(_) => anyhow::bail!("{what} timed out after {timeout:?} (child killed)"),
}
}
async fn raw_git(dir: &Path, args: &[&str]) -> Result<()> { async fn raw_git(dir: &Path, args: &[&str]) -> Result<()> {
let mut cmd = lifecycle::git_command(); let out = lifecycle::git_command()
cmd.current_dir(dir).args(args); .current_dir(dir)
let label = format!("git {} in {}", args.join(" "), dir.display()); .args(args)
let out = output_with_timeout(cmd, GIT_TIMEOUT, &label).await?; .output()
.await
.with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?;
if !out.status.success() { if !out.status.success() {
anyhow::bail!( anyhow::bail!(
"git {} failed: {}", "git {} failed: {}",

View file

@ -34,15 +34,6 @@ let
[credential "http://${config.services.hyperhive.forge.domain}"] [credential "http://${config.services.hyperhive.forge.domain}"]
helper = hive-forge helper = hive-forge
username = core username = core
[http]
# Abort a stalled fetch instead of blocking startup indefinitely. If a
# git+http transfer (e.g. `nix flake lock` of the forge-hosted config
# inputs) drops below 1 KiB/s for 30s -- forge unreachable / not ready
# on cold boot -- git fails fast rather than hanging the whole daemon.
# Pairs with GIT_TERMINAL_PROMPT=0 (no credential-prompt hang) and the
# migration shellout timeout in migrate.rs.
lowSpeedLimit = 1024
lowSpeedTime = 30
''; '';
# git credential helper for hive-core's authenticated fetches of the # git credential helper for hive-core's authenticated fetches of the

View file

@ -18,15 +18,6 @@ in
# the writable StateDirectory. # the writable StateDirectory.
HOME = "/var/lib/hyperhive"; HOME = "/var/lib/hyperhive";
HYPERHIVE_GIT = "${pkgs.git}/bin/git"; HYPERHIVE_GIT = "${pkgs.git}/bin/git";
# Never let git block on an interactive credential prompt. hive-core is a
# TTY-less system user, so a prompt (e.g. the forge credential helper
# returns nothing because the forge isn't reachable yet on cold boot)
# would hang forever — this is what froze the whole daemon during startup
# migration's `nix flake lock` of the forge-hosted config inputs. With
# this set, git fails fast instead of prompting. Paired with the git http
# low-speed abort in `safeDirGitconfig` (bounds a stalled transfer) and
# the 120s migration shellout timeout in migrate.rs.
GIT_TERMINAL_PROMPT = "0";
# No HIVE_STATIC_DIR: the gateway static-serves the dashboard dist # No HIVE_STATIC_DIR: the gateway static-serves the dashboard dist
# (see the hive-gateway module); this router is API-only. # (see the hive-gateway module); this router is API-only.
# Path to the base agent frontend dist. hive-c0re's # Path to the base agent frontend dist. hive-c0re's