Compare commits

...
5 changed files with 90 additions and 18 deletions

View file

@ -47,10 +47,17 @@ pub(super) async fn git_commit(dir: &Path, message: &str) -> Result<()> {
/// Spawn `git` honoring the `HYPERHIVE_GIT` env var (absolute path baked in
/// by the NixOS module), falling back to bare `git` (PATH lookup) otherwise.
///
/// `kill_on_drop(true)`: if the caller's future is dropped before the child
/// exits — e.g. a `tokio::time::timeout` around startup migration fires — the
/// git child is killed instead of orphaned (left retrying an unreachable
/// forge). No-op on normal completion, where the child has already exited.
#[must_use]
pub fn git_command() -> Command {
let exe = std::env::var("HYPERHIVE_GIT").unwrap_or_else(|_| "git".into());
Command::new(exe)
let mut cmd = Command::new(exe);
cmd.kill_on_drop(true);
cmd
}
pub async fn git(dir: &Path, args: &[&str]) -> Result<()> {

View file

@ -1391,6 +1391,10 @@ async fn nix_output(dir: &Path, args: &[&str]) -> Result<std::process::Output> {
Command::new("nix")
.current_dir(dir)
.args(nix_argv(args))
// Kill the nix child if the caller's future is dropped (e.g. the
// startup-migration timeout around `sync_agents` fires) rather than
// orphaning it against an unreachable forge. No-op on normal exit.
.kill_on_drop(true)
.output()
.await
.with_context(|| format!("nix {} in {}", args.join(" "), dir.display()))

View file

@ -6,6 +6,7 @@
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result};
use tokio::process::Command;
@ -17,6 +18,18 @@ use crate::tool_groups;
const KILL_SWITCH: &str = "HIVE_SKIP_META_MIGRATION";
/// Per-shellout timeouts for the blocking startup migration. `run` is
/// awaited *before* the daemon starts serving (main.rs), so any child
/// process that wedges here freezes the whole daemon — admin socket +
/// dashboard included — with no diagnostics: a git/container shellout was
/// observed blocked for 86min under a concurrent `nixos-rebuild`. Every
/// shellout now runs under a timeout that kills the child on elapse, so a
/// stuck migration degrades to a logged warning instead of a hung boot.
/// Git ops are quick; `nixos-container update` can legitimately trigger a
/// nix build, so it gets a much longer budget.
const GIT_TIMEOUT: Duration = Duration::from_mins(2);
const CONTAINER_TIMEOUT: Duration = Duration::from_mins(10);
/// Substring that identifies the *current* agent flake boilerplate.
/// Bumped whenever the template changes so the startup migration
/// re-renders existing agents onto the new shape. Today the marker
@ -46,28 +59,40 @@ pub async fn run(coord: &Arc<Coordinator>) -> Result<()> {
// Phase 0: move harness-owned files out of state/ into harness/.
// Idempotent — rename is a no-op if the source doesn't exist and
// the destination already does.
tracing::debug!("migration: phase 0 (harness files)");
for name in &names {
migrate_harness_files(name);
}
// Phase 1 + 2: per-agent applied + proposed.
tracing::debug!("migration: phase 1+2 (applied + proposed repos)");
for name in &names {
tracing::debug!(%name, "migration: applied+proposed");
if let Err(e) = migrate_applied_repo(name).await {
tracing::warn!(%name, error = ?e, "migration: applied repo rewrite failed");
}
if let Err(e) =
lifecycle::setup_proposed(&Coordinator::agent_proposed_dir(name), name).await
{
tracing::warn!(%name, error = ?e, "migration: setup_proposed failed");
let proposed_dir = Coordinator::agent_proposed_dir(name);
let proposed = lifecycle::setup_proposed(&proposed_dir, name);
match tokio::time::timeout(GIT_TIMEOUT, proposed).await {
Ok(Err(e)) => tracing::warn!(%name, error = ?e, "migration: setup_proposed failed"),
Err(_) => {
tracing::warn!(%name, timeout = ?GIT_TIMEOUT, "migration: setup_proposed timed out — skipping");
}
Ok(Ok(())) => {}
}
}
// Phase 3: meta repo.
tracing::debug!("migration: phase 3 (meta sync_agents)");
let agents = lifecycle::agents_for_meta_listing()
.await
.unwrap_or_default();
if let Err(e) = meta::sync_agents(&coord.hive_env(), &agents).await {
tracing::warn!(error = ?e, "migration: meta sync_agents failed");
match tokio::time::timeout(GIT_TIMEOUT, meta::sync_agents(&coord.hive_env(), &agents)).await {
Ok(Err(e)) => tracing::warn!(error = ?e, "migration: meta sync_agents failed"),
Err(_) => {
tracing::warn!(timeout = ?GIT_TIMEOUT, "migration: meta sync_agents timed out — skipping");
}
Ok(Ok(())) => {}
}
// Phase 4: container repoint, guarded by marker.
@ -75,6 +100,7 @@ pub async fn run(coord: &Arc<Coordinator>) -> Result<()> {
tracing::debug!("migration: phase 4 marker present, skipping repoint");
return Ok(());
}
tracing::debug!("migration: phase 4 (container repoint)");
let mut all_ok = true;
for name in &names {
// Mark Rebuilding so the crash watcher skips this container
@ -301,11 +327,14 @@ async fn migrate_applied_repo(name: &str) -> Result<()> {
async fn repoint_container(name: &str) -> Result<()> {
let container = lifecycle::container_name(name);
let flake_ref = format!("{}#{name}", crate::paths::meta_root().display());
let out = Command::new("nixos-container")
.args(["update", &container, "--flake", &flake_ref])
.output()
.await
.with_context(|| format!("nixos-container update {container}"))?;
let mut cmd = Command::new("nixos-container");
cmd.args(["update", &container, "--flake", &flake_ref]);
let out = output_with_timeout(
cmd,
CONTAINER_TIMEOUT,
&format!("nixos-container update {container}"),
)
.await?;
if !out.status.success() {
anyhow::bail!(
"nixos-container update {container} exited {}: {}",
@ -346,13 +375,27 @@ fn backfill_manager_tool_groups(names: &[String]) {
}
}
/// Run a command to completion under a timeout, capturing its output. On
/// timeout the child is killed (`kill_on_drop`) and an error is returned,
/// so a wedged shellout can never freeze startup migration. `what` is a
/// human label surfaced in the timeout error + `with_context`.
async fn output_with_timeout(
mut cmd: Command,
timeout: Duration,
what: &str,
) -> Result<std::process::Output> {
cmd.kill_on_drop(true);
match tokio::time::timeout(timeout, cmd.output()).await {
Ok(r) => r.with_context(|| format!("run {what}")),
Err(_) => anyhow::bail!("{what} timed out after {timeout:?} (child killed)"),
}
}
async fn raw_git(dir: &Path, args: &[&str]) -> Result<()> {
let out = lifecycle::git_command()
.current_dir(dir)
.args(args)
.output()
.await
.with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?;
let mut cmd = lifecycle::git_command();
cmd.current_dir(dir).args(args);
let label = format!("git {} in {}", args.join(" "), dir.display());
let out = output_with_timeout(cmd, GIT_TIMEOUT, &label).await?;
if !out.status.success() {
anyhow::bail!(
"git {} failed: {}",

View file

@ -34,6 +34,15 @@ let
[credential "http://${config.services.hyperhive.forge.domain}"]
helper = hive-forge
username = core
[http]
# Abort a stalled fetch instead of blocking startup indefinitely. If a
# git+http transfer (e.g. `nix flake lock` of the forge-hosted config
# inputs) drops below 1 KiB/s for 30s -- forge unreachable / not ready
# on cold boot -- git fails fast rather than hanging the whole daemon.
# Pairs with GIT_TERMINAL_PROMPT=0 (no credential-prompt hang) and the
# migration shellout timeout in migrate.rs.
lowSpeedLimit = 1024
lowSpeedTime = 30
'';
# git credential helper for hive-core's authenticated fetches of the

View file

@ -18,6 +18,15 @@ in
# the writable StateDirectory.
HOME = "/var/lib/hyperhive";
HYPERHIVE_GIT = "${pkgs.git}/bin/git";
# Never let git block on an interactive credential prompt. hive-core is a
# TTY-less system user, so a prompt (e.g. the forge credential helper
# returns nothing because the forge isn't reachable yet on cold boot)
# would hang forever — this is what froze the whole daemon during startup
# migration's `nix flake lock` of the forge-hosted config inputs. With
# this set, git fails fast instead of prompting. Paired with the git http
# low-speed abort in `safeDirGitconfig` (bounds a stalled transfer) and
# the 120s migration shellout timeout in migrate.rs.
GIT_TERMINAL_PROMPT = "0";
# No HIVE_STATIC_DIR: the gateway static-serves the dashboard dist
# (see the hive-gateway module); this router is API-only.
# Path to the base agent frontend dist. hive-c0re's