Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6bf839cfe4 | ||
|
|
40cf7f29de | ||
|
|
618d774467 | ||
|
|
d00b349102 |
5 changed files with 90 additions and 18 deletions
|
|
@ -47,10 +47,17 @@ pub(super) async fn git_commit(dir: &Path, message: &str) -> Result<()> {
|
|||
|
||||
/// Spawn `git` honoring the `HYPERHIVE_GIT` env var (absolute path baked in
|
||||
/// by the NixOS module), falling back to bare `git` (PATH lookup) otherwise.
|
||||
///
|
||||
/// `kill_on_drop(true)`: if the caller's future is dropped before the child
|
||||
/// exits — e.g. a `tokio::time::timeout` around startup migration fires — the
|
||||
/// git child is killed instead of orphaned (left retrying an unreachable
|
||||
/// forge). No-op on normal completion, where the child has already exited.
|
||||
#[must_use]
|
||||
pub fn git_command() -> Command {
|
||||
let exe = std::env::var("HYPERHIVE_GIT").unwrap_or_else(|_| "git".into());
|
||||
Command::new(exe)
|
||||
let mut cmd = Command::new(exe);
|
||||
cmd.kill_on_drop(true);
|
||||
cmd
|
||||
}
|
||||
|
||||
pub async fn git(dir: &Path, args: &[&str]) -> Result<()> {
|
||||
|
|
|
|||
|
|
@ -1391,6 +1391,10 @@ async fn nix_output(dir: &Path, args: &[&str]) -> Result<std::process::Output> {
|
|||
Command::new("nix")
|
||||
.current_dir(dir)
|
||||
.args(nix_argv(args))
|
||||
// Kill the nix child if the caller's future is dropped (e.g. the
|
||||
// startup-migration timeout around `sync_agents` fires) rather than
|
||||
// orphaning it against an unreachable forge. No-op on normal exit.
|
||||
.kill_on_drop(true)
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("nix {} in {}", args.join(" "), dir.display()))
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use tokio::process::Command;
|
||||
|
|
@ -17,6 +18,18 @@ use crate::tool_groups;
|
|||
|
||||
const KILL_SWITCH: &str = "HIVE_SKIP_META_MIGRATION";
|
||||
|
||||
/// Per-shellout timeouts for the blocking startup migration. `run` is
|
||||
/// awaited *before* the daemon starts serving (main.rs), so any child
|
||||
/// process that wedges here freezes the whole daemon — admin socket +
|
||||
/// dashboard included — with no diagnostics: a git/container shellout was
|
||||
/// observed blocked for 86min under a concurrent `nixos-rebuild`. Every
|
||||
/// shellout now runs under a timeout that kills the child on elapse, so a
|
||||
/// stuck migration degrades to a logged warning instead of a hung boot.
|
||||
/// Git ops are quick; `nixos-container update` can legitimately trigger a
|
||||
/// nix build, so it gets a much longer budget.
|
||||
const GIT_TIMEOUT: Duration = Duration::from_mins(2);
|
||||
const CONTAINER_TIMEOUT: Duration = Duration::from_mins(10);
|
||||
|
||||
/// Substring that identifies the *current* agent flake boilerplate.
|
||||
/// Bumped whenever the template changes so the startup migration
|
||||
/// re-renders existing agents onto the new shape. Today the marker
|
||||
|
|
@ -46,28 +59,40 @@ pub async fn run(coord: &Arc<Coordinator>) -> Result<()> {
|
|||
// Phase 0: move harness-owned files out of state/ into harness/.
|
||||
// Idempotent — rename is a no-op if the source doesn't exist and
|
||||
// the destination already does.
|
||||
tracing::debug!("migration: phase 0 (harness files)");
|
||||
for name in &names {
|
||||
migrate_harness_files(name);
|
||||
}
|
||||
|
||||
// Phase 1 + 2: per-agent applied + proposed.
|
||||
tracing::debug!("migration: phase 1+2 (applied + proposed repos)");
|
||||
for name in &names {
|
||||
tracing::debug!(%name, "migration: applied+proposed");
|
||||
if let Err(e) = migrate_applied_repo(name).await {
|
||||
tracing::warn!(%name, error = ?e, "migration: applied repo rewrite failed");
|
||||
}
|
||||
if let Err(e) =
|
||||
lifecycle::setup_proposed(&Coordinator::agent_proposed_dir(name), name).await
|
||||
{
|
||||
tracing::warn!(%name, error = ?e, "migration: setup_proposed failed");
|
||||
let proposed_dir = Coordinator::agent_proposed_dir(name);
|
||||
let proposed = lifecycle::setup_proposed(&proposed_dir, name);
|
||||
match tokio::time::timeout(GIT_TIMEOUT, proposed).await {
|
||||
Ok(Err(e)) => tracing::warn!(%name, error = ?e, "migration: setup_proposed failed"),
|
||||
Err(_) => {
|
||||
tracing::warn!(%name, timeout = ?GIT_TIMEOUT, "migration: setup_proposed timed out — skipping");
|
||||
}
|
||||
Ok(Ok(())) => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 3: meta repo.
|
||||
tracing::debug!("migration: phase 3 (meta sync_agents)");
|
||||
let agents = lifecycle::agents_for_meta_listing()
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
if let Err(e) = meta::sync_agents(&coord.hive_env(), &agents).await {
|
||||
tracing::warn!(error = ?e, "migration: meta sync_agents failed");
|
||||
match tokio::time::timeout(GIT_TIMEOUT, meta::sync_agents(&coord.hive_env(), &agents)).await {
|
||||
Ok(Err(e)) => tracing::warn!(error = ?e, "migration: meta sync_agents failed"),
|
||||
Err(_) => {
|
||||
tracing::warn!(timeout = ?GIT_TIMEOUT, "migration: meta sync_agents timed out — skipping");
|
||||
}
|
||||
Ok(Ok(())) => {}
|
||||
}
|
||||
|
||||
// Phase 4: container repoint, guarded by marker.
|
||||
|
|
@ -75,6 +100,7 @@ pub async fn run(coord: &Arc<Coordinator>) -> Result<()> {
|
|||
tracing::debug!("migration: phase 4 marker present, skipping repoint");
|
||||
return Ok(());
|
||||
}
|
||||
tracing::debug!("migration: phase 4 (container repoint)");
|
||||
let mut all_ok = true;
|
||||
for name in &names {
|
||||
// Mark Rebuilding so the crash watcher skips this container
|
||||
|
|
@ -301,11 +327,14 @@ async fn migrate_applied_repo(name: &str) -> Result<()> {
|
|||
async fn repoint_container(name: &str) -> Result<()> {
|
||||
let container = lifecycle::container_name(name);
|
||||
let flake_ref = format!("{}#{name}", crate::paths::meta_root().display());
|
||||
let out = Command::new("nixos-container")
|
||||
.args(["update", &container, "--flake", &flake_ref])
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("nixos-container update {container}"))?;
|
||||
let mut cmd = Command::new("nixos-container");
|
||||
cmd.args(["update", &container, "--flake", &flake_ref]);
|
||||
let out = output_with_timeout(
|
||||
cmd,
|
||||
CONTAINER_TIMEOUT,
|
||||
&format!("nixos-container update {container}"),
|
||||
)
|
||||
.await?;
|
||||
if !out.status.success() {
|
||||
anyhow::bail!(
|
||||
"nixos-container update {container} exited {}: {}",
|
||||
|
|
@ -346,13 +375,27 @@ fn backfill_manager_tool_groups(names: &[String]) {
|
|||
}
|
||||
}
|
||||
|
||||
/// Run a command to completion under a timeout, capturing its output. On
|
||||
/// timeout the child is killed (`kill_on_drop`) and an error is returned,
|
||||
/// so a wedged shellout can never freeze startup migration. `what` is a
|
||||
/// human label surfaced in the timeout error + `with_context`.
|
||||
async fn output_with_timeout(
|
||||
mut cmd: Command,
|
||||
timeout: Duration,
|
||||
what: &str,
|
||||
) -> Result<std::process::Output> {
|
||||
cmd.kill_on_drop(true);
|
||||
match tokio::time::timeout(timeout, cmd.output()).await {
|
||||
Ok(r) => r.with_context(|| format!("run {what}")),
|
||||
Err(_) => anyhow::bail!("{what} timed out after {timeout:?} (child killed)"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn raw_git(dir: &Path, args: &[&str]) -> Result<()> {
|
||||
let out = lifecycle::git_command()
|
||||
.current_dir(dir)
|
||||
.args(args)
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?;
|
||||
let mut cmd = lifecycle::git_command();
|
||||
cmd.current_dir(dir).args(args);
|
||||
let label = format!("git {} in {}", args.join(" "), dir.display());
|
||||
let out = output_with_timeout(cmd, GIT_TIMEOUT, &label).await?;
|
||||
if !out.status.success() {
|
||||
anyhow::bail!(
|
||||
"git {} failed: {}",
|
||||
|
|
|
|||
|
|
@ -34,6 +34,15 @@ let
|
|||
[credential "http://${config.services.hyperhive.forge.domain}"]
|
||||
helper = hive-forge
|
||||
username = core
|
||||
[http]
|
||||
# Abort a stalled fetch instead of blocking startup indefinitely. If a
|
||||
# git+http transfer (e.g. `nix flake lock` of the forge-hosted config
|
||||
# inputs) drops below 1 KiB/s for 30s -- forge unreachable / not ready
|
||||
# on cold boot -- git fails fast rather than hanging the whole daemon.
|
||||
# Pairs with GIT_TERMINAL_PROMPT=0 (no credential-prompt hang) and the
|
||||
# migration shellout timeout in migrate.rs.
|
||||
lowSpeedLimit = 1024
|
||||
lowSpeedTime = 30
|
||||
'';
|
||||
|
||||
# git credential helper for hive-core's authenticated fetches of the
|
||||
|
|
|
|||
|
|
@ -18,6 +18,15 @@ in
|
|||
# the writable StateDirectory.
|
||||
HOME = "/var/lib/hyperhive";
|
||||
HYPERHIVE_GIT = "${pkgs.git}/bin/git";
|
||||
# Never let git block on an interactive credential prompt. hive-core is a
|
||||
# TTY-less system user, so a prompt (e.g. the forge credential helper
|
||||
# returns nothing because the forge isn't reachable yet on cold boot)
|
||||
# would hang forever — this is what froze the whole daemon during startup
|
||||
# migration's `nix flake lock` of the forge-hosted config inputs. With
|
||||
# this set, git fails fast instead of prompting. Paired with the git http
|
||||
# low-speed abort in `safeDirGitconfig` (bounds a stalled transfer) and
|
||||
# the 120s migration shellout timeout in migrate.rs.
|
||||
GIT_TERMINAL_PROMPT = "0";
|
||||
# No HIVE_STATIC_DIR: the gateway static-serves the dashboard dist
|
||||
# (see the hive-gateway module); this router is API-only.
|
||||
# Path to the base agent frontend dist. hive-c0re's
|
||||
|
|
|
|||
Loading…
Reference in a new issue