meta nix/git calls under META_LOCK time out instead of wedging every deploy
A hung forge fetch inside `nix flake update hyperhive` (or any other meta git/nix child) now fails after its deadline with an error naming the command, the child is killed, and META_LOCK, BuildSlot and MetaWindow are released. The boot sweep already treats that failure as a warning and fans out the rebuilds. A wait on META_LOCK past 60s logs a warning naming the waiter and the holder.
This commit is contained in:
atlas
mara
parent
8d70afd93e
commit
7cde4e7b15
6 changed files with 270 additions and 58 deletions
|
|
@ -5,10 +5,13 @@
|
|||
//! `docs/agent-lifecycle/config-changes.md::Meta flake`.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Mutex as StdMutex, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use tokio::process::Command;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::{Mutex, MutexGuard};
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::coordinator::HiveEnv;
|
||||
use crate::lifecycle;
|
||||
|
|
@ -27,6 +30,73 @@ const GIT_EMAIL: &str = "c0re@hyperhive.local";
|
|||
/// take turns instead of colliding.
|
||||
static META_LOCK: Mutex<()> = Mutex::const_new(());
|
||||
|
||||
/// Name of the operation holding [`META_LOCK`], empty when free. Read only to
|
||||
/// name the holder in a stalled-waiter warning.
|
||||
static META_HOLDER: StdMutex<&'static str> = StdMutex::new("");
|
||||
|
||||
/// A wait on [`META_LOCK`] longer than this logs a warning, then again every
|
||||
/// interval, naming the waiter and the holder.
|
||||
const LOCK_WAIT_WARN: Duration = Duration::from_mins(1);
|
||||
|
||||
/// [`META_LOCK`] guard that records its holder in [`META_HOLDER`].
|
||||
struct MetaGuard<'a> {
|
||||
_lock: MutexGuard<'a, ()>,
|
||||
holder: &'a StdMutex<&'static str>,
|
||||
}
|
||||
|
||||
impl Drop for MetaGuard<'_> {
|
||||
// Runs before the field drop releases the mutex, so the next holder's
|
||||
// name is never overwritten by this one's clear.
|
||||
fn drop(&mut self) {
|
||||
*self.holder.lock().unwrap_or_else(PoisonError::into_inner) = "";
|
||||
}
|
||||
}
|
||||
|
||||
/// Take [`META_LOCK`] for `op`, warning while the wait drags on.
|
||||
async fn lock_meta(op: &'static str) -> MetaGuard<'static> {
|
||||
lock_notifying(
|
||||
&META_LOCK,
|
||||
&META_HOLDER,
|
||||
op,
|
||||
LOCK_WAIT_WARN,
|
||||
|holder, waited| {
|
||||
tracing::warn!(
|
||||
op,
|
||||
holder,
|
||||
waited_secs = waited.as_secs(),
|
||||
"meta_lock: still waiting for META_LOCK"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Take `lock`, calling `on_stall(holder, waited)` each time `interval` passes
|
||||
/// without getting it.
|
||||
async fn lock_notifying<'a>(
|
||||
lock: &'a Mutex<()>,
|
||||
holder: &'a StdMutex<&'static str>,
|
||||
op: &'static str,
|
||||
interval: Duration,
|
||||
mut on_stall: impl FnMut(&'static str, Duration),
|
||||
) -> MetaGuard<'a> {
|
||||
let start = Instant::now();
|
||||
let guard = loop {
|
||||
match tokio::time::timeout(interval, lock.lock()).await {
|
||||
Ok(guard) => break guard,
|
||||
Err(_) => on_stall(
|
||||
*holder.lock().unwrap_or_else(PoisonError::into_inner),
|
||||
start.elapsed(),
|
||||
),
|
||||
}
|
||||
};
|
||||
*holder.lock().unwrap_or_else(PoisonError::into_inner) = op;
|
||||
MetaGuard {
|
||||
_lock: guard,
|
||||
holder,
|
||||
}
|
||||
}
|
||||
|
||||
// Exclusivity for meta-repo *windows* that span multiple `META_LOCK`
|
||||
// acquisitions is **not** a mutex in this module. `META_LOCK` above
|
||||
// serializes individual git ops but cannot keep another op out of a window
|
||||
|
|
@ -80,7 +150,7 @@ async fn stage_generated_meta_files(dir: &std::path::Path) -> Result<()> {
|
|||
/// rendered contents differ from disk; an unchanged `flake.nix` is a
|
||||
/// no-op.
|
||||
pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
|
||||
let _guard = META_LOCK.lock().await;
|
||||
let _guard = lock_meta("sync_agents").await;
|
||||
// Before anything is written: a hive without a forge URL would deploy a
|
||||
// whole fleet of agents that silently never log in, because the agent
|
||||
// option treats "unset" as "no forge configured" rather than erroring.
|
||||
|
|
@ -215,7 +285,7 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
|
|||
/// (no separate finalize) because rebuild has no failure-revert
|
||||
/// semantics. The input follows the url `sync_agents` rendered just before.
|
||||
pub async fn lock_update_for_rebuild(name: &str) -> Result<()> {
|
||||
let _guard = META_LOCK.lock().await;
|
||||
let _guard = lock_meta("lock_update_for_rebuild").await;
|
||||
let dir = crate::paths::meta_root();
|
||||
let input = format!("agent-{name}");
|
||||
nix(&dir, &rebuild_relock_args(&input)).await?;
|
||||
|
|
@ -249,7 +319,7 @@ fn rebuild_relock_args(input: &str) -> [&str; 3] {
|
|||
/// real input keys (e.g. via `inputs_view()` snapshotted from the lock
|
||||
/// file) when targeting specific inputs.
|
||||
pub async fn lock_update(inputs: &[String]) -> Result<()> {
|
||||
let _guard = META_LOCK.lock().await;
|
||||
let _guard = lock_meta("lock_update").await;
|
||||
let dir = crate::paths::meta_root();
|
||||
let mut args: Vec<&str> = vec!["flake", "update"];
|
||||
for i in inputs {
|
||||
|
|
@ -274,7 +344,7 @@ pub async fn lock_update(inputs: &[String]) -> Result<()> {
|
|||
/// rev, commit if the lock changed. Cheaper than `sync_agents`
|
||||
/// because the per-agent inputs aren't touched.
|
||||
pub async fn lock_update_hyperhive() -> Result<()> {
|
||||
let _guard = META_LOCK.lock().await;
|
||||
let _guard = lock_meta("lock_update_hyperhive").await;
|
||||
let dir = crate::paths::meta_root();
|
||||
nix(&dir, &["flake", "update", "hyperhive"]).await?;
|
||||
if !paths_dirty(&dir, &["flake.lock"]).await? {
|
||||
|
|
@ -292,7 +362,7 @@ pub async fn lock_update_hyperhive() -> Result<()> {
|
|||
///
|
||||
/// Removing a file or a git stage/commit step fails.
|
||||
pub async fn remove_permission_files() -> Result<()> {
|
||||
let _guard = META_LOCK.lock().await;
|
||||
let _guard = lock_meta("remove_permission_files").await;
|
||||
let dir = crate::paths::meta_root();
|
||||
let mut removed = Vec::new();
|
||||
for file in crate::workers::permissions_import::META_FILES {
|
||||
|
|
@ -924,12 +994,11 @@ fn render_flake(
|
|||
async fn git_staged_names(dir: &Path) -> Result<Vec<String>> {
|
||||
// `--diff-filter=ACM` skips deleted entries — we only care about
|
||||
// additions and modifications for message-building purposes.
|
||||
let out = lifecycle::git_command()
|
||||
.current_dir(dir)
|
||||
.args(["diff", "--cached", "--name-only", "--diff-filter=ACM"])
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("git diff --cached in {}", dir.display()))?;
|
||||
let out = git_output(
|
||||
dir,
|
||||
&["diff", "--cached", "--name-only", "--diff-filter=ACM"],
|
||||
)
|
||||
.await?;
|
||||
if out.status.success() {
|
||||
let names = String::from_utf8_lossy(&out.stdout)
|
||||
.lines()
|
||||
|
|
@ -940,12 +1009,7 @@ async fn git_staged_names(dir: &Path) -> Result<Vec<String>> {
|
|||
return Ok(names);
|
||||
}
|
||||
// Fallback for initial commit (no HEAD yet): parse `git status --porcelain`.
|
||||
let st = lifecycle::git_command()
|
||||
.current_dir(dir)
|
||||
.args(["status", "--porcelain"])
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("git status in {}", dir.display()))?;
|
||||
let st = git_output(dir, &["status", "--porcelain"]).await?;
|
||||
let names = String::from_utf8_lossy(&st.stdout)
|
||||
.lines()
|
||||
.filter(|l| l.starts_with("A ") || l.starts_with("M "))
|
||||
|
|
@ -955,13 +1019,19 @@ async fn git_staged_names(dir: &Path) -> Result<Vec<String>> {
|
|||
Ok(names)
|
||||
}
|
||||
|
||||
/// Run `git <args>` in `dir` within [`lifecycle::GIT_LOCAL_DEADLINE`]; the caller
|
||||
/// reads the exit status.
|
||||
async fn git_output(dir: &Path, args: &[&str]) -> Result<std::process::Output> {
|
||||
lifecycle::run_with_deadline(
|
||||
lifecycle::git_command().current_dir(dir).args(args),
|
||||
lifecycle::GIT_LOCAL_DEADLINE,
|
||||
&format!("git {} in {}", args.join(" "), dir.display()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn git(dir: &Path, args: &[&str]) -> Result<()> {
|
||||
let out = lifecycle::git_command()
|
||||
.current_dir(dir)
|
||||
.args(args)
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?;
|
||||
let out = git_output(dir, args).await?;
|
||||
if !out.status.success() {
|
||||
bail!(
|
||||
"git {} failed ({}): {}",
|
||||
|
|
@ -1021,12 +1091,7 @@ async fn git_commit_paths(dir: &Path, message: &str, paths: &[&str]) -> Result<(
|
|||
async fn paths_dirty(dir: &Path, paths: &[&str]) -> Result<bool> {
|
||||
let mut args = vec!["diff", "--quiet", "HEAD", "--"];
|
||||
args.extend_from_slice(paths);
|
||||
let out = lifecycle::git_command()
|
||||
.current_dir(dir)
|
||||
.args(&args)
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("git diff --quiet in {}", dir.display()))?;
|
||||
let out = git_output(dir, &args).await?;
|
||||
// Exit 0 = no differences; 1 = differences; anything else (e.g.
|
||||
// no HEAD yet on a fresh repo) → treat as dirty so the commit
|
||||
// path runs and surfaces real errors loudly.
|
||||
|
|
@ -1046,17 +1111,28 @@ fn nix_argv<'a>(args: &[&'a str]) -> Vec<&'a str> {
|
|||
|
||||
/// Run `nix <args>` in `dir` (flakes enabled), capturing combined output.
|
||||
/// The core of [`nix`].
|
||||
///
|
||||
/// Every call site in this module is `flake lock` or `flake update`, which
|
||||
/// fetch inputs from the forge, so the whole function runs under
|
||||
/// [`lifecycle::FETCH_DEADLINE`]. A call that builds or evaluates a system
|
||||
/// needs its own bound, not this one.
|
||||
async fn nix_output(dir: &Path, args: &[&str]) -> Result<std::process::Output> {
|
||||
Command::new("nix")
|
||||
.current_dir(dir)
|
||||
.args(nix_argv(args))
|
||||
// Kill the nix child if the caller's future is dropped (e.g. the
|
||||
// startup-migration timeout around `sync_agents` fires) rather than
|
||||
// orphaning it against an unreachable forge. No-op on normal exit.
|
||||
.kill_on_drop(true)
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("nix {} in {}", args.join(" "), dir.display()))
|
||||
nix_output_with("nix".as_ref(), dir, args, lifecycle::FETCH_DEADLINE).await
|
||||
}
|
||||
|
||||
/// [`nix_output`] with the program and deadline injectable for tests.
|
||||
async fn nix_output_with(
|
||||
program: &std::ffi::OsStr,
|
||||
dir: &Path,
|
||||
args: &[&str],
|
||||
deadline: Duration,
|
||||
) -> Result<std::process::Output> {
|
||||
lifecycle::run_with_deadline(
|
||||
Command::new(program).current_dir(dir).args(nix_argv(args)),
|
||||
deadline,
|
||||
&format!("nix {} in {}", args.join(" "), dir.display()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Turn a finished nix [`Output`](std::process::Output) into `Result<()>`,
|
||||
|
|
@ -2092,3 +2168,61 @@ mod tests {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod lock_wait_tests {
|
||||
use super::*;
|
||||
|
||||
/// A waiter behind a held lock is told, per interval, who holds it; once
|
||||
/// the holder drops, it gets the lock and becomes the recorded holder.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn stalled_waiter_reports_holder_then_acquires() {
|
||||
let lock = Mutex::new(());
|
||||
let holder = StdMutex::new("");
|
||||
let first = lock_notifying(&lock, &holder, "first", Duration::from_mins(1), |_, _| {
|
||||
panic!("an uncontended lock must not report a stall")
|
||||
})
|
||||
.await;
|
||||
assert_eq!(*holder.lock().unwrap(), "first");
|
||||
|
||||
let stalls = std::cell::RefCell::new(Vec::new());
|
||||
let waiter = lock_notifying(&lock, &holder, "second", Duration::from_mins(1), |h, w| {
|
||||
stalls.borrow_mut().push((h, w.as_secs()));
|
||||
});
|
||||
tokio::pin!(waiter);
|
||||
// 150s of virtual time: two intervals elapse while `first` is held.
|
||||
tokio::select! {
|
||||
_ = &mut waiter => panic!("lock is held"),
|
||||
() = tokio::time::sleep(Duration::from_secs(150)) => {}
|
||||
}
|
||||
drop(first);
|
||||
assert_eq!(*holder.lock().unwrap(), "");
|
||||
let second = waiter.await;
|
||||
assert_eq!(*holder.lock().unwrap(), "second");
|
||||
drop(second);
|
||||
assert_eq!(*stalls.borrow(), vec![("first", 60), ("first", 120)]);
|
||||
}
|
||||
/// A hung nix child comes back as a timeout error naming the command. The
|
||||
/// deadline and program are injected; the production const is untouched.
|
||||
#[tokio::test]
|
||||
async fn nix_output_times_out_naming_the_command() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let fake = dir.path().join("nix");
|
||||
std::fs::write(&fake, "#!/bin/sh\nexec sleep 30\n").expect("write");
|
||||
std::fs::set_permissions(&fake, std::os::unix::fs::PermissionsExt::from_mode(0o755))
|
||||
.expect("chmod");
|
||||
let err = nix_output_with(
|
||||
fake.as_os_str(),
|
||||
dir.path(),
|
||||
&["flake", "update", "hyperhive"],
|
||||
Duration::from_millis(300),
|
||||
)
|
||||
.await
|
||||
.expect_err("hung nix must time out");
|
||||
let msg = format!("{err:#}");
|
||||
assert!(
|
||||
msg.contains("nix flake update hyperhive in") && msg.contains("timed out after 0.3s"),
|
||||
"{msg}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue