diff --git a/docs/turn-loop/config.md b/docs/turn-loop/config.md index 902b6a12..9dace98b 100644 --- a/docs/turn-loop/config.md +++ b/docs/turn-loop/config.md @@ -202,7 +202,12 @@ services.hyperhive.agent.claudePluginsAutoUpdate = false; # defaul - **`claudePlugins`** — list of plugin specs passed to `claude plugin install `. The harness installs each spec on every boot, assuming `install` is idempotent; failures log a warning - but don't abort boot. Defaults to Anthropic's `skill-creator` (so every + but don't abort boot. Boot waits for the marketplace adds, the optional + update and the installs (serial `claude plugin` calls). The harness kills + and logs any call that hasn't exited within 2 minutes, and caps the whole + sequence at 4 minutes: after that the remaining calls fail at once with + the same warning and the web UI starts. Defaults to Anthropic's + `skill-creator` (so every agent can author, refine, and evaluate its own skills) plus hyperhive's own `base` plugin — skills that apply to every agent regardless of role (currently just `state-hygiene`) — all without diff --git a/hive-agent/src/plugins.rs b/hive-agent/src/plugins.rs index 5a1a3b09..3c544ae7 100644 --- a/hive-agent/src/plugins.rs +++ b/hive-agent/src/plugins.rs @@ -11,18 +11,60 @@ //! plugin specs resolve against current index data. Marketplace update //! failures are non-fatal — stale index is better than no install attempt. +use std::io; +use std::process::Output; +use std::time::{Duration, Instant}; + use tokio::process::Command; const PLUGINS_PATH: &str = "/etc/hyperhive/claude-plugins.json"; const MARKETPLACES_PATH: &str = "/etc/hyperhive/claude-marketplaces.json"; const AUTO_UPDATE_PATH: &str = "/etc/hyperhive/claude-plugins-auto-update.json"; +/// Per-call deadline for `claude plugin …`; a git clone over a slow link fits, a hung fetch does not. +const PLUGIN_CALL_TIMEOUT: Duration = Duration::from_mins(2); + +/// Total budget for the whole serial sequence in [`install_configured`]. The +/// harness awaits it before serving the web UI, so this is the longest a +/// blackholed fetch path can delay boot, however many marketplaces and plugins +/// are configured. +const PLUGIN_BOOT_BUDGET: Duration = Duration::from_mins(4); + +/// The deadline for the next call: [`PLUGIN_CALL_TIMEOUT`], cut to what is +/// left of the boot budget ending at `boot_end` (zero once it is spent). +fn call_budget(now: Instant, boot_end: Instant) -> Duration { + PLUGIN_CALL_TIMEOUT.min(boot_end.saturating_duration_since(now)) +} + +/// Run `claude plugin ` under [`call_budget`]. On timeout the child is +/// killed (`kill_on_drop`) and a `TimedOut` error is returned. Only the +/// `claude` process is killed; a `git` it forked is not. +async fn run_claude_plugin(args: &[&str], boot_end: Instant) -> io::Result { + let mut cmd = Command::new("claude"); + cmd.arg("plugin").args(args); + run_bounded(cmd, call_budget(Instant::now(), boot_end)).await +} + +/// Run `cmd` to completion within `deadline`; the child is killed when the +/// deadline passes. +async fn run_bounded(mut cmd: Command, deadline: Duration) -> io::Result { + let call = cmd.kill_on_drop(true).output(); + tokio::time::timeout(deadline, call) + .await + .unwrap_or_else(|_| { + Err(io::Error::new( + io::ErrorKind::TimedOut, + format!("no exit within {}s", deadline.as_secs_f32()), + )) + }) +} + /// Add every marketplace from `/etc/hyperhive/claude-marketplaces.json` /// via `claude plugin marketplace add `. Idempotent: re-add of /// an existing marketplace is treated as success (claude prints an /// "already exists" message and exits non-zero on some versions). /// Required before any `@` install can resolve. -async fn add_marketplaces() { +async fn add_marketplaces(boot_end: Instant) { let Ok(raw) = tokio::fs::read_to_string(MARKETPLACES_PATH).await else { return; }; @@ -34,11 +76,7 @@ async fn add_marketplaces() { } }; for source in sources { - match Command::new("claude") - .args(["plugin", "marketplace", "add", &source]) - .output() - .await - { + match run_claude_plugin(&["marketplace", "add", &source], boot_end).await { Ok(out) if out.status.success() => { tracing::info!(source = %source, "claude plugin marketplace add ok"); } @@ -56,7 +94,7 @@ async fn add_marketplaces() { } } Err(e) => { - tracing::warn!(source = %source, error = ?e, "claude plugin marketplace add spawn failed"); + tracing::warn!(source = %source, error = ?e, "claude plugin marketplace add spawn failed or timed out"); } } } @@ -73,12 +111,8 @@ async fn auto_update_enabled() -> bool { /// Update all configured plugin marketplaces. Non-fatal — logs a warning /// on failure but does not abort the install sequence. -async fn update_marketplaces() { - match Command::new("claude") - .args(["plugin", "marketplace", "update"]) - .output() - .await - { +async fn update_marketplaces(boot_end: Instant) { + match run_claude_plugin(&["marketplace", "update"], boot_end).await { Ok(out) if out.status.success() => { tracing::info!("claude plugin marketplace update ok"); } @@ -90,7 +124,7 @@ async fn update_marketplaces() { ); } Err(e) => { - tracing::warn!(error = ?e, "claude plugin marketplace update spawn failed (non-fatal)"); + tracing::warn!(error = ?e, "claude plugin marketplace update spawn failed or timed out (non-fatal)"); } } } @@ -111,18 +145,15 @@ pub async fn install_configured() { if specs.is_empty() { return; } - add_marketplaces().await; + let boot_end = Instant::now() + PLUGIN_BOOT_BUDGET; + add_marketplaces(boot_end).await; if auto_update_enabled().await { - update_marketplaces().await; + update_marketplaces(boot_end).await; } else { tracing::debug!("claudePluginsAutoUpdate=false, skipping marketplace update"); } for spec in specs { - match Command::new("claude") - .args(["plugin", "install", &spec]) - .output() - .await - { + match run_claude_plugin(&["install", &spec], boot_end).await { Ok(out) if out.status.success() => { tracing::info!(spec = %spec, "claude plugin install ok"); } @@ -136,8 +167,50 @@ pub async fn install_configured() { ); } Err(e) => { - tracing::warn!(spec = %spec, error = ?e, "claude plugin install spawn failed"); + tracing::warn!(spec = %spec, error = ?e, "claude plugin install spawn failed or timed out"); } } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn run_bounded_times_out_a_hung_child() { + let mut cmd = Command::new("sleep"); + cmd.arg("30"); + let err = run_bounded(cmd, Duration::from_millis(100)) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::TimedOut); + } + + #[tokio::test] + async fn run_bounded_returns_output_of_a_prompt_child() { + let cmd = Command::new("true"); + let out = run_bounded(cmd, Duration::from_secs(5)).await; + assert!(out.unwrap().status.success()); + } + + #[test] + fn serial_calls_never_exceed_the_boot_budget() { + let start = Instant::now(); + let boot_end = start + PLUGIN_BOOT_BUDGET; + let mut now = start; + // Worst case: every call hangs until its own deadline. + for _ in 0..50 { + now += call_budget(now, boot_end); + } + assert_eq!(now - start, PLUGIN_BOOT_BUDGET); + assert_eq!(call_budget(now, boot_end), Duration::ZERO); + } + + #[test] + fn a_call_gets_the_full_timeout_while_budget_remains() { + let start = Instant::now(); + let boot_end = start + PLUGIN_BOOT_BUDGET; + assert_eq!(call_budget(start, boot_end), PLUGIN_CALL_TIMEOUT); + } +}