From 8be90ae8f2aebe5fdda98667d53fbd3c1942f454 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 16 Oct 2026 09:28:09 +0200 Subject: [PATCH 1/3] hive-agent: bound each claude plugin call with a deadline The claude plugin marketplace add/update and install calls fetch over the network and are awaited before the web UI is spawned. One hung fetch stopped the agent from booting at all, against the module's own promise that plugin failures never block boot. Each call now runs under a 2 minute deadline; on expiry the child is killed, a warning is logged and boot continues, as for any other plugin failure. --- hive-agent/src/plugins.rs | 75 +++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/hive-agent/src/plugins.rs b/hive-agent/src/plugins.rs index 5a1a3b09..20e015c2 100644 --- a/hive-agent/src/plugins.rs +++ b/hive-agent/src/plugins.rs @@ -11,12 +11,41 @@ //! plugin specs resolve against current index data. Marketplace update //! failures are non-fatal — stale index is better than no install attempt. +use std::io; +use std::process::Output; +use std::time::Duration; + use tokio::process::Command; const PLUGINS_PATH: &str = "/etc/hyperhive/claude-plugins.json"; const MARKETPLACES_PATH: &str = "/etc/hyperhive/claude-marketplaces.json"; const AUTO_UPDATE_PATH: &str = "/etc/hyperhive/claude-plugins-auto-update.json"; +/// Per-call deadline for `claude plugin …`; a git clone over a slow link fits, a hung fetch does not. +const PLUGIN_CALL_TIMEOUT: Duration = Duration::from_mins(2); + +/// Run `claude plugin ` under [`PLUGIN_CALL_TIMEOUT`]. On timeout the +/// child is killed (`kill_on_drop`) and a `TimedOut` error is returned. +async fn run_claude_plugin(args: &[&str]) -> io::Result { + let mut cmd = Command::new("claude"); + cmd.arg("plugin").args(args); + run_bounded(cmd, PLUGIN_CALL_TIMEOUT).await +} + +/// Run `cmd` to completion within `deadline`; the child is killed when the +/// deadline passes. +async fn run_bounded(mut cmd: Command, deadline: Duration) -> io::Result { + let call = cmd.kill_on_drop(true).output(); + tokio::time::timeout(deadline, call) + .await + .unwrap_or_else(|_| { + Err(io::Error::new( + io::ErrorKind::TimedOut, + format!("no exit within {}s", deadline.as_secs_f32()), + )) + }) +} + /// Add every marketplace from `/etc/hyperhive/claude-marketplaces.json` /// via `claude plugin marketplace add `. Idempotent: re-add of /// an existing marketplace is treated as success (claude prints an @@ -34,11 +63,7 @@ async fn add_marketplaces() { } }; for source in sources { - match Command::new("claude") - .args(["plugin", "marketplace", "add", &source]) - .output() - .await - { + match run_claude_plugin(&["marketplace", "add", &source]).await { Ok(out) if out.status.success() => { tracing::info!(source = %source, "claude plugin marketplace add ok"); } @@ -56,7 +81,7 @@ async fn add_marketplaces() { } } Err(e) => { - tracing::warn!(source = %source, error = ?e, "claude plugin marketplace add spawn failed"); + tracing::warn!(source = %source, error = ?e, "claude plugin marketplace add spawn failed or timed out"); } } } @@ -74,11 +99,7 @@ async fn auto_update_enabled() -> bool { /// Update all configured plugin marketplaces. Non-fatal — logs a warning /// on failure but does not abort the install sequence. async fn update_marketplaces() { - match Command::new("claude") - .args(["plugin", "marketplace", "update"]) - .output() - .await - { + match run_claude_plugin(&["marketplace", "update"]).await { Ok(out) if out.status.success() => { tracing::info!("claude plugin marketplace update ok"); } @@ -90,7 +111,7 @@ async fn update_marketplaces() { ); } Err(e) => { - tracing::warn!(error = ?e, "claude plugin marketplace update spawn failed (non-fatal)"); + tracing::warn!(error = ?e, "claude plugin marketplace update spawn failed or timed out (non-fatal)"); } } } @@ -118,11 +139,7 @@ pub async fn install_configured() { tracing::debug!("claudePluginsAutoUpdate=false, skipping marketplace update"); } for spec in specs { - match Command::new("claude") - .args(["plugin", "install", &spec]) - .output() - .await - { + match run_claude_plugin(&["install", &spec]).await { Ok(out) if out.status.success() => { tracing::info!(spec = %spec, "claude plugin install ok"); } @@ -136,8 +153,30 @@ pub async fn install_configured() { ); } Err(e) => { - tracing::warn!(spec = %spec, error = ?e, "claude plugin install spawn failed"); + tracing::warn!(spec = %spec, error = ?e, "claude plugin install spawn failed or timed out"); } } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn run_bounded_times_out_a_hung_child() { + let mut cmd = Command::new("sleep"); + cmd.arg("30"); + let err = run_bounded(cmd, Duration::from_millis(100)) + .await + .unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::TimedOut); + } + + #[tokio::test] + async fn run_bounded_returns_output_of_a_prompt_child() { + let cmd = Command::new("true"); + let out = run_bounded(cmd, Duration::from_secs(5)).await; + assert!(out.unwrap().status.success()); + } +} From aa6ca8764af64d33236512df2ab146e3ea3109f4 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 16 Oct 2026 09:58:04 +0200 Subject: [PATCH 2/3] hive-agent: cap the whole plugin install sequence, not just each call The calls run serially before the web UI is served, so a per-call deadline alone left N x 2 min of boot delay (8 min with the default two marketplaces and two plugins, 10 with auto-update) when the fetch path is blackholed. One shared 4 minute budget now bounds the sequence: each call gets min(2 min, what is left), and calls after the budget is spent fail at once with the same warning. Chosen over spawning the install: boot keeps installing plugins before the first turn, as today, and the diff is one Instant threaded through the three call sites. docs/turn-loop/config.md states the 2 minute and 4 minute bounds. --- docs/turn-loop/config.md | 7 ++++- hive-agent/src/plugins.rs | 58 +++++++++++++++++++++++++++++++-------- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/docs/turn-loop/config.md b/docs/turn-loop/config.md index 902b6a12..abe34166 100644 --- a/docs/turn-loop/config.md +++ b/docs/turn-loop/config.md @@ -202,7 +202,12 @@ services.hyperhive.agent.claudePluginsAutoUpdate = false; # defaul - **`claudePlugins`** — list of plugin specs passed to `claude plugin install `. The harness installs each spec on every boot, assuming `install` is idempotent; failures log a warning - but don't abort boot. Defaults to Anthropic's `skill-creator` (so every + but don't abort boot. Boot waits for the + marketplace adds, the optional update and the installs (serial + `claude plugin` calls): each call is killed and logged if it has not + exited within 2 minutes, and the whole sequence is capped at 4 minutes, + after which the remaining calls fail at once with the same warning and the + web UI starts. Defaults to Anthropic's `skill-creator` (so every agent can author, refine, and evaluate its own skills) plus hyperhive's own `base` plugin — skills that apply to every agent regardless of role (currently just `state-hygiene`) — all without diff --git a/hive-agent/src/plugins.rs b/hive-agent/src/plugins.rs index 20e015c2..3c544ae7 100644 --- a/hive-agent/src/plugins.rs +++ b/hive-agent/src/plugins.rs @@ -13,7 +13,7 @@ use std::io; use std::process::Output; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::process::Command; @@ -24,12 +24,25 @@ const AUTO_UPDATE_PATH: &str = "/etc/hyperhive/claude-plugins-auto-update.json"; /// Per-call deadline for `claude plugin …`; a git clone over a slow link fits, a hung fetch does not. const PLUGIN_CALL_TIMEOUT: Duration = Duration::from_mins(2); -/// Run `claude plugin ` under [`PLUGIN_CALL_TIMEOUT`]. On timeout the -/// child is killed (`kill_on_drop`) and a `TimedOut` error is returned. -async fn run_claude_plugin(args: &[&str]) -> io::Result { +/// Total budget for the whole serial sequence in [`install_configured`]. The +/// harness awaits it before serving the web UI, so this is the longest a +/// blackholed fetch path can delay boot, however many marketplaces and plugins +/// are configured. +const PLUGIN_BOOT_BUDGET: Duration = Duration::from_mins(4); + +/// The deadline for the next call: [`PLUGIN_CALL_TIMEOUT`], cut to what is +/// left of the boot budget ending at `boot_end` (zero once it is spent). +fn call_budget(now: Instant, boot_end: Instant) -> Duration { + PLUGIN_CALL_TIMEOUT.min(boot_end.saturating_duration_since(now)) +} + +/// Run `claude plugin ` under [`call_budget`]. On timeout the child is +/// killed (`kill_on_drop`) and a `TimedOut` error is returned. Only the +/// `claude` process is killed; a `git` it forked is not. +async fn run_claude_plugin(args: &[&str], boot_end: Instant) -> io::Result { let mut cmd = Command::new("claude"); cmd.arg("plugin").args(args); - run_bounded(cmd, PLUGIN_CALL_TIMEOUT).await + run_bounded(cmd, call_budget(Instant::now(), boot_end)).await } /// Run `cmd` to completion within `deadline`; the child is killed when the @@ -51,7 +64,7 @@ async fn run_bounded(mut cmd: Command, deadline: Duration) -> io::Result /// an existing marketplace is treated as success (claude prints an /// "already exists" message and exits non-zero on some versions). /// Required before any `@` install can resolve. -async fn add_marketplaces() { +async fn add_marketplaces(boot_end: Instant) { let Ok(raw) = tokio::fs::read_to_string(MARKETPLACES_PATH).await else { return; }; @@ -63,7 +76,7 @@ async fn add_marketplaces() { } }; for source in sources { - match run_claude_plugin(&["marketplace", "add", &source]).await { + match run_claude_plugin(&["marketplace", "add", &source], boot_end).await { Ok(out) if out.status.success() => { tracing::info!(source = %source, "claude plugin marketplace add ok"); } @@ -98,8 +111,8 @@ async fn auto_update_enabled() -> bool { /// Update all configured plugin marketplaces. Non-fatal — logs a warning /// on failure but does not abort the install sequence. -async fn update_marketplaces() { - match run_claude_plugin(&["marketplace", "update"]).await { +async fn update_marketplaces(boot_end: Instant) { + match run_claude_plugin(&["marketplace", "update"], boot_end).await { Ok(out) if out.status.success() => { tracing::info!("claude plugin marketplace update ok"); } @@ -132,14 +145,15 @@ pub async fn install_configured() { if specs.is_empty() { return; } - add_marketplaces().await; + let boot_end = Instant::now() + PLUGIN_BOOT_BUDGET; + add_marketplaces(boot_end).await; if auto_update_enabled().await { - update_marketplaces().await; + update_marketplaces(boot_end).await; } else { tracing::debug!("claudePluginsAutoUpdate=false, skipping marketplace update"); } for spec in specs { - match run_claude_plugin(&["install", &spec]).await { + match run_claude_plugin(&["install", &spec], boot_end).await { Ok(out) if out.status.success() => { tracing::info!(spec = %spec, "claude plugin install ok"); } @@ -179,4 +193,24 @@ mod tests { let out = run_bounded(cmd, Duration::from_secs(5)).await; assert!(out.unwrap().status.success()); } + + #[test] + fn serial_calls_never_exceed_the_boot_budget() { + let start = Instant::now(); + let boot_end = start + PLUGIN_BOOT_BUDGET; + let mut now = start; + // Worst case: every call hangs until its own deadline. + for _ in 0..50 { + now += call_budget(now, boot_end); + } + assert_eq!(now - start, PLUGIN_BOOT_BUDGET); + assert_eq!(call_budget(now, boot_end), Duration::ZERO); + } + + #[test] + fn a_call_gets_the_full_timeout_while_budget_remains() { + let start = Instant::now(); + let boot_end = start + PLUGIN_BOOT_BUDGET; + assert_eq!(call_budget(start, boot_end), PLUGIN_CALL_TIMEOUT); + } } From d67c86677bc49670f4e1b192ca46d870f69855a3 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 16 Oct 2026 11:30:04 +0200 Subject: [PATCH 3/3] docs/turn-loop: write the claudePlugins deadline sentence in active voice vale's write-good.Passive and Microsoft.Contractions rules (error level) flagged 'is killed', 'is capped' and 'has not'. Same bounds stated: 2 minutes per call, 4 minutes for the sequence. --- docs/turn-loop/config.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/turn-loop/config.md b/docs/turn-loop/config.md index abe34166..9dace98b 100644 --- a/docs/turn-loop/config.md +++ b/docs/turn-loop/config.md @@ -202,12 +202,12 @@ services.hyperhive.agent.claudePluginsAutoUpdate = false; # defaul - **`claudePlugins`** — list of plugin specs passed to `claude plugin install `. The harness installs each spec on every boot, assuming `install` is idempotent; failures log a warning - but don't abort boot. Boot waits for the - marketplace adds, the optional update and the installs (serial - `claude plugin` calls): each call is killed and logged if it has not - exited within 2 minutes, and the whole sequence is capped at 4 minutes, - after which the remaining calls fail at once with the same warning and the - web UI starts. Defaults to Anthropic's `skill-creator` (so every + but don't abort boot. Boot waits for the marketplace adds, the optional + update and the installs (serial `claude plugin` calls). The harness kills + and logs any call that hasn't exited within 2 minutes, and caps the whole + sequence at 4 minutes: after that the remaining calls fail at once with + the same warning and the web UI starts. Defaults to Anthropic's + `skill-creator` (so every agent can author, refine, and evaluate its own skills) plus hyperhive's own `base` plugin — skills that apply to every agent regardless of role (currently just `state-hygiene`) — all without