diff --git a/Cargo.lock b/Cargo.lock index 83e62455..f0d437c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1489,6 +1489,22 @@ dependencies = [ "url", ] +[[package]] +name = "hive-agent-mcp" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "clap", + "hive-sh4re", + "rmcp", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "hive-agent-wake" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index c5442811..14695861 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "3" members = [ "hive-ag3nt", + "hive-agent-mcp", "hive-agent-wake", "hive-bash-mcp", "hive-c0re", diff --git a/hive-ag3nt/src/lib.rs b/hive-ag3nt/src/lib.rs index 61a81535..b933f409 100644 --- a/hive-ag3nt/src/lib.rs +++ b/hive-ag3nt/src/lib.rs @@ -10,9 +10,7 @@ pub mod harness_state; pub mod identity; pub mod login; pub mod login_session; -pub mod mcp; pub mod mcp_config; -pub mod mcp_loose_ends; pub mod paths; pub mod plugins; pub mod prompt; diff --git a/hive-ag3nt/src/mcp_config.rs b/hive-ag3nt/src/mcp_config.rs index 3275bece..8b3223f1 100644 --- a/hive-ag3nt/src/mcp_config.rs +++ b/hive-ag3nt/src/mcp_config.rs @@ -2,10 +2,9 @@ //! set into the `--allowedTools` / `--tools` argument strings and renders the //! `--mcp-config` blob claude reads at spawn (built-in hyperhive server + //! any `hyperhive.extraMcpServers`). Pure config-string generation consumed by -//! [`crate::turn`] when it builds the claude command — it never touches the -//! running MCP server ([`crate::mcp`]). The `send` allow-list check -//! ([`check_send_allowed`]) lives here too since it's driven by the same -//! `/etc/hyperhive/*.json` operator config. +//! [`crate::turn`] when it builds the claude command. It never touches the +//! running MCP server (a separate binary) — the `send` allow-list check that +//! server enforces lives alongside it in the `hive-agent-mcp` crate. /// Name of the hyperhive MCP server inside claude's view. Claude prefixes /// tools as `mcp____` (e.g. `mcp__hyperhive__send`). @@ -251,59 +250,6 @@ pub fn builtin_tools_arg() -> String { /// `mcp____` pattern in `--allowedTools`. const EXTRA_MCP_PATH: &str = "/etc/hyperhive/extra-mcp.json"; -/// Where the NixOS module writes the per-agent send allow-list (see -/// `nix/templates/harness/`). Empty list = unrestricted (the -/// default). Non-empty list constrains `mcp__hyperhive__send`'s `to` -/// field; the manager is always implicitly permitted regardless of -/// the list contents. -const SEND_ALLOW_PATH: &str = "/etc/hyperhive/send-allow.json"; - -/// Enforce the per-agent send allow-list. Returns `Ok` when the -/// recipient is permitted (no list configured, `` sentinel -/// always allowed, or `to` is in the list); returns `Err(refusal)` -/// with a claude-readable string when blocked ��� the harness surfaces -/// the refusal as the tool result so claude knows the message didn't -/// land and can react (e.g. route via `` instead). -pub fn check_send_allowed(to: &str) -> Result<(), String> { - if to == hive_sh4re::PARENT_RECIPIENT { - // Always allow `` — the allow-list constrains peer - // chatter, not the structural reporting line; the operator - // can rewire who the parent IS via `set_parent` without - // having to remember to update the per-agent allow-list. - // The broker resolves the sentinel to the real parent label - // on the host side per topology.json (falls back to `operator` - // for root agents). - return Ok(()); - } - let Ok(raw) = std::fs::read_to_string(SEND_ALLOW_PATH) else { - return Ok(()); // file missing → no policy configured → unrestricted - }; - let allow: Vec = match serde_json::from_str(&raw) { - Ok(v) => v, - Err(e) => { - tracing::warn!( - path = SEND_ALLOW_PATH, - error = ?e, - "send allow-list parse failed; falling back to unrestricted", - ); - return Ok(()); - } - }; - if allow.is_empty() { - return Ok(()); // empty list = unrestricted (back-compat) - } - if allow.iter().any(|n| n == to) { - return Ok(()); - } - Err(format!( - "send refused: recipient '{to}' not in hyperhive.allowedRecipients \ - (configured in agent.nix). Allowed: {allow:?}. Your structural \ - parent is always reachable — route through `send(to: \"{}\", …)` \ - if you need to reach someone outside the allow-list.", - hive_sh4re::PARENT_RECIPIENT - )) -} - #[derive(Debug, serde::Deserialize)] struct ExtraMcpServer { command: String, diff --git a/hive-ag3nt/src/serve_common.rs b/hive-ag3nt/src/serve_common.rs index 5178147b..c3fdcf6f 100644 --- a/hive-ag3nt/src/serve_common.rs +++ b/hive-ag3nt/src/serve_common.rs @@ -5,7 +5,6 @@ //! stay in the binary because they touch the request enum variants directly. use crate::events::Bus; -use crate::mcp::REDELIVERY_HINT; use crate::turn::{TurnError, TurnOutcome}; use crate::turn_stats::TurnStatRow; pub use hive_sh4re::wire_time::now_unix; @@ -23,38 +22,20 @@ pub fn format_wake_prompt( unread: u64, redelivered: bool, ) -> String { - let banner = if redelivered { REDELIVERY_HINT } else { "" }; + let banner = if redelivered { + hive_sh4re::REDELIVERY_HINT + } else { + "" + }; let tag = if id > 0 { format!("[msg #{id}] ") } else { String::new() }; - let pending = pending_hint(unread); + let pending = hive_sh4re::pending_hint(unread); format!("{banner}{tag}Incoming message from `{from}`:\n---\n{body}\n---{pending}") } -/// Shared "(N more message(s) pending …)" advisory appended after both the -/// wake prompt body and the `recv` tool result whenever the inbox still has -/// queued messages once the current message/batch is popped. Returns an empty -/// string when `remaining == 0`. The leading `\n\n` separates it from the -/// preceding body/message block, and the suggested `max` is clamped to the -/// server-side recv cap so the hint never asks for more than one round-trip -/// can deliver. One builder so the wake prompt and the in-turn recv result -/// stay identical. -#[must_use] -pub fn pending_hint(remaining: u64) -> String { - if remaining == 0 { - return String::new(); - } - let batch = remaining.min(u64::from(hive_sh4re::RECV_BATCH_MAX)); - format!( - "\n\n({remaining} more message(s) pending in your inbox — call `mcp__hyperhive__recv` \ - with `max: {batch}` to drain the next batch before acting. If the \ - backlog is stale/already handled, `ack_until(up_to: )` \ - clears everything up to that id in one call instead.)" - ) -} - /// Field-named args for [`build_row`]. Mirrors the turn-stats row /// columns; `outcome` and `bus` borrow for the duration of the call. pub struct TurnRowArgs<'a> { diff --git a/hive-agent-mcp/Cargo.toml b/hive-agent-mcp/Cargo.toml new file mode 100644 index 00000000..531840e1 --- /dev/null +++ b/hive-agent-mcp/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "hive-agent-mcp" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "hive-agent-mcp" +path = "src/main.rs" + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +axum.workspace = true +clap.workspace = true +hive-sh4re.workspace = true +rmcp.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true diff --git a/hive-agent-mcp/src/client.rs b/hive-agent-mcp/src/client.rs new file mode 100644 index 00000000..c175cd11 --- /dev/null +++ b/hive-agent-mcp/src/client.rs @@ -0,0 +1,152 @@ +use std::path::Path; +use std::time::Duration; + +use anyhow::{Result, anyhow}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::UnixStream; + +/// Backoff schedule between attempts. Five entries → up to 5 retries on +/// top of the initial attempt; total wall-clock cap = 2+4+8+16+30 = 60s. +/// Sized to ride out a hive-c0re restart (systemd usually has the unix +/// socket back inside ~5s) without the agent-side claude session having +/// to handle the transient itself — burning tokens on a tool-error retry +/// loop is more expensive than 60s of in-harness sleep. +const RETRY_BACKOFFS_MS: &[u64] = &[2_000, 4_000, 8_000, 16_000, 30_000]; + +/// Send `req` over the unix socket and decode the single-line JSON +/// response, retrying transient connect/IO failures on the backoff +/// schedule above and reporting how many retries it took past the initial +/// attempt (0 = succeeded first try). MCP tool handlers +/// use this so they can append a one-line hint to the tool result when +/// retries happened — that way claude knows the prior socket flake +/// wasn't a content error and shouldn't trigger an LLM-level retry of +/// its own. +/// +/// # Errors +/// +/// Returns an error if all retries are exhausted, or on a fatal protocol +/// error (serialization / deserialization failure). +/// +/// # Panics +/// +/// Panics if `RETRY_BACKOFFS_MS.len()` does not fit in a `u32`, which +/// cannot happen with the current compile-time constant. +pub async fn request_retried(socket: &Path, req: &Req) -> Result<(Resp, u32)> +where + Req: Serialize + ?Sized, + Resp: DeserializeOwned, +{ + let mut last_err: Option = None; + let max_retries = u32::try_from(RETRY_BACKOFFS_MS.len()).unwrap(); + for attempt in 0..=max_retries { + match try_once::(socket, req).await { + Ok(resp) => return Ok((resp, attempt)), + Err(RequestError::Fatal(e)) => return Err(e), + Err(RequestError::Transient(e)) => { + if attempt < max_retries { + let sleep_ms = RETRY_BACKOFFS_MS[attempt as usize]; + tracing::warn!( + attempt = attempt + 1, + sleep_ms, + error = %e, + "hive socket attempt failed; retrying" + ); + last_err = Some(e); + tokio::time::sleep(Duration::from_millis(sleep_ms)).await; + } else { + last_err = Some(e); + } + } + } + } + // Reaching here means the final attempt returned `Transient`, which always + // sets `last_err` — so this is infallible. + Err(last_err.expect("a transient failure on the final attempt set last_err")) +} + +/// Transient = connect / IO error worth a retry (server restart, broken +/// pipe). Fatal = serialization / deserialization / protocol error +/// where retrying would just repeat the same failure. +enum RequestError { + Transient(anyhow::Error), + Fatal(anyhow::Error), +} + +async fn try_once(socket: &Path, req: &Req) -> Result +where + Req: Serialize + ?Sized, + Resp: DeserializeOwned, +{ + let stream = match UnixStream::connect(socket).await { + Ok(stream) => stream, + Err(e) => { + // A refused or missing socket usually means hive-c0re is + // mid-restart (operator redeploy / rebuild) — the socket is + // recreated on its boot and `request_retried` rides it out. When + // the error *does* surface (retries exhausted, or a non-retried + // caller) add that context so claude reads it as a likely + // transient rather than a hard failure worth escalating. + let restarting = matches!( + e.kind(), + std::io::ErrorKind::ConnectionRefused | std::io::ErrorKind::NotFound + ); + let mut err = anyhow::Error::new(e).context(format!("connect to {}", socket.display())); + if restarting { + err = err.context( + "hive-c0re may be restarting (e.g. an operator redeploy); \ + the harness already retried ~60s before surfacing this", + ); + } + return Err(RequestError::Transient(err)); + } + }; + let (read, mut write) = stream.into_split(); + + let mut payload = serde_json::to_string(req).map_err(|e| RequestError::Fatal(e.into()))?; + payload.push('\n'); + write + .write_all(payload.as_bytes()) + .await + .map_err(|e| RequestError::Transient(e.into()))?; + write + .flush() + .await + .map_err(|e| RequestError::Transient(e.into()))?; + + let mut reader = BufReader::new(read); + let mut line = String::new(); + let read_bytes = reader + .read_line(&mut line) + .await + .map_err(|e| RequestError::Transient(e.into()))?; + if read_bytes == 0 || line.is_empty() { + return Err(RequestError::Transient(anyhow!( + "server closed connection without responding" + ))); + } + serde_json::from_str(line.trim()).map_err(|e| RequestError::Fatal(e.into())) +} + +#[cfg(test)] +mod tests { + use super::{RequestError, try_once}; + + /// A connect to a non-existent socket path (ENOENT → `NotFound`) is + /// classified transient AND annotated with the "hive-c0re is restarting" + /// hint, so a surfaced tool error reads as the expected transient. + #[tokio::test] + async fn missing_socket_connect_is_transient_with_restart_hint() { + let bogus = std::path::Path::new("/nonexistent/hive/mcp.sock"); + match try_once::<(), serde_json::Value>(bogus, &()).await { + Err(RequestError::Transient(e)) => { + let msg = format!("{e:#}"); + assert!(msg.contains("restarting"), "missing restart hint: {msg}"); + assert!(msg.contains("connect to"), "missing connect context: {msg}"); + } + Err(RequestError::Fatal(e)) => panic!("expected transient, got fatal: {e:#}"), + Ok(_) => panic!("expected connect failure to a non-existent socket"), + } + } +} diff --git a/hive-ag3nt/src/mcp_loose_ends.rs b/hive-agent-mcp/src/loose_ends.rs similarity index 100% rename from hive-ag3nt/src/mcp_loose_ends.rs rename to hive-agent-mcp/src/loose_ends.rs diff --git a/hive-ag3nt/src/bin/hive-agent-mcp.rs b/hive-agent-mcp/src/main.rs similarity index 74% rename from hive-ag3nt/src/bin/hive-agent-mcp.rs rename to hive-agent-mcp/src/main.rs index baee732b..07d2e746 100644 --- a/hive-ag3nt/src/bin/hive-agent-mcp.rs +++ b/hive-agent-mcp/src/main.rs @@ -6,12 +6,25 @@ //! sole transport — there is no stdio mode. Sibling of `hive-agent` (the //! serve loop that renders the `--mcp-config` blob pointing here) and //! `hive-agent-wake`. +//! +//! Standalone bin crate: the MCP surface (`mcp/`) plus its small support +//! modules (socket client, send allow-list, loose-end scanner, path +//! resolver) live here rather than in the `hive-ag3nt` harness lib, so the +//! server binary doesn't link the whole turn loop. use std::path::PathBuf; use anyhow::Result; use clap::Parser; -use hive_ag3nt::{DEFAULT_SOCKET, mcp}; + +mod client; +mod loose_ends; +mod mcp; +mod paths; +mod send_allow; + +/// Per-agent MCP socket, bind-mounted from the host into every container. +const DEFAULT_SOCKET: &str = "/run/hive/mcp.sock"; #[derive(Parser)] #[command(name = "hive-agent-mcp", about = "hyperhive MCP server")] diff --git a/hive-ag3nt/src/mcp/args.rs b/hive-agent-mcp/src/mcp/args.rs similarity index 100% rename from hive-ag3nt/src/mcp/args.rs rename to hive-agent-mcp/src/mcp/args.rs diff --git a/hive-ag3nt/src/mcp/mod.rs b/hive-agent-mcp/src/mcp/mod.rs similarity index 99% rename from hive-ag3nt/src/mcp/mod.rs rename to hive-agent-mcp/src/mcp/mod.rs index 563ce71f..d5837327 100644 --- a/hive-ag3nt/src/mcp/mod.rs +++ b/hive-agent-mcp/src/mcp/mod.rs @@ -32,9 +32,7 @@ pub use args::{ RequestInitConfigArgs, RequestSchedulePromptArgs, RestartArgs, SendArgs, SetStatusArgs, StartArgs, UpdateArgs, UpdateMetaInputsArgs, }; -pub use render::{ - IDLE_WAIT_HINT, REDELIVERY_HINT, annotate_retries, format_ack, format_agent_meta, format_recv, -}; +pub use render::{annotate_retries, format_ack, format_agent_meta, format_recv}; use render::{ format_matrix_summary, loose_end_kind_label, matrix_unread_summary, parse_loose_end_kind, @@ -149,7 +147,7 @@ impl AgentServer { let to = args.to.clone(); // Check per-agent allow-list (hyperhive.allowedRecipients). When no // policy file is present (e.g. manager containers) the check is a no-op. - if let Err(refusal) = crate::mcp_config::check_send_allowed(&to) { + if let Err(refusal) = crate::send_allow::check_send_allowed(&to) { return run_tool_envelope("send", log, async move { refusal }).await; } run_tool_envelope("send", log, async move { @@ -340,7 +338,7 @@ impl AgentServer { // Append loose-end items published by external MCP daemons // (e.g. active bash tasks from hive-bash-mcp). Generic — no // per-MCP knowledge needed here. - let mcp_items = crate::mcp_loose_ends::collect(); + let mcp_items = crate::loose_ends::collect(); if !mcp_items.is_empty() { use std::fmt::Write as _; let n = mcp_items.len(); diff --git a/hive-ag3nt/src/mcp/render.rs b/hive-agent-mcp/src/mcp/render.rs similarity index 96% rename from hive-ag3nt/src/mcp/render.rs rename to hive-agent-mcp/src/mcp/render.rs index fdc172a8..db1a6eb2 100644 --- a/hive-ag3nt/src/mcp/render.rs +++ b/hive-agent-mcp/src/mcp/render.rs @@ -100,7 +100,11 @@ fn render_recv_messages( } let mut out = if messages.len() == 1 { let m = &messages[0]; - let banner = if m.redelivered { REDELIVERY_HINT } else { "" }; + let banner = if m.redelivered { + hive_sh4re::REDELIVERY_HINT + } else { + "" + }; format!("{banner}{}from: {}\n\n{}", msg_id_tag(m.id), m.from, m.body) } else { let n = messages.len(); @@ -109,7 +113,11 @@ fn render_recv_messages( if i > 0 { out.push_str("\n---\n\n"); } - let banner = if m.redelivered { REDELIVERY_HINT } else { "" }; + let banner = if m.redelivered { + hive_sh4re::REDELIVERY_HINT + } else { + "" + }; let _ = write!( out, "{banner}{}from: {}\n\n{}", @@ -120,7 +128,7 @@ fn render_recv_messages( } out }; - out.push_str(&crate::serve_common::pending_hint(remaining)); + out.push_str(&hive_sh4re::pending_hint(remaining)); out } @@ -136,13 +144,6 @@ fn msg_id_tag(id: i64) -> String { } } -/// Header prepended to message bodies that were popped by a prior -/// harness session, never acked (turn crash / OOM / restart), and -/// resurfaced by `RequeueInflight` on this session's boot. Same -/// string surfaces in the wake prompt (see the bin loops) and the -/// in-turn `recv` tool result so claude sees the warning either way. -pub const REDELIVERY_HINT: &str = "[redelivered after harness restart — may already be handled]\n"; - /// Appended to the `recv` empty result when the agent parked on a /// long-poll (`wait_seconds > 0`) that timed out with nothing new. /// Nudges the model to spend the idle time on other useful work diff --git a/hive-agent-mcp/src/paths.rs b/hive-agent-mcp/src/paths.rs new file mode 100644 index 00000000..4d9efa19 --- /dev/null +++ b/hive-agent-mcp/src/paths.rs @@ -0,0 +1,19 @@ +//! Per-agent path resolution for this MCP server. Self-contained (mirrors +//! the sibling helper daemons' own `paths.rs`) so the bin does not link the +//! harness lib just to resolve the agent's state dir. Only the one helper +//! the MCP server needs lives here. + +use std::path::PathBuf; + +/// Durable state directory for the current agent. Reads `HYPERHIVE_STATE_DIR` +/// first (always set by the meta flake to `/agents/{label}/state`); falls back +/// to the same pattern derived from `HIVE_LABEL` for dev/test environments +/// where the env var may not be set. +#[must_use] +pub fn state_dir() -> PathBuf { + if let Some(p) = std::env::var_os("HYPERHIVE_STATE_DIR") { + return PathBuf::from(p); + } + let label = std::env::var("HIVE_LABEL").unwrap_or_default(); + PathBuf::from(format!("/agents/{label}/state")) +} diff --git a/hive-agent-mcp/src/send_allow.rs b/hive-agent-mcp/src/send_allow.rs new file mode 100644 index 00000000..d7a18ad6 --- /dev/null +++ b/hive-agent-mcp/src/send_allow.rs @@ -0,0 +1,58 @@ +//! Per-agent `send` allow-list enforcement. Driven by the operator config at +//! `/etc/hyperhive/send-allow.json` (written by the NixOS harness module); the +//! MCP server calls [`check_send_allowed`] before forwarding a `send` tool call +//! so a blocked recipient surfaces as a claude-readable tool result rather than +//! a silent drop. + +/// Where the NixOS module writes the per-agent send allow-list (see +/// `nix/templates/harness/`). Empty list = unrestricted (the +/// default). Non-empty list constrains `mcp__hyperhive__send`'s `to` +/// field; the manager is always implicitly permitted regardless of +/// the list contents. +const SEND_ALLOW_PATH: &str = "/etc/hyperhive/send-allow.json"; + +/// Enforce the per-agent send allow-list. Returns `Ok` when the +/// recipient is permitted (no list configured, `` sentinel +/// always allowed, or `to` is in the list); returns `Err(refusal)` +/// with a claude-readable string when blocked — the harness surfaces +/// the refusal as the tool result so claude knows the message didn't +/// land and can react (e.g. route via `` instead). +pub fn check_send_allowed(to: &str) -> Result<(), String> { + if to == hive_sh4re::PARENT_RECIPIENT { + // Always allow `` — the allow-list constrains peer + // chatter, not the structural reporting line; the operator + // can rewire who the parent IS via `set_parent` without + // having to remember to update the per-agent allow-list. + // The broker resolves the sentinel to the real parent label + // on the host side per topology.json (falls back to `operator` + // for root agents). + return Ok(()); + } + let Ok(raw) = std::fs::read_to_string(SEND_ALLOW_PATH) else { + return Ok(()); // file missing → no policy configured → unrestricted + }; + let allow: Vec = match serde_json::from_str(&raw) { + Ok(v) => v, + Err(e) => { + tracing::warn!( + path = SEND_ALLOW_PATH, + error = ?e, + "send allow-list parse failed; falling back to unrestricted", + ); + return Ok(()); + } + }; + if allow.is_empty() { + return Ok(()); // empty list = unrestricted (back-compat) + } + if allow.iter().any(|n| n == to) { + return Ok(()); + } + Err(format!( + "send refused: recipient '{to}' not in hyperhive.allowedRecipients \ + (configured in agent.nix). Allowed: {allow:?}. Your structural \ + parent is always reachable — route through `send(to: \"{}\", …)` \ + if you need to reach someone outside the allow-list.", + hive_sh4re::PARENT_RECIPIENT + )) +} diff --git a/hive-sh4re/src/lib.rs b/hive-sh4re/src/lib.rs index 8963dc61..f77b636e 100644 --- a/hive-sh4re/src/lib.rs +++ b/hive-sh4re/src/lib.rs @@ -18,6 +18,34 @@ pub mod wire_time; /// constant instead of a scattered magic value. pub const RECV_BATCH_MAX: u32 = 5; +/// Banner prepended to a wake prompt / `recv` result when the message was +/// redelivered after a harness restart (the turn that first drove it never +/// acked). Shared between the harness serve loop (wake prompt) and the MCP +/// server (`recv` tool result) so both surfaces phrase it identically. +pub const REDELIVERY_HINT: &str = "[redelivered after harness restart — may already be handled]\n"; + +/// Shared "(N more message(s) pending …)" advisory appended after both the +/// wake prompt body and the `recv` tool result whenever the inbox still has +/// queued messages once the current message/batch is popped. Returns an empty +/// string when `remaining == 0`. The leading `\n\n` separates it from the +/// preceding body/message block, and the suggested `max` is clamped to the +/// server-side recv cap so the hint never asks for more than one round-trip +/// can deliver. One builder so the wake prompt (harness serve loop) and the +/// in-turn recv result (MCP server) stay identical. +#[must_use] +pub fn pending_hint(remaining: u64) -> String { + if remaining == 0 { + return String::new(); + } + let batch = remaining.min(u64::from(RECV_BATCH_MAX)); + format!( + "\n\n({remaining} more message(s) pending in your inbox — call `mcp__hyperhive__recv` \ + with `max: {batch}` to drain the next batch before acting. If the \ + backlog is stale/already handled, `ack_until(up_to: )` \ + clears everything up to that id in one call instead.)" + ) +} + /// One row in the approval queue. `commit_ref` is overloaded per /// `kind` — see `docs/approvals.md::Approval kinds (wire shapes)` /// for the encoding table and lifecycle.