Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/hive-subagent-mcp/src/swarm_term.rs

580 lines
21 KiB
Rust

//! Publishing each subagent's terminal rows onto the swarm queue, as this
//! agent.
//!
//! Every line a subagent writes passes through `session`'s sink, which hands
//! it to [`Publisher::offer`]. The sink runs synchronously on the subagent's
//! output reader, so `offer` only queues. One task drains the queue: it
//! classifies each `stream-json` event into the rows the agent's own terminal
//! carries (`hive_sh4re::stream_enrich`) and publishes them on
//! `$SWARM.term.<agent>.sub.<subagent>` (`swarm_wire::subagent_term`).
//!
//! **Output only.** Nothing here subscribes: a subagent takes no input from
//! the swarm.
//!
//! **As the agent, never as the hive.** The connection presents the agent's
//! own queue credential, read from the swarm secret store under the agent's
//! store identity: the same credential and identity the harness uses, since
//! both daemons run as the agent's user. The hive's shared client is never
//! tried, because the queue grants it nothing under these subjects.
//!
//! **Publish only.** The rows land in `term-sub-<agent>`, the stream
//! `swarm-controller` creates for this agent; this daemon never opens it. A
//! row published while that stream does not exist yet still reaches a live
//! subscriber, but the swarm does not list the subagent from it.
//!
//! **Run state too.** The same task publishes each subagent's running or
//! finished state to `$KV.subagent-status.<agent>.<subagent>`, with a heartbeat.
//!
//! **Best-effort.** A subagent's run never waits on the queue: no store, a
//! refused credential, a full queue and a failed publish are each a log
//! line, and the run goes on.
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use hive_sh4re::term_msg::{ClassifyCtx, Level, TermMsg, fit, iso8601_utc};
use swarm_queue_client::agent_publisher::{Queue, Target};
use swarm_wire::subagent_status::{self, State as RunState, Status};
use swarm_wire::subagent_term;
use tokio::sync::mpsc;
/// Lines waiting for the publish task. Past this, `offer` drops the line
/// rather than block a subagent's output reader.
const QUEUE_DEPTH: usize = 4096;
/// Room left under the server's payload limit for subject, headers and
/// framing, as in `hive-agent`'s `swarm_term`.
const HEADROOM: usize = 1024;
/// One line of a subagent's output, as the runtime's sink hands it over.
pub enum Output {
/// A `stream-json` event.
Event(serde_json::Value),
/// A stdout line that was not JSON.
Stdout(String),
/// A stderr line.
Stderr(String),
}
/// The model and effort a subagent was started with, published as headers on
/// each of its rows (`swarm_wire::subagent_term`). `None` is the harness's
/// default, which has no name to publish.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RunInfo {
pub model: Option<String>,
pub effort: Option<String>,
}
impl RunInfo {
/// The headers for one row: only the values there are, and only ones a
/// header can hold. The values come from the agent's start request, so one
/// with a line break (which `async_nats` panics on) or an absurd length is
/// left out rather than published.
fn headers(&self) -> async_nats::HeaderMap {
let mut headers = async_nats::HeaderMap::new();
if let Some(model) = header_value(self.model.as_deref()) {
headers.insert(subagent_term::HEADER_MODEL, model);
}
if let Some(effort) = header_value(self.effort.as_deref()) {
headers.insert(subagent_term::HEADER_EFFORT, effort);
}
headers
}
}
/// The longest model or effort name published. Real ones are a few
/// characters; the value rides on every row and is shown in the UI.
const MAX_VALUE_LEN: usize = 64;
/// `value` trimmed, as a header value, or `None` when it is empty, too long, or
/// not one a header can carry.
fn header_value(value: Option<&str>) -> Option<async_nats::HeaderValue> {
let value = value?.trim();
if value.is_empty() || value.len() > MAX_VALUE_LEN {
return None;
}
value.parse().ok()
}
/// A change in whether a subagent is running, stamped when it happened.
enum StatusEvent {
Running {
subagent: String,
info: Arc<RunInfo>,
at: i64,
},
Finished {
subagent: String,
exit: Option<i32>,
at: i64,
},
}
struct Line {
subagent: String,
info: Arc<RunInfo>,
output: Output,
/// Unix seconds when the sink saw the line.
ts: i64,
}
/// The sink's handle on the publish task.
pub struct Publisher {
tx: mpsc::Sender<Line>,
/// Unbounded, unlike `tx`: a run sends two of these, and a lost
/// `Finished` would leave the task heartbeating a subagent that ended.
status: mpsc::UnboundedSender<StatusEvent>,
dropped: Arc<AtomicU64>,
}
impl Publisher {
/// Start the publish task, or return `None` when this daemon has no
/// queue address or no store to read the agent's credential from. Logs
/// which, once.
#[must_use]
pub fn spawn() -> Option<Self> {
let target = match Target::from_lookup(|k| std::env::var(k).ok()) {
Ok(Some(target)) => target,
Ok(None) => {
tracing::info!(
"no swarm queue address or no secret store for this agent; subagent \
terminals are not published"
);
return None;
}
Err(e) => {
tracing::warn!(
error = format!("{e:#}"),
"this agent's secret store coordinates are incomplete; subagent \
terminals are not published"
);
return None;
}
};
swarm_secret_client::install_crypto_provider();
tracing::info!(
url = %target.url,
stream = %subagent_term::stream_name(&target.agent),
"publishing subagent terminals to the swarm queue as this agent"
);
let (publisher, rx, status_rx) = Self::channel(QUEUE_DEPTH);
tokio::spawn(run(rx, status_rx, target, Arc::clone(&publisher.dropped)));
Some(publisher)
}
fn channel(
depth: usize,
) -> (
Self,
mpsc::Receiver<Line>,
mpsc::UnboundedReceiver<StatusEvent>,
) {
let (tx, rx) = mpsc::channel(depth);
let (status, status_rx) = mpsc::unbounded_channel();
let publisher = Self {
tx,
status,
dropped: Arc::new(AtomicU64::new(0)),
};
(publisher, rx, status_rx)
}
/// `subagent` started a run, on the model and effort `info` names.
pub fn running(&self, subagent: &str, info: &Arc<RunInfo>) {
// A closed receiver means the task is gone; there is nobody to tell.
let _ = self.status.send(StatusEvent::Running {
subagent: subagent.to_owned(),
info: Arc::clone(info),
at: unix_now(),
});
}
/// `subagent`'s run ended, with `exit` as its process's exit status, or
/// `None` when a signal killed it.
pub fn finished(&self, subagent: &str, exit: Option<i32>) {
let _ = self.status.send(StatusEvent::Finished {
subagent: subagent.to_owned(),
exit,
at: unix_now(),
});
}
/// Queue `output` from `subagent` for publishing, with the `info` its rows
/// carry as headers. Never blocks: a full queue drops the line and counts
/// it.
pub fn offer(&self, subagent: &str, info: &Arc<RunInfo>, output: Output) {
let ts = unix_now();
let line = Line {
subagent: subagent.to_owned(),
info: Arc::clone(info),
output,
ts,
};
if self.tx.try_send(line).is_err() {
self.dropped.fetch_add(1, Ordering::Relaxed);
}
}
}
/// Unix seconds now.
fn unix_now() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX))
}
async fn run(
mut rx: mpsc::Receiver<Line>,
mut status_rx: mpsc::UnboundedReceiver<StatusEvent>,
target: Target,
dropped: Arc<AtomicU64>,
) {
let target = Arc::new(target);
let mut ctxs: HashMap<String, ClassifyCtx> = HashMap::new();
let mut queue = Queue::new("subagent terminal");
let mut runs: HashMap<String, Status> = HashMap::new();
let period = Duration::from_secs(subagent_status::HEARTBEAT_S.unsigned_abs());
let mut beat = tokio::time::interval_at(tokio::time::Instant::now() + period, period);
loop {
tokio::select! {
line = rx.recv() => {
let Some(line) = line else { return };
let missed = dropped.swap(0, Ordering::Relaxed);
if missed > 0 {
tracing::warn!(missed, "subagent terminal: queue full, lines dropped");
}
publish_line(&line, &mut ctxs, &mut queue, &target).await;
}
Some(event) = status_rx.recv() => {
let (subagent, status) = apply(&mut runs, event);
publish_status(&mut queue, &target, &subagent, &status).await;
}
_ = beat.tick() => {
for (subagent, status) in refresh(&mut runs, unix_now()) {
publish_status(&mut queue, &target, &subagent, &status).await;
}
}
}
}
}
/// Publish the rows one line of output makes.
async fn publish_line(
line: &Line,
ctxs: &mut HashMap<String, ClassifyCtx>,
queue: &mut Queue,
target: &Arc<Target>,
) {
let ctx = ctxs.entry(line.subagent.clone()).or_default();
let rows = classify(&line.output, line.ts, ctx);
if rows.is_empty() {
return;
}
let Some(client) = queue.ready(target).await else {
return;
};
let subject = subagent_term::subject(&target.agent, &line.subagent);
let headers = line.info.headers();
for row in rows {
publish(&client, &subject, &headers, row).await;
}
}
/// Fold `event` into the runs the task is keeping alive, and return the status
/// to publish for it. A finished subagent leaves the map, so it is not
/// heartbeated again; its model and effort carry over to the final status.
fn apply(runs: &mut HashMap<String, Status>, event: StatusEvent) -> (String, Status) {
match event {
StatusEvent::Running { subagent, info, at } => {
let status = Status {
state: RunState::Running,
since: at,
exit: None,
model: info.model.clone(),
effort: info.effort.clone(),
ts: at,
};
runs.insert(subagent.clone(), status.clone());
(subagent, status)
}
StatusEvent::Finished { subagent, exit, at } => {
let previous = runs.remove(&subagent);
let status = Status {
state: RunState::Finished,
since: at,
exit,
model: previous.as_ref().and_then(|p| p.model.clone()),
effort: previous.and_then(|p| p.effort),
ts: at,
};
(subagent, status)
}
}
}
/// Stamp every running subagent's status with `ts`, the heartbeat, and return
/// them to publish.
fn refresh(runs: &mut HashMap<String, Status>, ts: i64) -> Vec<(String, Status)> {
runs.iter_mut()
.map(|(subagent, status)| {
status.ts = ts;
(subagent.clone(), status.clone())
})
.collect()
}
/// Publish `status` as `subagent`'s value in the status bucket. A KV put is a
/// publish to the key's subject, which the agent's credential may do and the
/// controller's bucket captures. Every failure is a log line.
async fn publish_status(queue: &mut Queue, target: &Arc<Target>, subagent: &str, status: &Status) {
let Some(client) = queue.ready(target).await else {
return;
};
if let Err(e) = swarm_queue_client::ensure_connected(&client) {
tracing::warn!(error = %swarm_queue_client::chain(&e), "subagent status: publish skipped");
return;
}
let payload = match serde_json::to_vec(status) {
Ok(payload) => payload,
Err(e) => {
tracing::warn!(error = %e, "subagent status: serialising failed, not published");
return;
}
};
let subject = subagent_status::subject(&target.agent, subagent);
if let Err(e) = client.publish(subject, payload.into()).await {
tracing::warn!(error = %e, subagent, "subagent status: publish failed");
}
}
/// The rows one line of output renders as, each stamped with `ts`.
fn classify(output: &Output, ts: i64, ctx: &mut ClassifyCtx) -> Vec<TermMsg> {
let rows = match output {
Output::Event(v) => hive_sh4re::stream_enrich::classify_stream_value(v, ctx),
Output::Stdout(line) => vec![TermMsg::new(Level::Debug, line.clone())],
Output::Stderr(line) => vec![TermMsg::new(Level::Warn, format!("stderr: {line}"))],
};
let ts = iso8601_utc(ts);
rows.into_iter().map(|m| m.at(ts.as_str())).collect()
}
/// Offer one row. Every failure is terminal for that row and for nothing else.
async fn publish(
client: &async_nats::Client,
subject: &str,
headers: &async_nats::HeaderMap,
msg: TermMsg,
) {
// Unconnected, the client buffers rather than fails, and reports the
// library's default payload limit rather than the server's.
if let Err(e) = swarm_queue_client::ensure_connected(client) {
tracing::warn!(error = %swarm_queue_client::chain(&e), "subagent terminal: publish skipped");
return;
}
let limit = swarm_queue_client::max_payload(client).saturating_sub(HEADROOM);
let summary = msg.summary.clone();
let Some(msg) = fit(msg, limit) else {
tracing::warn!(
summary,
limit,
"subagent terminal: row does not fit even without its body, dropped"
);
return;
};
let payload = match serde_json::to_vec(&msg) {
Ok(payload) => payload,
Err(e) => {
tracing::warn!(error = %e, "subagent terminal: serialising failed, row dropped");
return;
}
};
let sent = client
.publish_with_headers(subject.to_owned(), headers.clone(), payload.into())
.await;
if let Err(e) = sent {
tracing::warn!(error = %e, "subagent terminal: publish failed, row dropped");
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Each subagent publishes on its own subject under this agent's family,
/// the one the queue grants the agent.
#[test]
fn a_subagent_publishes_under_the_agents_own_family() {
assert_eq!(
subagent_term::subject("iris", "scout"),
"$SWARM.term.iris.sub.scout"
);
assert_eq!(
subagent_term::subject("iris", "w-2"),
"$SWARM.term.iris.sub.w-2"
);
}
/// The sink must never wait on the publisher: a full queue drops the line
/// and counts it.
#[test]
fn a_full_queue_drops_and_counts_rather_than_blocks() {
let (publisher, _rx, _status) = Publisher::channel(1);
let info = Arc::new(RunInfo::default());
publisher.offer("scout", &info, Output::Stdout("one".to_owned()));
publisher.offer("scout", &info, Output::Stdout("two".to_owned()));
assert_eq!(publisher.dropped.load(Ordering::Relaxed), 1);
}
fn event_running(subagent: &str, at: i64) -> StatusEvent {
StatusEvent::Running {
subagent: subagent.to_owned(),
info: Arc::new(RunInfo {
model: Some("opus".to_owned()),
effort: None,
}),
at,
}
}
/// A start is `running` since then with the run's model; the end is
/// `finished` with the exit status and the same model, and the subagent
/// is not heartbeated afterwards.
#[test]
fn a_run_is_running_then_finished_with_its_exit_and_model() {
let mut runs = HashMap::new();
let (name, started) = apply(&mut runs, event_running("scout", 100));
assert_eq!(name, "scout");
assert_eq!(started.state, RunState::Running);
assert_eq!((started.since, started.ts, started.exit), (100, 100, None));
assert_eq!(started.model.as_deref(), Some("opus"));
let finished = StatusEvent::Finished {
subagent: "scout".to_owned(),
exit: Some(0),
at: 160,
};
let (_, ended) = apply(&mut runs, finished);
assert_eq!(ended.state, RunState::Finished);
assert_eq!((ended.since, ended.ts, ended.exit), (160, 160, Some(0)));
assert_eq!(ended.model.as_deref(), Some("opus"));
assert!(refresh(&mut runs, 220).is_empty(), "nothing left to beat");
}
/// The heartbeat moves `ts` and nothing else, so a reader tells a live
/// writer from a dead one by `ts` while `since` still says when it began.
#[test]
fn the_heartbeat_refreshes_ts_and_keeps_since() {
let mut runs = HashMap::new();
apply(&mut runs, event_running("scout", 100));
apply(&mut runs, event_running("other", 130));
let mut beat = refresh(&mut runs, 160);
beat.sort_by(|a, b| a.0.cmp(&b.0));
assert_eq!(beat.len(), 2);
assert_eq!(
(beat[0].0.as_str(), beat[0].1.since, beat[0].1.ts),
("other", 130, 160)
);
assert_eq!(
(beat[1].0.as_str(), beat[1].1.since, beat[1].1.ts),
("scout", 100, 160)
);
assert!(beat.iter().all(|(_, s)| s.state == RunState::Running));
}
/// A lost end would leave the task heartbeating a subagent that ended, so
/// the status events are never dropped for a full queue.
#[test]
fn status_events_are_not_bounded_by_the_line_queue() {
let (publisher, _rx, mut status) = Publisher::channel(1);
let info = Arc::new(RunInfo::default());
for _ in 0..10 {
publisher.running("scout", &info);
publisher.finished("scout", Some(0));
}
let mut seen = 0;
while status.try_recv().is_ok() {
seen += 1;
}
assert_eq!(seen, 20);
}
/// The list endpoint reads these two headers off a subagent's newest row.
#[test]
fn a_row_carries_the_model_and_effort_the_subagent_runs() {
let info = RunInfo {
model: Some("opus".to_owned()),
effort: Some("high".to_owned()),
};
let headers = info.headers();
assert_eq!(
headers
.get(subagent_term::HEADER_MODEL)
.map(ToString::to_string),
Some("opus".to_owned())
);
assert_eq!(
headers
.get(subagent_term::HEADER_EFFORT)
.map(ToString::to_string),
Some("high".to_owned())
);
}
/// A subagent on the default model names none, and an empty header would
/// read as a model called "".
#[test]
fn a_value_that_is_not_set_is_not_published() {
let info = RunInfo {
model: None,
effort: Some(String::new()),
};
assert!(info.headers().is_empty());
}
/// The values are the caller's. A line break would panic `async_nats`, in
/// the task that publishes every subagent's rows.
#[test]
fn a_value_a_header_cannot_hold_is_left_out_and_never_panics() {
for bad in ["a\nb", "high\r\nX: y", &"x".repeat(MAX_VALUE_LEN + 1)] {
let info = RunInfo {
model: Some(bad.to_owned()),
effort: Some(bad.to_owned()),
};
assert!(info.headers().is_empty(), "{bad:?}");
}
let padded = RunInfo {
model: Some(" opus\n".to_owned()),
effort: None,
};
assert_eq!(
padded
.headers()
.get(subagent_term::HEADER_MODEL)
.map(ToString::to_string),
Some("opus".to_owned())
);
}
#[test]
fn every_kind_of_line_renders_as_stamped_rows() {
let mut ctx = ClassifyCtx::default();
let ts = 1_789_302_903;
let event = serde_json::json!({
"type": "assistant",
"message": { "content": [{ "type": "text", "text": "looking" }] }
});
let rows = classify(&Output::Event(event), ts, &mut ctx);
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].ts, "2026-09-13T12:35:03Z");
let rows = classify(&Output::Stderr("boom".to_owned()), ts, &mut ctx);
assert_eq!(rows[0].level, Level::Warn);
assert_eq!(rows[0].summary, "stderr: boom");
let rows = classify(&Output::Stdout("chatter".to_owned()), ts, &mut ctx);
assert_eq!(rows[0].level, Level::Debug);
assert_eq!(rows[0].ts, "2026-09-13T12:35:03Z");
}
}