580 lines
21 KiB
Rust
580 lines
21 KiB
Rust
//! Publishing each subagent's terminal rows onto the swarm queue, as this
|
|
//! agent.
|
|
//!
|
|
//! Every line a subagent writes passes through `session`'s sink, which hands
|
|
//! it to [`Publisher::offer`]. The sink runs synchronously on the subagent's
|
|
//! output reader, so `offer` only queues. One task drains the queue: it
|
|
//! classifies each `stream-json` event into the rows the agent's own terminal
|
|
//! carries (`hive_sh4re::stream_enrich`) and publishes them on
|
|
//! `$SWARM.term.<agent>.sub.<subagent>` (`swarm_wire::subagent_term`).
|
|
//!
|
|
//! **Output only.** Nothing here subscribes: a subagent takes no input from
|
|
//! the swarm.
|
|
//!
|
|
//! **As the agent, never as the hive.** The connection presents the agent's
|
|
//! own queue credential, read from the swarm secret store under the agent's
|
|
//! store identity: the same credential and identity the harness uses, since
|
|
//! both daemons run as the agent's user. The hive's shared client is never
|
|
//! tried, because the queue grants it nothing under these subjects.
|
|
//!
|
|
//! **Publish only.** The rows land in `term-sub-<agent>`, the stream
|
|
//! `swarm-controller` creates for this agent; this daemon never opens it. A
|
|
//! row published while that stream does not exist yet still reaches a live
|
|
//! subscriber, but the swarm does not list the subagent from it.
|
|
//!
|
|
//! **Run state too.** The same task publishes each subagent's running or
|
|
//! finished state to `$KV.subagent-status.<agent>.<subagent>`, with a heartbeat.
|
|
//!
|
|
//! **Best-effort.** A subagent's run never waits on the queue: no store, a
|
|
//! refused credential, a full queue and a failed publish are each a log
|
|
//! line, and the run goes on.
|
|
|
|
use std::collections::HashMap;
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicU64, Ordering};
|
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
|
|
|
use hive_sh4re::term_msg::{ClassifyCtx, Level, TermMsg, fit, iso8601_utc};
|
|
use swarm_queue_client::agent_publisher::{Queue, Target};
|
|
use swarm_wire::subagent_status::{self, State as RunState, Status};
|
|
use swarm_wire::subagent_term;
|
|
use tokio::sync::mpsc;
|
|
|
|
/// Lines waiting for the publish task. Past this, `offer` drops the line
|
|
/// rather than block a subagent's output reader.
|
|
const QUEUE_DEPTH: usize = 4096;
|
|
|
|
/// Room left under the server's payload limit for subject, headers and
|
|
/// framing, as in `hive-agent`'s `swarm_term`.
|
|
const HEADROOM: usize = 1024;
|
|
|
|
/// One line of a subagent's output, as the runtime's sink hands it over.
|
|
pub enum Output {
|
|
/// A `stream-json` event.
|
|
Event(serde_json::Value),
|
|
/// A stdout line that was not JSON.
|
|
Stdout(String),
|
|
/// A stderr line.
|
|
Stderr(String),
|
|
}
|
|
|
|
/// The model and effort a subagent was started with, published as headers on
|
|
/// each of its rows (`swarm_wire::subagent_term`). `None` is the harness's
|
|
/// default, which has no name to publish.
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
pub struct RunInfo {
|
|
pub model: Option<String>,
|
|
pub effort: Option<String>,
|
|
}
|
|
|
|
impl RunInfo {
|
|
/// The headers for one row: only the values there are, and only ones a
|
|
/// header can hold. The values come from the agent's start request, so one
|
|
/// with a line break (which `async_nats` panics on) or an absurd length is
|
|
/// left out rather than published.
|
|
fn headers(&self) -> async_nats::HeaderMap {
|
|
let mut headers = async_nats::HeaderMap::new();
|
|
if let Some(model) = header_value(self.model.as_deref()) {
|
|
headers.insert(subagent_term::HEADER_MODEL, model);
|
|
}
|
|
if let Some(effort) = header_value(self.effort.as_deref()) {
|
|
headers.insert(subagent_term::HEADER_EFFORT, effort);
|
|
}
|
|
headers
|
|
}
|
|
}
|
|
|
|
/// The longest model or effort name published. Real ones are a few
|
|
/// characters; the value rides on every row and is shown in the UI.
|
|
const MAX_VALUE_LEN: usize = 64;
|
|
|
|
/// `value` trimmed, as a header value, or `None` when it is empty, too long, or
|
|
/// not one a header can carry.
|
|
fn header_value(value: Option<&str>) -> Option<async_nats::HeaderValue> {
|
|
let value = value?.trim();
|
|
if value.is_empty() || value.len() > MAX_VALUE_LEN {
|
|
return None;
|
|
}
|
|
value.parse().ok()
|
|
}
|
|
|
|
/// A change in whether a subagent is running, stamped when it happened.
|
|
enum StatusEvent {
|
|
Running {
|
|
subagent: String,
|
|
info: Arc<RunInfo>,
|
|
at: i64,
|
|
},
|
|
Finished {
|
|
subagent: String,
|
|
exit: Option<i32>,
|
|
at: i64,
|
|
},
|
|
}
|
|
|
|
struct Line {
|
|
subagent: String,
|
|
info: Arc<RunInfo>,
|
|
output: Output,
|
|
/// Unix seconds when the sink saw the line.
|
|
ts: i64,
|
|
}
|
|
|
|
/// The sink's handle on the publish task.
|
|
pub struct Publisher {
|
|
tx: mpsc::Sender<Line>,
|
|
/// Unbounded, unlike `tx`: a run sends two of these, and a lost
|
|
/// `Finished` would leave the task heartbeating a subagent that ended.
|
|
status: mpsc::UnboundedSender<StatusEvent>,
|
|
dropped: Arc<AtomicU64>,
|
|
}
|
|
|
|
impl Publisher {
|
|
/// Start the publish task, or return `None` when this daemon has no
|
|
/// queue address or no store to read the agent's credential from. Logs
|
|
/// which, once.
|
|
#[must_use]
|
|
pub fn spawn() -> Option<Self> {
|
|
let target = match Target::from_lookup(|k| std::env::var(k).ok()) {
|
|
Ok(Some(target)) => target,
|
|
Ok(None) => {
|
|
tracing::info!(
|
|
"no swarm queue address or no secret store for this agent; subagent \
|
|
terminals are not published"
|
|
);
|
|
return None;
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(
|
|
error = format!("{e:#}"),
|
|
"this agent's secret store coordinates are incomplete; subagent \
|
|
terminals are not published"
|
|
);
|
|
return None;
|
|
}
|
|
};
|
|
swarm_secret_client::install_crypto_provider();
|
|
tracing::info!(
|
|
url = %target.url,
|
|
stream = %subagent_term::stream_name(&target.agent),
|
|
"publishing subagent terminals to the swarm queue as this agent"
|
|
);
|
|
let (publisher, rx, status_rx) = Self::channel(QUEUE_DEPTH);
|
|
tokio::spawn(run(rx, status_rx, target, Arc::clone(&publisher.dropped)));
|
|
Some(publisher)
|
|
}
|
|
|
|
fn channel(
|
|
depth: usize,
|
|
) -> (
|
|
Self,
|
|
mpsc::Receiver<Line>,
|
|
mpsc::UnboundedReceiver<StatusEvent>,
|
|
) {
|
|
let (tx, rx) = mpsc::channel(depth);
|
|
let (status, status_rx) = mpsc::unbounded_channel();
|
|
let publisher = Self {
|
|
tx,
|
|
status,
|
|
dropped: Arc::new(AtomicU64::new(0)),
|
|
};
|
|
(publisher, rx, status_rx)
|
|
}
|
|
|
|
/// `subagent` started a run, on the model and effort `info` names.
|
|
pub fn running(&self, subagent: &str, info: &Arc<RunInfo>) {
|
|
// A closed receiver means the task is gone; there is nobody to tell.
|
|
let _ = self.status.send(StatusEvent::Running {
|
|
subagent: subagent.to_owned(),
|
|
info: Arc::clone(info),
|
|
at: unix_now(),
|
|
});
|
|
}
|
|
|
|
/// `subagent`'s run ended, with `exit` as its process's exit status, or
|
|
/// `None` when a signal killed it.
|
|
pub fn finished(&self, subagent: &str, exit: Option<i32>) {
|
|
let _ = self.status.send(StatusEvent::Finished {
|
|
subagent: subagent.to_owned(),
|
|
exit,
|
|
at: unix_now(),
|
|
});
|
|
}
|
|
|
|
/// Queue `output` from `subagent` for publishing, with the `info` its rows
|
|
/// carry as headers. Never blocks: a full queue drops the line and counts
|
|
/// it.
|
|
pub fn offer(&self, subagent: &str, info: &Arc<RunInfo>, output: Output) {
|
|
let ts = unix_now();
|
|
let line = Line {
|
|
subagent: subagent.to_owned(),
|
|
info: Arc::clone(info),
|
|
output,
|
|
ts,
|
|
};
|
|
if self.tx.try_send(line).is_err() {
|
|
self.dropped.fetch_add(1, Ordering::Relaxed);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Unix seconds now.
|
|
fn unix_now() -> i64 {
|
|
SystemTime::now()
|
|
.duration_since(UNIX_EPOCH)
|
|
.map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX))
|
|
}
|
|
|
|
async fn run(
|
|
mut rx: mpsc::Receiver<Line>,
|
|
mut status_rx: mpsc::UnboundedReceiver<StatusEvent>,
|
|
target: Target,
|
|
dropped: Arc<AtomicU64>,
|
|
) {
|
|
let target = Arc::new(target);
|
|
let mut ctxs: HashMap<String, ClassifyCtx> = HashMap::new();
|
|
let mut queue = Queue::new("subagent terminal");
|
|
let mut runs: HashMap<String, Status> = HashMap::new();
|
|
let period = Duration::from_secs(subagent_status::HEARTBEAT_S.unsigned_abs());
|
|
let mut beat = tokio::time::interval_at(tokio::time::Instant::now() + period, period);
|
|
loop {
|
|
tokio::select! {
|
|
line = rx.recv() => {
|
|
let Some(line) = line else { return };
|
|
let missed = dropped.swap(0, Ordering::Relaxed);
|
|
if missed > 0 {
|
|
tracing::warn!(missed, "subagent terminal: queue full, lines dropped");
|
|
}
|
|
publish_line(&line, &mut ctxs, &mut queue, &target).await;
|
|
}
|
|
Some(event) = status_rx.recv() => {
|
|
let (subagent, status) = apply(&mut runs, event);
|
|
publish_status(&mut queue, &target, &subagent, &status).await;
|
|
}
|
|
_ = beat.tick() => {
|
|
for (subagent, status) in refresh(&mut runs, unix_now()) {
|
|
publish_status(&mut queue, &target, &subagent, &status).await;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Publish the rows one line of output makes.
|
|
async fn publish_line(
|
|
line: &Line,
|
|
ctxs: &mut HashMap<String, ClassifyCtx>,
|
|
queue: &mut Queue,
|
|
target: &Arc<Target>,
|
|
) {
|
|
let ctx = ctxs.entry(line.subagent.clone()).or_default();
|
|
let rows = classify(&line.output, line.ts, ctx);
|
|
if rows.is_empty() {
|
|
return;
|
|
}
|
|
let Some(client) = queue.ready(target).await else {
|
|
return;
|
|
};
|
|
let subject = subagent_term::subject(&target.agent, &line.subagent);
|
|
let headers = line.info.headers();
|
|
for row in rows {
|
|
publish(&client, &subject, &headers, row).await;
|
|
}
|
|
}
|
|
|
|
/// Fold `event` into the runs the task is keeping alive, and return the status
|
|
/// to publish for it. A finished subagent leaves the map, so it is not
|
|
/// heartbeated again; its model and effort carry over to the final status.
|
|
fn apply(runs: &mut HashMap<String, Status>, event: StatusEvent) -> (String, Status) {
|
|
match event {
|
|
StatusEvent::Running { subagent, info, at } => {
|
|
let status = Status {
|
|
state: RunState::Running,
|
|
since: at,
|
|
exit: None,
|
|
model: info.model.clone(),
|
|
effort: info.effort.clone(),
|
|
ts: at,
|
|
};
|
|
runs.insert(subagent.clone(), status.clone());
|
|
(subagent, status)
|
|
}
|
|
StatusEvent::Finished { subagent, exit, at } => {
|
|
let previous = runs.remove(&subagent);
|
|
let status = Status {
|
|
state: RunState::Finished,
|
|
since: at,
|
|
exit,
|
|
model: previous.as_ref().and_then(|p| p.model.clone()),
|
|
effort: previous.and_then(|p| p.effort),
|
|
ts: at,
|
|
};
|
|
(subagent, status)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Stamp every running subagent's status with `ts`, the heartbeat, and return
|
|
/// them to publish.
|
|
fn refresh(runs: &mut HashMap<String, Status>, ts: i64) -> Vec<(String, Status)> {
|
|
runs.iter_mut()
|
|
.map(|(subagent, status)| {
|
|
status.ts = ts;
|
|
(subagent.clone(), status.clone())
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Publish `status` as `subagent`'s value in the status bucket. A KV put is a
|
|
/// publish to the key's subject, which the agent's credential may do and the
|
|
/// controller's bucket captures. Every failure is a log line.
|
|
async fn publish_status(queue: &mut Queue, target: &Arc<Target>, subagent: &str, status: &Status) {
|
|
let Some(client) = queue.ready(target).await else {
|
|
return;
|
|
};
|
|
if let Err(e) = swarm_queue_client::ensure_connected(&client) {
|
|
tracing::warn!(error = %swarm_queue_client::chain(&e), "subagent status: publish skipped");
|
|
return;
|
|
}
|
|
let payload = match serde_json::to_vec(status) {
|
|
Ok(payload) => payload,
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "subagent status: serialising failed, not published");
|
|
return;
|
|
}
|
|
};
|
|
let subject = subagent_status::subject(&target.agent, subagent);
|
|
if let Err(e) = client.publish(subject, payload.into()).await {
|
|
tracing::warn!(error = %e, subagent, "subagent status: publish failed");
|
|
}
|
|
}
|
|
|
|
/// The rows one line of output renders as, each stamped with `ts`.
|
|
fn classify(output: &Output, ts: i64, ctx: &mut ClassifyCtx) -> Vec<TermMsg> {
|
|
let rows = match output {
|
|
Output::Event(v) => hive_sh4re::stream_enrich::classify_stream_value(v, ctx),
|
|
Output::Stdout(line) => vec![TermMsg::new(Level::Debug, line.clone())],
|
|
Output::Stderr(line) => vec![TermMsg::new(Level::Warn, format!("stderr: {line}"))],
|
|
};
|
|
let ts = iso8601_utc(ts);
|
|
rows.into_iter().map(|m| m.at(ts.as_str())).collect()
|
|
}
|
|
|
|
/// Offer one row. Every failure is terminal for that row and for nothing else.
|
|
async fn publish(
|
|
client: &async_nats::Client,
|
|
subject: &str,
|
|
headers: &async_nats::HeaderMap,
|
|
msg: TermMsg,
|
|
) {
|
|
// Unconnected, the client buffers rather than fails, and reports the
|
|
// library's default payload limit rather than the server's.
|
|
if let Err(e) = swarm_queue_client::ensure_connected(client) {
|
|
tracing::warn!(error = %swarm_queue_client::chain(&e), "subagent terminal: publish skipped");
|
|
return;
|
|
}
|
|
let limit = swarm_queue_client::max_payload(client).saturating_sub(HEADROOM);
|
|
let summary = msg.summary.clone();
|
|
let Some(msg) = fit(msg, limit) else {
|
|
tracing::warn!(
|
|
summary,
|
|
limit,
|
|
"subagent terminal: row does not fit even without its body, dropped"
|
|
);
|
|
return;
|
|
};
|
|
let payload = match serde_json::to_vec(&msg) {
|
|
Ok(payload) => payload,
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "subagent terminal: serialising failed, row dropped");
|
|
return;
|
|
}
|
|
};
|
|
let sent = client
|
|
.publish_with_headers(subject.to_owned(), headers.clone(), payload.into())
|
|
.await;
|
|
if let Err(e) = sent {
|
|
tracing::warn!(error = %e, "subagent terminal: publish failed, row dropped");
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
/// Each subagent publishes on its own subject under this agent's family,
|
|
/// the one the queue grants the agent.
|
|
#[test]
|
|
fn a_subagent_publishes_under_the_agents_own_family() {
|
|
assert_eq!(
|
|
subagent_term::subject("iris", "scout"),
|
|
"$SWARM.term.iris.sub.scout"
|
|
);
|
|
assert_eq!(
|
|
subagent_term::subject("iris", "w-2"),
|
|
"$SWARM.term.iris.sub.w-2"
|
|
);
|
|
}
|
|
|
|
/// The sink must never wait on the publisher: a full queue drops the line
|
|
/// and counts it.
|
|
#[test]
|
|
fn a_full_queue_drops_and_counts_rather_than_blocks() {
|
|
let (publisher, _rx, _status) = Publisher::channel(1);
|
|
let info = Arc::new(RunInfo::default());
|
|
publisher.offer("scout", &info, Output::Stdout("one".to_owned()));
|
|
publisher.offer("scout", &info, Output::Stdout("two".to_owned()));
|
|
assert_eq!(publisher.dropped.load(Ordering::Relaxed), 1);
|
|
}
|
|
|
|
fn event_running(subagent: &str, at: i64) -> StatusEvent {
|
|
StatusEvent::Running {
|
|
subagent: subagent.to_owned(),
|
|
info: Arc::new(RunInfo {
|
|
model: Some("opus".to_owned()),
|
|
effort: None,
|
|
}),
|
|
at,
|
|
}
|
|
}
|
|
|
|
/// A start is `running` since then with the run's model; the end is
|
|
/// `finished` with the exit status and the same model, and the subagent
|
|
/// is not heartbeated afterwards.
|
|
#[test]
|
|
fn a_run_is_running_then_finished_with_its_exit_and_model() {
|
|
let mut runs = HashMap::new();
|
|
let (name, started) = apply(&mut runs, event_running("scout", 100));
|
|
assert_eq!(name, "scout");
|
|
assert_eq!(started.state, RunState::Running);
|
|
assert_eq!((started.since, started.ts, started.exit), (100, 100, None));
|
|
assert_eq!(started.model.as_deref(), Some("opus"));
|
|
|
|
let finished = StatusEvent::Finished {
|
|
subagent: "scout".to_owned(),
|
|
exit: Some(0),
|
|
at: 160,
|
|
};
|
|
let (_, ended) = apply(&mut runs, finished);
|
|
assert_eq!(ended.state, RunState::Finished);
|
|
assert_eq!((ended.since, ended.ts, ended.exit), (160, 160, Some(0)));
|
|
assert_eq!(ended.model.as_deref(), Some("opus"));
|
|
assert!(refresh(&mut runs, 220).is_empty(), "nothing left to beat");
|
|
}
|
|
|
|
/// The heartbeat moves `ts` and nothing else, so a reader tells a live
|
|
/// writer from a dead one by `ts` while `since` still says when it began.
|
|
#[test]
|
|
fn the_heartbeat_refreshes_ts_and_keeps_since() {
|
|
let mut runs = HashMap::new();
|
|
apply(&mut runs, event_running("scout", 100));
|
|
apply(&mut runs, event_running("other", 130));
|
|
let mut beat = refresh(&mut runs, 160);
|
|
beat.sort_by(|a, b| a.0.cmp(&b.0));
|
|
assert_eq!(beat.len(), 2);
|
|
assert_eq!(
|
|
(beat[0].0.as_str(), beat[0].1.since, beat[0].1.ts),
|
|
("other", 130, 160)
|
|
);
|
|
assert_eq!(
|
|
(beat[1].0.as_str(), beat[1].1.since, beat[1].1.ts),
|
|
("scout", 100, 160)
|
|
);
|
|
assert!(beat.iter().all(|(_, s)| s.state == RunState::Running));
|
|
}
|
|
|
|
/// A lost end would leave the task heartbeating a subagent that ended, so
|
|
/// the status events are never dropped for a full queue.
|
|
#[test]
|
|
fn status_events_are_not_bounded_by_the_line_queue() {
|
|
let (publisher, _rx, mut status) = Publisher::channel(1);
|
|
let info = Arc::new(RunInfo::default());
|
|
for _ in 0..10 {
|
|
publisher.running("scout", &info);
|
|
publisher.finished("scout", Some(0));
|
|
}
|
|
let mut seen = 0;
|
|
while status.try_recv().is_ok() {
|
|
seen += 1;
|
|
}
|
|
assert_eq!(seen, 20);
|
|
}
|
|
|
|
/// The list endpoint reads these two headers off a subagent's newest row.
|
|
#[test]
|
|
fn a_row_carries_the_model_and_effort_the_subagent_runs() {
|
|
let info = RunInfo {
|
|
model: Some("opus".to_owned()),
|
|
effort: Some("high".to_owned()),
|
|
};
|
|
let headers = info.headers();
|
|
assert_eq!(
|
|
headers
|
|
.get(subagent_term::HEADER_MODEL)
|
|
.map(ToString::to_string),
|
|
Some("opus".to_owned())
|
|
);
|
|
assert_eq!(
|
|
headers
|
|
.get(subagent_term::HEADER_EFFORT)
|
|
.map(ToString::to_string),
|
|
Some("high".to_owned())
|
|
);
|
|
}
|
|
|
|
/// A subagent on the default model names none, and an empty header would
|
|
/// read as a model called "".
|
|
#[test]
|
|
fn a_value_that_is_not_set_is_not_published() {
|
|
let info = RunInfo {
|
|
model: None,
|
|
effort: Some(String::new()),
|
|
};
|
|
assert!(info.headers().is_empty());
|
|
}
|
|
|
|
/// The values are the caller's. A line break would panic `async_nats`, in
|
|
/// the task that publishes every subagent's rows.
|
|
#[test]
|
|
fn a_value_a_header_cannot_hold_is_left_out_and_never_panics() {
|
|
for bad in ["a\nb", "high\r\nX: y", &"x".repeat(MAX_VALUE_LEN + 1)] {
|
|
let info = RunInfo {
|
|
model: Some(bad.to_owned()),
|
|
effort: Some(bad.to_owned()),
|
|
};
|
|
assert!(info.headers().is_empty(), "{bad:?}");
|
|
}
|
|
let padded = RunInfo {
|
|
model: Some(" opus\n".to_owned()),
|
|
effort: None,
|
|
};
|
|
assert_eq!(
|
|
padded
|
|
.headers()
|
|
.get(subagent_term::HEADER_MODEL)
|
|
.map(ToString::to_string),
|
|
Some("opus".to_owned())
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn every_kind_of_line_renders_as_stamped_rows() {
|
|
let mut ctx = ClassifyCtx::default();
|
|
let ts = 1_789_302_903;
|
|
let event = serde_json::json!({
|
|
"type": "assistant",
|
|
"message": { "content": [{ "type": "text", "text": "looking" }] }
|
|
});
|
|
let rows = classify(&Output::Event(event), ts, &mut ctx);
|
|
assert_eq!(rows.len(), 1);
|
|
assert_eq!(rows[0].ts, "2026-09-13T12:35:03Z");
|
|
|
|
let rows = classify(&Output::Stderr("boom".to_owned()), ts, &mut ctx);
|
|
assert_eq!(rows[0].level, Level::Warn);
|
|
assert_eq!(rows[0].summary, "stderr: boom");
|
|
|
|
let rows = classify(&Output::Stdout("chatter".to_owned()), ts, &mut ctx);
|
|
assert_eq!(rows[0].level, Level::Debug);
|
|
assert_eq!(rows[0].ts, "2026-09-13T12:35:03Z");
|
|
}
|
|
}
|