//! Publishing each subagent's terminal rows onto the swarm queue, as this //! agent. //! //! Every line a subagent writes passes through `session`'s sink, which hands //! it to [`Publisher::offer`]. The sink runs synchronously on the subagent's //! output reader, so `offer` only queues. One task drains the queue: it //! classifies each `stream-json` event into the rows the agent's own terminal //! carries (`hive_sh4re::stream_enrich`) and publishes them on //! `$SWARM.term..sub.` (`swarm_wire::subagent_term`). //! //! **Output only.** Nothing here subscribes: a subagent takes no input from //! the swarm. //! //! **As the agent, never as the hive.** The connection presents the agent's //! own queue credential, read from the swarm secret store under the agent's //! store identity: the same credential and identity the harness uses, since //! both daemons run as the agent's user. The hive's shared client is never //! tried, because the queue grants it nothing under these subjects. //! //! **Publish only.** The rows land in `term-sub-`, the stream //! `swarm-controller` creates for this agent; this daemon never opens it. A //! row published while that stream does not exist yet still reaches a live //! subscriber, but the swarm does not list the subagent from it. //! //! **Run state too.** The same task publishes each subagent's running or //! finished state to `$KV.subagent-status..`, with a heartbeat. //! //! **Best-effort.** A subagent's run never waits on the queue: no store, a //! refused credential, a full queue and a failed publish are each a log //! line, and the run goes on. use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use hive_sh4re::term_msg::{ClassifyCtx, Level, TermMsg, fit, iso8601_utc}; use swarm_queue_client::agent_publisher::{Queue, Target}; use swarm_wire::subagent_status::{self, State as RunState, Status}; use swarm_wire::subagent_term; use tokio::sync::mpsc; /// Lines waiting for the publish task. Past this, `offer` drops the line /// rather than block a subagent's output reader. const QUEUE_DEPTH: usize = 4096; /// Room left under the server's payload limit for subject, headers and /// framing, as in `hive-agent`'s `swarm_term`. const HEADROOM: usize = 1024; /// One line of a subagent's output, as the runtime's sink hands it over. pub enum Output { /// A `stream-json` event. Event(serde_json::Value), /// A stdout line that was not JSON. Stdout(String), /// A stderr line. Stderr(String), } /// The model and effort a subagent was started with, published as headers on /// each of its rows (`swarm_wire::subagent_term`). `None` is the harness's /// default, which has no name to publish. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct RunInfo { pub model: Option, pub effort: Option, } impl RunInfo { /// The headers for one row: only the values there are, and only ones a /// header can hold. The values come from the agent's start request, so one /// with a line break (which `async_nats` panics on) or an absurd length is /// left out rather than published. fn headers(&self) -> async_nats::HeaderMap { let mut headers = async_nats::HeaderMap::new(); if let Some(model) = header_value(self.model.as_deref()) { headers.insert(subagent_term::HEADER_MODEL, model); } if let Some(effort) = header_value(self.effort.as_deref()) { headers.insert(subagent_term::HEADER_EFFORT, effort); } headers } } /// The longest model or effort name published. Real ones are a few /// characters; the value rides on every row and is shown in the UI. const MAX_VALUE_LEN: usize = 64; /// `value` trimmed, as a header value, or `None` when it is empty, too long, or /// not one a header can carry. fn header_value(value: Option<&str>) -> Option { let value = value?.trim(); if value.is_empty() || value.len() > MAX_VALUE_LEN { return None; } value.parse().ok() } /// A change in whether a subagent is running, stamped when it happened. enum StatusEvent { Running { subagent: String, info: Arc, at: i64, }, Finished { subagent: String, exit: Option, at: i64, }, } struct Line { subagent: String, info: Arc, output: Output, /// Unix seconds when the sink saw the line. ts: i64, } /// The sink's handle on the publish task. pub struct Publisher { tx: mpsc::Sender, /// Unbounded, unlike `tx`: a run sends two of these, and a lost /// `Finished` would leave the task heartbeating a subagent that ended. status: mpsc::UnboundedSender, dropped: Arc, } impl Publisher { /// Start the publish task, or return `None` when this daemon has no /// queue address or no store to read the agent's credential from. Logs /// which, once. #[must_use] pub fn spawn() -> Option { let target = match Target::from_lookup(|k| std::env::var(k).ok()) { Ok(Some(target)) => target, Ok(None) => { tracing::info!( "no swarm queue address or no secret store for this agent; subagent \ terminals are not published" ); return None; } Err(e) => { tracing::warn!( error = format!("{e:#}"), "this agent's secret store coordinates are incomplete; subagent \ terminals are not published" ); return None; } }; swarm_secret_client::install_crypto_provider(); tracing::info!( url = %target.url, stream = %subagent_term::stream_name(&target.agent), "publishing subagent terminals to the swarm queue as this agent" ); let (publisher, rx, status_rx) = Self::channel(QUEUE_DEPTH); tokio::spawn(run(rx, status_rx, target, Arc::clone(&publisher.dropped))); Some(publisher) } fn channel( depth: usize, ) -> ( Self, mpsc::Receiver, mpsc::UnboundedReceiver, ) { let (tx, rx) = mpsc::channel(depth); let (status, status_rx) = mpsc::unbounded_channel(); let publisher = Self { tx, status, dropped: Arc::new(AtomicU64::new(0)), }; (publisher, rx, status_rx) } /// `subagent` started a run, on the model and effort `info` names. pub fn running(&self, subagent: &str, info: &Arc) { // A closed receiver means the task is gone; there is nobody to tell. let _ = self.status.send(StatusEvent::Running { subagent: subagent.to_owned(), info: Arc::clone(info), at: unix_now(), }); } /// `subagent`'s run ended, with `exit` as its process's exit status, or /// `None` when a signal killed it. pub fn finished(&self, subagent: &str, exit: Option) { let _ = self.status.send(StatusEvent::Finished { subagent: subagent.to_owned(), exit, at: unix_now(), }); } /// Queue `output` from `subagent` for publishing, with the `info` its rows /// carry as headers. Never blocks: a full queue drops the line and counts /// it. pub fn offer(&self, subagent: &str, info: &Arc, output: Output) { let ts = unix_now(); let line = Line { subagent: subagent.to_owned(), info: Arc::clone(info), output, ts, }; if self.tx.try_send(line).is_err() { self.dropped.fetch_add(1, Ordering::Relaxed); } } } /// Unix seconds now. fn unix_now() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX)) } async fn run( mut rx: mpsc::Receiver, mut status_rx: mpsc::UnboundedReceiver, target: Target, dropped: Arc, ) { let target = Arc::new(target); let mut ctxs: HashMap = HashMap::new(); let mut queue = Queue::new("subagent terminal"); let mut runs: HashMap = HashMap::new(); let period = Duration::from_secs(subagent_status::HEARTBEAT_S.unsigned_abs()); let mut beat = tokio::time::interval_at(tokio::time::Instant::now() + period, period); loop { tokio::select! { line = rx.recv() => { let Some(line) = line else { return }; let missed = dropped.swap(0, Ordering::Relaxed); if missed > 0 { tracing::warn!(missed, "subagent terminal: queue full, lines dropped"); } publish_line(&line, &mut ctxs, &mut queue, &target).await; } Some(event) = status_rx.recv() => { let (subagent, status) = apply(&mut runs, event); publish_status(&mut queue, &target, &subagent, &status).await; } _ = beat.tick() => { for (subagent, status) in refresh(&mut runs, unix_now()) { publish_status(&mut queue, &target, &subagent, &status).await; } } } } } /// Publish the rows one line of output makes. async fn publish_line( line: &Line, ctxs: &mut HashMap, queue: &mut Queue, target: &Arc, ) { let ctx = ctxs.entry(line.subagent.clone()).or_default(); let rows = classify(&line.output, line.ts, ctx); if rows.is_empty() { return; } let Some(client) = queue.ready(target).await else { return; }; let subject = subagent_term::subject(&target.agent, &line.subagent); let headers = line.info.headers(); for row in rows { publish(&client, &subject, &headers, row).await; } } /// Fold `event` into the runs the task is keeping alive, and return the status /// to publish for it. A finished subagent leaves the map, so it is not /// heartbeated again; its model and effort carry over to the final status. fn apply(runs: &mut HashMap, event: StatusEvent) -> (String, Status) { match event { StatusEvent::Running { subagent, info, at } => { let status = Status { state: RunState::Running, since: at, exit: None, model: info.model.clone(), effort: info.effort.clone(), ts: at, }; runs.insert(subagent.clone(), status.clone()); (subagent, status) } StatusEvent::Finished { subagent, exit, at } => { let previous = runs.remove(&subagent); let status = Status { state: RunState::Finished, since: at, exit, model: previous.as_ref().and_then(|p| p.model.clone()), effort: previous.and_then(|p| p.effort), ts: at, }; (subagent, status) } } } /// Stamp every running subagent's status with `ts`, the heartbeat, and return /// them to publish. fn refresh(runs: &mut HashMap, ts: i64) -> Vec<(String, Status)> { runs.iter_mut() .map(|(subagent, status)| { status.ts = ts; (subagent.clone(), status.clone()) }) .collect() } /// Publish `status` as `subagent`'s value in the status bucket. A KV put is a /// publish to the key's subject, which the agent's credential may do and the /// controller's bucket captures. Every failure is a log line. async fn publish_status(queue: &mut Queue, target: &Arc, subagent: &str, status: &Status) { let Some(client) = queue.ready(target).await else { return; }; if let Err(e) = swarm_queue_client::ensure_connected(&client) { tracing::warn!(error = %swarm_queue_client::chain(&e), "subagent status: publish skipped"); return; } let payload = match serde_json::to_vec(status) { Ok(payload) => payload, Err(e) => { tracing::warn!(error = %e, "subagent status: serialising failed, not published"); return; } }; let subject = subagent_status::subject(&target.agent, subagent); if let Err(e) = client.publish(subject, payload.into()).await { tracing::warn!(error = %e, subagent, "subagent status: publish failed"); } } /// The rows one line of output renders as, each stamped with `ts`. fn classify(output: &Output, ts: i64, ctx: &mut ClassifyCtx) -> Vec { let rows = match output { Output::Event(v) => hive_sh4re::stream_enrich::classify_stream_value(v, ctx), Output::Stdout(line) => vec![TermMsg::new(Level::Debug, line.clone())], Output::Stderr(line) => vec![TermMsg::new(Level::Warn, format!("stderr: {line}"))], }; let ts = iso8601_utc(ts); rows.into_iter().map(|m| m.at(ts.as_str())).collect() } /// Offer one row. Every failure is terminal for that row and for nothing else. async fn publish( client: &async_nats::Client, subject: &str, headers: &async_nats::HeaderMap, msg: TermMsg, ) { // Unconnected, the client buffers rather than fails, and reports the // library's default payload limit rather than the server's. if let Err(e) = swarm_queue_client::ensure_connected(client) { tracing::warn!(error = %swarm_queue_client::chain(&e), "subagent terminal: publish skipped"); return; } let limit = swarm_queue_client::max_payload(client).saturating_sub(HEADROOM); let summary = msg.summary.clone(); let Some(msg) = fit(msg, limit) else { tracing::warn!( summary, limit, "subagent terminal: row does not fit even without its body, dropped" ); return; }; let payload = match serde_json::to_vec(&msg) { Ok(payload) => payload, Err(e) => { tracing::warn!(error = %e, "subagent terminal: serialising failed, row dropped"); return; } }; let sent = client .publish_with_headers(subject.to_owned(), headers.clone(), payload.into()) .await; if let Err(e) = sent { tracing::warn!(error = %e, "subagent terminal: publish failed, row dropped"); } } #[cfg(test)] mod tests { use super::*; /// Each subagent publishes on its own subject under this agent's family, /// the one the queue grants the agent. #[test] fn a_subagent_publishes_under_the_agents_own_family() { assert_eq!( subagent_term::subject("iris", "scout"), "$SWARM.term.iris.sub.scout" ); assert_eq!( subagent_term::subject("iris", "w-2"), "$SWARM.term.iris.sub.w-2" ); } /// The sink must never wait on the publisher: a full queue drops the line /// and counts it. #[test] fn a_full_queue_drops_and_counts_rather_than_blocks() { let (publisher, _rx, _status) = Publisher::channel(1); let info = Arc::new(RunInfo::default()); publisher.offer("scout", &info, Output::Stdout("one".to_owned())); publisher.offer("scout", &info, Output::Stdout("two".to_owned())); assert_eq!(publisher.dropped.load(Ordering::Relaxed), 1); } fn event_running(subagent: &str, at: i64) -> StatusEvent { StatusEvent::Running { subagent: subagent.to_owned(), info: Arc::new(RunInfo { model: Some("opus".to_owned()), effort: None, }), at, } } /// A start is `running` since then with the run's model; the end is /// `finished` with the exit status and the same model, and the subagent /// is not heartbeated afterwards. #[test] fn a_run_is_running_then_finished_with_its_exit_and_model() { let mut runs = HashMap::new(); let (name, started) = apply(&mut runs, event_running("scout", 100)); assert_eq!(name, "scout"); assert_eq!(started.state, RunState::Running); assert_eq!((started.since, started.ts, started.exit), (100, 100, None)); assert_eq!(started.model.as_deref(), Some("opus")); let finished = StatusEvent::Finished { subagent: "scout".to_owned(), exit: Some(0), at: 160, }; let (_, ended) = apply(&mut runs, finished); assert_eq!(ended.state, RunState::Finished); assert_eq!((ended.since, ended.ts, ended.exit), (160, 160, Some(0))); assert_eq!(ended.model.as_deref(), Some("opus")); assert!(refresh(&mut runs, 220).is_empty(), "nothing left to beat"); } /// The heartbeat moves `ts` and nothing else, so a reader tells a live /// writer from a dead one by `ts` while `since` still says when it began. #[test] fn the_heartbeat_refreshes_ts_and_keeps_since() { let mut runs = HashMap::new(); apply(&mut runs, event_running("scout", 100)); apply(&mut runs, event_running("other", 130)); let mut beat = refresh(&mut runs, 160); beat.sort_by(|a, b| a.0.cmp(&b.0)); assert_eq!(beat.len(), 2); assert_eq!( (beat[0].0.as_str(), beat[0].1.since, beat[0].1.ts), ("other", 130, 160) ); assert_eq!( (beat[1].0.as_str(), beat[1].1.since, beat[1].1.ts), ("scout", 100, 160) ); assert!(beat.iter().all(|(_, s)| s.state == RunState::Running)); } /// A lost end would leave the task heartbeating a subagent that ended, so /// the status events are never dropped for a full queue. #[test] fn status_events_are_not_bounded_by_the_line_queue() { let (publisher, _rx, mut status) = Publisher::channel(1); let info = Arc::new(RunInfo::default()); for _ in 0..10 { publisher.running("scout", &info); publisher.finished("scout", Some(0)); } let mut seen = 0; while status.try_recv().is_ok() { seen += 1; } assert_eq!(seen, 20); } /// The list endpoint reads these two headers off a subagent's newest row. #[test] fn a_row_carries_the_model_and_effort_the_subagent_runs() { let info = RunInfo { model: Some("opus".to_owned()), effort: Some("high".to_owned()), }; let headers = info.headers(); assert_eq!( headers .get(subagent_term::HEADER_MODEL) .map(ToString::to_string), Some("opus".to_owned()) ); assert_eq!( headers .get(subagent_term::HEADER_EFFORT) .map(ToString::to_string), Some("high".to_owned()) ); } /// A subagent on the default model names none, and an empty header would /// read as a model called "". #[test] fn a_value_that_is_not_set_is_not_published() { let info = RunInfo { model: None, effort: Some(String::new()), }; assert!(info.headers().is_empty()); } /// The values are the caller's. A line break would panic `async_nats`, in /// the task that publishes every subagent's rows. #[test] fn a_value_a_header_cannot_hold_is_left_out_and_never_panics() { for bad in ["a\nb", "high\r\nX: y", &"x".repeat(MAX_VALUE_LEN + 1)] { let info = RunInfo { model: Some(bad.to_owned()), effort: Some(bad.to_owned()), }; assert!(info.headers().is_empty(), "{bad:?}"); } let padded = RunInfo { model: Some(" opus\n".to_owned()), effort: None, }; assert_eq!( padded .headers() .get(subagent_term::HEADER_MODEL) .map(ToString::to_string), Some("opus".to_owned()) ); } #[test] fn every_kind_of_line_renders_as_stamped_rows() { let mut ctx = ClassifyCtx::default(); let ts = 1_789_302_903; let event = serde_json::json!({ "type": "assistant", "message": { "content": [{ "type": "text", "text": "looking" }] } }); let rows = classify(&Output::Event(event), ts, &mut ctx); assert_eq!(rows.len(), 1); assert_eq!(rows[0].ts, "2026-09-13T12:35:03Z"); let rows = classify(&Output::Stderr("boom".to_owned()), ts, &mut ctx); assert_eq!(rows[0].level, Level::Warn); assert_eq!(rows[0].summary, "stderr: boom"); let rows = classify(&Output::Stdout("chatter".to_owned()), ts, &mut ctx); assert_eq!(rows[0].level, Level::Debug); assert_eq!(rows[0].ts, "2026-09-13T12:35:03Z"); } }