mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm: list subagents from their published run state

Browse source 
The subagents list reads each subagent's status from the subagent-status
bucket and reports its state, when it began and the exit status. A subagent
with no status keeps the newest-row lookup, so an older daemon lists as before.
The swarm UI uses the state when there is one and the row-age guess otherwise.
This commit is contained in:
iris 2026-11-15 14:51:37 +01:00 • committed by mara
commit e9bc28fe2f
1 changed files with 309 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

176
swarm-controller/src/subagent_term.rs
View file

@ -133,6 +133,40 @@ pub(crate) struct SubagentEntry {
pub model: Option<String>,
/// Its reasoning effort, from the `Swarm-Effort` header, with the same `None`.
pub effort: Option<String>,
/// Whether it is running, from the status its daemon published. `None`
/// for a subagent that has published none (an older daemon), which a
/// reader has to judge from `last_row_unix`.
pub state: Option<SubagentState>,
/// When the subagent entered its published state, unix seconds. `None`
/// with `state`.
pub since_unix: Option<i64>,
/// The process's exit status once it finished, when it exited by itself.
pub exit: Option<i32>,
}
/// What the status a subagent's daemon published says about it, as of the
/// request.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub(crate) enum SubagentState {
/// A run is in progress and the daemon reported in recently.
Running,
/// The last run ended.
Finished,
/// The status says running, but the daemon has not reported in for three
/// heartbeats: it died or lost the queue.
Stale,
}
impl SubagentState {
fn of(status: &swarm_wire::subagent_status::Status, now_s: i64) -> Self {
use swarm_wire::subagent_status::State;
match status.state {
State::Finished => Self::Finished,
State::Running if status.is_running(now_s) => Self::Running,
State::Running => Self::Stale,
}
}
}
#[utoipa::path(
@ -159,7 +193,7 @@ pub(crate) async fn list_subagents(
) -> Result<Json<Vec<SubagentEntry>>, problem_details::ProblemDetails> {
let agent = ident(&agent).map_err(bad_request)?;
let client = connected_client(&state).map_err(|e| unavailable(&e))?;
let js = async_nats::jetstream::new(client);
let js = async_nats::jetstream::new(client.clone());
let stream_name = subagent_term::stream_name(&agent);
let stream = match js.get_stream(&stream_name).await {
Ok(stream) => stream,
@ -177,14 +211,28 @@ pub(crate) async fn list_subagents(
// Owned names and `async move`, not a closure over `&names`: a borrowed
// item type trips a "`FnOnce` is not general enough" inference error where
// `routes!` checks this fn as a handler.
let statuses = read_statuses(&client, &agent, &names).await;
let (stream_ref, agent_ref) = (&stream, &agent);
let lookups: Vec<Result<NewestRow, String>> = futures_util::stream::iter(names.clone())
.map(|name| async move { newest_row(stream_ref, agent_ref, &name).await })
.buffered(LOOKUP_CONCURRENCY)
.collect()
.await;
let failed = lookups.iter().filter(|l| l.is_err()).count();
if let Some(Err(error)) = lookups.iter().find(|l| l.is_err()) {
// A subagent with a published status needs no newest-row lookup: the
// status carries what the row's time and headers were read for.
let lookups: Vec<Option<Result<NewestRow, String>>> = futures_util::stream::iter(
names
.iter()
.cloned()
.zip(statuses.iter().map(Option::is_some)),
)
.map(|(name, has_status)| async move {
if has_status {
None
} else {
Some(newest_row(stream_ref, agent_ref, &name).await)
}
})
.buffered(LOOKUP_CONCURRENCY)
.collect()
.await;
let failed = lookups.iter().flatten().filter(|l| l.is_err()).count();
if let Some(Err(error)) = lookups.iter().flatten().find(|l| l.is_err()) {
tracing::warn!(
%agent,
failed,
@ -193,22 +241,116 @@ pub(crate) async fn list_subagents(
"listing subagents: reading the newest row failed (first error shown)"
);
}
let now_s = unix_now();
let entries = names
.into_iter()
.zip(statuses)
.zip(lookups)
.map(|(name, lookup)| {
let row = lookup.ok();
SubagentEntry {
name,
last_row_unix: row.as_ref().map(|r| r.unix),
model: row.as_ref().and_then(|r| r.model.clone()),
effort: row.and_then(|r| r.effort),
}
})
.map(|((name, status), lookup)| entry_of(name, status, lookup, now_s))
.collect();
Ok(Json(entries))
}
/// One list entry. A published status gives the state, when it began and the
/// model and effort; without one, the newest row gives the time and the
/// headers give the model and effort.
fn entry_of(
name: String,
status: Option<swarm_wire::subagent_status::Status>,
lookup: Option<Result<NewestRow, String>>,
now_s: i64,
) -> SubagentEntry {
if let Some(status) = status {
return SubagentEntry {
name,
last_row_unix: None,
state: Some(SubagentState::of(&status, now_s)),
since_unix: Some(status.since),
exit: status.exit,
model: status.model,
effort: status.effort,
};
}
let row = lookup.and_then(Result::ok);
SubagentEntry {
name,
last_row_unix: row.as_ref().map(|r| r.unix),
model: row.as_ref().and_then(|r| r.model.clone()),
effort: row.and_then(|r| r.effort),
state: None,
since_unix: None,
exit: None,
}
}
/// Seconds since the unix epoch, by this host's clock.
fn unix_now() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX))
}
/// The status each of `names` last published, in the same order: `None` for a
/// subagent with none, and for every subagent when the bucket cannot be read
/// (an older swarm, or the queue failing), so the list falls back to the rows.
async fn read_statuses(
client: &async_nats::Client,
agent: &str,
names: &[String],
) -> Vec<Option<swarm_wire::subagent_status::Status>> {
let js = async_nats::jetstream::new(client.clone());
let store = match js.get_key_value(swarm_wire::subagent_status::BUCKET).await {
Ok(store) => store,
Err(e) => {
tracing::debug!(
%agent,
error = %swarm_queue_client::chain(&e),
"listing subagents: the status bucket is not readable; using the rows"
);
return vec![None; names.len()];
}
};
// Owned items and `async move`, not a closure over borrows: a borrowed item
// type trips the same "`FnOnce` is not general enough" error as above.
let items: Vec<(async_nats::jetstream::kv::Store, String, String)> = names
.iter()
.map(|name| (store.clone(), agent.to_owned(), name.clone()))
.collect();
futures_util::stream::iter(items)
.map(|(store, agent, name)| async move { read_status(&store, &agent, &name).await })
.buffered(LOOKUP_CONCURRENCY)
.collect()
.await
}
/// One subagent's published status, or `None` when it has none or it could not
/// be read (each failure but a missing key is a log line).
async fn read_status(
store: &async_nats::jetstream::kv::Store,
agent: &str,
name: &str,
) -> Option<swarm_wire::subagent_status::Status> {
let key = swarm_wire::subagent_status::key(agent, name);
match tokio::time::timeout(LOOKUP_TIMEOUT, store.get(&key)).await {
Ok(Ok(Some(value))) => match serde_json::from_slice(&value) {
Ok(status) => Some(status),
Err(e) => {
tracing::warn!(%agent, subagent = %name, error = %e, "listing subagents: the status is not readable");
None
}
},
Ok(Ok(None)) => None,
Ok(Err(e)) => {
tracing::warn!(%agent, subagent = %name, error = %swarm_queue_client::chain(&e), "listing subagents: reading the status failed");
None
}
Err(_) => {
tracing::warn!(%agent, subagent = %name, "listing subagents: no answer reading the status");
None
}
}
}
/// What a subagent's newest row says: when it was published, and the model and
/// effort its headers name.
#[derive(Debug, PartialEq, Eq)]