swarm: cancel an agent's turn from the swarm queue
A separate $SWARM.control.<agent> subject carries commands, apart from
the text subject. swarm-controller exposes POST /api/agents/{name}/cancel
as a NATS request; the harness answers whether a turn was actually
stopped, through the same cancel path as the dashboard button. The agent
credential gets subscribe on its control subject and allow_responses to
answer; only the controller may publish there.
Refs #3803
This commit is contained in:
parent
e7ed28b382
commit
fefc565794
11 changed files with 558 additions and 69 deletions
|
|
@ -15,23 +15,90 @@
|
|||
//! **Own credential only.** The queue grants subscribe on this subject to the
|
||||
//! agent's own credential alone.
|
||||
//!
|
||||
//! **Core NATS, no reply.** A message sent while this task is not subscribed
|
||||
//! is gone; the controller sends it as a request so the operator gets a 503
|
||||
//! for that instead. No agent grant covers a reply subject, so nothing here
|
||||
//! answers the request.
|
||||
//! **Core NATS, no reply to input.** A message sent while this task is not
|
||||
//! subscribed is gone; the controller sends it as a request so the operator
|
||||
//! gets a 503 for that instead. Input is not answered: the controller reads a
|
||||
//! request that times out as "a subscriber held it".
|
||||
//!
|
||||
//! **Commands are a separate subject and are answered.** The swarm UI's
|
||||
//! cancel arrives on [`swarm_queue_client::agent_control_subject`], never on
|
||||
//! the input subject, so nothing a person types can be read as a command. The
|
||||
//! reply says whether a turn was actually stopped; the agent's grant allows
|
||||
//! answering only the request it was just sent (`allow_responses`).
|
||||
|
||||
use futures_util::{Stream, StreamExt as _};
|
||||
use hive_sh4re::inbox::DeliveredMessage;
|
||||
use tokio::sync::mpsc::UnboundedSender;
|
||||
|
||||
/// Start the subscriber, if this agent has a label to name its subject with.
|
||||
pub fn spawn(tx: UnboundedSender<DeliveredMessage>) {
|
||||
use crate::events::Bus;
|
||||
|
||||
/// Start the subscribers, if this agent has a label to name its subjects with.
|
||||
pub fn spawn(
|
||||
tx: UnboundedSender<DeliveredMessage>,
|
||||
bus: Bus,
|
||||
interrupted: std::sync::Arc<std::sync::atomic::AtomicBool>,
|
||||
) {
|
||||
let agent = crate::identity::label();
|
||||
if agent.is_empty() {
|
||||
tracing::warn!("this agent has no label; not taking input from the swarm queue");
|
||||
return;
|
||||
}
|
||||
tokio::spawn(run(tx, agent));
|
||||
tokio::spawn(run(tx, agent.clone()));
|
||||
tokio::spawn(run_control(agent, bus, interrupted));
|
||||
}
|
||||
|
||||
/// Answer each control command for `agent` until the subscription ends.
|
||||
async fn run_control(
|
||||
agent: String,
|
||||
bus: Bus,
|
||||
interrupted: std::sync::Arc<std::sync::atomic::AtomicBool>,
|
||||
) {
|
||||
let client = crate::swarm_queue::client().await;
|
||||
let subject = swarm_queue_client::agent_control_subject(&agent);
|
||||
let mut subscriber = match client.subscribe(subject.clone()).await {
|
||||
Ok(subscriber) => subscriber,
|
||||
Err(e) => {
|
||||
tracing::warn!(subject, error = %e, "swarm control: subscribe failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
tracing::info!(subject, "taking control commands from the swarm queue");
|
||||
while let Some(msg) = subscriber.next().await {
|
||||
let Some(reply) = msg.reply else {
|
||||
// Sent without a reply subject: nobody to tell, and the
|
||||
// controller always asks for one.
|
||||
tracing::warn!("swarm control: command without a reply subject dropped");
|
||||
continue;
|
||||
};
|
||||
let Some(answer) = act(&msg.payload, |command| match command {
|
||||
swarm_queue_client::AgentControl::Cancel => {
|
||||
crate::web_ui::cancel_turn(&bus, &interrupted, "swarm UI: cancel")
|
||||
}
|
||||
}) else {
|
||||
continue;
|
||||
};
|
||||
let payload = serde_json::to_vec(&answer).expect("a reply encodes");
|
||||
if let Err(e) = client.publish(reply, payload.into()).await {
|
||||
tracing::warn!(error = %e, "swarm control: answering failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The answer to a command payload, running `cancel` for a cancel; `None`
|
||||
/// (logged) when the payload is not a command, which gets no answer.
|
||||
fn act(
|
||||
payload: &[u8],
|
||||
cancel: impl FnOnce(swarm_queue_client::AgentControl) -> bool,
|
||||
) -> Option<swarm_queue_client::AgentControlReply> {
|
||||
match serde_json::from_slice::<swarm_queue_client::AgentControl>(payload) {
|
||||
Ok(command) => Some(swarm_queue_client::AgentControlReply {
|
||||
acted: cancel(command),
|
||||
}),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "swarm control: undecodable command dropped");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(tx: UnboundedSender<DeliveredMessage>, agent: String) {
|
||||
|
|
@ -88,8 +155,8 @@ fn delivered(payload: &[u8]) -> Option<DeliveredMessage> {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::forward;
|
||||
use swarm_queue_client::AgentInput;
|
||||
use super::{act, forward};
|
||||
use swarm_queue_client::{AgentControl, AgentControlReply, AgentInput};
|
||||
|
||||
fn payload(from: &str, body: &str) -> Vec<u8> {
|
||||
serde_json::to_vec(&AgentInput {
|
||||
|
|
@ -136,4 +203,21 @@ mod tests {
|
|||
let payloads = futures_util::stream::repeat(payload("mara", "hi"));
|
||||
forward(payloads, &tx).await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cancel_is_answered_with_whether_it_acted() {
|
||||
for acted in [true, false] {
|
||||
let got = act(br#""cancel""#, |c| {
|
||||
assert_eq!(c, AgentControl::Cancel);
|
||||
acted
|
||||
});
|
||||
assert_eq!(got, Some(AgentControlReply { acted }));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_payload_that_is_not_a_command_is_not_answered_or_acted_on() {
|
||||
let got = act(b"pause", |_| panic!("ran a command that was not one"));
|
||||
assert_eq!(got, None);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue