vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm: named scheduled prompts for one or more agents, fired in each agent

Browse source 
swarm-controller keeps every scheduled prompt (id, name, agents, body,
anchor, interval, paused) as one document under one key of a new
`schedule-definitions` KV bucket, so each change is a single
compare-and-set and a prompt exists for all of its agents or none. It
writes each agent's key of a new `agent-schedules` bucket as the prompts
naming that agent, without names or other agents, after every change and
on a one-minute reconcile pass. It is the only writer of both, behind
GET/POST /api/schedules and PATCH/DELETE /api/schedules/{id}.
`swarmctl schedule add|list|edit|pause|resume|rm` drives those routes over
the controller socket.

An agent's own queue credential is granted one more subject, the direct
get of its own key in `agent-schedules`. A KV watch is not granted: its
consumer create subject carries no key, so it would cover every agent's.

hive-agent's new schedule ticker reads that key every 10 seconds and fires
due slots as the same `schedule:<id>` todo hive-c0re's worker pushes. Fire
times are anchor + k*interval; the last fired slot per schedule id is kept
in the state db, so a body edit or list refresh never moves or refires a
slot, downtime fires the latest missed slot once, a restart refires
nothing, a one-shot fires once, and paused slots are skipped.

Refs #4914
This commit is contained in:
atlas 2026-10-04 16:29:44 +02:00 • committed by mara
commit a0b1846006
24 changed files with 2617 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

39
swarmctl/src/agent.rs
View file

@ -184,8 +184,23 @@ pub(crate) async fn post<Req: Serialize, Resp: serde::de::DeserializeOwned>(
request: &Req,
what: &str,
) -> Result<Resp> {
let body =
serde_json::to_vec(request).with_context(|| format!("serialising the {what} request"))?;
send(socket, hyper::Method::POST, uri, Some(request), what).await
}
/// One request to `uri` over the controller's unix socket, with `request` as
/// its JSON body when there is one. A `204` decodes as JSON `null`, so a
/// caller expecting no body asks for `()`.
pub(crate) async fn send<Req: Serialize, Resp: serde::de::DeserializeOwned>(
socket: &Path,
method: hyper::Method,
uri: &str,
request: Option<&Req>,
what: &str,
) -> Result<Resp> {
let body = request
.map(serde_json::to_vec)
.transpose()
.with_context(|| format!("serialising the {what} request"))?;
let stream = UnixStream::connect(socket)
.await
@ -193,19 +208,22 @@ pub(crate) async fn post<Req: Serialize, Resp: serde::de::DeserializeOwned>(
let (mut sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(stream))
.await
.with_context(|| format!("HTTP handshake with {}", socket.display()))?;
// Drives the connection's I/O; a one-shot POST has nothing to do with
// Drives the connection's I/O; a one-shot request has nothing to do with
// the join handle, and a connection that dies surfaces as an error on
// `send_request` or on reading the body.
tokio::spawn(conn);
let req = hyper::Request::builder()
.method(hyper::Method::POST)
let mut req = hyper::Request::builder()
.method(method)
.uri(uri)
// A unix socket has no authority of its own, but HTTP/1.1 requires
// the header; the controller routes on the path alone.
.header(hyper::header::HOST, "localhost")
.header(hyper::header::CONTENT_TYPE, "application/json")
.body(Full::new(bytes::Bytes::from(body)))
.header(hyper::header::HOST, "localhost");
if body.is_some() {
req = req.header(hyper::header::CONTENT_TYPE, "application/json");
}
let req = req
.body(Full::new(bytes::Bytes::from(body.unwrap_or_default())))
.with_context(|| format!("building the {what} request"))?;
let resp = sender
@ -223,6 +241,11 @@ pub(crate) async fn post<Req: Serialize, Resp: serde::de::DeserializeOwned>(
if !status.is_success() {
bail!("{}", describe_error(status, &body));
}
let body = if status == hyper::StatusCode::NO_CONTENT {
bytes::Bytes::from_static(b"null")
} else {
body
};
serde_json::from_slice(&body).with_context(|| {
format!(
"swarm-controller answered {status} but its body is not a {what} response: {}",

175
swarmctl/src/main.rs
View file

@ -29,6 +29,7 @@
mod agent;
mod forge;
mod schedule;
mod users;
use std::fs::{self, File, Permissions};
@ -119,6 +120,16 @@ enum Verb {
#[command(subcommand)]
command: AgentVerb,
},
/// Manage scheduled prompts.
///
/// The swarm keeps each prompt, and every agent it names fires it in its
/// own container. Fire times are the schedule's start time plus whole
/// intervals: changing the body never moves them, and an agent that was
/// down fires the latest missed one once.
Schedule {
#[command(subcommand)]
command: ScheduleVerb,
},
/// Manage subjects in the swarm's SSO provider.
User {
#[command(subcommand)]
@ -276,6 +287,119 @@ struct ForgeMakeAdminArgs {
controller_socket: Option<PathBuf>,
}
#[derive(Subcommand)]
enum ScheduleVerb {
/// Add a scheduled prompt and print it with its new id.
///
/// Without `--every` it fires once. Without `--at` its first fire is now.
Add(ScheduleAddArgs),
/// List scheduled prompts, one row each, with their agents and next fire
/// time.
List(ScheduleListArgs),
/// Change a scheduled prompt.
///
/// Only the flags given change, and the id stays. `--at` and `--every`
/// move the fire times; `--name`, `--body` and the agent flags don't.
Edit(ScheduleEditArgs),
/// Stop a scheduled prompt firing. Times that pass while it's paused
/// aren't fired when it resumes.
Pause(ScheduleIdArgs),
/// Let a paused scheduled prompt fire again, from its next time on.
Resume(ScheduleIdArgs),
/// Remove a scheduled prompt from every agent it names.
Rm(ScheduleIdArgs),
}
#[derive(Args)]
struct ScheduleAddArgs {
/// The prompt's title.
#[arg(long, value_name = "TEXT")]
name: String,
/// Agent to fire it in. Repeatable; at least one.
#[arg(long = "agent", value_name = "AGENT", required = true)]
agents: Vec<String>,
/// The prompt, delivered to each agent verbatim.
#[arg(long, value_name = "TEXT")]
body: String,
/// First fire time: an RFC 3339 time with its offset, such as
/// `2026-10-05T09:00:00+02:00`, or unix seconds. Defaults to now.
#[arg(long, value_name = "TIME")]
at: Option<String>,
/// Time between fires: a count with a unit of `s`, `m`, `h` or `d`,
/// such as `1h` or `7d`. Leave out for a one-shot.
#[arg(long, value_name = "DURATION")]
every: Option<String>,
/// Add it paused.
#[arg(long)]
paused: bool,
#[command(flatten)]
socket: ControllerSocketArg,
}
#[derive(Args)]
struct ScheduleListArgs {
/// Show only the prompts that fire in this agent.
#[arg(long, value_name = "AGENT")]
agent: Option<String>,
#[command(flatten)]
socket: ControllerSocketArg,
}
#[derive(Args)]
struct ScheduleEditArgs {
/// The prompt's id, as `schedule list` shows it.
id: String,
/// New title.
#[arg(long, value_name = "TEXT")]
name: Option<String>,
/// New prompt text.
#[arg(long, value_name = "TEXT")]
body: Option<String>,
/// New first fire time, in the form `schedule add --at` takes.
#[arg(long, value_name = "TIME")]
at: Option<String>,
/// New time between fires, in the form `schedule add --every` takes.
#[arg(long, value_name = "DURATION", conflicts_with = "once")]
every: Option<String>,
/// Make it a one-shot.
#[arg(long)]
once: bool,
/// Agent to start firing it in. Repeatable.
#[arg(long = "add-agent", value_name = "AGENT")]
add_agents: Vec<String>,
/// Agent to stop firing it in. Repeatable. Fails if the agent isn't a
/// target, or if no agent would remain.
#[arg(long = "remove-agent", value_name = "AGENT")]
remove_agents: Vec<String>,
#[command(flatten)]
socket: ControllerSocketArg,
}
#[derive(Args)]
struct ScheduleIdArgs {
/// The prompt's id, as `schedule list` shows it.
id: String,
#[command(flatten)]
socket: ControllerSocketArg,
}
#[derive(Args)]
struct ControllerSocketArg {
/// swarm-controller's unix socket.
///
/// Supplied by the nix module that installs this binary, from the same
/// `socketPath` option the daemon binds; falls back to
/// `SWARM_CONTROLLER_SOCKET`.
#[arg(long, value_name = "PATH")]
controller_socket: Option<PathBuf>,
}
impl ControllerSocketArg {
fn resolve(self) -> Result<PathBuf> {
path_from(self.controller_socket, "SWARM_CONTROLLER_SOCKET")
}
}
#[derive(Subcommand)]
enum UserVerb {
/// Add a user, generating a password for them.
@ -379,6 +503,8 @@ fn main() -> Result<()> {
let socket = path_from(args.controller_socket, "SWARM_CONTROLLER_SOCKET")?;
forge::make_admin(&socket, &args.name)
}
// Same socket-resolution reasoning as `Create` above.
Verb::Schedule { command } => schedule(command),
// Resolved lazily, inside the one arm that actually touches the
// deployment env vars — see the `MarkdownDocs` doc comment above
// for why an unconditional resolve up front would be wrong.
@ -418,6 +544,38 @@ fn main() -> Result<()> {
}
}
fn schedule(command: ScheduleVerb) -> Result<()> {
match command {
ScheduleVerb::Add(a) => schedule::add(
&a.socket.resolve()?,
&schedule::AddFlags {
name: &a.name,
agents: &a.agents,
body: &a.body,
at: a.at.as_deref(),
every: a.every.as_deref(),
paused: a.paused,
},
),
ScheduleVerb::List(a) => schedule::list(&a.socket.resolve()?, a.agent.as_deref()),
ScheduleVerb::Edit(a) => {
let edit = schedule::EditRequest::new(schedule::EditFlags {
name: a.name,
body: a.body,
at: a.at.as_deref(),
every: a.every.as_deref(),
once: a.once,
add_agents: a.add_agents,
remove_agents: a.remove_agents,
})?;
schedule::edit(&a.socket.resolve()?, &a.id, &edit)
}
ScheduleVerb::Pause(a) => schedule::set_paused(&a.socket.resolve()?, &a.id, true),
ScheduleVerb::Resume(a) => schedule::set_paused(&a.socket.resolve()?, &a.id, false),
ScheduleVerb::Rm(a) => schedule::remove(&a.socket.resolve()?, &a.id),
}
}
fn user_add(paths: &Paths, args: AddArgs) -> Result<()> {
users::validate_username(&args.username)?;
@ -713,6 +871,23 @@ mod tests {
Cli::command().debug_assert();
}
/// A schedule with no agent would fire nowhere, so `--agent` is required
/// at parse time, and repeats collect into the target list.
#[test]
fn schedule_add_takes_one_or_more_agents() {
let base = ["swarmctl", "schedule", "add", "--name", "n", "--body", "b"];
assert!(Cli::try_parse_from(base).is_err());
let cli = Cli::try_parse_from(base.iter().copied().chain(["--agent", "a", "--agent", "b"]))
.expect("parses");
let Verb::Schedule {
command: ScheduleVerb::Add(args),
} = cli.command
else {
panic!("parsed as another verb");
};
assert_eq!(args.agents, ["a", "b"]);
}
/// `--hive` carries the address the agent is deployed to and the
/// endpoint has no default for it, so omitting it has to fail at parse
/// time rather than reach the controller as an empty string.

416
swarmctl/src/schedule.rs Normal file
View file

@ -0,0 +1,416 @@
//! `swarmctl schedule` — the operator's scheduled prompts, through
//! swarm-controller's `/api/schedules` routes.
//!
//! Each verb waits for the controller's answer, which is the list or the
//! schedule as stored. Each agent the schedule names picks a change up on its
//! next read of its schedule key and fires in its own container.
//!
//! The shapes below mirror the controller's own `ScheduleView` and request
//! bodies, for the reason [`crate::agent`]'s module doc gives.
use std::path::Path;
use anyhow::{Context as _, Result, bail};
use serde::{Deserialize, Serialize};
use crate::agent::{parse_ident, send};
/// One schedule as the controller shows it.
#[derive(Debug, Deserialize)]
struct ScheduleView {
id: String,
name: String,
agents: Vec<String>,
body: String,
anchor_unix: i64,
interval_seconds: Option<u64>,
paused: bool,
next_fire_unix: Option<i64>,
}
/// Body of `POST /api/schedules`.
#[derive(Serialize)]
struct AddRequest<'a> {
name: &'a str,
agents: Vec<String>,
body: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
anchor_unix: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
interval_seconds: Option<u64>,
paused: bool,
}
/// Body of `PATCH /api/schedules/{id}`. An absent field is left as it is.
#[derive(Default, Serialize)]
pub(crate) struct EditRequest {
#[serde(skip_serializing_if = "Option::is_none")]
name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
body: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
anchor_unix: Option<i64>,
#[serde(skip_serializing_if = "IntervalEdit::is_keep")]
interval_seconds: IntervalEdit,
#[serde(skip_serializing_if = "Option::is_none")]
paused: Option<bool>,
#[serde(skip_serializing_if = "Vec::is_empty")]
add_agents: Vec<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
remove_agents: Vec<String>,
}
/// What an edit does to the interval. `Set(None)` is sent as `null`, which
/// makes a one-shot; `Keep` is not sent at all.
#[derive(Clone, Copy, Default)]
enum IntervalEdit {
#[default]
Keep,
Set(Option<u64>),
}
impl IntervalEdit {
fn is_keep(&self) -> bool {
matches!(self, Self::Keep)
}
}
impl Serialize for IntervalEdit {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
match self {
Self::Keep | Self::Set(None) => s.serialize_none(),
Self::Set(Some(seconds)) => s.serialize_u64(*seconds),
}
}
}
/// What `schedule edit` was asked to change.
pub(crate) struct EditFlags<'a> {
pub name: Option<String>,
pub body: Option<String>,
pub at: Option<&'a str>,
pub every: Option<&'a str>,
pub once: bool,
pub add_agents: Vec<String>,
pub remove_agents: Vec<String>,
}
impl EditRequest {
/// The edit `schedule edit` asks for. `once` and `every` are exclusive
/// in the CLI, so at most one of them is set here.
pub(crate) fn new(flags: EditFlags<'_>) -> Result<Self> {
let interval_seconds = match (flags.every, flags.once) {
(Some(every), _) => IntervalEdit::Set(Some(parse_every(every)?)),
(None, true) => IntervalEdit::Set(None),
(None, false) => IntervalEdit::Keep,
};
let edit = Self {
name: flags.name,
body: flags.body,
anchor_unix: flags.at.map(parse_at).transpose()?,
interval_seconds,
paused: None,
add_agents: agents(&flags.add_agents)?,
remove_agents: agents(&flags.remove_agents)?,
};
if edit.name.is_none()
&& edit.body.is_none()
&& edit.anchor_unix.is_none()
&& edit.interval_seconds.is_keep()
&& edit.add_agents.is_empty()
&& edit.remove_agents.is_empty()
{
bail!(
"nothing to change: pass --name, --body, --at, --every, --once, --add-agent \
or --remove-agent"
);
}
Ok(edit)
}
fn paused(paused: bool) -> Self {
Self {
paused: Some(paused),
..Self::default()
}
}
}
fn agents(names: &[String]) -> Result<Vec<String>> {
names.iter().map(|n| parse_ident(n, "agent name")).collect()
}
fn runtime() -> Result<tokio::runtime::Runtime> {
tokio::runtime::Builder::new_current_thread()
.enable_io()
.build()
.context("starting a tokio runtime for the controller request")
}
const COLLECTION: &str = "/api/schedules";
/// The schedule's path. The id is checked like an agent name, which every id
/// the controller mints satisfies, so it cannot reshape the URI.
fn member(id: &str) -> Result<String> {
let id = parse_ident(id, "schedule id")?;
Ok(format!("{COLLECTION}/{id}"))
}
/// What `schedule add` was asked for.
pub(crate) struct AddFlags<'a> {
pub name: &'a str,
pub agents: &'a [String],
pub body: &'a str,
pub at: Option<&'a str>,
pub every: Option<&'a str>,
pub paused: bool,
}
/// Run `swarmctl schedule add`.
pub(crate) fn add(socket: &Path, flags: &AddFlags<'_>) -> Result<()> {
let request = AddRequest {
name: flags.name,
agents: agents(flags.agents)?,
body: flags.body,
anchor_unix: flags.at.map(parse_at).transpose()?,
interval_seconds: flags.every.map(parse_every).transpose()?,
paused: flags.paused,
};
let added: ScheduleView = runtime()?.block_on(send(
socket,
hyper::Method::POST,
COLLECTION,
Some(&request),
"schedule-add",
))?;
println!("added schedule {}", added.id);
println!("{}", render(&added));
Ok(())
}
/// Run `swarmctl schedule list`, optionally only the schedules naming `agent`.
pub(crate) fn list(socket: &Path, agent: Option<&str>) -> Result<()> {
let agent = agent.map(|a| parse_ident(a, "agent name")).transpose()?;
let schedules: Vec<ScheduleView> = runtime()?.block_on(send(
socket,
hyper::Method::GET,
COLLECTION,
None::<&()>,
"schedule-list",
))?;
let shown: Vec<&ScheduleView> = schedules
.iter()
.filter(|s| agent.as_ref().is_none_or(|a| s.agents.contains(a)))
.collect();
if shown.is_empty() {
println!("no schedules");
}
for s in shown {
println!("{}", render(s));
}
Ok(())
}
/// Run `swarmctl schedule edit`.
pub(crate) fn edit(socket: &Path, id: &str, edit: &EditRequest) -> Result<()> {
let edited: ScheduleView = runtime()?.block_on(send(
socket,
hyper::Method::PATCH,
&member(id)?,
Some(edit),
"schedule-edit",
))?;
println!("{}", render(&edited));
Ok(())
}
/// Run `swarmctl schedule pause` or `resume`.
pub(crate) fn set_paused(socket: &Path, id: &str, paused: bool) -> Result<()> {
edit(socket, id, &EditRequest::paused(paused))
}
/// Run `swarmctl schedule rm`.
pub(crate) fn remove(socket: &Path, id: &str) -> Result<()> {
let uri = member(id)?;
runtime()?.block_on(send::<(), ()>(
socket,
hyper::Method::DELETE,
&uri,
None,
"schedule-rm",
))?;
println!("removed schedule {id}");
Ok(())
}
/// `--at`: unix seconds, or an RFC 3339 time with its offset.
fn parse_at(value: &str) -> Result<i64> {
if let Ok(unix) = value.parse::<i64>() {
return Ok(unix);
}
chrono::DateTime::parse_from_rfc3339(value)
.map(|t| t.timestamp())
.with_context(|| {
format!(
"--at {value:?} is neither unix seconds nor an RFC 3339 time such as \
2026-10-05T09:00:00+02:00"
)
})
}
/// `--every`: a positive count of seconds, minutes, hours or days, such as
/// `90s`, `15m`, `1h` or `7d`. A bare number is seconds.
fn parse_every(value: &str) -> Result<u64> {
let (digits, unit) = match value.char_indices().last() {
Some((i, c)) if c.is_ascii_alphabetic() => (&value[..i], c),
_ => (value, 's'),
};
let scale = match unit {
's' => 1,
'm' => 60,
'h' => 3600,
'd' => 86_400,
_ => bail!("--every {value:?}: the unit must be s, m, h or d"),
};
let count: u64 = digits
.parse()
.with_context(|| format!("--every {value:?} is not a count followed by s, m, h or d"))?;
match count.checked_mul(scale) {
Some(0) => bail!("--every {value:?} is zero; leave --every out for a one-shot"),
Some(seconds) => Ok(seconds),
None => bail!("--every {value:?} is too long"),
}
}
fn utc(unix: i64) -> String {
chrono::DateTime::from_timestamp(unix, 0).map_or_else(
|| unix.to_string(),
|t| t.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
)
}
/// One row per schedule: id, name, cadence, next fire, agents, and the
/// body's first line.
fn render(s: &ScheduleView) -> String {
let cadence = match s.interval_seconds {
Some(i) => format!("every {}", every_text(i)),
None => "once".to_owned(),
};
let next = match (s.paused, s.next_fire_unix) {
(true, _) => "paused".to_owned(),
(false, Some(t)) => format!("next {}", utc(t)),
(false, None) => format!("fired at {}", utc(s.anchor_unix)),
};
let body = s.body.lines().next().unwrap_or_default();
format!(
"{} {:?} {cadence} from {} {next} agents: {} | {body}",
s.id,
s.name,
utc(s.anchor_unix),
s.agents.join(", ")
)
}
fn every_text(seconds: u64) -> String {
for (unit, scale) in [("d", 86_400), ("h", 3600), ("m", 60)] {
if seconds.is_multiple_of(scale) {
return format!("{}{unit}", seconds / scale);
}
}
format!("{seconds}s")
}
#[cfg(test)]
mod tests {
use super::*;
fn flags() -> EditFlags<'static> {
EditFlags {
name: None,
body: None,
at: None,
every: None,
once: false,
add_agents: Vec::new(),
remove_agents: Vec::new(),
}
}
#[test]
fn every_takes_a_count_and_a_unit() {
assert_eq!(parse_every("90").unwrap(), 90);
assert_eq!(parse_every("90s").unwrap(), 90);
assert_eq!(parse_every("15m").unwrap(), 900);
assert_eq!(parse_every("1h").unwrap(), 3600);
assert_eq!(parse_every("7d").unwrap(), 604_800);
assert!(parse_every("0h").is_err());
assert!(parse_every("1w").is_err());
assert!(parse_every("h").is_err());
assert!(parse_every("").is_err());
}
#[test]
fn at_takes_unix_seconds_or_rfc3339() {
assert_eq!(parse_at("1700000000").unwrap(), 1_700_000_000);
assert_eq!(parse_at("1970-01-01T01:00:00+01:00").unwrap(), 0);
assert!(parse_at("tomorrow").is_err());
}
/// The edit body says exactly what the flags changed: a `--once` sends an
/// explicit `null` interval, and an unset flag sends nothing.
#[test]
fn an_edit_sends_only_what_was_asked() {
let body = |e: &EditRequest| serde_json::to_string(e).unwrap();
let e = EditRequest::new(EditFlags {
name: Some("hi".to_owned()),
..flags()
})
.unwrap();
assert_eq!(body(&e), r#"{"name":"hi"}"#);
let e = EditRequest::new(EditFlags {
once: true,
..flags()
})
.unwrap();
assert_eq!(body(&e), r#"{"interval_seconds":null}"#);
let e = EditRequest::new(EditFlags {
at: Some("5"),
every: Some("1m"),
add_agents: vec!["iris".to_owned()],
remove_agents: vec!["atlas".to_owned()],
..flags()
})
.unwrap();
assert_eq!(
body(&e),
r#"{"anchor_unix":5,"interval_seconds":60,"add_agents":["iris"],"remove_agents":["atlas"]}"#
);
assert_eq!(body(&EditRequest::paused(true)), r#"{"paused":true}"#);
assert!(EditRequest::new(flags()).is_err());
assert!(
EditRequest::new(EditFlags {
add_agents: vec!["Not An Ident".to_owned()],
..flags()
})
.is_err()
);
}
/// The controller's list item, as the wire shape documents it, decodes
/// and renders as one row naming every agent.
#[test]
fn a_listed_schedule_renders_as_one_row() {
let s: ScheduleView = serde_json::from_str(
r#"{"id":"abc","name":"standup","agents":["atlas","iris"],"body":"post it\nmore",
"anchor_unix":0,"interval_seconds":86400,"paused":false,"next_fire_unix":86400}"#,
)
.unwrap();
let row = render(&s);
assert!(!row.contains('\n'), "{row}");
assert!(row.contains("\"standup\""), "{row}");
assert!(row.contains("every 1d"), "{row}");
assert!(row.contains("next 1970-01-02T00:00:00Z"), "{row}");
assert!(row.contains("agents: atlas, iris"), "{row}");
assert!(row.ends_with("| post it"), "{row}");
}
}