Watch
0
0
Fork
You've already forked hyperhive
0

archetype flows behave like the built-ins: switch off per archetype or per agent, board cards, duplicate

The document's disabled list takes archetype:<archetype>/<flow> (off for every member) and the same with @<agent> (off for one member); a key naming nothing is ignored. GET /api/notify/flows lists the archetype flows with their members, the board draws a card for each with a switch, a checkbox per agent and duplicate, and the dry run counts their hits. The flows keep following the archetype.
This commit is contained in:
atlas 2026-11-11 19:08:19 +01:00 • committed by mara
commit 68c24643f6
6 changed files with 908 additions and 102 deletions

View file

@ -46,7 +46,7 @@ pub async fn serve(client: async_nats::Client) {
tokio::spawn(async move {
let Some(reply) = request.reply else { return };
let answer = if dry_run {
dry_run_answer(&js, &request.payload).await
dry_run_answer(&client, &js, &request.payload).await
} else {
deliveries_answer(&js, &request.payload).await
};
@ -91,7 +91,32 @@ async fn deliveries_answer(js: &jetstream::Context, payload: &[u8]) -> Vec<u8> {
)
}
async fn dry_run_answer(js: &jetstream::Context, payload: &[u8]) -> Vec<u8> {
/// The archetypes' flows as they run under `document`'s switches. A bucket that
/// cannot be read leaves them out of the counts.
async fn archetype_flows(
client: &async_nats::Client,
js: &jetstream::Context,
document: &swarm_wire::notify_flows::Document,
) -> Vec<crate::flow::ArchetypeFlow> {
let found = async {
let roster = crate::roster_of(js).await.map_err(|e| format!("{e:#}"))?;
crate::archetype_flows::load(client, &roster).await
}
.await;
match found {
Ok(found) => crate::archetype_flows::enabled(&found, &document.disabled_builtins),
Err(why) => {
tracing::warn!(%why, "notify router: a dry run without the archetype flows");
Vec::new()
}
}
}
async fn dry_run_answer(
client: &async_nats::Client,
js: &jetstream::Context,
payload: &[u8],
) -> Vec<u8> {
body(
async {
let request: DryRunRequest = serde_json::from_slice(payload)?;
@ -100,6 +125,7 @@ async fn dry_run_answer(js: &jetstream::Context, payload: &[u8]) -> Vec<u8> {
.check()
.map_err(|why| anyhow::anyhow!("the document fails its check: {why}"))?;
let (flows, _) = request.document.active();
let archetype = archetype_flows(client, js, &request.document).await;
let mut events: Vec<(String, Vec<u8>)> =
log::newest(js, forge_events::STREAM, DRY_RUN_EVENTS)
.await?
@ -109,7 +135,7 @@ async fn dry_run_answer(js: &jetstream::Context, payload: &[u8]) -> Vec<u8> {
})
.collect();
events.reverse();
Ok(dry_run::run(&flows, &events))
Ok(dry_run::run(&flows, &archetype, &events))
}
.await,
)

View file

@ -6,6 +6,10 @@
//! `agent-archetype` key (`swarm_queue_client::agent_archetype`) names that
//! archetype.
//!
//! Each runs under its key (`archetype_flow_key`), and the document's
//! `disabled_builtins` switches it off for the archetype or for one member
//! ([`enabled`]).
//!
//! The router reads both buckets again every [`REFRESH`], for the agents of the
//! roster only, so an agent that is not in the swarm is no member. Each read it
//! cannot make or decode leaves the flows it already has; before any read
@ -16,7 +20,7 @@ use std::time::{Duration, Instant};
use async_nats::jetstream::kv;
use swarm_queue_client::agent_archetype::{self, Read};
use swarm_queue_client::archetypes::{self, SourceRecord};
use swarm_queue_client::archetypes::{self, SourceRecord, flows_of};
use swarm_wire::agent_archetype::ArchetypeRef;
use crate::flow::ArchetypeFlow;
@ -119,49 +123,63 @@ async fn read(
Ok((refs, records))
}
/// The flows to run: each flow of each archetype that has an agent, with those
/// agents, and why any flow was left out. A flow's id is prefixed with its
/// archetype's name, so it cannot be a built-in's or the document's, and no two
/// archetypes' collide.
/// The flows to run, see [`flows_of`], each under its key
/// (`swarm_wire::notify_flows::archetype_flow_key`) as its id: it cannot be a built-in's or the
/// document's, and no two archetypes' collide.
fn build(
refs: &BTreeMap<String, ArchetypeRef>,
records: &BTreeMap<String, SourceRecord>,
) -> (Vec<ArchetypeFlow>, Vec<String>) {
let mut agents: BTreeMap<(&str, &str), Vec<String>> = BTreeMap::new();
for (agent, r) in refs {
agents
.entry((r.source.as_str(), r.name.as_str()))
.or_default()
.push(agent.clone());
}
let mut active = Vec::new();
let mut skipped = Vec::new();
for ((source, name), members) in agents {
let Some(meta) = records
.get(source)
.and_then(|record| record.archetypes.get(name))
else {
continue;
};
let mut ids = BTreeSet::new();
for flow in meta.flows.iter().filter(|f| f.enabled) {
if !ids.insert(flow.id.as_str()) {
skipped.push(format!("{name}: flow {}: its id is used twice", flow.id));
} else if let Err(why) = flow.validate_archetype() {
skipped.push(format!("{name}: flow {}: {why}", flow.id));
} else {
let mut flow = flow.clone();
flow.id = format!("{name}:{}", flow.id);
active.push(ArchetypeFlow {
flow,
members: members.clone(),
});
let (found, skipped) = flows_of(refs, records);
let active = found
.into_iter()
.map(|found| {
let mut flow = found.flow.clone();
flow.id = found.key();
ArchetypeFlow {
flow,
members: found.members,
}
}
}
})
.collect();
(active, skipped)
}
/// `flows` less what `disabled`, the document's `disabled_builtins`, switches
/// off: a flow whose key is in it, and a member whose per-agent key is. A flow
/// left with no member is dropped. A key naming nothing there is ignored.
pub fn enabled(flows: &[ArchetypeFlow], disabled: &[String]) -> Vec<ArchetypeFlow> {
flows
.iter()
.filter(|a| !disabled.contains(&a.flow.id))
.filter_map(|a| {
let members: Vec<String> = a
.members
.iter()
.filter(|m| !disabled.contains(&format!("{}@{m}", a.flow.id)))
.cloned()
.collect();
(!members.is_empty()).then(|| ArchetypeFlow {
flow: a.flow.clone(),
members,
})
})
.collect()
}
/// The flows the archetypes carry right now for `roster`, read from the
/// buckets. A bucket that cannot be read is an error.
pub async fn load(
client: &async_nats::Client,
roster: &BTreeSet<String>,
) -> Result<Vec<ArchetypeFlow>, String> {
let (refs, records) = read(client, roster).await?;
let (active, skipped) = build(&refs, &records);
for why in &skipped {
tracing::error!(%why, "notify router: skipping an archetype flow");
}
Ok(active)
}
#[cfg(test)]
mod tests {
use swarm_queue_client::archetypes::ArchetypeMeta;
@ -271,8 +289,8 @@ mod tests {
assert_eq!(
members(&built),
[
("reviewer:new-issues", &carol[..]),
("triage:new-issues", &alice_bob[..])
("archetype:reviewer/new-issues", &carol[..]),
("archetype:triage/new-issues", &alice_bob[..])
]
);
}
@ -299,7 +317,7 @@ mod tests {
.iter()
.map(|a| a.flow.id.as_str())
.collect::<Vec<_>>(),
["triage:good"]
["archetype:triage/good"]
);
assert_eq!(built.1.len(), 2, "{:?}", built.1);
assert!(built.1[0].contains("named"), "{:?}", built.1);
@ -356,9 +374,91 @@ mod tests {
assert_eq!(
members(&(active, skipped)),
[(
"triage:new-issues",
"archetype:triage/new-issues",
&["alice".to_owned(), "bob".to_owned()][..]
)]
);
}
fn two_flows() -> Vec<ArchetypeFlow> {
let records = record(&[
("triage", vec![flow("new-issues", Recipients::Members)]),
("reviewer", vec![flow("new-issues", Recipients::Members)]),
]);
build(
&refs(&[
("alice", "triage"),
("bob", "triage"),
("carol", "reviewer"),
]),
&records,
)
.0
}
fn who(flows: &[ArchetypeFlow]) -> Vec<(String, Vec<&str>)> {
flows
.iter()
.map(|a| {
(
a.flow.id.clone(),
a.members.iter().map(String::as_str).collect(),
)
})
.collect()
}
fn off(keys: &[&str]) -> Vec<String> {
keys.iter().map(|k| (*k).to_owned()).collect()
}
/// A flow's key switches it off for its archetype, the key with `@agent`
/// for that agent alone, and a key for something that is gone does nothing.
#[test]
fn the_disabled_list_switches_off_a_flow_or_one_agent_of_it() {
let flows = two_flows();
// Control: nothing off.
assert_eq!(
who(&enabled(&flows, &[])),
[
("archetype:reviewer/new-issues".to_owned(), vec!["carol"]),
(
"archetype:triage/new-issues".to_owned(),
vec!["alice", "bob"]
),
]
);
// The archetype's key: the other archetype's flow of the same id stays.
assert_eq!(
who(&enabled(&flows, &off(&["archetype:triage/new-issues"]))),
[("archetype:reviewer/new-issues".to_owned(), vec!["carol"])]
);
// One agent's key: the other members still get the flow.
assert_eq!(
who(&enabled(
&flows,
&off(&["archetype:triage/new-issues@alice"])
)),
[
("archetype:reviewer/new-issues".to_owned(), vec!["carol"]),
("archetype:triage/new-issues".to_owned(), vec!["bob"]),
]
);
// Every member off drops the flow; stale keys and built-in ids change nothing.
assert_eq!(
who(&enabled(
&flows,
&off(&[
"archetype:reviewer/new-issues@carol",
"archetype:triage/gone",
"archetype:triage/new-issues@dave",
"closed"
])
)),
[(
"archetype:triage/new-issues".to_owned(),
vec!["alice", "bob"]
)]
);
}
}

View file

@ -12,12 +12,18 @@ use swarm_wire::notify_flows::Flow;
use swarm_wire::router_api::{DryRun, FlowHits, NodeHits};
use crate::event::{Delivery, Snapshot, fields};
use crate::flow::ArchetypeFlow;
use crate::{flow, route};
/// `flows` over `events`, oldest first, each a forge event type and body.
/// An event that names no issue or PR is not counted.
pub fn run(flows: &[Flow], events: &[(String, Vec<u8>)]) -> DryRun {
let mut hits: Vec<BTreeMap<&str, u64>> = flows
/// `flows` and the archetypes' `archetype` flows over `events`, oldest first,
/// each a forge event type and body. An event that names no issue or PR is not
/// counted.
pub fn run(flows: &[Flow], archetype: &[ArchetypeFlow], events: &[(String, Vec<u8>)]) -> DryRun {
let all: Vec<&Flow> = flows
.iter()
.chain(archetype.iter().map(|a| &a.flow))
.collect();
let mut hits: Vec<BTreeMap<&str, u64>> = all
.iter()
.map(|f| f.nodes.iter().map(|n| (n.id.as_str(), 0)).collect())
.collect();
@ -31,19 +37,19 @@ pub fn run(flows: &[Flow], events: &[(String, Vec<u8>)]) -> DryRun {
let key = delivery.state_key();
let before = snapshots.remove(&key).unwrap_or_default();
let event_fields = fields(event_type, &delivery, &before);
for (flow, counts) in flows.iter().zip(&mut hits) {
for (flow, counts) in all.iter().zip(&mut hits) {
for id in flow::reached(flow, event_type, &event_fields) {
if let Some(count) = counts.get_mut(id) {
*count += 1;
}
}
}
let routed = route::route(event_type, &delivery, &before, flows, &[], |_| true);
let routed = route::route(event_type, &delivery, &before, flows, archetype, |_| true);
snapshots.insert(key, routed.next);
}
DryRun {
events: run,
flows: flows
flows: all
.iter()
.zip(hits)
.map(|(flow, counts)| FlowHits {
@ -102,7 +108,7 @@ mod tests {
("issues".to_owned(), body("closed", 5, &[])),
("issues".to_owned(), b"not json".to_vec()),
];
let r = run(&flows, &events);
let r = run(&flows, &[], &events);
assert_eq!(r.events, 4);
// The source sees all three assign events, the match and the sink both
// assigned ones, though the second adds nobody.
@ -160,7 +166,7 @@ mod tests {
edges: vec![edge("source", "atlas")],
};
let assign = ("issue_assign".to_owned(), body("assigned", 5, &["atlas"]));
let r = run(&[flow], &[assign.clone(), assign]);
let r = run(&[flow], &[], &[assign.clone(), assign]);
assert_eq!(count(&r, "added", "source"), 2);
assert_eq!(count(&r, "added", "atlas"), 1);
}
@ -173,11 +179,43 @@ mod tests {
..Document::default()
};
let (flows, _) = doc.active();
let r = run(&flows, &[]);
let r = run(&flows, &[], &[]);
assert_eq!(r.events, 0);
let listed: Vec<&str> = r.flows.iter().map(|f| f.flow.as_str()).collect();
assert!(!listed.contains(&"closed"));
assert!(listed.contains(&"assigned"));
assert_eq!(listed.len(), builtins().len() - 1);
}
/// An archetype's flow is counted next to the document's, under its key.
#[test]
fn an_archetype_flow_is_counted_under_its_key() {
let (flows, _) = Document::default().active();
let mut flow = builtins().remove(0);
flow.id = "archetype:triage/new".to_owned();
flow.nodes[2].kind =
swarm_wire::notify_flows::NodeKind::Todo(swarm_wire::notify_flows::TodoSink {
to: swarm_wire::notify_flows::Recipients::Members,
op: swarm_wire::router_todo::Op::Upsert,
key: "forge:{{ repo }}#{{ number }}:triage".to_owned(),
prefix: false,
text: "triage".to_owned(),
});
let archetype = [ArchetypeFlow {
flow,
members: vec!["alice".to_owned()],
}];
let events = vec![("issue_assign".to_owned(), body("assigned", 5, &["atlas"]))];
let r = run(&flows, &archetype, &events);
assert_eq!(count(&r, "archetype:triage/new", "source"), 1);
assert_eq!(count(&r, "archetype:triage/new", "sink"), 1);
assert_eq!(count(&r, "assigned", "sink"), 1, "the document's flow too");
// Control: with no archetype flow it has no entry.
assert!(
run(&flows, &[], &events)
.flows
.iter()
.all(|f| f.flow != "archetype:triage/new")
);
}
}

View file

@ -239,12 +239,14 @@ impl Router {
self.flows.refresh(&self.js).await;
let roster = self.roster().await.map_err(Skip::Retry)?;
self.archetype_flows.refresh(&self.nats, &roster).await;
let archetype_flows =
archetype_flows::enabled(&self.archetype_flows.active, &self.flows.disabled);
let routed = route::route(
&event_type,
&delivery,
&snapshot,
&self.flows.active,
&self.archetype_flows.active,
&archetype_flows,
|a| roster.contains(a),
);
let flows: BTreeSet<&str> = routed
@ -364,32 +366,37 @@ impl Router {
{
return Ok(agents.clone());
}
let bucket = swarm_wire::agent_status::BUCKET;
let stream = self
.js
.get_stream(format!("KV_{bucket}"))
.await
.context("opening the agent-status bucket")?;
let subjects: Vec<(String, usize)> = stream
.info_with_subjects(format!("$KV.{bucket}.>"))
.await
.context("listing agent-status keys")?
.try_collect()
.await
.context("listing agent-status keys")?;
let prefix = format!("$KV.{bucket}.");
let agents: BTreeSet<String> = subjects
.into_iter()
.filter_map(|(subject, _)| {
let key = subject.strip_prefix(&prefix)?;
swarm_wire::agent_status::split_key(key).map(|(_, agent)| agent.to_owned())
})
.collect();
let agents = roster_of(&self.js).await?;
self.roster = Some((Instant::now(), agents.clone()));
Ok(agents)
}
}
/// The agents of this swarm, read from the agent-status bucket.
pub(crate) async fn roster_of(js: &jetstream::Context) -> Result<BTreeSet<String>> {
let bucket = swarm_wire::agent_status::BUCKET;
let stream = js
.get_stream(format!("KV_{bucket}"))
.await
.context("opening the agent-status bucket")?;
let subjects: Vec<(String, usize)> = stream
.info_with_subjects(format!("$KV.{bucket}.>"))
.await
.context("listing agent-status keys")?
.try_collect()
.await
.context("listing agent-status keys")?;
let prefix = format!("$KV.{bucket}.");
let agents: BTreeSet<String> = subjects
.into_iter()
.filter_map(|(subject, _)| {
let key = subject.strip_prefix(&prefix)?;
swarm_wire::agent_status::split_key(key).map(|(_, agent)| agent.to_owned())
})
.collect();
Ok(agents)
}
/// Why an event was not handled.
enum Skip {
/// It can never be handled; dropped for good.

View file

@ -738,4 +738,85 @@ mod tests {
]
);
}
/// An archetype edit changes what its member receives at the next read of
/// the archetype record, with no flow document stored at all.
#[tokio::test]
async fn an_archetype_edit_changes_what_a_member_receives() {
use swarm_queue_client::agent_archetype::{self, ArchetypeRef};
use swarm_queue_client::archetypes::{self, ArchetypeMeta, SourceRecord};
let nats = crate::test_nats::TestNats::start();
let client = nats.client().await;
let records = archetypes::open_or_create(&client).await.expect("records");
let choices = agent_archetype::open_or_create(&client)
.await
.expect("choices");
let choice = ArchetypeRef {
source: "hyperhive".to_owned(),
name: "triage".to_owned(),
};
choices
.put("alice", serde_json::to_vec(&choice).unwrap().into())
.await
.expect("put the choice");
let roster = std::collections::BTreeSet::from(["alice".to_owned()]);
let publish = |text: &'static str| {
let records = records.clone();
async move {
let mut flow = members_flow();
flow.id = "new".to_owned();
if let NodeKind::Todo(sink) = &mut flow.nodes[2].kind {
sink.text = text.to_owned();
}
let record = SourceRecord {
source: "hyperhive".to_owned(),
archetypes: [(
"triage".to_owned(),
ArchetypeMeta {
description: "d".to_owned(),
flows: vec![flow],
..ArchetypeMeta::default()
},
)]
.into(),
refused: std::collections::BTreeMap::default(),
};
records
.put("hyperhive", serde_json::to_vec(&record).unwrap().into())
.await
.expect("put the record");
}
};
let received = || async {
let flows = crate::archetype_flows::load(&client, &roster)
.await
.expect("a read");
let routed = super::route(
"issue_assign",
&delivery("assigned", &[]),
&Snapshot::default(),
&[],
&flows,
|_| true,
);
routed
.actions
.iter()
.map(|a| (a.agent.clone(), a.op.summary.clone()))
.collect::<Vec<_>>()
};
publish("first wording").await;
assert_eq!(
received().await,
[("alice".to_owned(), "first wording".to_owned())]
);
publish("second wording").await;
assert_eq!(
received().await,
[("alice".to_owned(), "second wording".to_owned())]
);
}
}

View file

@ -41,6 +41,9 @@ pub enum Read {
/// The flows in force, and which revision of the key they came from.
pub struct Flows {
pub active: Vec<Flow>,
/// The document's `disabled_builtins`, which also switches off archetype
/// flows ([`crate::archetype_flows::enabled`]).
pub disabled: Vec<String>,
/// The revision `active` was read from; `None` for the built-ins alone.
revision: Option<u64>,
/// The last problem logged, so a standing one is logged once.
@ -53,6 +56,7 @@ impl Flows {
pub fn builtins() -> Self {
Self {
active: Document::default().active().0,
disabled: Vec::new(),
revision: None,
problem: None,
checked: None,
@ -70,6 +74,7 @@ impl Flows {
);
}
self.active = Document::default().active().0;
self.disabled.clear();
self.problem = None;
}
Read::Document { revision, .. } if self.revision == Some(revision) => {}
@ -92,6 +97,7 @@ impl Flows {
"notify router: running the flows document"
);
self.active = active;
self.disabled = doc.disabled_builtins;
self.revision = Some(revision);
self.problem = None;
}
@ -285,4 +291,22 @@ mod tests {
});
assert_eq!(ids(&f), BUILTIN_IDS);
}
/// The document's disabled list is kept whole, archetype keys among the
/// built-in ids, and goes with the document: a deleted one clears it.
#[test]
fn the_disabled_list_includes_archetype_keys_and_goes_with_the_document() {
let mut f = Flows::builtins();
assert!(f.disabled.is_empty());
f.apply(at(1, doc(&["closed", "archetype:triage/new@alice"], &[])));
assert_eq!(f.disabled, ["closed", "archetype:triage/new@alice"]);
f.apply(junk(2, b"not json"));
assert_eq!(
f.disabled.len(),
2,
"a document it cannot read changes nothing"
);
f.apply(Read::Absent);
assert!(f.disabled.is_empty());
}
}