|
|
|
|
@ -291,28 +291,11 @@ struct MetaDeps {
|
|
|
|
|
const NEW_AGENT_WANTED_STATE: swarm_wire::wanted::AgentState =
|
|
|
|
|
swarm_wire::wanted::AgentState::Paused;
|
|
|
|
|
|
|
|
|
|
/// Run a claimed node and log one warn when it fails, whatever its kind.
|
|
|
|
|
pub(crate) async fn run_swarm_node(
|
|
|
|
|
id: hive_jobq::NodeId,
|
|
|
|
|
kind: SwarmNodeKind,
|
|
|
|
|
builder: hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
|
|
|
deps: WorkerDeps,
|
|
|
|
|
) -> (
|
|
|
|
|
hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
|
|
|
hive_jobq::scheduler::Outcome,
|
|
|
|
|
) {
|
|
|
|
|
let node_kind = hive_jobq_wire::WireNode::label(&kind);
|
|
|
|
|
let (builder, outcome) = run_node_work(kind, builder, deps).await;
|
|
|
|
|
if let hive_jobq::scheduler::Outcome::Failed(error) = &outcome {
|
|
|
|
|
tracing::warn!(node = id.get(), kind = %node_kind, %error, "swarm jobq: node failed");
|
|
|
|
|
}
|
|
|
|
|
(builder, outcome)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Run a claimed node's actual work. Mirrors `hive-c0re/src/job_queue/
|
|
|
|
|
/// exec.rs::run_node`'s role exactly — the one place a `SwarmNodeKind`
|
|
|
|
|
/// variant turns into a real effect.
|
|
|
|
|
async fn run_node_work(
|
|
|
|
|
pub(crate) async fn run_swarm_node(
|
|
|
|
|
_id: hive_jobq::NodeId,
|
|
|
|
|
kind: SwarmNodeKind,
|
|
|
|
|
builder: hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
|
|
|
deps: WorkerDeps,
|
|
|
|
|
@ -448,7 +431,7 @@ async fn forge_node<T>(
|
|
|
|
|
/// together.
|
|
|
|
|
const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed";
|
|
|
|
|
|
|
|
|
|
/// The `MintAgentIdentity` arm, lifted out so `run_node_work` stays under
|
|
|
|
|
/// The `MintAgentIdentity` arm, lifted out so `run_swarm_node` stays under
|
|
|
|
|
/// `clippy::too_many_lines`.
|
|
|
|
|
pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome {
|
|
|
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
|
@ -463,7 +446,7 @@ pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// The `MintAgentForgeToken` arm, lifted out so `run_node_work` stays under
|
|
|
|
|
/// The `MintAgentForgeToken` arm, lifted out so `run_swarm_node` stays under
|
|
|
|
|
/// `clippy::too_many_lines`.
|
|
|
|
|
async fn mint_forge_token(
|
|
|
|
|
forge: Option<Arc<forge::Client>>,
|
|
|
|
|
@ -698,7 +681,7 @@ fn finish_update_meta(
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// The `RecordFanout` arm, lifted out so `run_node_work` stays under the line limit.
|
|
|
|
|
/// The `RecordFanout` arm, lifted out so `run_swarm_node` stays under the line limit.
|
|
|
|
|
async fn record_fanout(
|
|
|
|
|
queue: Option<&async_nats::Client>,
|
|
|
|
|
pin: &str,
|
|
|
|
|
@ -1467,54 +1450,18 @@ pub(crate) fn queue_builder_forge_token_mint(
|
|
|
|
|
.map_err(|e| anyhow::anyhow!("{e}"))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// How often [`spawn_meta_update_timer`] looks at the last `UpdateMeta`.
|
|
|
|
|
const META_UPDATE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(10);
|
|
|
|
|
|
|
|
|
|
/// Look at the last `UpdateMeta` now and every [`META_UPDATE_INTERVAL`] after,
|
|
|
|
|
/// and queue one when none has run yet or the latest did not succeed, so a
|
|
|
|
|
/// boot run that failed or hung is retried. The first tick is the boot run.
|
|
|
|
|
/// Once a run has succeeded a tick adds nothing: the graph keeps every node,
|
|
|
|
|
/// so a clock-driven node per tick would grow it without bound.
|
|
|
|
|
pub(crate) fn spawn_meta_update_timer(
|
|
|
|
|
sched: Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
|
|
|
|
|
) {
|
|
|
|
|
tokio::spawn(async move {
|
|
|
|
|
let mut ticker = tokio::time::interval(META_UPDATE_INTERVAL);
|
|
|
|
|
loop {
|
|
|
|
|
ticker.tick().await;
|
|
|
|
|
if let Err(e) = queue_meta_update(&sched) {
|
|
|
|
|
tracing::warn!(error = %format!("{e:#}"), "update_meta: queueing failed");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Insert one `UpdateMeta` job when none is in flight and the latest one, by
|
|
|
|
|
/// node id, is not `Done` (or none exists). Returns whether a job was inserted.
|
|
|
|
|
fn queue_meta_update(
|
|
|
|
|
/// Insert one `UpdateMeta` job unless one is already pending, which reads the
|
|
|
|
|
/// same heads when it runs. Returns whether a job was inserted.
|
|
|
|
|
pub(crate) fn queue_meta_update(
|
|
|
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
|
|
|
) -> Result<bool> {
|
|
|
|
|
let mut sched = sched
|
|
|
|
|
.lock()
|
|
|
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
|
|
|
let mut update_metas = sched
|
|
|
|
|
.graph()
|
|
|
|
|
.nodes()
|
|
|
|
|
.filter(|n| matches!(n.payload, SwarmNodeKind::UpdateMeta))
|
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
if update_metas.iter().any(|n| {
|
|
|
|
|
matches!(
|
|
|
|
|
n.state,
|
|
|
|
|
hive_jobq::State::Pending | hive_jobq::State::Running | hive_jobq::State::Finishing
|
|
|
|
|
)
|
|
|
|
|
}) {
|
|
|
|
|
return Ok(false);
|
|
|
|
|
}
|
|
|
|
|
update_metas.sort_by_key(|n| n.id);
|
|
|
|
|
if update_metas
|
|
|
|
|
.last()
|
|
|
|
|
.is_some_and(|n| n.state == hive_jobq::State::Done)
|
|
|
|
|
{
|
|
|
|
|
let pending = sched.graph().nodes().any(|n| {
|
|
|
|
|
n.state == hive_jobq::State::Pending && matches!(n.payload, SwarmNodeKind::UpdateMeta)
|
|
|
|
|
});
|
|
|
|
|
if pending {
|
|
|
|
|
return Ok(false);
|
|
|
|
|
}
|
|
|
|
|
sched
|
|
|
|
|
@ -3126,88 +3073,6 @@ mod tests {
|
|
|
|
|
labels.sort();
|
|
|
|
|
assert_eq!(labels, ["create_forge_user", "mint_agent_forge_token"]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// A scheduler holding one `UpdateMeta` that ran and ended `outcome`.
|
|
|
|
|
async fn after_one_update_meta(
|
|
|
|
|
outcome: hive_jobq::scheduler::Outcome,
|
|
|
|
|
) -> std::sync::Arc<
|
|
|
|
|
std::sync::Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
|
|
|
> {
|
|
|
|
|
let sched =
|
|
|
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
|
|
|
hive_jobq::Graph::new(),
|
|
|
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
|
|
|
)));
|
|
|
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
|
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move { (b, outcome) })
|
|
|
|
|
.expect("runnable")
|
|
|
|
|
.await
|
|
|
|
|
.1
|
|
|
|
|
.expect("no growth");
|
|
|
|
|
sched
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn update_metas(
|
|
|
|
|
sched: &std::sync::Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
|
|
|
) -> usize {
|
|
|
|
|
sched
|
|
|
|
|
.lock()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.graph()
|
|
|
|
|
.nodes()
|
|
|
|
|
.filter(|n| matches!(n.payload, SwarmNodeKind::UpdateMeta))
|
|
|
|
|
.count()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// With none run yet the tick queues one, and queues no second while that
|
|
|
|
|
/// one is pending or running.
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
async fn the_meta_timer_queues_first_and_skips_while_one_is_in_flight() {
|
|
|
|
|
let sched =
|
|
|
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
|
|
|
hive_jobq::Graph::new(),
|
|
|
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
|
|
|
)));
|
|
|
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
|
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
|
|
|
assert_eq!(update_metas(&sched), 1, "a pending one blocks the tick");
|
|
|
|
|
|
|
|
|
|
let running = hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move {
|
|
|
|
|
(b, hive_jobq::scheduler::Outcome::Failed("late".to_owned()))
|
|
|
|
|
})
|
|
|
|
|
.expect("runnable");
|
|
|
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
|
|
|
assert_eq!(update_metas(&sched), 1, "a running one blocks the tick");
|
|
|
|
|
running.await.1.expect("no growth");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Once the latest run succeeded, ticks add no node.
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
async fn the_meta_timer_adds_nothing_after_a_success() {
|
|
|
|
|
let sched = after_one_update_meta(hive_jobq::scheduler::Outcome::Done).await;
|
|
|
|
|
for _ in 0..3 {
|
|
|
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
|
|
|
}
|
|
|
|
|
assert_eq!(update_metas(&sched), 1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// A failed latest run is retried; a success after it ends the retries.
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
async fn the_meta_timer_retries_after_a_failure_until_a_success() {
|
|
|
|
|
let sched =
|
|
|
|
|
after_one_update_meta(hive_jobq::scheduler::Outcome::Failed("no".to_owned())).await;
|
|
|
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
|
|
|
assert_eq!(update_metas(&sched), 2);
|
|
|
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move {
|
|
|
|
|
(b, hive_jobq::scheduler::Outcome::Done)
|
|
|
|
|
})
|
|
|
|
|
.expect("runnable")
|
|
|
|
|
.await
|
|
|
|
|
.1
|
|
|
|
|
.expect("no growth");
|
|
|
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
|
|
|
assert_eq!(update_metas(&sched), 2);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
@ -3217,7 +3082,7 @@ mod mint_identity_tests {
|
|
|
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
|
use tracing_subscriber::layer::SubscriberExt as _;
|
|
|
|
|
|
|
|
|
|
use super::{SwarmNodeKind, WorkerDeps, mint_identity, run_swarm_node};
|
|
|
|
|
use super::mint_identity;
|
|
|
|
|
|
|
|
|
|
/// One event's level and its fields, each by name.
|
|
|
|
|
type Event = (tracing::Level, Vec<(String, String)>);
|
|
|
|
|
@ -3274,62 +3139,4 @@ mod mint_identity_tests {
|
|
|
|
|
assert_eq!(field("agent"), "a/b");
|
|
|
|
|
assert_eq!(field("error"), error);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// A node of any kind whose work returns `Failed` logs one warn naming the
|
|
|
|
|
/// node's id, its kind and the error, from the one place that runs nodes.
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
async fn a_failed_node_logs_its_kind_id_and_error() {
|
|
|
|
|
let events = Events::default();
|
|
|
|
|
let _guard =
|
|
|
|
|
tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
|
|
|
|
|
let sched = Arc::new(Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
|
|
|
hive_jobq::Graph::new(),
|
|
|
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
|
|
|
)));
|
|
|
|
|
let ids = sched
|
|
|
|
|
.lock()
|
|
|
|
|
.unwrap()
|
|
|
|
|
.insert_job(None, |b| {
|
|
|
|
|
vec![
|
|
|
|
|
b.node(SwarmNodeKind::SetAgentWanted {
|
|
|
|
|
hive: "h".to_owned(),
|
|
|
|
|
agent: "a".to_owned(),
|
|
|
|
|
})
|
|
|
|
|
.guid(),
|
|
|
|
|
]
|
|
|
|
|
})
|
|
|
|
|
.expect("inserted");
|
|
|
|
|
let deps = WorkerDeps {
|
|
|
|
|
auth: None,
|
|
|
|
|
forge: super::Late::default(),
|
|
|
|
|
queue: None,
|
|
|
|
|
wanted: None,
|
|
|
|
|
matrix_homeserver: None,
|
|
|
|
|
};
|
|
|
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
|
|
|
run_swarm_node(id, kind, builder, deps)
|
|
|
|
|
})
|
|
|
|
|
.expect("runnable")
|
|
|
|
|
.await
|
|
|
|
|
.1
|
|
|
|
|
.expect("no growth declared");
|
|
|
|
|
|
|
|
|
|
let got = events.0.lock().expect("unpoisoned").clone();
|
|
|
|
|
let [(level, fields)] = got.as_slice() else {
|
|
|
|
|
panic!("one event, got {got:?}");
|
|
|
|
|
};
|
|
|
|
|
assert_eq!(*level, tracing::Level::WARN);
|
|
|
|
|
let field = |name: &str| {
|
|
|
|
|
fields
|
|
|
|
|
.iter()
|
|
|
|
|
.find(|(n, _)| n == name)
|
|
|
|
|
.map_or_else(|| panic!("no {name} in {fields:?}"), |(_, v)| v.clone())
|
|
|
|
|
};
|
|
|
|
|
assert_eq!(field("kind"), "set_agent_wanted");
|
|
|
|
|
assert_eq!(field("node"), ids[0].get().to_string());
|
|
|
|
|
assert!(
|
|
|
|
|
field("error").contains("no swarm queue is configured"),
|
|
|
|
|
"{fields:?}"
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|