diff --git a/docs/swarm/builder.md b/docs/swarm/builder.md index ab7fff0bd..6f0274e0b 100644 --- a/docs/swarm/builder.md +++ b/docs/swarm/builder.md @@ -143,7 +143,9 @@ running rev (`services.hyperhive.c0re.hyperhiveRev`, from `self.rev`): The controller commits `flake.nix` and the returned `flake.lock` to `internal/meta` in one commit, and gives each hive's forge account read on the -repo. It re-renders when it starts, before it deploys a merged config PR, and +repo. It re-renders when it starts, and again every ten minutes while the latest +update hasn't succeeded (a tick adds nothing while one is pending or running, +or after a success), before it deploys a merged config PR, and before the first deploy of a new agent. Those deploys run whether or not the update succeeded: a hive that builds from `internal/meta` does so only while it pins the agent's current config (see diff --git a/swarm-controller/README.md b/swarm-controller/README.md index 10a8ec55f..f58c94398 100644 --- a/swarm-controller/README.md +++ b/swarm-controller/README.md @@ -22,7 +22,9 @@ The swarm's control plane. The swarm UI and `swarmctl` are its clients. reported takes its hive and declared state in `/api/agents/status` from the one hive whose declaration places it. - **Job graph** — a `hive-jobq` scheduler, served at `GET /api/jobq/graph`. - Every provisioning step below runs as a node in it. + Every provisioning step below runs as a node in it. A node that fails logs + one `warn` line, `swarm jobq: node failed`, with its `node` id, `kind` and + `error`. - **Agent creation** — `POST /api/agents` queues the SSO identity (through `swarm-authelia-bridge`), forge user, config repo, store identity, forge token and matrix account, declares the agent `paused`, then sends its hive diff --git a/swarm-controller/src/jobq.rs b/swarm-controller/src/jobq.rs index 18e508257..f66e64020 100644 --- a/swarm-controller/src/jobq.rs +++ b/swarm-controller/src/jobq.rs @@ -291,11 +291,28 @@ struct MetaDeps { const NEW_AGENT_WANTED_STATE: swarm_wire::wanted::AgentState = swarm_wire::wanted::AgentState::Paused; +/// Run a claimed node and log one warn when it fails, whatever its kind. +pub(crate) async fn run_swarm_node( + id: hive_jobq::NodeId, + kind: SwarmNodeKind, + builder: hive_jobq::builder::JobBuilder, + deps: WorkerDeps, +) -> ( + hive_jobq::builder::JobBuilder, + hive_jobq::scheduler::Outcome, +) { + let node_kind = hive_jobq_wire::WireNode::label(&kind); + let (builder, outcome) = run_node_work(kind, builder, deps).await; + if let hive_jobq::scheduler::Outcome::Failed(error) = &outcome { + tracing::warn!(node = id.get(), kind = %node_kind, %error, "swarm jobq: node failed"); + } + (builder, outcome) +} + /// Run a claimed node's actual work. Mirrors `hive-c0re/src/job_queue/ /// exec.rs::run_node`'s role exactly — the one place a `SwarmNodeKind` /// variant turns into a real effect. -pub(crate) async fn run_swarm_node( - _id: hive_jobq::NodeId, +async fn run_node_work( kind: SwarmNodeKind, builder: hive_jobq::builder::JobBuilder, deps: WorkerDeps, @@ -431,7 +448,7 @@ async fn forge_node( /// together. const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed"; -/// The `MintAgentIdentity` arm, lifted out so `run_swarm_node` stays under +/// The `MintAgentIdentity` arm, lifted out so `run_node_work` stays under /// `clippy::too_many_lines`. pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome { use hive_jobq::scheduler::Outcome; @@ -446,7 +463,7 @@ pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome } } -/// The `MintAgentForgeToken` arm, lifted out so `run_swarm_node` stays under +/// The `MintAgentForgeToken` arm, lifted out so `run_node_work` stays under /// `clippy::too_many_lines`. async fn mint_forge_token( forge: Option>, @@ -681,7 +698,7 @@ fn finish_update_meta( } } -/// The `RecordFanout` arm, lifted out so `run_swarm_node` stays under the line limit. +/// The `RecordFanout` arm, lifted out so `run_node_work` stays under the line limit. async fn record_fanout( queue: Option<&async_nats::Client>, pin: &str, @@ -1450,18 +1467,54 @@ pub(crate) fn queue_builder_forge_token_mint( .map_err(|e| anyhow::anyhow!("{e}")) } -/// Insert one `UpdateMeta` job unless one is already pending, which reads the -/// same heads when it runs. Returns whether a job was inserted. -pub(crate) fn queue_meta_update( +/// How often [`spawn_meta_update_timer`] looks at the last `UpdateMeta`. +const META_UPDATE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(10); + +/// Look at the last `UpdateMeta` now and every [`META_UPDATE_INTERVAL`] after, +/// and queue one when none has run yet or the latest did not succeed, so a +/// boot run that failed or hung is retried. The first tick is the boot run. +/// Once a run has succeeded a tick adds nothing: the graph keeps every node, +/// so a clock-driven node per tick would grow it without bound. +pub(crate) fn spawn_meta_update_timer( + sched: Arc>>, +) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(META_UPDATE_INTERVAL); + loop { + ticker.tick().await; + if let Err(e) = queue_meta_update(&sched) { + tracing::warn!(error = %format!("{e:#}"), "update_meta: queueing failed"); + } + } + }); +} + +/// Insert one `UpdateMeta` job when none is in flight and the latest one, by +/// node id, is not `Done` (or none exists). Returns whether a job was inserted. +fn queue_meta_update( sched: &Mutex>, ) -> Result { let mut sched = sched .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - let pending = sched.graph().nodes().any(|n| { - n.state == hive_jobq::State::Pending && matches!(n.payload, SwarmNodeKind::UpdateMeta) - }); - if pending { + let mut update_metas = sched + .graph() + .nodes() + .filter(|n| matches!(n.payload, SwarmNodeKind::UpdateMeta)) + .collect::>(); + if update_metas.iter().any(|n| { + matches!( + n.state, + hive_jobq::State::Pending | hive_jobq::State::Running | hive_jobq::State::Finishing + ) + }) { + return Ok(false); + } + update_metas.sort_by_key(|n| n.id); + if update_metas + .last() + .is_some_and(|n| n.state == hive_jobq::State::Done) + { return Ok(false); } sched @@ -3073,6 +3126,88 @@ mod tests { labels.sort(); assert_eq!(labels, ["create_forge_user", "mint_agent_forge_token"]); } + + /// A scheduler holding one `UpdateMeta` that ran and ended `outcome`. + async fn after_one_update_meta( + outcome: hive_jobq::scheduler::Outcome, + ) -> std::sync::Arc< + std::sync::Mutex>, + > { + let sched = + std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ))); + assert!(super::queue_meta_update(&sched).expect("tick")); + hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move { (b, outcome) }) + .expect("runnable") + .await + .1 + .expect("no growth"); + sched + } + + fn update_metas( + sched: &std::sync::Mutex>, + ) -> usize { + sched + .lock() + .unwrap() + .graph() + .nodes() + .filter(|n| matches!(n.payload, SwarmNodeKind::UpdateMeta)) + .count() + } + + /// With none run yet the tick queues one, and queues no second while that + /// one is pending or running. + #[tokio::test] + async fn the_meta_timer_queues_first_and_skips_while_one_is_in_flight() { + let sched = + std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ))); + assert!(super::queue_meta_update(&sched).expect("tick")); + assert!(!super::queue_meta_update(&sched).expect("tick")); + assert_eq!(update_metas(&sched), 1, "a pending one blocks the tick"); + + let running = hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move { + (b, hive_jobq::scheduler::Outcome::Failed("late".to_owned())) + }) + .expect("runnable"); + assert!(!super::queue_meta_update(&sched).expect("tick")); + assert_eq!(update_metas(&sched), 1, "a running one blocks the tick"); + running.await.1.expect("no growth"); + } + + /// Once the latest run succeeded, ticks add no node. + #[tokio::test] + async fn the_meta_timer_adds_nothing_after_a_success() { + let sched = after_one_update_meta(hive_jobq::scheduler::Outcome::Done).await; + for _ in 0..3 { + assert!(!super::queue_meta_update(&sched).expect("tick")); + } + assert_eq!(update_metas(&sched), 1); + } + + /// A failed latest run is retried; a success after it ends the retries. + #[tokio::test] + async fn the_meta_timer_retries_after_a_failure_until_a_success() { + let sched = + after_one_update_meta(hive_jobq::scheduler::Outcome::Failed("no".to_owned())).await; + assert!(super::queue_meta_update(&sched).expect("tick")); + assert_eq!(update_metas(&sched), 2); + hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move { + (b, hive_jobq::scheduler::Outcome::Done) + }) + .expect("runnable") + .await + .1 + .expect("no growth"); + assert!(!super::queue_meta_update(&sched).expect("tick")); + assert_eq!(update_metas(&sched), 2); + } } #[cfg(test)] @@ -3082,7 +3217,7 @@ mod mint_identity_tests { use hive_jobq::scheduler::Outcome; use tracing_subscriber::layer::SubscriberExt as _; - use super::mint_identity; + use super::{SwarmNodeKind, WorkerDeps, mint_identity, run_swarm_node}; /// One event's level and its fields, each by name. type Event = (tracing::Level, Vec<(String, String)>); @@ -3139,4 +3274,62 @@ mod mint_identity_tests { assert_eq!(field("agent"), "a/b"); assert_eq!(field("error"), error); } + + /// A node of any kind whose work returns `Failed` logs one warn naming the + /// node's id, its kind and the error, from the one place that runs nodes. + #[tokio::test] + async fn a_failed_node_logs_its_kind_id_and_error() { + let events = Events::default(); + let _guard = + tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone())); + let sched = Arc::new(Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ))); + let ids = sched + .lock() + .unwrap() + .insert_job(None, |b| { + vec![ + b.node(SwarmNodeKind::SetAgentWanted { + hive: "h".to_owned(), + agent: "a".to_owned(), + }) + .guid(), + ] + }) + .expect("inserted"); + let deps = WorkerDeps { + auth: None, + forge: super::Late::default(), + queue: None, + wanted: None, + matrix_homeserver: None, + }; + hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { + run_swarm_node(id, kind, builder, deps) + }) + .expect("runnable") + .await + .1 + .expect("no growth declared"); + + let got = events.0.lock().expect("unpoisoned").clone(); + let [(level, fields)] = got.as_slice() else { + panic!("one event, got {got:?}"); + }; + assert_eq!(*level, tracing::Level::WARN); + let field = |name: &str| { + fields + .iter() + .find(|(n, _)| n == name) + .map_or_else(|| panic!("no {name} in {fields:?}"), |(_, v)| v.clone()) + }; + assert_eq!(field("kind"), "set_agent_wanted"); + assert_eq!(field("node"), ids[0].get().to_string()); + assert!( + field("error").contains("no swarm queue is configured"), + "{fields:?}" + ); + } } diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs index 3e8de038b..7fe1dabb4 100644 --- a/swarm-controller/src/startup.rs +++ b/swarm-controller/src/startup.rs @@ -288,10 +288,8 @@ fn spawn_forge_workers( ) { // `internal/meta` appears, catches up with every config head that moved // while this daemon was down, and takes this daemon's rev as the hyperhive - // pin when that is newer. - if let Err(e) = crate::jobq::queue_meta_update(jobq) { - tracing::warn!(error = %format!("{e:#}"), "update_meta: queueing failed"); - } + // pin when that is newer. The timer's first tick is this boot run. + crate::jobq::spawn_meta_update_timer(Arc::clone(jobq)); forge::objects::spawn(Arc::clone(&client)); if let (Some(bridge), Some(queue)) = (auth, queue) { forge::operator_profiles::spawn(Arc::clone(&client), bridge, queue);