diff --git a/docs/swarm/builder.md b/docs/swarm/builder.md index 6f0274e0b..ab7fff0bd 100644 --- a/docs/swarm/builder.md +++ b/docs/swarm/builder.md @@ -143,9 +143,7 @@ running rev (`services.hyperhive.c0re.hyperhiveRev`, from `self.rev`): The controller commits `flake.nix` and the returned `flake.lock` to `internal/meta` in one commit, and gives each hive's forge account read on the -repo. It re-renders when it starts, and again every ten minutes while the latest -update hasn't succeeded (a tick adds nothing while one is pending or running, -or after a success), before it deploys a merged config PR, and +repo. It re-renders when it starts, before it deploys a merged config PR, and before the first deploy of a new agent. Those deploys run whether or not the update succeeded: a hive that builds from `internal/meta` does so only while it pins the agent's current config (see diff --git a/swarm-controller/README.md b/swarm-controller/README.md index f58c94398..10a8ec55f 100644 --- a/swarm-controller/README.md +++ b/swarm-controller/README.md @@ -22,9 +22,7 @@ The swarm's control plane. The swarm UI and `swarmctl` are its clients. reported takes its hive and declared state in `/api/agents/status` from the one hive whose declaration places it. - **Job graph** — a `hive-jobq` scheduler, served at `GET /api/jobq/graph`. - Every provisioning step below runs as a node in it. A node that fails logs - one `warn` line, `swarm jobq: node failed`, with its `node` id, `kind` and - `error`. + Every provisioning step below runs as a node in it. - **Agent creation** — `POST /api/agents` queues the SSO identity (through `swarm-authelia-bridge`), forge user, config repo, store identity, forge token and matrix account, declares the agent `paused`, then sends its hive diff --git a/swarm-controller/src/jobq.rs b/swarm-controller/src/jobq.rs index f66e64020..18e508257 100644 --- a/swarm-controller/src/jobq.rs +++ b/swarm-controller/src/jobq.rs @@ -291,28 +291,11 @@ struct MetaDeps { const NEW_AGENT_WANTED_STATE: swarm_wire::wanted::AgentState = swarm_wire::wanted::AgentState::Paused; -/// Run a claimed node and log one warn when it fails, whatever its kind. -pub(crate) async fn run_swarm_node( - id: hive_jobq::NodeId, - kind: SwarmNodeKind, - builder: hive_jobq::builder::JobBuilder, - deps: WorkerDeps, -) -> ( - hive_jobq::builder::JobBuilder, - hive_jobq::scheduler::Outcome, -) { - let node_kind = hive_jobq_wire::WireNode::label(&kind); - let (builder, outcome) = run_node_work(kind, builder, deps).await; - if let hive_jobq::scheduler::Outcome::Failed(error) = &outcome { - tracing::warn!(node = id.get(), kind = %node_kind, %error, "swarm jobq: node failed"); - } - (builder, outcome) -} - /// Run a claimed node's actual work. Mirrors `hive-c0re/src/job_queue/ /// exec.rs::run_node`'s role exactly — the one place a `SwarmNodeKind` /// variant turns into a real effect. -async fn run_node_work( +pub(crate) async fn run_swarm_node( + _id: hive_jobq::NodeId, kind: SwarmNodeKind, builder: hive_jobq::builder::JobBuilder, deps: WorkerDeps, @@ -448,7 +431,7 @@ async fn forge_node( /// together. const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed"; -/// The `MintAgentIdentity` arm, lifted out so `run_node_work` stays under +/// The `MintAgentIdentity` arm, lifted out so `run_swarm_node` stays under /// `clippy::too_many_lines`. pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome { use hive_jobq::scheduler::Outcome; @@ -463,7 +446,7 @@ pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome } } -/// The `MintAgentForgeToken` arm, lifted out so `run_node_work` stays under +/// The `MintAgentForgeToken` arm, lifted out so `run_swarm_node` stays under /// `clippy::too_many_lines`. async fn mint_forge_token( forge: Option>, @@ -698,7 +681,7 @@ fn finish_update_meta( } } -/// The `RecordFanout` arm, lifted out so `run_node_work` stays under the line limit. +/// The `RecordFanout` arm, lifted out so `run_swarm_node` stays under the line limit. async fn record_fanout( queue: Option<&async_nats::Client>, pin: &str, @@ -1467,54 +1450,18 @@ pub(crate) fn queue_builder_forge_token_mint( .map_err(|e| anyhow::anyhow!("{e}")) } -/// How often [`spawn_meta_update_timer`] looks at the last `UpdateMeta`. -const META_UPDATE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(10); - -/// Look at the last `UpdateMeta` now and every [`META_UPDATE_INTERVAL`] after, -/// and queue one when none has run yet or the latest did not succeed, so a -/// boot run that failed or hung is retried. The first tick is the boot run. -/// Once a run has succeeded a tick adds nothing: the graph keeps every node, -/// so a clock-driven node per tick would grow it without bound. -pub(crate) fn spawn_meta_update_timer( - sched: Arc>>, -) { - tokio::spawn(async move { - let mut ticker = tokio::time::interval(META_UPDATE_INTERVAL); - loop { - ticker.tick().await; - if let Err(e) = queue_meta_update(&sched) { - tracing::warn!(error = %format!("{e:#}"), "update_meta: queueing failed"); - } - } - }); -} - -/// Insert one `UpdateMeta` job when none is in flight and the latest one, by -/// node id, is not `Done` (or none exists). Returns whether a job was inserted. -fn queue_meta_update( +/// Insert one `UpdateMeta` job unless one is already pending, which reads the +/// same heads when it runs. Returns whether a job was inserted. +pub(crate) fn queue_meta_update( sched: &Mutex>, ) -> Result { let mut sched = sched .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - let mut update_metas = sched - .graph() - .nodes() - .filter(|n| matches!(n.payload, SwarmNodeKind::UpdateMeta)) - .collect::>(); - if update_metas.iter().any(|n| { - matches!( - n.state, - hive_jobq::State::Pending | hive_jobq::State::Running | hive_jobq::State::Finishing - ) - }) { - return Ok(false); - } - update_metas.sort_by_key(|n| n.id); - if update_metas - .last() - .is_some_and(|n| n.state == hive_jobq::State::Done) - { + let pending = sched.graph().nodes().any(|n| { + n.state == hive_jobq::State::Pending && matches!(n.payload, SwarmNodeKind::UpdateMeta) + }); + if pending { return Ok(false); } sched @@ -3126,88 +3073,6 @@ mod tests { labels.sort(); assert_eq!(labels, ["create_forge_user", "mint_agent_forge_token"]); } - - /// A scheduler holding one `UpdateMeta` that ran and ended `outcome`. - async fn after_one_update_meta( - outcome: hive_jobq::scheduler::Outcome, - ) -> std::sync::Arc< - std::sync::Mutex>, - > { - let sched = - std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ))); - assert!(super::queue_meta_update(&sched).expect("tick")); - hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move { (b, outcome) }) - .expect("runnable") - .await - .1 - .expect("no growth"); - sched - } - - fn update_metas( - sched: &std::sync::Mutex>, - ) -> usize { - sched - .lock() - .unwrap() - .graph() - .nodes() - .filter(|n| matches!(n.payload, SwarmNodeKind::UpdateMeta)) - .count() - } - - /// With none run yet the tick queues one, and queues no second while that - /// one is pending or running. - #[tokio::test] - async fn the_meta_timer_queues_first_and_skips_while_one_is_in_flight() { - let sched = - std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ))); - assert!(super::queue_meta_update(&sched).expect("tick")); - assert!(!super::queue_meta_update(&sched).expect("tick")); - assert_eq!(update_metas(&sched), 1, "a pending one blocks the tick"); - - let running = hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move { - (b, hive_jobq::scheduler::Outcome::Failed("late".to_owned())) - }) - .expect("runnable"); - assert!(!super::queue_meta_update(&sched).expect("tick")); - assert_eq!(update_metas(&sched), 1, "a running one blocks the tick"); - running.await.1.expect("no growth"); - } - - /// Once the latest run succeeded, ticks add no node. - #[tokio::test] - async fn the_meta_timer_adds_nothing_after_a_success() { - let sched = after_one_update_meta(hive_jobq::scheduler::Outcome::Done).await; - for _ in 0..3 { - assert!(!super::queue_meta_update(&sched).expect("tick")); - } - assert_eq!(update_metas(&sched), 1); - } - - /// A failed latest run is retried; a success after it ends the retries. - #[tokio::test] - async fn the_meta_timer_retries_after_a_failure_until_a_success() { - let sched = - after_one_update_meta(hive_jobq::scheduler::Outcome::Failed("no".to_owned())).await; - assert!(super::queue_meta_update(&sched).expect("tick")); - assert_eq!(update_metas(&sched), 2); - hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move { - (b, hive_jobq::scheduler::Outcome::Done) - }) - .expect("runnable") - .await - .1 - .expect("no growth"); - assert!(!super::queue_meta_update(&sched).expect("tick")); - assert_eq!(update_metas(&sched), 2); - } } #[cfg(test)] @@ -3217,7 +3082,7 @@ mod mint_identity_tests { use hive_jobq::scheduler::Outcome; use tracing_subscriber::layer::SubscriberExt as _; - use super::{SwarmNodeKind, WorkerDeps, mint_identity, run_swarm_node}; + use super::mint_identity; /// One event's level and its fields, each by name. type Event = (tracing::Level, Vec<(String, String)>); @@ -3274,62 +3139,4 @@ mod mint_identity_tests { assert_eq!(field("agent"), "a/b"); assert_eq!(field("error"), error); } - - /// A node of any kind whose work returns `Failed` logs one warn naming the - /// node's id, its kind and the error, from the one place that runs nodes. - #[tokio::test] - async fn a_failed_node_logs_its_kind_id_and_error() { - let events = Events::default(); - let _guard = - tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone())); - let sched = Arc::new(Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ))); - let ids = sched - .lock() - .unwrap() - .insert_job(None, |b| { - vec![ - b.node(SwarmNodeKind::SetAgentWanted { - hive: "h".to_owned(), - agent: "a".to_owned(), - }) - .guid(), - ] - }) - .expect("inserted"); - let deps = WorkerDeps { - auth: None, - forge: super::Late::default(), - queue: None, - wanted: None, - matrix_homeserver: None, - }; - hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { - run_swarm_node(id, kind, builder, deps) - }) - .expect("runnable") - .await - .1 - .expect("no growth declared"); - - let got = events.0.lock().expect("unpoisoned").clone(); - let [(level, fields)] = got.as_slice() else { - panic!("one event, got {got:?}"); - }; - assert_eq!(*level, tracing::Level::WARN); - let field = |name: &str| { - fields - .iter() - .find(|(n, _)| n == name) - .map_or_else(|| panic!("no {name} in {fields:?}"), |(_, v)| v.clone()) - }; - assert_eq!(field("kind"), "set_agent_wanted"); - assert_eq!(field("node"), ids[0].get().to_string()); - assert!( - field("error").contains("no swarm queue is configured"), - "{fields:?}" - ); - } } diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs index 7fe1dabb4..3e8de038b 100644 --- a/swarm-controller/src/startup.rs +++ b/swarm-controller/src/startup.rs @@ -288,8 +288,10 @@ fn spawn_forge_workers( ) { // `internal/meta` appears, catches up with every config head that moved // while this daemon was down, and takes this daemon's rev as the hyperhive - // pin when that is newer. The timer's first tick is this boot run. - crate::jobq::spawn_meta_update_timer(Arc::clone(jobq)); + // pin when that is newer. + if let Err(e) = crate::jobq::queue_meta_update(jobq) { + tracing::warn!(error = %format!("{e:#}"), "update_meta: queueing failed"); + } forge::objects::spawn(Arc::clone(&client)); if let (Some(bridge), Some(queue)) = (auth, queue) { forge::operator_profiles::spawn(Arc::clone(&client), bridge, queue);