diff --git a/docs/4-implementation/swarm/builder.md b/docs/4-implementation/swarm/builder.md index 2f946ce3..045008d0 100644 --- a/docs/4-implementation/swarm/builder.md +++ b/docs/4-implementation/swarm/builder.md @@ -287,7 +287,8 @@ merged PR's (a new agent, the pin fan-out), leave no comment. When the forge refuses the comment, the controller logs a warning and the node ends as before. The controller exports the gauge `swarm.deploys.held`: the agents whose latest -build failed, with the label `reason` set to `failed` or `unavailable`. The Grafana +build failed, or whose latest deploy a failed facts copy skipped, with the label +`reason` set to `failed` or `unavailable` (a failed copy is `unavailable`). The Grafana alert `hh-deploys-held` fires when the sum stays above zero for 30 minutes, so a dead builder or a config that fails to build shows in Grafana before anyone notices an agent not updating. diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 8c83d57f..ba7c5ff9 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -1762,7 +1762,8 @@ in # stay in the grouping so the alert names the holder. # Agents whose deploy the build gate holds back, from the gauge # swarm-controller exports (`held_deploys.rs`): a build that - # fails, or that keeps getting no answer from the builder. + # fails, or that keeps getting no answer from the builder. A deploy + # whose facts copy to its hive fails is held too. # Thirty minutes covers a builder restart and the controller's # ten-minute retries. (metricThresholdRule { @@ -1771,7 +1772,7 @@ in expr = ''sum({__name__="swarm.deploys.held"})''; threshold = 0; for = "30m"; - summary = "swarm-controller is holding back the deploy of one or more agents because the builder's build of internal/meta fails or gets no answer; the failed `build_meta` node on the Jobs page says which"; + summary = "swarm-controller is holding back the deploy of one or more agents because the builder's build of internal/meta fails or gets no answer, or the cache facts can't be copied to its hive; the failed `build_meta` or `copy_attic_facts` node on the Jobs page says which"; }) (metricThresholdRule { uid = "hh-bao-client-cert-lifetime"; diff --git a/swarm-controller/src/held_deploys.rs b/swarm-controller/src/held_deploys.rs index fa578d19..ab60cbcd 100644 --- a/swarm-controller/src/held_deploys.rs +++ b/swarm-controller/src/held_deploys.rs @@ -2,15 +2,17 @@ //! the build gate, exported on the jobq exporter. Grafana alerts when it stays //! above zero (`hh-deploys-held`, `nix/host-modules/swarm-grafana.nix`). //! -//! An agent counts when its latest `BuildMeta` node is `Failed`, once per agent -//! under one of two `reason` labels: +//! An agent counts when its latest `BuildMeta` node is `Failed`, or when its +//! latest `TriggerDeploy` is `Skipped` behind a `CopyAtticFacts` that `Failed`. +//! It counts once, under one of two `reason` labels: //! - `failed`: `nix` failed, its timeout included. //! - `unavailable`: the build got no verdict (no builder, a forge error, no //! `internal/meta` head, a missing read grant, a closure the builder built but -//! did not push to the cache). +//! didn't push to the cache), or the cache facts couldn't be copied to the hive. //! -//! The ten-minute timer builds either again. A pending, running or finished -//! build does not count, so the value falls to zero when a build succeeds. +//! The ten-minute timer declares either again. A pending, running or finished +//! build, or a deploy past its copy, does not count, so the value falls to zero +//! when the retry succeeds. use std::sync::{Arc, Mutex}; @@ -25,11 +27,14 @@ use crate::meta_build; /// The gauge's name. The Grafana rule selects it by this name. pub(crate) const METRIC: &str = "swarm.deploys.held"; -/// Held agents as `(failed, unavailable)`. -fn count(graph: &hive_jobq::Graph) -> (u64, u64) { +/// The latest node of each agent whose payload `pick` returns the agent of. +fn latest_per_agent( + graph: &hive_jobq::Graph, + pick: impl Fn(&SwarmNodeKind) -> Option<&String>, +) -> std::collections::BTreeMap<&String, &hive_jobq::Node> { let mut latest = std::collections::BTreeMap::new(); for n in graph.nodes() { - if let SwarmNodeKind::BuildMeta { agent, .. } = &n.payload + if let Some(agent) = pick(&n.payload) && latest .get(agent) .is_none_or(|l: &&hive_jobq::Node<_, _>| l.id < n.id) @@ -38,15 +43,45 @@ fn count(graph: &hive_jobq::Graph) -> (u64, u6 } } latest - .into_values() - .filter(|n| n.state == State::Failed) - .fold((0, 0), |(failed, unavailable), n| { - if n.error.as_deref().is_some_and(meta_build::is_no_verdict) { - (failed, unavailable + 1) +} + +/// Held agents as `(failed, unavailable)`. +fn count(graph: &hive_jobq::Graph) -> (u64, u64) { + let builds = latest_per_agent(graph, |p| match p { + SwarmNodeKind::BuildMeta { agent, .. } => Some(agent), + _ => None, + }); + let deploys = latest_per_agent(graph, |p| match p { + SwarmNodeKind::TriggerDeploy { agent, .. } => Some(agent), + _ => None, + }); + let copy_failed = |deploy: &hive_jobq::Node| { + deploy.state == State::Skipped + && deploy.deps.iter().any(|d| match d { + hive_jobq::Dep::Node { id, .. } => graph.node(*id).is_some_and(|dep| { + matches!(dep.payload, SwarmNodeKind::CopyAtticFacts { .. }) + && dep.state == State::Failed + }), + hive_jobq::Dep::Resource { .. } => false, + }) + }; + let (mut failed, mut unavailable) = (0, 0); + for (agent, build) in &builds { + if build.state == State::Failed { + if build + .error + .as_deref() + .is_some_and(meta_build::is_no_verdict) + { + unavailable += 1; } else { - (failed + 1, unavailable) + failed += 1; } - }) + } else if deploys.get(agent).is_some_and(|d| copy_failed(d)) { + unavailable += 1; + } + } + (failed, unavailable) } /// Register the gauge on `provider`, reading `jobq` at each export. Nothing when @@ -142,6 +177,87 @@ mod tests { assert_eq!(count(held.lock().unwrap().graph()), (1, 0)); } + /// A scheduler where one declaration of `agent`'s build, facts copy and + /// deploy ran, the build and the copy ending as given. + async fn declared(jobq: &Jobq, agent: &str, build: Outcome, copy: Outcome) { + jobq.lock() + .unwrap() + .insert_job(None, |b| { + let build = b.node(SwarmNodeKind::BuildMeta { + agent: agent.to_owned(), + pr: None, + }); + let copy = b.node(SwarmNodeKind::CopyAtticFacts { + hive: "pr1ma".to_owned(), + }); + let deploy = b + .node(SwarmNodeKind::TriggerDeploy { + hive: "pr1ma".to_owned(), + agent: agent.to_owned(), + rev: None, + }) + .after_ok(build) + .after_ok(copy); + vec![deploy.guid()] + }) + .expect("insert"); + while let Some(runner) = Scheduler::claim_next(jobq, { + let (build, copy) = (build.clone(), copy.clone()); + move |_, kind, b| async move { + let outcome = match kind { + SwarmNodeKind::BuildMeta { .. } => build, + SwarmNodeKind::CopyAtticFacts { .. } => copy, + _ => Outcome::Done, + }; + (b, outcome) + } + }) { + runner.await.1.expect("no growth"); + } + } + + fn empty_jobq() -> Jobq { + Arc::new(Mutex::new(Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ))) + } + + /// A deploy the facts copy held back counts as unavailable, and a green + /// build with a green copy does not (the control that the deploy runs). + #[tokio::test] + async fn a_failed_facts_copy_holds_its_agent_as_unavailable() { + let jobq = empty_jobq(); + declared( + &jobq, + "atlas", + Outcome::Done, + Outcome::Failed("denied".into()), + ) + .await; + declared(&jobq, "argus", Outcome::Done, Outcome::Done).await; + assert_eq!(count(jobq.lock().unwrap().graph()), (0, 1)); + } + + /// An agent whose build and copy both failed counts once, under the build's + /// reason; a retry that goes through releases it. + #[tokio::test] + async fn a_failed_build_and_copy_count_once_and_a_good_retry_releases() { + let jobq = empty_jobq(); + declared(&jobq, "atlas", failed(), Outcome::Failed("denied".into())).await; + assert_eq!(count(jobq.lock().unwrap().graph()), (1, 0)); + declared( + &jobq, + "atlas", + Outcome::Done, + Outcome::Failed("denied".into()), + ) + .await; + assert_eq!(count(jobq.lock().unwrap().graph()), (0, 1)); + declared(&jobq, "atlas", Outcome::Done, Outcome::Done).await; + assert_eq!(count(jobq.lock().unwrap().graph()), (0, 0)); + } + /// The gauge reads the graph at export time and labels both reasons, zero /// included, so a rule sees the series before anything is held. #[tokio::test]