retry every failed meta build each round, a timeout included, and drop the attempt count

swarm-controller/src/held_deploys.rs

@@ -4,13 +4,12 @@
 //!
 //! An agent counts when its latest `BuildMeta` node is `Failed`, once per agent
 //! under one of two `reason` labels:
-//! - `red`: `nix` exited with an error twice. Nothing retries it; a config merge
-//!   builds again.
-//! - `retrying`: the build gave no verdict or met its first `nix` error, so the
-//!   ten-minute timer builds it again.
+//! - `failed`: `nix` failed, its timeout included.
+//! - `unavailable`: the build got no verdict (no builder, a forge error, no
+//!   `internal/meta` head, a missing read grant).
 //!
-//! A pending, running or finished build does not count, so the value falls to
-//! zero when the builder answers.
+//! The ten-minute timer builds either again. A pending, running or finished
+//! build does not count, so the value falls to zero when a build succeeds.
 
 use std::sync::{Arc, Mutex};
 
@@ -25,7 +24,7 @@ use crate::meta_build;
 /// The gauge's name. The Grafana rule selects it by this name.
 pub(crate) const METRIC: &str = "swarm.deploys.held";
 
-/// Held agents as `(red, retrying)`.
+/// Held agents as `(failed, unavailable)`.
 fn count(graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>) -> (u64, u64) {
     let mut latest = std::collections::BTreeMap::new();
     for n in graph.nodes() {
@@ -40,11 +39,11 @@ fn count(graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>) -> (u64, u6
     latest
         .into_values()
         .filter(|n| n.state == State::Failed)
-        .fold((0, 0), |(red, retrying), n| {
-            if n.error.as_deref().is_some_and(meta_build::is_retried) {
-                (red, retrying + 1)
+        .fold((0, 0), |(failed, unavailable), n| {
+            if n.error.as_deref().is_some_and(meta_build::is_no_verdict) {
+                (failed, unavailable + 1)
             } else {
-                (red + 1, retrying)
+                (failed + 1, unavailable)
             }
         })
 }
@@ -64,13 +63,13 @@ pub(crate) fn register(
         .u64_observable_gauge(METRIC)
         .with_description("agents whose deploy the build gate is holding back, by reason")
         .with_callback(move |obs| {
-            let (red, retrying) = count(
+            let (failed, unavailable) = count(
                 jobq.lock()
                     .unwrap_or_else(std::sync::PoisonError::into_inner)
                     .graph(),
             );
-            obs.observe(red, &[KeyValue::new("reason", "red")]);
-            obs.observe(retrying, &[KeyValue::new("reason", "retrying")]);
+            obs.observe(failed, &[KeyValue::new("reason", "failed")]);
+            obs.observe(unavailable, &[KeyValue::new("reason", "unavailable")]);
         })
         .build();
 }
@@ -97,7 +96,6 @@ mod tests {
                 .append(
                     SwarmNodeKind::BuildMeta {
                         agent: (*agent).to_owned(),
-                        attempt: 0,
                     },
                     Vec::new(),
                     None,
@@ -113,19 +111,19 @@ mod tests {
         jobq
     }
 
-    fn red() -> Outcome {
+    fn failed() -> Outcome {
         Outcome::Failed("error: attribute missing".to_owned())
     }
 
-    fn retrying() -> Outcome {
-        Outcome::Failed(format!("{}no builder", meta_build::RETRY_PREFIX))
+    fn unavailable() -> Outcome {
+        Outcome::Failed(format!("{}no builder", meta_build::NO_VERDICT_PREFIX))
     }
 
     #[tokio::test]
     async fn a_failed_latest_build_holds_its_agent_by_reason() {
         let jobq = after(&[
-            ("atlas", red()),
-            ("argus", retrying()),
+            ("atlas", failed()),
+            ("argus", unavailable()),
             ("iris", Outcome::Done),
         ])
         .await;
@@ -136,9 +134,9 @@ mod tests {
     /// releases it, and a red one after a green one holds it.
     #[tokio::test]
     async fn only_the_latest_build_of_an_agent_counts() {
-        let released = after(&[("atlas", red()), ("atlas", Outcome::Done)]).await;
+        let released = after(&[("atlas", failed()), ("atlas", Outcome::Done)]).await;
         assert_eq!(count(released.lock().unwrap().graph()), (0, 0));
-        let held = after(&[("atlas", Outcome::Done), ("atlas", red())]).await;
+        let held = after(&[("atlas", Outcome::Done), ("atlas", failed())]).await;
         assert_eq!(count(held.lock().unwrap().graph()), (1, 0));
     }
 
@@ -146,7 +144,7 @@ mod tests {
     /// included, so a rule sees the series before anything is held.
     #[tokio::test]
     async fn the_gauge_exports_both_reasons() {
-        let jobq = after(&[("atlas", red())]).await;
+        let jobq = after(&[("atlas", failed())]).await;
         let exporter = InMemoryMetricExporter::default();
         let reader = PeriodicReader::builder(exporter.clone()).build();
         let provider = SdkMeterProvider::builder().with_reader(reader).build();
@@ -174,6 +172,9 @@ mod tests {
             }
         }
         points.sort();
-        assert_eq!(points, [("red".to_owned(), 1), ("retrying".to_owned(), 0)]);
+        assert_eq!(
+            points,
+            [("failed".to_owned(), 1), ("unavailable".to_owned(), 0)]
+        );
     }
 }