swarm: a failed facts copy counts in the held-deploys gauge

docs/4-implementation/swarm/builder.md

@@ -287,7 +287,8 @@ merged PR's (a new agent, the pin fan-out), leave no comment. When the forge
 refuses the comment, the controller logs a warning and the node ends as before.
 
 The controller exports the gauge `swarm.deploys.held`: the agents whose latest
-build failed, with the label `reason` set to `failed` or `unavailable`. The Grafana
+build failed, or whose latest deploy a failed facts copy skipped, with the label
+`reason` set to `failed` or `unavailable` (a failed copy is `unavailable`). The Grafana
 alert `hh-deploys-held` fires when the sum stays above zero for 30 minutes, so
 a dead builder or a config that fails to build shows in Grafana before anyone notices an agent
 not updating.

nix/host-modules/swarm-grafana.nix

@@ -1762,7 +1762,8 @@ in
                     # stay in the grouping so the alert names the holder.
                     # Agents whose deploy the build gate holds back, from the gauge
                     # swarm-controller exports (`held_deploys.rs`): a build that
-                    # fails, or that keeps getting no answer from the builder.
+                    # fails, or that keeps getting no answer from the builder. A deploy
+                    # whose facts copy to its hive fails is held too.
                     # Thirty minutes covers a builder restart and the controller's
                     # ten-minute retries.
                     (metricThresholdRule {
@@ -1771,7 +1772,7 @@ in
                       expr = ''sum({__name__="swarm.deploys.held"})'';
                       threshold = 0;
                       for = "30m";
-                      summary = "swarm-controller is holding back the deploy of one or more agents because the builder's build of internal/meta fails or gets no answer; the failed `build_meta` node on the Jobs page says which";
+                      summary = "swarm-controller is holding back the deploy of one or more agents because the builder's build of internal/meta fails or gets no answer, or the cache facts can't be copied to its hive; the failed `build_meta` or `copy_attic_facts` node on the Jobs page says which";
                     })
                     (metricThresholdRule {
                       uid = "hh-bao-client-cert-lifetime";

swarm-controller/src/held_deploys.rs

@@ -2,15 +2,17 @@
 //! the build gate, exported on the jobq exporter. Grafana alerts when it stays
 //! above zero (`hh-deploys-held`, `nix/host-modules/swarm-grafana.nix`).
 //!
-//! An agent counts when its latest `BuildMeta` node is `Failed`, once per agent
+//! An agent counts when its latest `BuildMeta` node is `Failed`, or when its
-//! under one of two `reason` labels:
+//! latest `TriggerDeploy` is `Skipped` behind a `CopyAtticFacts` that `Failed`.
+//! It counts once, under one of two `reason` labels:
 //! - `failed`: `nix` failed, its timeout included.
 //! - `unavailable`: the build got no verdict (no builder, a forge error, no
 //!   `internal/meta` head, a missing read grant, a closure the builder built but
-//!   did not push to the cache).
+//!   didn't push to the cache), or the cache facts couldn't be copied to the hive.
 //!
-//! The ten-minute timer builds either again. A pending, running or finished
+//! The ten-minute timer declares either again. A pending, running or finished
-//! build does not count, so the value falls to zero when a build succeeds.
+//! build, or a deploy past its copy, does not count, so the value falls to zero
+//! when the retry succeeds.
 
 use std::sync::{Arc, Mutex};
 
@@ -25,11 +27,14 @@ use crate::meta_build;
 /// The gauge's name. The Grafana rule selects it by this name.
 pub(crate) const METRIC: &str = "swarm.deploys.held";
 
-/// Held agents as `(failed, unavailable)`.
+/// The latest node of each agent whose payload `pick` returns the agent of.
-fn count(graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>) -> (u64, u64) {
+fn latest_per_agent(
+    graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
+    pick: impl Fn(&SwarmNodeKind) -> Option<&String>,
+) -> std::collections::BTreeMap<&String, &hive_jobq::Node<SwarmNodeKind, SwarmResourceKind>> {
     let mut latest = std::collections::BTreeMap::new();
     for n in graph.nodes() {
-        if let SwarmNodeKind::BuildMeta { agent, .. } = &n.payload
+        if let Some(agent) = pick(&n.payload)
             && latest
                 .get(agent)
                 .is_none_or(|l: &&hive_jobq::Node<_, _>| l.id < n.id)
@@ -38,15 +43,45 @@ fn count(graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>) -> (u64, u6
         }
     }
     latest
-        .into_values()
+}
-        .filter(|n| n.state == State::Failed)
+
-        .fold((0, 0), |(failed, unavailable), n| {
+/// Held agents as `(failed, unavailable)`.
-            if n.error.as_deref().is_some_and(meta_build::is_no_verdict) {
+fn count(graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>) -> (u64, u64) {
-                (failed, unavailable + 1)
+    let builds = latest_per_agent(graph, |p| match p {
+        SwarmNodeKind::BuildMeta { agent, .. } => Some(agent),
+        _ => None,
+    });
+    let deploys = latest_per_agent(graph, |p| match p {
+        SwarmNodeKind::TriggerDeploy { agent, .. } => Some(agent),
+        _ => None,
+    });
+    let copy_failed = |deploy: &hive_jobq::Node<SwarmNodeKind, SwarmResourceKind>| {
+        deploy.state == State::Skipped
+            && deploy.deps.iter().any(|d| match d {
+                hive_jobq::Dep::Node { id, .. } => graph.node(*id).is_some_and(|dep| {
+                    matches!(dep.payload, SwarmNodeKind::CopyAtticFacts { .. })
+                        && dep.state == State::Failed
+                }),
+                hive_jobq::Dep::Resource { .. } => false,
+            })
+    };
+    let (mut failed, mut unavailable) = (0, 0);
+    for (agent, build) in &builds {
+        if build.state == State::Failed {
+            if build
+                .error
+                .as_deref()
+                .is_some_and(meta_build::is_no_verdict)
+            {
+                unavailable += 1;
             } else {
-                (failed + 1, unavailable)
+                failed += 1;
             }
-        })
+        } else if deploys.get(agent).is_some_and(|d| copy_failed(d)) {
+            unavailable += 1;
+        }
+    }
+    (failed, unavailable)
 }
 
 /// Register the gauge on `provider`, reading `jobq` at each export. Nothing when
@@ -142,6 +177,87 @@ mod tests {
         assert_eq!(count(held.lock().unwrap().graph()), (1, 0));
     }
 
+    /// A scheduler where one declaration of `agent`'s build, facts copy and
+    /// deploy ran, the build and the copy ending as given.
+    async fn declared(jobq: &Jobq, agent: &str, build: Outcome, copy: Outcome) {
+        jobq.lock()
+            .unwrap()
+            .insert_job(None, |b| {
+                let build = b.node(SwarmNodeKind::BuildMeta {
+                    agent: agent.to_owned(),
+                    pr: None,
+                });
+                let copy = b.node(SwarmNodeKind::CopyAtticFacts {
+                    hive: "pr1ma".to_owned(),
+                });
+                let deploy = b
+                    .node(SwarmNodeKind::TriggerDeploy {
+                        hive: "pr1ma".to_owned(),
+                        agent: agent.to_owned(),
+                        rev: None,
+                    })
+                    .after_ok(build)
+                    .after_ok(copy);
+                vec![deploy.guid()]
+            })
+            .expect("insert");
+        while let Some(runner) = Scheduler::claim_next(jobq, {
+            let (build, copy) = (build.clone(), copy.clone());
+            move |_, kind, b| async move {
+                let outcome = match kind {
+                    SwarmNodeKind::BuildMeta { .. } => build,
+                    SwarmNodeKind::CopyAtticFacts { .. } => copy,
+                    _ => Outcome::Done,
+                };
+                (b, outcome)
+            }
+        }) {
+            runner.await.1.expect("no growth");
+        }
+    }
+
+    fn empty_jobq() -> Jobq {
+        Arc::new(Mutex::new(Scheduler::new(
+            hive_jobq::Graph::new(),
+            hive_jobq::resources::ResourceTable::new(),
+        )))
+    }
+
+    /// A deploy the facts copy held back counts as unavailable, and a green
+    /// build with a green copy does not (the control that the deploy runs).
+    #[tokio::test]
+    async fn a_failed_facts_copy_holds_its_agent_as_unavailable() {
+        let jobq = empty_jobq();
+        declared(
+            &jobq,
+            "atlas",
+            Outcome::Done,
+            Outcome::Failed("denied".into()),
+        )
+        .await;
+        declared(&jobq, "argus", Outcome::Done, Outcome::Done).await;
+        assert_eq!(count(jobq.lock().unwrap().graph()), (0, 1));
+    }
+
+    /// An agent whose build and copy both failed counts once, under the build's
+    /// reason; a retry that goes through releases it.
+    #[tokio::test]
+    async fn a_failed_build_and_copy_count_once_and_a_good_retry_releases() {
+        let jobq = empty_jobq();
+        declared(&jobq, "atlas", failed(), Outcome::Failed("denied".into())).await;
+        assert_eq!(count(jobq.lock().unwrap().graph()), (1, 0));
+        declared(
+            &jobq,
+            "atlas",
+            Outcome::Done,
+            Outcome::Failed("denied".into()),
+        )
+        .await;
+        assert_eq!(count(jobq.lock().unwrap().graph()), (0, 1));
+        declared(&jobq, "atlas", Outcome::Done, Outcome::Done).await;
+        assert_eq!(count(jobq.lock().unwrap().graph()), (0, 0));
+    }
+
     /// The gauge reads the graph at export time and labels both reasons, zero
     /// included, so a rule sees the series before anything is held.
     #[tokio::test]