mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm: a failed facts copy counts in the held-deploys gauge

Browse source
This commit is contained in:
atlas 2026-11-07 12:28:43 +01:00
commit 1b6d7f3c99
1 changed files with 136 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

146
swarm-controller/src/held_deploys.rs
View file

@ -2,15 +2,17 @@
//! the build gate, exported on the jobq exporter. Grafana alerts when it stays
//! above zero (`hh-deploys-held`, `nix/host-modules/swarm-grafana.nix`).
//!
//! An agent counts when its latest `BuildMeta` node is `Failed`, once per agent
//! under one of two `reason` labels:
//! An agent counts when its latest `BuildMeta` node is `Failed`, or when its
//! latest `TriggerDeploy` is `Skipped` behind a `CopyAtticFacts` that `Failed`.
//! It counts once, under one of two `reason` labels:
//! - `failed`: `nix` failed, its timeout included.
//! - `unavailable`: the build got no verdict (no builder, a forge error, no
//! `internal/meta` head, a missing read grant, a closure the builder built but
//! did not push to the cache).
//! didn't push to the cache), or the cache facts couldn't be copied to the hive.
//!
//! The ten-minute timer builds either again. A pending, running or finished
//! build does not count, so the value falls to zero when a build succeeds.
//! The ten-minute timer declares either again. A pending, running or finished
//! build, or a deploy past its copy, does not count, so the value falls to zero
//! when the retry succeeds.
use std::sync::{Arc, Mutex};
@ -25,11 +27,14 @@ use crate::meta_build;
/// The gauge's name. The Grafana rule selects it by this name.
pub(crate) const METRIC: &str = "swarm.deploys.held";
/// Held agents as `(failed, unavailable)`.
fn count(graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>) -> (u64, u64) {
/// The latest node of each agent whose payload `pick` returns the agent of.
fn latest_per_agent(
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
pick: impl Fn(&SwarmNodeKind) -> Option<&String>,
) -> std::collections::BTreeMap<&String, &hive_jobq::Node<SwarmNodeKind, SwarmResourceKind>> {
let mut latest = std::collections::BTreeMap::new();
for n in graph.nodes() {
if let SwarmNodeKind::BuildMeta { agent, .. } = &n.payload
if let Some(agent) = pick(&n.payload)
&& latest
.get(agent)
.is_none_or(|l: &&hive_jobq::Node<_, _>| l.id < n.id)
@ -38,15 +43,45 @@ fn count(graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>) -> (u64, u6
}
}
latest
.into_values()
.filter(|n| n.state == State::Failed)
.fold((0, 0), |(failed, unavailable), n| {
if n.error.as_deref().is_some_and(meta_build::is_no_verdict) {
(failed, unavailable + 1)
}
/// Held agents as `(failed, unavailable)`.
fn count(graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>) -> (u64, u64) {
let builds = latest_per_agent(graph, |p| match p {
SwarmNodeKind::BuildMeta { agent, .. } => Some(agent),
_ => None,
});
let deploys = latest_per_agent(graph, |p| match p {
SwarmNodeKind::TriggerDeploy { agent, .. } => Some(agent),
_ => None,
});
let copy_failed = |deploy: &hive_jobq::Node<SwarmNodeKind, SwarmResourceKind>| {
deploy.state == State::Skipped
&& deploy.deps.iter().any(|d| match d {
hive_jobq::Dep::Node { id, .. } => graph.node(*id).is_some_and(|dep| {
matches!(dep.payload, SwarmNodeKind::CopyAtticFacts { .. })
&& dep.state == State::Failed
}),
hive_jobq::Dep::Resource { .. } => false,
})
};
let (mut failed, mut unavailable) = (0, 0);
for (agent, build) in &builds {
if build.state == State::Failed {
if build
.error
.as_deref()
.is_some_and(meta_build::is_no_verdict)
{
unavailable += 1;
} else {
(failed + 1, unavailable)
failed += 1;
}
})
} else if deploys.get(agent).is_some_and(|d| copy_failed(d)) {
unavailable += 1;
}
}
(failed, unavailable)
}
/// Register the gauge on `provider`, reading `jobq` at each export. Nothing when
@ -142,6 +177,87 @@ mod tests {
assert_eq!(count(held.lock().unwrap().graph()), (1, 0));
}
/// A scheduler where one declaration of `agent`'s build, facts copy and
/// deploy ran, the build and the copy ending as given.
async fn declared(jobq: &Jobq, agent: &str, build: Outcome, copy: Outcome) {
jobq.lock()
.unwrap()
.insert_job(None, |b| {
let build = b.node(SwarmNodeKind::BuildMeta {
agent: agent.to_owned(),
pr: None,
});
let copy = b.node(SwarmNodeKind::CopyAtticFacts {
hive: "pr1ma".to_owned(),
});
let deploy = b
.node(SwarmNodeKind::TriggerDeploy {
hive: "pr1ma".to_owned(),
agent: agent.to_owned(),
rev: None,
})
.after_ok(build)
.after_ok(copy);
vec![deploy.guid()]
})
.expect("insert");
while let Some(runner) = Scheduler::claim_next(jobq, {
let (build, copy) = (build.clone(), copy.clone());
move |_, kind, b| async move {
let outcome = match kind {
SwarmNodeKind::BuildMeta { .. } => build,
SwarmNodeKind::CopyAtticFacts { .. } => copy,
_ => Outcome::Done,
};
(b, outcome)
}
}) {
runner.await.1.expect("no growth");
}
}
fn empty_jobq() -> Jobq {
Arc::new(Mutex::new(Scheduler::new(
hive_jobq::Graph::new(),
hive_jobq::resources::ResourceTable::new(),
)))
}
/// A deploy the facts copy held back counts as unavailable, and a green
/// build with a green copy does not (the control that the deploy runs).
#[tokio::test]
async fn a_failed_facts_copy_holds_its_agent_as_unavailable() {
let jobq = empty_jobq();
declared(
&jobq,
"atlas",
Outcome::Done,
Outcome::Failed("denied".into()),
)
.await;
declared(&jobq, "argus", Outcome::Done, Outcome::Done).await;
assert_eq!(count(jobq.lock().unwrap().graph()), (0, 1));
}
/// An agent whose build and copy both failed counts once, under the build's
/// reason; a retry that goes through releases it.
#[tokio::test]
async fn a_failed_build_and_copy_count_once_and_a_good_retry_releases() {
let jobq = empty_jobq();
declared(&jobq, "atlas", failed(), Outcome::Failed("denied".into())).await;
assert_eq!(count(jobq.lock().unwrap().graph()), (1, 0));
declared(
&jobq,
"atlas",
Outcome::Done,
Outcome::Failed("denied".into()),
)
.await;
assert_eq!(count(jobq.lock().unwrap().graph()), (0, 1));
declared(&jobq, "atlas", Outcome::Done, Outcome::Done).await;
assert_eq!(count(jobq.lock().unwrap().graph()), (0, 0));
}
/// The gauge reads the graph at export time and labels both reasons, zero
/// included, so a rule sees the series before anything is held.
#[tokio::test]