vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

comment once on a merged config pr when its swarm build fails

Browse source
This commit is contained in:
atlas 2026-10-26 21:00:51 +01:00 • committed by mara
commit 0bc5a7e475
1 changed files with 460 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

87
swarm-controller/src/jobq.rs
View file

@ -116,7 +116,10 @@ pub(crate) enum SwarmNodeKind {
///
/// Carries no rev: it reads the head when it runs, which in most DAGs is
/// after the `UpdateMeta` that commits it.
BuildMeta { agent: String },
///
/// `pr` is the merged config PR the build belongs to, if any: a red `nix`
/// build is reported on it once.
BuildMeta { agent: String, pr: Option<u64> },
/// Re-mint `agent`'s queue secret if it is still stored and old enough.
/// See `agent_renewal`.
///
@ -202,10 +205,12 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
| SwarmNodeKind::MintAgentIdentity { agent }
| SwarmNodeKind::MintAgentForgeToken { agent }
| SwarmNodeKind::MintAgentMatrixAccount { agent }
| SwarmNodeKind::BuildMeta { agent }
| SwarmNodeKind::RenewAgentQueueCredential { agent } => {
serde_json::json!({ "agent": agent })
}
SwarmNodeKind::BuildMeta { agent, pr } => {
serde_json::json!({ "agent": agent, "pr": pr })
}
SwarmNodeKind::MintHiveSenderToken { hive }
| SwarmNodeKind::MintHiveForgeToken { hive } => serde_json::json!({ "hive": hive }),
SwarmNodeKind::MintClassifierForgeToken
@ -414,7 +419,7 @@ async fn run_node_work(
SwarmNodeKind::BuildAgent { agent, rev, pr } => {
build_agent(&deps, forge, agent, rev, pr).await
}
SwarmNodeKind::BuildMeta { agent } => build_meta(&deps, forge, agent).await,
SwarmNodeKind::BuildMeta { agent, pr } => build_meta(&deps, forge, agent, pr).await,
SwarmNodeKind::RenewAgentQueueCredential { agent } => renew_queue_credential(&agent).await,
SwarmNodeKind::SetAgentWanted { hive, agent } => {
set_agent_wanted(deps.wanted.as_deref(), &hive, &agent).await
@ -602,8 +607,9 @@ async fn build_meta(
deps: &WorkerDeps,
forge: Option<Arc<forge::Client>>,
agent: String,
pr: Option<u64>,
) -> hive_jobq::scheduler::Outcome {
meta_build::run(forge.as_deref(), deps.queue.as_ref(), agent).await
meta_build::run(forge.as_deref(), deps.queue.as_ref(), agent, pr).await
}
/// What an `UpdateMeta` pass decided: the deploys to send, and the placements
@ -746,11 +752,13 @@ fn gated_deploy<'a>(
hive: &str,
agent: &str,
rev: Option<String>,
pr: Option<u64>,
update: Option<hive_jobq::builder::NodeGuid>,
) -> hive_jobq::builder::NodeRef<'a, SwarmNodeKind, SwarmResourceKind> {
let mut build = b
.node(SwarmNodeKind::BuildMeta {
agent: agent.to_owned(),
pr,
})
.needs(SwarmResourceKind::Builder);
if let Some(update) = update {
@ -785,7 +793,7 @@ fn declare_fan_out(
}
let deploys: Vec<hive_jobq::builder::NodeGuid> = placements
.iter()
.map(|(hive, agent)| gated_deploy(grown, hive, agent, None, None).guid())
.map(|(hive, agent)| gated_deploy(grown, hive, agent, None, None, None).guid())
.collect();
if let FanOut::Record { pin } = fan_out {
let mut record = grown.node(SwarmNodeKind::RecordFanout { pin: pin.clone() });
@ -1173,7 +1181,7 @@ pub(crate) async fn queue_merged_config_deploy(
state: &AppState,
merged: config_pr::MergedConfigPr,
) {
let config_pr::MergedConfigPr { agent, rev } = merged;
let config_pr::MergedConfigPr { agent, rev, pr } = merged;
let declared = match read_declarations(state, state.hives.iter()).await {
Ok(declared) => declared,
Err((hive, e)) => {
@ -1197,7 +1205,7 @@ pub(crate) async fn queue_merged_config_deploy(
.jobq
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert_job(None, |b| merged_deploy_job(b, &hive, &agent, &rev));
.insert_job(None, |b| merged_deploy_job(b, &hive, &agent, &rev, pr));
match inserted {
Ok(_) => tracing::info!(%agent, %rev, %hive, "config-pr merge: deploy queued"),
Err(e) => {
@ -1218,12 +1226,13 @@ fn merged_deploy_job(
hive: &str,
agent: &str,
rev: &str,
pr: u64,
) -> Vec<hive_jobq::builder::NodeGuid> {
let update_meta = b
.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta);
let root = update_meta.guid();
let _deploy = gated_deploy(b, hive, agent, Some(rev.to_owned()), Some(root));
let _deploy = gated_deploy(b, hive, agent, Some(rev.to_owned()), Some(pr), Some(root));
vec![root]
}
@ -1347,7 +1356,7 @@ pub(crate) fn declare_agent_job(
// queue at all fails this node — but `TriggerDeploy` has nothing to
// publish to on such a host either, so the strict edge cancels a node
// that could not have succeeded anyway.
let trigger_deploy = gated_deploy(b, hive, agent, None, Some(update_meta.guid()))
let trigger_deploy = gated_deploy(b, hive, agent, None, None, Some(update_meta.guid()))
.after_any(mint_identity)
.after_any(mint_forge_token)
// `after_any` for the same reason: the container reads this token
@ -1561,6 +1570,9 @@ struct SkippedDeploy {
hive: String,
agent: String,
rev: Option<String>,
/// The merged config PR the failed build belonged to, so the build
/// declared again reports on it too.
pr: Option<u64>,
cause: SkipCause,
}
@ -1592,14 +1604,15 @@ fn skipped_deploys(
let SwarmNodeKind::TriggerDeploy { hive, agent, rev } = &n.payload else {
return None;
};
let failed = |is: fn(&SwarmNodeKind) -> bool| {
n.deps.iter().any(|d| match d {
hive_jobq::Dep::Node { id, .. } => graph.node(*id).is_some_and(|dep| {
is(&dep.payload) && dep.state == hive_jobq::State::Failed
}),
hive_jobq::Dep::Resource { .. } => false,
let dep_nodes = || {
n.deps.iter().filter_map(|d| match d {
hive_jobq::Dep::Node { id, .. } => graph.node(*id),
hive_jobq::Dep::Resource { .. } => None,
})
};
let failed = |is: fn(&SwarmNodeKind) -> bool| {
dep_nodes().any(|dep| is(&dep.payload) && dep.state == hive_jobq::State::Failed)
};
let cause = if failed(|p| matches!(p, SwarmNodeKind::UpdateMeta)) {
SkipCause::FailedUpdate
} else if failed(|p| matches!(p, SwarmNodeKind::BuildMeta { .. })) {
@ -1607,10 +1620,15 @@ fn skipped_deploys(
} else {
return None;
};
let pr = dep_nodes().find_map(|dep| match &dep.payload {
SwarmNodeKind::BuildMeta { pr, .. } => *pr,
_ => None,
});
Some(SkippedDeploy {
hive: hive.clone(),
agent: agent.clone(),
rev: rev.clone(),
pr,
cause,
})
})
@ -1666,7 +1684,7 @@ fn queue_meta_update(
});
let deploys: Vec<_> = skipped
.into_iter()
.map(|s| gated_deploy(b, &s.hive, &s.agent, s.rev, update).guid())
.map(|s| gated_deploy(b, &s.hive, &s.agent, s.rev, s.pr, update).guid())
.collect();
update.map_or(deploys, |update| vec![update])
})
@ -2294,7 +2312,7 @@ mod tests {
}
fn merged_dag(b: &TestBuilder) {
let _ = super::merged_deploy_job(b, "pr1ma", "atlas", "abc");
let _ = super::merged_deploy_job(b, "pr1ma", "atlas", "abc", 7);
}
/// A deploy needs the meta update to succeed: its edge refuses a failed
@ -2890,7 +2908,7 @@ mod tests {
.filter(|n| n.payload.label() == label)
.filter(|n| n.state == State::Pending)
.map(|n| match &n.payload {
SwarmNodeKind::BuildMeta { agent }
SwarmNodeKind::BuildMeta { agent, .. }
| SwarmNodeKind::TriggerDeploy { agent, .. } => agent.clone(),
other => panic!("not a build or a deploy: {other:?}"),
})
@ -3811,12 +3829,27 @@ mod tests {
nodes.into_iter().map(|(_, rev)| rev).collect()
}
/// The `pr` of `agent`'s builds, oldest first.
fn build_prs(sched: &SharedSched, agent: &str) -> Vec<Option<u64>> {
let guard = sched.lock().unwrap();
let mut nodes: Vec<_> = guard
.graph()
.nodes()
.filter_map(|n| match &n.payload {
SwarmNodeKind::BuildMeta { agent: a, pr } if a == agent => Some((n.id, *pr)),
_ => None,
})
.collect();
nodes.sort_by_key(|(id, _)| *id);
nodes.into_iter().map(|(_, pr)| pr).collect()
}
fn merged_atlas() -> SharedSched {
let sched = fresh_sched();
sched
.lock()
.unwrap()
.insert_job(None, |b| merged_deploy_job(b, "pr1ma", "atlas", "abc"))
.insert_job(None, |b| merged_deploy_job(b, "pr1ma", "atlas", "abc", 7))
.expect("the graph inserts");
sched
}
@ -3865,11 +3898,27 @@ mod tests {
.iter()
.all(|rev| rev.as_deref() == Some("abc"))
);
// So does the PR it belongs to: every retried build reports on it too.
assert_eq!(build_prs(&sched, "atlas"), [Some(7); 4]);
assert!(
!queue_meta_update(&sched).expect("tick"),
"the success ends the retries"
);
}
/// Only a merged config PR's build carries a PR: the creation DAG's does
/// not, so its failure has nobody to comment on.
#[test]
fn only_a_merged_prs_build_carries_its_pr() {
let sched = merged_atlas();
sched
.lock()
.unwrap()
.insert_job(None, |b| declare_agent_job(b, "iris", "pr1ma", false))
.expect("the graph inserts");
assert_eq!(build_prs(&sched, "atlas"), [Some(7)]);
assert_eq!(build_prs(&sched, "iris"), [None]);
}
}
#[cfg(test)]