vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

build a failed meta build once more, and treat a timeout or missing grant as no verdict

Browse source
This commit is contained in:
atlas 2026-10-26 17:53:39 +01:00
commit 99d2ed5b9e
1 changed files with 621 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

200
swarm-controller/src/jobq.rs
View file

@ -115,8 +115,10 @@ pub(crate) enum SwarmNodeKind {
/// runs only when the build is green. See `meta_build`.
///
/// Carries no rev: it reads the head when it runs, which in most DAGs is
/// after the `UpdateMeta` that commits it.
BuildMeta { agent: String },
/// after the `UpdateMeta` that commits it. `attempt` is how many builds of
/// this deploy already ended in an error from `nix`; the build after the
/// second such error is red.
BuildMeta { agent: String, attempt: u32 },
/// Re-mint `agent`'s queue secret if it is still stored and old enough.
/// See `agent_renewal`.
///
@ -202,7 +204,6 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
| SwarmNodeKind::MintAgentIdentity { agent }
| SwarmNodeKind::MintAgentForgeToken { agent }
| SwarmNodeKind::MintAgentMatrixAccount { agent }
| SwarmNodeKind::BuildMeta { agent }
| SwarmNodeKind::RenewAgentQueueCredential { agent } => {
serde_json::json!({ "agent": agent })
}
@ -215,6 +216,9 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
SwarmNodeKind::BuildAgent { agent, rev, pr } => {
serde_json::json!({ "agent": agent, "rev": rev, "pr": pr })
}
SwarmNodeKind::BuildMeta { agent, attempt } => {
serde_json::json!({ "agent": agent, "attempt": attempt })
}
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => {
serde_json::json!({ "agent": agent, "hive": hive, "rev": rev })
}
@ -414,16 +418,13 @@ async fn run_node_work(
SwarmNodeKind::BuildAgent { agent, rev, pr } => {
build_agent(&deps, forge, agent, rev, pr).await
}
SwarmNodeKind::BuildMeta { agent } => build_meta(&deps, forge, agent).await,
SwarmNodeKind::BuildMeta { agent, attempt } => {
build_meta(&deps, forge, agent, attempt).await
}
SwarmNodeKind::RenewAgentQueueCredential { agent } => renew_queue_credential(&agent).await,
SwarmNodeKind::SetAgentWanted { hive, agent } => match deps.wanted {
None => Outcome::Failed(
"no swarm queue is configured on this host, so no wanted-state \
declaration can be written"
.to_owned(),
),
Some(writer) => declare_new_agent(&writer, &hive, &agent).await,
},
SwarmNodeKind::SetAgentWanted { hive, agent } => {
set_agent_wanted(deps.wanted.as_deref(), &hive, &agent).await
}
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => match deps.queue {
None => Outcome::Failed(
"no swarm queue is configured on this host, so no hive can be told to deploy"
@ -607,8 +608,9 @@ async fn build_meta(
deps: &WorkerDeps,
forge: Option<Arc<forge::Client>>,
agent: String,
attempt: u32,
) -> hive_jobq::scheduler::Outcome {
meta_build::run(forge.as_deref(), deps.queue.as_ref(), agent).await
meta_build::run(forge.as_deref(), deps.queue.as_ref(), agent, attempt).await
}
/// What an `UpdateMeta` pass decided: the deploys to send, and the placements
@ -746,16 +748,21 @@ async fn record_fanout(
/// parent. The build waits on it, since the build reads the head that update
/// commits. The deploy keeps its own edge to it too: `skipped_deploys` finds
/// the deploys a failed update skipped by that edge.
///
/// `attempt` is the build's `BuildMeta::attempt`: 0 for a deploy no build has
/// failed for yet.
fn gated_deploy<'a>(
b: &'a hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
hive: &str,
agent: &str,
rev: Option<String>,
update: Option<hive_jobq::builder::NodeGuid>,
attempt: u32,
) -> hive_jobq::builder::NodeRef<'a, SwarmNodeKind, SwarmResourceKind> {
let mut build = b
.node(SwarmNodeKind::BuildMeta {
agent: agent.to_owned(),
attempt,
})
.needs(SwarmResourceKind::Builder);
if let Some(update) = update {
@ -790,7 +797,7 @@ fn declare_fan_out(
}
let deploys: Vec<hive_jobq::builder::NodeGuid> = placements
.iter()
.map(|(hive, agent)| gated_deploy(grown, hive, agent, None, None).guid())
.map(|(hive, agent)| gated_deploy(grown, hive, agent, None, None, 0).guid())
.collect();
if let FanOut::Record { pin } = fan_out {
let mut record = grown.node(SwarmNodeKind::RecordFanout { pin: pin.clone() });
@ -967,6 +974,23 @@ async fn mint_hive_sender(homeserver: Option<&str>, hive: &str) -> hive_jobq::sc
}
}
/// The `SetAgentWanted` arm, lifted out so `run_node_work` stays under the line
/// limit: it fails without a wanted-state writer.
async fn set_agent_wanted(
wanted: Option<&wanted::WantedWriter>,
hive: &str,
agent: &str,
) -> hive_jobq::scheduler::Outcome {
match wanted {
None => hive_jobq::scheduler::Outcome::Failed(
"no swarm queue is configured on this host, so no wanted-state \
declaration can be written"
.to_owned(),
),
Some(writer) => declare_new_agent(writer, hive, agent).await,
}
}
/// Declare a brand-new agent at [`NEW_AGENT_WANTED_STATE`] — unless it turns
/// out not to be new: an agent that already has a declaration (other than
/// `Destroyed`, which this treats as reusable) is left alone, so a retried
@ -1211,7 +1235,7 @@ fn merged_deploy_job(
.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta);
let root = update_meta.guid();
let _deploy = gated_deploy(b, hive, agent, Some(rev.to_owned()), Some(root));
let _deploy = gated_deploy(b, hive, agent, Some(rev.to_owned()), Some(root), 0);
vec![root]
}
@ -1335,7 +1359,7 @@ pub(crate) fn declare_agent_job(
// queue at all fails this node — but `TriggerDeploy` has nothing to
// publish to on such a host either, so the strict edge cancels a node
// that could not have succeeded anyway.
let trigger_deploy = gated_deploy(b, hive, agent, None, Some(update_meta.guid()))
let trigger_deploy = gated_deploy(b, hive, agent, None, Some(update_meta.guid()), 0)
.after_any(mint_identity)
.after_any(mint_forge_token)
// `after_any` for the same reason: the container reads this token
@ -1540,7 +1564,8 @@ pub(crate) fn spawn_meta_update_timer(
enum SkipCause {
/// It waited on an `UpdateMeta` that `Failed`.
FailedUpdate,
/// It waited on a `BuildMeta` that gave no verdict ([`meta_build::is_retried`]).
/// It waited on a `BuildMeta` that gave no verdict or whose first `nix`
/// error asks for another build ([`meta_build::is_retried`]).
UnfinishedBuild,
}
@ -1550,16 +1575,18 @@ struct SkippedDeploy {
agent: String,
rev: Option<String>,
cause: SkipCause,
/// The `BuildMeta::attempt` of the build that replaces the failed one.
attempt: u32,
}
/// The deploys the timer declares again: for each agent, its latest
/// `TriggerDeploy` when the scheduler skipped it because an `UpdateMeta` it
/// waited on `Failed`, or because the `BuildMeta` it waited on gave no verdict.
/// A deploy skipped for another reason is left alone. A red build is a verdict
/// on the config, which a config merge answers with a build of its own. The
/// pause declaration or the config seeding failing means the update that would
/// unblock the deploy was skipped too, and deploying past those is the race
/// those edges close.
/// waited on `Failed`, or because the `BuildMeta` it waited on gave no verdict
/// or met its first `nix` error. A deploy skipped for another reason is left
/// alone. A red build, the second `nix` error, is a verdict on the config, which
/// a config merge answers with a build of its own. The pause declaration or the
/// config seeding failing means the update that would unblock the deploy was
/// skipped too, and deploying past those is the race those edges close.
fn skipped_deploys(
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
) -> Vec<SkippedDeploy> {
@ -1581,24 +1608,28 @@ fn skipped_deploys(
let SwarmNodeKind::TriggerDeploy { hive, agent, rev } = &n.payload else {
return None;
};
let waited_on =
|wanted: fn(&hive_jobq::Node<SwarmNodeKind, SwarmResourceKind>) -> bool| {
n.deps.iter().any(|d| match d {
hive_jobq::Dep::Node { id, .. } => graph.node(*id).is_some_and(wanted),
hive_jobq::Dep::Resource { .. } => false,
})
};
let cause = if waited_on(|dep| {
let deps = || {
n.deps.iter().filter_map(|d| match d {
hive_jobq::Dep::Node { id, .. } => graph.node(*id),
hive_jobq::Dep::Resource { .. } => None,
})
};
let failed_update = deps().any(|dep| {
matches!(dep.payload, SwarmNodeKind::UpdateMeta)
&& dep.state == hive_jobq::State::Failed
}) {
SkipCause::FailedUpdate
} else if waited_on(|dep| {
matches!(dep.payload, SwarmNodeKind::BuildMeta { .. })
&& dep.state == hive_jobq::State::Failed
&& dep.error.as_deref().is_some_and(meta_build::is_retried)
}) {
SkipCause::UnfinishedBuild
});
let unfinished_build = deps().find_map(|dep| match (&dep.payload, &dep.error) {
(SwarmNodeKind::BuildMeta { attempt, .. }, Some(error))
if dep.state == hive_jobq::State::Failed && meta_build::is_retried(error) =>
{
Some(meta_build::next_attempt(*attempt, error))
}
_ => None,
});
let (cause, attempt) = if failed_update {
(SkipCause::FailedUpdate, 0)
} else if let Some(attempt) = unfinished_build {
(SkipCause::UnfinishedBuild, attempt)
} else {
return None;
};
@ -1607,6 +1638,7 @@ fn skipped_deploys(
agent: agent.clone(),
rev: rev.clone(),
cause,
attempt,
})
})
.collect()
@ -1662,7 +1694,7 @@ fn queue_meta_update(
});
let deploys: Vec<_> = skipped
.into_iter()
.map(|s| gated_deploy(b, &s.hive, &s.agent, s.rev, update).guid())
.map(|s| gated_deploy(b, &s.hive, &s.agent, s.rev, update, s.attempt).guid())
.collect();
update.map_or(deploys, |update| vec![update])
})
@ -2357,7 +2389,7 @@ mod tests {
hive_jobq::Dep::Node { id, when } => graph
.node(*id)
.filter(|n| {
matches!(&n.payload, SwarmNodeKind::BuildMeta { agent: a } if a == agent)
matches!(&n.payload, SwarmNodeKind::BuildMeta { agent: a, .. } if a == agent)
})
.map(|n| (n, *when)),
hive_jobq::Dep::Resource { .. } => None,
@ -2641,7 +2673,7 @@ mod tests {
{
Outcome::Failed("the hive did not take it".to_owned())
}
SwarmNodeKind::BuildMeta { agent }
SwarmNodeKind::BuildMeta { agent, .. }
if failing_build.as_ref().is_some_and(|(a, _)| *a == agent) =>
{
Outcome::Failed(failing_build.map(|(_, error)| error).expect("matched"))
@ -2902,7 +2934,7 @@ mod tests {
.filter(|n| n.payload.label() == label)
.filter(|n| n.state == State::Pending)
.map(|n| match &n.payload {
SwarmNodeKind::BuildMeta { agent }
SwarmNodeKind::BuildMeta { agent, .. }
| SwarmNodeKind::TriggerDeploy { agent, .. } => agent.clone(),
other => panic!("not a build or a deploy: {other:?}"),
})
@ -3786,7 +3818,9 @@ mod tests {
let error = error.clone();
move |_id, kind, builder| async move {
let outcome = match kind {
SwarmNodeKind::BuildMeta { agent: a } if a == agent => Outcome::Failed(error),
SwarmNodeKind::BuildMeta { agent: a, .. } if a == agent => {
Outcome::Failed(error)
}
_ => Outcome::Done,
};
(builder, outcome)
@ -3885,6 +3919,84 @@ mod tests {
assert!(!queue_meta_update(&sched).expect("tick"));
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
}
/// The `attempt` of `agent`'s builds, oldest first.
fn build_attempts(sched: &SharedSched, agent: &str) -> Vec<u32> {
let guard = sched.lock().unwrap();
let mut nodes: Vec<_> = guard
.graph()
.nodes()
.filter_map(|n| match &n.payload {
SwarmNodeKind::BuildMeta { agent: a, attempt } if a == agent => {
Some((n.id, *attempt))
}
_ => None,
})
.collect();
nodes.sort_by_key(|(id, _)| *id);
nodes.into_iter().map(|(_, attempt)| attempt).collect()
}
fn merged_atlas() -> SharedSched {
let sched = fresh_sched();
sched
.lock()
.unwrap()
.insert_job(None, |b| merged_deploy_job(b, "pr1ma", "atlas", "abc"))
.expect("the graph inserts");
sched
}
/// `nix` failing once is built again on the next tick, as the second
/// attempt; failing again is red, and the tick declares nothing more.
#[tokio::test]
async fn a_first_nix_error_is_built_once_more_and_the_second_is_red() {
use hive_jobq::State;
let sched = merged_atlas();
let first = format!("{}error: fetch failed", meta_build::FIRST_FAILURE_PREFIX);
drain_failing_build(&sched, "atlas", first).await;
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
assert_eq!(build_attempts(&sched, "atlas"), [0]);
assert!(queue_meta_update(&sched).expect("tick"));
assert_eq!(build_attempts(&sched, "atlas"), [0, 1]);
drain_failing_build(&sched, "atlas", "error: attribute missing".to_owned()).await;
assert_eq!(
deploys_of(&sched, "atlas"),
[State::Skipped, State::Skipped]
);
assert!(!queue_meta_update(&sched).expect("tick"));
assert_eq!(build_attempts(&sched, "atlas"), [0, 1]);
}
/// A build that passes after a first `nix` error sends the deploy, once.
#[tokio::test]
async fn a_build_that_passes_after_a_first_nix_error_deploys() {
use hive_jobq::State;
let sched = merged_atlas();
let first = format!("{}error: fetch failed", meta_build::FIRST_FAILURE_PREFIX);
drain_failing_build(&sched, "atlas", first).await;
assert!(queue_meta_update(&sched).expect("tick"));
drain(&sched, false, false).await;
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped, State::Done]);
assert!(!queue_meta_update(&sched).expect("tick"));
}
/// A build with no verdict keeps its attempt, however many ticks it takes,
/// so a builder that is away does not use up the second `nix` attempt.
#[tokio::test]
async fn a_build_with_no_verdict_keeps_its_attempt() {
let sched = merged_atlas();
let away = format!("{}no builder", meta_build::RETRY_PREFIX);
drain_failing_build(&sched, "atlas", away.clone()).await;
assert!(queue_meta_update(&sched).expect("tick"));
drain_failing_build(&sched, "atlas", away).await;
assert!(queue_meta_update(&sched).expect("tick"));
assert_eq!(build_attempts(&sched, "atlas"), [0, 0, 0]);
}
}
#[cfg(test)]