vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

gate each deploy on a swarm build of the agent from internal/meta

Browse source
This commit is contained in:
atlas 2026-10-26 16:40:06 +01:00
commit 5e64048e53
6 changed files with 1570 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

3
swarm-controller/src/builder.rs
View file

@ -4,7 +4,8 @@
//! swarm UI's builds page for that build.
//!
//! Nothing is pushed, and the closure the builder made stays in its host's
//! store. A red status blocks nothing; the merge deploys as before.
//! store. A red status blocks nothing; the merge deploys as before. The build
//! that does gate a deploy is `meta_build`'s.
//!
//! Builds share one `SwarmResourceKind::Builder` slot, so one runs at a time,
//! and a newer push replaces its PR's build while that build is still

645
swarm-controller/src/jobq.rs
View file

@ -10,8 +10,8 @@ use crate::routes::AppState;
use crate::routes::agents::{claiming_hives, read_declarations};
use crate::startup::PUBLIC_URL_ENV;
use crate::{
agent_identity, agent_renewal, auth, builder, config_pr, forge, matrix_account, meta_fanout,
meta_flake, wanted,
agent_identity, agent_renewal, auth, builder, config_pr, forge, matrix_account, meta_build,
meta_fanout, meta_flake, wanted,
};
/// Node payload for the swarm-level job graph. Named `Swarm*` rather than
@ -110,6 +110,13 @@ pub(crate) enum SwarmNodeKind {
/// Carries no hive: the check builds on the builder's own agent base,
/// not on any hive.
BuildAgent { agent: String, rev: String, pr: u64 },
/// Build `agent` from `internal/meta`'s head on the swarm builder, and
/// record the closure in `agent-closures`. The `TriggerDeploy` behind it
/// runs only when the build is green. See `meta_build`.
///
/// Carries no rev: it reads the head when it runs, which in most DAGs is
/// after the `UpdateMeta` that commits it.
BuildMeta { agent: String },
/// Re-mint `agent`'s queue secret if it is still stored and old enough.
/// See `agent_renewal`.
///
@ -136,15 +143,16 @@ pub(crate) enum SwarmNodeKind {
},
/// Pin the swarm meta flake (`internal/meta`): render, lock on the swarm
/// builder, commit. See `meta_flake`. When the pin `internal/meta` holds is
/// not the one `meta_fanout` recorded, a `TriggerDeploy` per placed agent
/// follows, then a `RecordFanout`.
/// not the one `meta_fanout` recorded, a `BuildMeta` and a `TriggerDeploy`
/// per placed agent follow, then a `RecordFanout`.
///
/// Carries nothing: it reads every head when it runs, so one pending node
/// covers every trigger queued behind it.
UpdateMeta,
/// Record `pin` in `meta_fanout` as sent to the hives. Runs only once every
/// `TriggerDeploy` declared beside it is done, so a failed or lost deploy
/// leaves the record stale and the next `UpdateMeta` sends them again.
/// Record `pin` in `meta_fanout` as sent to the hives. Runs once every
/// `TriggerDeploy` declared beside it is done or skipped by its build gate.
/// A deploy that failed or was lost leaves the record stale, and the next
/// `UpdateMeta` sends them again.
RecordFanout { pin: String },
}
@ -166,6 +174,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
SwarmNodeKind::MintBuilderForgeToken => "mint_builder_forge_token".to_owned(),
SwarmNodeKind::MintCiMetricsForgeToken => "mint_ci_metrics_forge_token".to_owned(),
SwarmNodeKind::BuildAgent { .. } => "build_agent".to_owned(),
SwarmNodeKind::BuildMeta { .. } => "build_meta".to_owned(),
SwarmNodeKind::RenewAgentQueueCredential { .. } => {
"renew_agent_queue_credential".to_owned()
}
@ -193,6 +202,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
| SwarmNodeKind::MintAgentIdentity { agent }
| SwarmNodeKind::MintAgentForgeToken { agent }
| SwarmNodeKind::MintAgentMatrixAccount { agent }
| SwarmNodeKind::BuildMeta { agent }
| SwarmNodeKind::RenewAgentQueueCredential { agent } => {
serde_json::json!({ "agent": agent })
}
@ -220,8 +230,9 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
/// has capacity 1: the scheduler's default, which no caller overrides.
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub(crate) enum SwarmResourceKind {
/// The swarm builder: one `BuildAgent` node runs at a time, and the rest
/// wait pending, where a newer push can replace them.
/// The swarm builder: one `BuildAgent` or `BuildMeta` node runs at a time,
/// and the rest wait pending. A newer push replaces its PR's pending
/// `BuildAgent`.
Builder,
/// `internal/meta`: one `UpdateMeta` commits at a time, so two never race
/// on the same file's blob.
@ -403,6 +414,7 @@ async fn run_node_work(
SwarmNodeKind::BuildAgent { agent, rev, pr } => {
build_agent(&deps, forge, agent, rev, pr).await
}
SwarmNodeKind::BuildMeta { agent } => build_meta(&deps, forge, agent).await,
SwarmNodeKind::RenewAgentQueueCredential { agent } => renew_queue_credential(&agent).await,
SwarmNodeKind::SetAgentWanted { hive, agent } => match deps.wanted {
None => Outcome::Failed(
@ -428,7 +440,7 @@ async fn run_node_work(
/// A forge-calling node's work, run under [`forge::FORGE_DEADLINE`]. Elapsing
/// is the node's own error, naming `node`, so it fails like any other refused
/// step instead of holding its executor slot.
async fn forge_node<T>(
pub(crate) async fn forge_node<T>(
node: &str,
work: impl std::future::Future<Output = Result<T>>,
) -> Result<T> {
@ -590,6 +602,15 @@ pub(crate) async fn build_agent(
.await
}
/// The `BuildMeta` arm, lifted out for `clippy::too_many_lines`.
async fn build_meta(
deps: &WorkerDeps,
forge: Option<Arc<forge::Client>>,
agent: String,
) -> hive_jobq::scheduler::Outcome {
meta_build::run(forge.as_deref(), deps.queue.as_ref(), agent).await
}
/// What an `UpdateMeta` pass decided: the deploys to send, and the placements
/// to send them to.
struct MetaPass {
@ -717,9 +738,48 @@ async fn record_fanout(
}
}
/// Declare one rebuild of each `(hive, agent)` placement, which nothing else
/// tells to build again, and for a [`FanOut::Record`] the `RecordFanout` that
/// waits on every one of them.
/// The build and the deploy every path declares for one agent: a `BuildMeta`
/// of `agent`, then the `TriggerDeploy`, which runs only when the build is
/// green (`after_ok`), so a red build leaves it `Skipped`.
///
/// `update` is the `UpdateMeta` the pair follows when it is not this pair's
/// parent. The build waits on it, since the build reads the head that update
/// commits. The deploy keeps its own edge to it too: `skipped_deploys` finds
/// the deploys a failed update skipped by that edge.
fn gated_deploy<'a>(
b: &'a hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
hive: &str,
agent: &str,
rev: Option<String>,
update: Option<hive_jobq::builder::NodeGuid>,
) -> hive_jobq::builder::NodeRef<'a, SwarmNodeKind, SwarmResourceKind> {
let mut build = b
.node(SwarmNodeKind::BuildMeta {
agent: agent.to_owned(),
})
.needs(SwarmResourceKind::Builder);
if let Some(update) = update {
build = build.after_ok(update);
}
let mut deploy = b
.node(SwarmNodeKind::TriggerDeploy {
hive: hive.to_owned(),
agent: agent.to_owned(),
rev,
})
.after_ok(build);
if let Some(update) = update {
deploy = deploy.after_ok(update);
}
deploy
}
/// Declare one build and rebuild of each `(hive, agent)` placement, which
/// nothing else tells to build again, and for a [`FanOut::Record`] the
/// `RecordFanout` that waits on every one of them. A deploy the build gate
/// skipped counts as sent: the pin is recorded, so one red agent does not
/// resend every healthy one on each retry. A deploy skipped because the build
/// gave no verdict is re-declared by `queue_meta_update`.
fn declare_fan_out(
grown: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
fan_out: &FanOut,
@ -730,20 +790,18 @@ fn declare_fan_out(
}
let deploys: Vec<hive_jobq::builder::NodeGuid> = placements
.iter()
.map(|(hive, agent)| {
grown
.node(SwarmNodeKind::TriggerDeploy {
hive: hive.clone(),
agent: agent.clone(),
rev: None,
})
.guid()
})
.map(|(hive, agent)| gated_deploy(grown, hive, agent, None, None).guid())
.collect();
if let FanOut::Record { pin } = fan_out {
let mut record = grown.node(SwarmNodeKind::RecordFanout { pin: pin.clone() });
for deploy in deploys {
record = record.after_ok(deploy);
record = record.on_outcome(
deploy,
&[
hive_jobq::TerminalState::Done,
hive_jobq::TerminalState::Skipped,
],
);
}
}
}
@ -1136,13 +1194,13 @@ pub(crate) async fn queue_merged_config_deploy(
}
}
/// The sub-DAG a merged config PR is: the meta update, then the deploy of
/// `rev` on `hive`.
/// The sub-DAG a merged config PR is: the meta update, the build of the agent
/// from the updated flake, then the deploy of `rev` on `hive`.
///
/// `after_ok`: a hive builds only from `internal/meta`, so a deploy asked for
/// after a failed update (no builder, an unreadable config repo) could only
/// fail its build. The failed update is `Failed` in the graph and the deploy
/// `Skipped`.
/// fail its build, and one asked for after a red build would fail the same
/// way. The failed node is `Failed` in the graph and the deploy `Skipped`.
fn merged_deploy_job(
b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
hive: &str,
@ -1153,13 +1211,7 @@ fn merged_deploy_job(
.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta);
let root = update_meta.guid();
let _deploy = b
.node(SwarmNodeKind::TriggerDeploy {
hive: hive.to_owned(),
agent: agent.to_owned(),
rev: Some(rev.to_owned()),
})
.after_ok(update_meta);
let _deploy = gated_deploy(b, hive, agent, Some(rev.to_owned()), Some(root));
vec![root]
}
@ -1252,8 +1304,9 @@ pub(crate) fn declare_agent_job(
agent: agent.to_owned(),
});
// The hive builds the agent from `internal/meta` only, so the deploy needs
// the update to succeed (`after_ok`): after a failed one (no builder, an
// unreadable config repo) the hive's build could only fail.
// the update and the swarm's build from it to succeed (`after_ok`): after a
// failed update (no builder, an unreadable config repo) or a red build, the
// hive's build could only fail.
let update_meta = b
.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta)
@ -1282,13 +1335,7 @@ pub(crate) fn declare_agent_job(
// queue at all fails this node — but `TriggerDeploy` has nothing to
// publish to on such a host either, so the strict edge cancels a node
// that could not have succeeded anyway.
let trigger_deploy = b
.node(SwarmNodeKind::TriggerDeploy {
hive: hive.to_owned(),
agent: agent.to_owned(),
rev: None,
})
.after_ok(update_meta)
let trigger_deploy = gated_deploy(b, hive, agent, None, Some(update_meta.guid()))
.after_any(mint_identity)
.after_any(mint_forge_token)
// `after_any` for the same reason: the container reads this token
@ -1488,15 +1535,34 @@ pub(crate) fn spawn_meta_update_timer(
});
}
/// The deploys a failed `UpdateMeta` skipped, as `(hive, agent, rev)`: for each
/// agent, its latest `TriggerDeploy` when it waited on an `UpdateMeta` that
/// `Failed` (so the scheduler skipped it). A deploy skipped for another reason
/// (the pause declaration or the config seeding failed) is left alone: the
/// update that would unblock it was skipped too, and deploying past those is
/// the race those edges close.
/// Why a deploy is `Skipped` and the timer should declare it again.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum SkipCause {
/// It waited on an `UpdateMeta` that `Failed`.
FailedUpdate,
/// It waited on a `BuildMeta` that gave no verdict ([`meta_build::is_retried`]).
UnfinishedBuild,
}
/// A deploy [`skipped_deploys`] found.
struct SkippedDeploy {
hive: String,
agent: String,
rev: Option<String>,
cause: SkipCause,
}
/// The deploys the timer declares again: for each agent, its latest
/// `TriggerDeploy` when the scheduler skipped it because an `UpdateMeta` it
/// waited on `Failed`, or because the `BuildMeta` it waited on gave no verdict.
/// A deploy skipped for another reason is left alone. A red build is a verdict
/// on the config, which a config merge answers with a build of its own. The
/// pause declaration or the config seeding failing means the update that would
/// unblock the deploy was skipped too, and deploying past those is the race
/// those edges close.
fn skipped_deploys(
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
) -> Vec<(String, String, Option<String>)> {
) -> Vec<SkippedDeploy> {
let mut latest = std::collections::BTreeMap::new();
for n in graph.nodes() {
if let SwarmNodeKind::TriggerDeploy { hive, agent, .. } = &n.payload {
@ -1511,30 +1577,52 @@ fn skipped_deploys(
}
latest
.into_values()
.filter(|n| {
n.deps.iter().any(|d| match d {
hive_jobq::Dep::Node { id, .. } => graph.node(*id).is_some_and(|dep| {
matches!(dep.payload, SwarmNodeKind::UpdateMeta)
&& dep.state == hive_jobq::State::Failed
}),
hive_jobq::Dep::Resource { .. } => false,
.filter_map(|n| {
let SwarmNodeKind::TriggerDeploy { hive, agent, rev } = &n.payload else {
return None;
};
let waited_on =
|wanted: fn(&hive_jobq::Node<SwarmNodeKind, SwarmResourceKind>) -> bool| {
n.deps.iter().any(|d| match d {
hive_jobq::Dep::Node { id, .. } => graph.node(*id).is_some_and(wanted),
hive_jobq::Dep::Resource { .. } => false,
})
};
let cause = if waited_on(|dep| {
matches!(dep.payload, SwarmNodeKind::UpdateMeta)
&& dep.state == hive_jobq::State::Failed
}) {
SkipCause::FailedUpdate
} else if waited_on(|dep| {
matches!(dep.payload, SwarmNodeKind::BuildMeta { .. })
&& dep.state == hive_jobq::State::Failed
&& dep.error.as_deref().is_some_and(meta_build::is_retried)
}) {
SkipCause::UnfinishedBuild
} else {
return None;
};
Some(SkippedDeploy {
hive: hive.clone(),
agent: agent.clone(),
rev: rev.clone(),
cause,
})
})
.filter_map(|n| match &n.payload {
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => {
Some((hive.clone(), agent.clone(), rev.clone()))
}
_ => None,
})
.collect()
}
/// Insert one `UpdateMeta` job when none is in flight and the latest one, by
/// node id, is not `Done` (or none exists), with a fresh `TriggerDeploy`
/// behind it (`after_ok`) for every deploy the failed run skipped
/// ([`skipped_deploys`]), so an agent created or merged while the update was
/// failing is deployed by the run that succeeds. Returns whether a job was
/// inserted.
/// Insert one job when the latest `UpdateMeta`, by node id, is not `Done` (or
/// none exists) and none is in flight: an `UpdateMeta`, with a fresh build and
/// `TriggerDeploy` behind it ([`gated_deploy`]) for every deploy the failed
/// run skipped ([`skipped_deploys`]), so an agent created or merged while the
/// update was failing is deployed by the run that succeeds.
///
/// When the latest `UpdateMeta` is `Done`, a deploy skipped because the builder
/// gave no verdict still gets a fresh build and deploy, with no update, so a
/// builder that was down or not yet updated is caught up once it answers. That
/// adds nodes on every tick while the builder stays away. Returns whether a job
/// was inserted.
fn queue_meta_update(
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
) -> Result<bool> {
@ -1555,24 +1643,28 @@ fn queue_meta_update(
return Ok(false);
}
update_metas.sort_by_key(|n| n.id);
if update_metas
let update_due = update_metas
.last()
.is_some_and(|n| n.state == hive_jobq::State::Done)
{
.is_none_or(|n| n.state != hive_jobq::State::Done);
let skipped: Vec<SkippedDeploy> = skipped_deploys(sched.graph())
.into_iter()
.filter(|s| update_due || s.cause == SkipCause::UnfinishedBuild)
.collect();
if !update_due && skipped.is_empty() {
return Ok(false);
}
let skipped = skipped_deploys(sched.graph());
sched
.insert_job(None, |b| {
let update = b
.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta);
for (hive, agent, rev) in skipped {
let _ = b
.node(SwarmNodeKind::TriggerDeploy { hive, agent, rev })
.after_ok(update);
}
vec![update.guid()]
let update = update_due.then(|| {
b.node(SwarmNodeKind::UpdateMeta)
.needs(SwarmResourceKind::Meta)
.guid()
});
let deploys: Vec<_> = skipped
.into_iter()
.map(|s| gated_deploy(b, &s.hive, &s.agent, s.rev, update).guid())
.collect();
update.map_or(deploys, |update| vec![update])
})
.map_err(|e| anyhow::anyhow!("{e}"))?;
Ok(true)
@ -2228,12 +2320,152 @@ mod tests {
));
}
/// A deploy needs the swarm's build of its agent to succeed: its edge
/// refuses a failed build, in both DAGs that declare an update. The mint
/// edge is the control that the lookup finds an edge that accepts one.
#[test]
fn a_deploy_needs_the_meta_build_to_succeed() {
for declare in [create_dag as fn(&TestBuilder), merged_dag] {
assert!(!edge_accepts_failure(
declare,
"trigger_deploy",
"build_meta"
));
assert!(!edge_accepts_failure(declare, "build_meta", "update_meta"));
}
assert!(edge_accepts_failure(
create_dag,
"trigger_deploy",
"mint_agent_identity"
));
}
/// Every `TriggerDeploy` in `graph` waits, strictly, on a `BuildMeta` of the
/// same agent that needs the builder. Panics, naming `path`, when one does
/// not; returns how many deploys it checked.
fn assert_every_deploy_is_gated(
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
path: &str,
) -> usize {
let mut checked = 0;
for deploy in graph.nodes() {
let SwarmNodeKind::TriggerDeploy { agent, .. } = &deploy.payload else {
continue;
};
let gate = deploy.deps.iter().find_map(|d| {
match d {
hive_jobq::Dep::Node { id, when } => graph
.node(*id)
.filter(|n| {
matches!(&n.payload, SwarmNodeKind::BuildMeta { agent: a } if a == agent)
})
.map(|n| (n, *when)),
hive_jobq::Dep::Resource { .. } => None,
}
});
let (build, when) =
gate.unwrap_or_else(|| panic!("{path}: {agent}'s deploy has no build"));
assert!(
!when.accepts(hive_jobq::TerminalState::Failed),
"{path}: {agent}'s deploy runs after a failed build"
);
assert!(
build.deps.iter().any(|d| matches!(
d,
hive_jobq::Dep::Resource {
name: SwarmResourceKind::Builder,
..
}
)),
"{path}: {agent}'s build does not need the builder"
);
checked += 1;
}
checked
}
/// The four paths that send a deploy all declare the same build and deploy
/// pair, so a path that forgets the gate fails here.
#[tokio::test]
async fn all_four_deploy_paths_declare_the_gated_pair() {
let graph_of = |declare: fn(&TestBuilder)| {
let mut sched = hive_jobq::scheduler::Scheduler::new(
hive_jobq::Graph::new(),
hive_jobq::resources::ResourceTable::new(),
);
sched
.insert_job(None, |b| {
declare(b);
Vec::new()
})
.expect("the graph inserts");
sched
};
assert_eq!(
assert_every_deploy_is_gated(graph_of(merged_dag).graph(), "merge"),
1
);
assert_eq!(
assert_every_deploy_is_gated(graph_of(create_dag).graph(), "creation"),
1
);
let pins = Arc::new(FakePins::default());
let (_, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
assert_eq!(
assert_every_deploy_is_gated(sched.lock().unwrap().graph(), "fan-out"),
3
);
let sched = bob_deployed_and_atlas_skipped().await;
assert!(queue_meta_update(&sched).expect("tick"));
// bob's and atlas's first deploys and atlas's retry: all gated.
assert_eq!(
assert_every_deploy_is_gated(sched.lock().unwrap().graph(), "timer retry"),
3
);
}
/// With the swarm's build red, the deploy never runs, in both DAGs that
/// declare an update. The first run in each pair is the control, where the
/// build is green and the deploy does run.
#[tokio::test]
async fn a_red_build_means_no_deploy_runs() {
use hive_jobq::State;
for (name, declare) in [
("merged", merged_dag as fn(&TestBuilder)),
("create", create_dag as fn(&TestBuilder)),
] {
let green = run_dag_failing(declare, None).await;
assert_eq!(state_of(&green, "build_meta"), State::Done, "{name}");
assert_eq!(state_of(&green, "trigger_deploy"), State::Done, "{name}");
let red = run_dag_failing(declare, Some("build_meta")).await;
assert_eq!(state_of(&red, "update_meta"), State::Done, "{name}");
assert_eq!(state_of(&red, "build_meta"), State::Failed, "{name}");
assert_eq!(
state_of(&red, "trigger_deploy"),
State::Skipped,
"{name}: the deploy must not run after a red build"
);
}
}
/// Every node of the DAG `declare` builds, run to the end with the meta
/// update failing or succeeding and every other node succeeding: the
/// final state of each, by label.
async fn run_dag(
declare: impl FnOnce(&TestBuilder),
update_meta_fails: bool,
) -> Vec<(String, hive_jobq::State)> {
run_dag_failing(declare, update_meta_fails.then_some("update_meta")).await
}
/// [`run_dag`], with the nodes labelled `failing` failing.
async fn run_dag_failing(
declare: impl FnOnce(&TestBuilder),
failing: Option<&'static str>,
) -> Vec<(String, hive_jobq::State)> {
use hive_jobq::scheduler::{Outcome, Scheduler};
use hive_jobq_wire::WireNode as _;
@ -2250,11 +2482,10 @@ mod tests {
.expect("the graph inserts");
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
while let Some(runner) = Scheduler::claim_next(&sched, |_id, kind, builder| async move {
let outcome = match kind {
SwarmNodeKind::UpdateMeta if update_meta_fails => {
Outcome::Failed("no builder".to_owned())
}
_ => Outcome::Done,
let outcome = if failing == Some(kind.label().as_str()) {
Outcome::Failed("no builder".to_owned())
} else {
Outcome::Done
};
(builder, outcome)
}) {
@ -2355,6 +2586,27 @@ mod tests {
placed: &[(&str, &str)],
failing_agent: Option<&'static str>,
crash_before_record: bool,
) -> (FanOut, Arc<Mutex<TestScheduler>>) {
pass_with(
pins,
report,
placed,
failing_agent,
crash_before_record,
None,
)
.await
}
/// [`pass`], with the build of one agent failing with the given error when
/// `failing_build` names it.
async fn pass_with(
pins: &Arc<FakePins>,
report: &meta_flake::Report,
placed: &[(&str, &str)],
failing_agent: Option<&'static str>,
crash_before_record: bool,
failing_build: Option<(&'static str, String)>,
) -> (FanOut, Arc<Mutex<TestScheduler>>) {
use hive_jobq::scheduler::{Outcome, Scheduler};
use meta_fanout::PinStore as _;
@ -2380,7 +2632,8 @@ mod tests {
.expect("the node just inserted is runnable");
update.await.1.expect("the grown nodes insert");
while let Some(runner) = Scheduler::claim_next(&sched, {
let (pins, sched) = (Arc::clone(pins), Arc::clone(&sched));
let (pins, sched, failing_build) =
(Arc::clone(pins), Arc::clone(&sched), failing_build.clone());
move |_id, kind, builder| async move {
let outcome = match kind {
SwarmNodeKind::TriggerDeploy { agent, .. }
@ -2388,6 +2641,11 @@ mod tests {
{
Outcome::Failed("the hive did not take it".to_owned())
}
SwarmNodeKind::BuildMeta { agent }
if failing_build.as_ref().is_some_and(|(a, _)| *a == agent) =>
{
Outcome::Failed(failing_build.map(|(_, error)| error).expect("matched"))
}
SwarmNodeKind::RecordFanout { .. } if crash_before_record => {
Outcome::Failed("the controller died".to_owned())
}
@ -2398,12 +2656,17 @@ mod tests {
.graph()
.nodes()
.filter(|n| matches!(n.payload, SwarmNodeKind::TriggerDeploy { .. }))
.all(|n| n.state == hive_jobq::State::Done);
.all(|n| {
matches!(
n.state,
hive_jobq::State::Done | hive_jobq::State::Skipped
)
});
if deploys_done {
pins.record(&pin).await.expect("recorded");
Outcome::Done
} else {
Outcome::Failed("recorded before the deploys were done".to_owned())
Outcome::Failed("recorded before the deploys were settled".to_owned())
}
}
_ => Outcome::Done,
@ -2509,10 +2772,10 @@ mod tests {
assert_eq!(fan_out, FanOut::None);
}
/// The record waits on every deploy of its pass, for all of them to be
/// done, not merely finished.
/// The record waits on every deploy of its pass to be settled: done, or
/// skipped by its build gate. A failed deploy still rules it out.
#[tokio::test]
async fn the_record_waits_on_every_deploy_being_done() {
async fn the_record_waits_on_every_deploy_being_done_or_skipped() {
let pins = Arc::new(FakePins::default());
let (_, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
let guard = sched.lock().unwrap();
@ -2533,7 +2796,15 @@ mod tests {
let mut deploys: Vec<_> = graph
.nodes()
.filter(|n| matches!(n.payload, SwarmNodeKind::TriggerDeploy { .. }))
.map(|n| (n.id, hive_jobq::DepWhen::AFTER_OK))
.map(|n| {
(
n.id,
hive_jobq::DepWhen::of(&[
hive_jobq::TerminalState::Done,
hive_jobq::TerminalState::Skipped,
]),
)
})
.collect();
deploys.sort_by_key(|(id, _)| id.get());
assert_eq!(deploys.len(), 3);
@ -2566,6 +2837,83 @@ mod tests {
assert_eq!(fan_out, FanOut::Record { pin: "p1".into() });
}
/// A red build for one agent skips that agent's deploy only, and the pin
/// is still recorded: the red agent is not sent again with every healthy
/// one on the next retry. The timer declares nothing for it either, since a
/// red build is a verdict on its config.
#[tokio::test]
async fn a_red_build_skips_its_deploy_and_the_pin_is_still_recorded() {
use hive_jobq::State;
let pins = Arc::new(FakePins::holding("p0"));
let (_, sched) = pass_with(
&pins,
&report("p1", true),
&PLACED,
None,
false,
Some(("argus", "error: builder for argus failed".to_owned())),
)
.await;
let mut builds = states_of(&sched, "build_meta");
builds.sort_by_key(|s| format!("{s:?}"));
assert_eq!(builds, [State::Done, State::Done, State::Failed]);
let mut deploys = states_of(&sched, "trigger_deploy");
deploys.sort_by_key(|s| format!("{s:?}"));
assert_eq!(deploys, [State::Done, State::Done, State::Skipped]);
assert_eq!(states_of(&sched, "record_fanout"), [State::Done]);
assert_eq!(pins.held().as_deref(), Some("p1"));
// The failed build rolls the update up `Failed`, so the timer queues an
// update, and only an update: no deploy is declared again.
assert!(queue_meta_update(&sched).expect("tick"));
assert_eq!(states_of(&sched, "trigger_deploy").len(), 3);
assert_eq!(states_of(&sched, "build_meta").len(), 3);
assert_eq!(states_of(&sched, "update_meta").len(), 2);
}
/// A build that gave no verdict skips its deploy and still records the pin,
/// and the timer declares a build and deploy for that agent alone.
#[tokio::test]
async fn an_unfinished_build_is_declared_again_for_that_agent_alone() {
use hive_jobq::State;
use hive_jobq_wire::WireNode as _;
let pins = Arc::new(FakePins::holding("p0"));
let error = format!("{}no swarm builder is listening", meta_build::RETRY_PREFIX);
let (_, sched) = pass_with(
&pins,
&report("p1", true),
&PLACED,
None,
false,
Some(("argus", error)),
)
.await;
assert_eq!(states_of(&sched, "record_fanout"), [State::Done]);
assert_eq!(pins.held().as_deref(), Some("p1"));
assert!(queue_meta_update(&sched).expect("tick"));
let guard = sched.lock().unwrap();
let again = |label: &str| {
let mut found: Vec<_> = guard
.graph()
.nodes()
.filter(|n| n.payload.label() == label)
.filter(|n| n.state == State::Pending)
.map(|n| match &n.payload {
SwarmNodeKind::BuildMeta { agent }
| SwarmNodeKind::TriggerDeploy { agent, .. } => agent.clone(),
other => panic!("not a build or a deploy: {other:?}"),
})
.collect();
found.sort();
found
};
assert_eq!(again("build_meta"), ["argus"]);
assert_eq!(again("trigger_deploy"), ["argus"]);
}
/// With nothing placed there is nothing to wait on, and the pin is
/// recorded all the same.
#[tokio::test]
@ -3428,6 +3776,115 @@ mod tests {
drain(&sched, false, false).await;
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
}
/// Run every runnable node; the build of `agent` fails with `error` and
/// everything else succeeds.
async fn drain_failing_build(sched: &SharedSched, agent: &'static str, error: String) {
use hive_jobq::scheduler::{Outcome, Scheduler};
while let Some(runner) = Scheduler::claim_next(sched, {
let error = error.clone();
move |_id, kind, builder| async move {
let outcome = match kind {
SwarmNodeKind::BuildMeta { agent: a } if a == agent => Outcome::Failed(error),
_ => Outcome::Done,
};
(builder, outcome)
}
}) {
runner.await.1.expect("no growth declared");
}
}
fn fresh_sched() -> SharedSched {
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
hive_jobq::Graph::new(),
hive_jobq::resources::ResourceTable::new(),
)))
}
/// The `rev` of `agent`'s deploys, oldest first.
fn deploy_revs(sched: &SharedSched, agent: &str) -> Vec<Option<String>> {
let guard = sched.lock().unwrap();
let mut nodes: Vec<_> = guard
.graph()
.nodes()
.filter_map(|n| match &n.payload {
SwarmNodeKind::TriggerDeploy { agent: a, rev, .. } if a == agent => {
Some((n.id, rev.clone()))
}
_ => None,
})
.collect();
nodes.sort_by_key(|(id, _)| *id);
nodes.into_iter().map(|(_, rev)| rev).collect()
}
/// A merged config's build that gave no verdict, with the update `Done`:
/// the deploy is skipped, and the timer declares a build and the same
/// deploy again, without an update, until the builder answers.
#[tokio::test]
async fn an_unfinished_build_is_retried_until_the_builder_answers() {
use hive_jobq::State;
let sched = fresh_sched();
sched
.lock()
.unwrap()
.insert_job(None, |b| merged_deploy_job(b, "pr1ma", "atlas", "abc"))
.expect("the graph inserts");
let unavailable = format!("{}no swarm builder is listening", meta_build::RETRY_PREFIX);
drain_failing_build(&sched, "atlas", unavailable.clone()).await;
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
assert_eq!(update_metas(&sched), 1);
assert!(queue_meta_update(&sched).expect("tick"));
assert_eq!(update_metas(&sched), 1, "the retry needs no update");
drain_failing_build(&sched, "atlas", unavailable).await;
assert_eq!(
deploys_of(&sched, "atlas"),
[State::Skipped, State::Skipped]
);
assert!(queue_meta_update(&sched).expect("tick"));
drain(&sched, false, false).await;
assert_eq!(
deploys_of(&sched, "atlas"),
[State::Skipped, State::Skipped, State::Done]
);
// The merged deploy's rev survives every retry.
assert_eq!(
deploy_revs(&sched, "atlas"),
[
Some("abc".to_owned()),
Some("abc".to_owned()),
Some("abc".to_owned())
]
);
assert!(
!queue_meta_update(&sched).expect("tick"),
"the answer ends the retries"
);
}
/// A red build is a verdict: the timer declares nothing for it. The test
/// above is the control that the same shape with an unfinished build is
/// declared again.
#[tokio::test]
async fn a_red_build_is_not_declared_again() {
use hive_jobq::State;
let sched = fresh_sched();
sched
.lock()
.unwrap()
.insert_job(None, |b| merged_deploy_job(b, "pr1ma", "atlas", "abc"))
.expect("the graph inserts");
drain_failing_build(&sched, "atlas", "error: builder for x failed".to_owned()).await;
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
assert!(!queue_meta_update(&sched).expect("tick"));
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
}
}
#[cfg(test)]

1
swarm-controller/src/main.rs
View file

@ -51,6 +51,7 @@ mod jobq;
mod linked_accounts;
mod matrix_account;
mod message_stream;
mod meta_build;
mod meta_fanout;
mod meta_flake;
mod notify_flows;

351
swarm-controller/src/meta_build.rs Normal file
View file

@ -0,0 +1,351 @@
//! The `BuildMeta` node: build one agent from `internal/meta` on the swarm
//! builder before its deploy is sent, and record the closure in
//! [`swarm_wire::agent_closures`].
//!
//! The node reads `internal/meta`'s head when it runs, because in most DAGs it
//! is declared before the `UpdateMeta` that commits that head. It sends the
//! builder the agent and the head's commit, and on a `Built` reply writes the
//! key itself: the builder holds no grant on the bucket. A `Built` reply and
//! nothing else writes, so a red build leaves the previous record.
//!
//! The node ends one of three ways. `Done` is a green build. A red build, the
//! builder's `Failed` reply, is `Failed` with `nix`'s tail: the deploy behind it
//! is skipped and the Jobs page shows why. Anything that stopped the build from
//! giving a verdict is `Failed` with [`RETRY_PREFIX`], which
//! `jobq::queue_meta_update` retries on the meta timer.
use std::fmt::Display;
use std::future::Future;
use anyhow::{Context as _, Result};
use hive_jobq::scheduler::Outcome;
use swarm_wire::agent_closures::AgentClosure;
use swarm_wire::builder::{BuildFromMeta, FromMetaReply, META_SUBJECT, REPLY_DEADLINE};
use crate::agent_renewal::unix_now;
use crate::forge::{self, objects::SHARED_ORG};
use crate::meta_flake::{META_REPO, MetaForge};
/// What a `Failed` outcome that is not a verdict on the config starts with.
pub(crate) const RETRY_PREFIX: &str = "not built, will be retried: ";
/// Whether a `Failed` node's `error` says the build gave no verdict.
pub(crate) fn is_retried(error: &str) -> bool {
error.starts_with(RETRY_PREFIX)
}
fn not_built(why: impl Display) -> Outcome {
Outcome::Failed(format!("{RETRY_PREFIX}{why}"))
}
/// Where `internal/meta`'s head is read: the forge in production, a fake in
/// the tests.
pub(crate) trait MetaHead {
/// The head commit; `None` when the repo has none.
fn head(&self) -> impl Future<Output = Result<Option<String>>> + Send;
}
impl MetaHead for forge::Client {
async fn head(&self) -> Result<Option<String>> {
let read = self.head_files(SHARED_ORG, META_REPO, None, &[]);
let head = crate::jobq::forge_node("build_meta", read).await?;
Ok(head.map(|h| h.commit))
}
}
/// Where a build is sent and its closure recorded: the queue in production, a
/// fake in the tests.
pub(crate) trait MetaBuilds {
/// `Err` is a request that never reached a builder or never came back.
fn build(&self, request: Vec<u8>)
-> impl Future<Output = Result<FromMetaReply, String>> + Send;
fn record(
&self,
agent: &str,
closure: &AgentClosure,
) -> impl Future<Output = Result<()>> + Send;
}
impl MetaBuilds for async_nats::Client {
async fn build(&self, request: Vec<u8>) -> Result<FromMetaReply, String> {
use async_nats::RequestErrorKind;
let sent = async_nats::Request::new()
.payload(request.into())
.timeout(Some(REPLY_DEADLINE));
match self.send_request(META_SUBJECT, sent).await {
Ok(msg) => serde_json::from_slice(&msg.payload)
.map_err(|e| format!("the builder's meta build reply did not decode: {e}")),
Err(e) => Err(match e.kind() {
RequestErrorKind::NoResponders => "no swarm builder is listening for meta \
builds: deploy.builder.enable is set on no host, or its builder predates \
the request"
.to_owned(),
RequestErrorKind::TimedOut => format!(
"the swarm builder sent no answer within {} minutes",
REPLY_DEADLINE.as_secs() / 60
),
_ => format!("sending the meta build to the swarm builder failed: {e}"),
}),
}
}
async fn record(&self, agent: &str, closure: &AgentClosure) -> Result<()> {
let store = swarm_queue_client::agent_closures::open_or_create(self)
.await
.context("opening the agent-closures bucket")?;
let body = serde_json::to_vec(closure).context("encoding the closure record")?;
store
.put(agent, body.into())
.await
.map(|_| ())
.with_context(|| format!("writing agent-closures/{agent}"))
}
}
/// Build `agent` from `internal/meta`'s head and record the closure.
pub(crate) async fn run(
head: Option<&impl MetaHead>,
queue: Option<&impl MetaBuilds>,
agent: String,
) -> Outcome {
let Some(head) = head else {
return not_built(
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token not in the \
swarm secret store), so internal/meta's head cannot be read",
);
};
let Some(queue) = queue else {
return not_built("no swarm queue is configured on this host, so no build can be sent");
};
let meta_rev = match head.head().await {
Ok(Some(rev)) => rev,
Ok(None) => return not_built("the swarm has not published internal/meta yet"),
Err(e) => return not_built(format!("{e:#}")),
};
let request = match serde_json::to_vec(&BuildFromMeta {
agent: agent.clone(),
meta_rev: meta_rev.clone(),
}) {
Ok(request) => request,
Err(e) => return not_built(format!("encoding the build request: {e}")),
};
match queue.build(request).await {
Err(why) => not_built(why),
Ok(FromMetaReply::Unavailable { reason }) => not_built(reason),
Ok(FromMetaReply::Failed { error_tail }) => Outcome::Failed(error_tail),
Ok(FromMetaReply::Built { out_path, system }) => {
let closure = AgentClosure {
meta_rev,
store_path: out_path.clone(),
system,
built_unix: unix_now(),
};
match queue.record(&agent, &closure).await {
Ok(()) => Outcome::Done,
Err(e) => not_built(format!("built {out_path}, but recording it failed: {e:#}")),
}
}
}
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use super::*;
const REV: &str = "0123456789abcdef0123456789abcdef01234567";
struct FakeHead(Result<Option<&'static str>, &'static str>);
impl MetaHead for FakeHead {
async fn head(&self) -> Result<Option<String>> {
match self.0 {
Ok(rev) => Ok(rev.map(str::to_owned)),
Err(e) => Err(anyhow::anyhow!(e)),
}
}
}
/// A queue that answers every build with `reply`, and keeps what it was
/// sent and what was recorded.
struct FakeQueue {
reply: Result<FromMetaReply, String>,
record_fails: bool,
sent: Mutex<Vec<BuildFromMeta>>,
recorded: Mutex<Vec<(String, AgentClosure)>>,
}
impl FakeQueue {
fn answering(reply: Result<FromMetaReply, String>) -> Self {
Self {
reply,
record_fails: false,
sent: Mutex::new(Vec::new()),
recorded: Mutex::new(Vec::new()),
}
}
fn built() -> Self {
Self::answering(Ok(FromMetaReply::Built {
out_path: "/nix/store/x-toplevel".to_owned(),
system: "x86_64-linux".to_owned(),
}))
}
}
impl MetaBuilds for FakeQueue {
async fn build(&self, request: Vec<u8>) -> Result<FromMetaReply, String> {
self.sent
.lock()
.unwrap()
.push(serde_json::from_slice(&request).expect("a build request"));
self.reply.clone()
}
async fn record(&self, agent: &str, closure: &AgentClosure) -> Result<()> {
anyhow::ensure!(!self.record_fails, "the bucket refused the write");
self.recorded
.lock()
.unwrap()
.push((agent.to_owned(), closure.clone()));
Ok(())
}
}
async fn run_with(head: &FakeHead, queue: &FakeQueue) -> Outcome {
run(Some(head), Some(queue), "atlas".to_owned()).await
}
#[tokio::test]
async fn a_green_build_of_the_head_is_recorded_for_the_agent() {
let queue = FakeQueue::built();
let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await;
assert_eq!(outcome, Outcome::Done);
assert_eq!(
*queue.sent.lock().unwrap(),
[BuildFromMeta {
agent: "atlas".to_owned(),
meta_rev: REV.to_owned(),
}]
);
let recorded = queue.recorded.lock().unwrap();
let [(agent, closure)] = recorded.as_slice() else {
panic!("one record: {recorded:?}")
};
assert_eq!(agent, "atlas");
assert_eq!(closure.meta_rev, REV);
assert_eq!(closure.store_path, "/nix/store/x-toplevel");
assert_eq!(closure.system, "x86_64-linux");
assert!(closure.built_unix > 0);
}
/// The builder said the build is red: the node fails with `nix`'s tail,
/// which is not a retried failure, and nothing is recorded. The green test
/// above is the control that a record is written when the build is green.
#[tokio::test]
async fn a_red_build_fails_with_its_tail_and_records_nothing() {
let queue = FakeQueue::answering(Ok(FromMetaReply::Failed {
error_tail: "error: builder for foo failed".to_owned(),
}));
let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await;
assert_eq!(
outcome,
Outcome::Failed("error: builder for foo failed".to_owned())
);
assert!(queue.recorded.lock().unwrap().is_empty());
}
#[tokio::test]
async fn a_builder_that_gave_no_verdict_is_a_retried_failure_and_records_nothing() {
for (name, reply) in [
(
"no responders",
Err("no swarm builder is listening for meta builds".to_owned()),
),
(
"unavailable",
Ok(FromMetaReply::Unavailable {
reason: "busy".to_owned(),
}),
),
] {
let queue = FakeQueue::answering(reply);
let outcome = run_with(&FakeHead(Ok(Some(REV))), &queue).await;
let Outcome::Failed(error) = outcome else {
panic!("{name}: not failed")
};
assert!(is_retried(&error), "{name}: {error}");
assert!(queue.recorded.lock().unwrap().is_empty(), "{name}");
}
}
#[tokio::test]
async fn a_closure_that_cannot_be_recorded_is_a_retried_failure() {
let queue = FakeQueue {
record_fails: true,
..FakeQueue::built()
};
let Outcome::Failed(error) = run_with(&FakeHead(Ok(Some(REV))), &queue).await else {
panic!("not failed")
};
assert!(is_retried(&error), "{error}");
assert!(error.contains("/nix/store/x-toplevel"), "{error}");
}
#[tokio::test]
async fn without_a_head_or_a_queue_nothing_is_sent_and_the_failure_is_retried() {
for (name, head, with_queue) in [
("no repo yet", Some(FakeHead(Ok(None))), true),
("forge error", Some(FakeHead(Err("forge down"))), true),
("no forge", None, true),
("no queue", Some(FakeHead(Ok(Some(REV)))), false),
] {
let queue = FakeQueue::built();
let outcome = run(
head.as_ref(),
with_queue.then_some(&queue),
"atlas".to_owned(),
)
.await;
let Outcome::Failed(error) = outcome else {
panic!("{name}: not failed")
};
assert!(is_retried(&error), "{name}: {error}");
assert!(queue.sent.lock().unwrap().is_empty(), "{name}");
}
}
#[test]
fn the_repo_path_the_builder_fetches_is_the_one_the_controller_writes() {
assert_eq!(
swarm_wire::builder::META_REPO_PATH,
format!("{SHARED_ORG}/{META_REPO}")
);
}
/// The bucket on a real server: a record written for an agent reads back
/// under its key, and a second build replaces it.
#[tokio::test]
async fn the_closure_round_trips_through_the_bucket() {
let nats = crate::test_nats::TestNats::start();
let client = nats.client().await;
let mut closure = AgentClosure {
meta_rev: REV.to_owned(),
store_path: "/nix/store/a".to_owned(),
system: "x86_64-linux".to_owned(),
built_unix: 1,
};
client.record("atlas", &closure).await.expect("written");
closure.store_path = "/nix/store/b".to_owned();
client.record("atlas", &closure).await.expect("replaced");
let store = swarm_queue_client::agent_closures::open_or_create(&client)
.await
.expect("opens");
let got = store.get("atlas").await.expect("reads").expect("present");
let got: AgentClosure = serde_json::from_slice(&got).expect("decodes");
assert_eq!(got, closure);
assert!(store.get("argus").await.expect("reads").is_none());
}
}

10
swarm-controller/src/meta_fanout.rs
View file

@ -3,10 +3,12 @@
//! One key ([`KEY`]) in [`BUCKET`], which only this daemon reads or writes.
//! The `UpdateMeta` node fans out whenever the pin `internal/meta` holds is
//! not the one recorded here, and the `RecordFanout` node writes it once every
//! `TriggerDeploy` of that fan-out is done. A pass that dies, or a deploy that
//! fails, before then leaves the record stale, so the next pass sends the
//! deploys again. A deploy is idempotent per pin, so sending twice is safe and
//! sending never is not.
//! `TriggerDeploy` of that fan-out is done or skipped by its build gate. A pass
//! that dies, or a deploy that fails, before then leaves the record stale, so
//! the next pass sends the deploys again. A deploy the gate skipped counts as
//! sent: `jobq::queue_meta_update` declares it again when its build gave no
//! verdict, and a red build waits for the next config merge. A deploy is
//! idempotent per pin, so sending twice is safe and sending never is not.
//!
//! With no record, as on the first pass of a controller that has never written
//! one, the pin is not the recorded one and the agents are sent a deploy.

16
swarm-controller/src/meta_flake.rs
View file

@ -215,7 +215,7 @@ pub(crate) struct Report {
}
/// Bring `internal/meta` up to the forge's current heads for `agents`, and
/// give each of `hives`' forge accounts read on it.
/// give each of `hives`' forge accounts, and the swarm builder's, read on it.
///
/// # Errors
/// A head that cannot be read, a lock the builder refuses, or a refused commit.
@ -229,10 +229,13 @@ pub(crate) async fn update(
hives: &[String],
) -> Result<Report> {
forge.ensure_meta_repo().await?;
for hive in hives {
let login = swarm_secret_client::forge::hive_login(hive);
let logins = hives
.iter()
.map(|hive| swarm_secret_client::forge::hive_login(hive))
.chain([crate::forge::builder_account::LOGIN.to_owned()]);
for login in logins {
if let Err(e) = forge.grant_meta_read(&login).await {
tracing::warn!(%hive, error = %format!("{e:#}"), "meta flake: read grant on internal/meta failed");
tracing::warn!(%login, error = %format!("{e:#}"), "meta flake: read grant on internal/meta failed");
}
}
let committed = forge
@ -753,7 +756,10 @@ mod tests {
);
assert_eq!(
*forge.grants.lock().unwrap(),
vec![swarm_secret_client::forge::hive_login("pr1ma")]
vec![
swarm_secret_client::forge::hive_login("pr1ma"),
crate::forge::builder_account::LOGIN.to_owned(),
]
);
}