mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm: a deploy goes out only once its closure is cached and the hive has its cache facts

Browse source
This commit is contained in:
atlas 2026-11-07 11:53:25 +01:00
commit 79c1b6d81d
5 changed files with 432 additions and 92 deletions
Download patch file Download diff file Expand all files Collapse all files

102
swarm-controller/src/attic_copy.rs
View file

@ -6,14 +6,19 @@
//! [`RECONCILE_INTERVAL`] this writes each fact that is stored to every hive's
//! `attic/` prefix, and leaves a copy that already holds the value alone.
//!
//! A deploy waits for the same copy: the `CopyAtticFacts` node ([`copy_node`])
//! copies to the one hive the deploy goes to, through the same [`copy_hive`] as
//! the pass, so a hive is told to deploy only once its facts are in its prefix.
//!
//! A fact that is not stored is not copied, and a copy whose source is gone
//! stays: the pass never deletes. A swarm with no cache stores none, so the
//! pass writes nothing.
//! stays: neither the pass nor the node deletes. A swarm with no cache stores
//! none, so both write nothing.
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result};
use hive_jobq::scheduler::Outcome;
use swarm_secret_client::SecretStore;
use swarm_secret_client::attic::{Fact, Leaf, hive_path, service_path};
@ -94,6 +99,51 @@ async fn copy_hive(
Ok(written)
}
/// The stored facts copied to `hive`; how many copies were written. The one
/// hive [`copy_node`] serves; [`copy_pass`] reads the sources once for all of
/// them instead.
///
/// # Errors
/// A source or a copy that cannot be read, or a copy that cannot be written.
pub(crate) async fn ensure_copied(store: &impl FactStore, hive: &str) -> Result<usize> {
copy_hive(store, hive, &sources(store).await?).await
}
/// The `CopyAtticFacts` node's work: [`ensure_copied`] for `hive`, `Failed`
/// with the cause when it errors. Written copies are logged; a hive whose
/// copies are current is `Done` without a write.
pub(crate) async fn copy_node(store: &impl FactStore, hive: &str) -> Outcome {
match ensure_copied(store, hive).await {
Ok(written) => {
if written > 0 {
tracing::info!(hive, written, "attic facts: copied for a deploy");
}
Outcome::Done
}
Err(e) => Outcome::Failed(format!(
"the cache facts could not be copied to {hive}: {e:#}"
)),
}
}
/// [`copy_node`] against the swarm's store, within [`PASS_DEADLINE`].
pub(crate) async fn copy_node_now(hive: &str) -> Outcome {
let work = async {
match crate::store::connect().await {
Ok(store) => copy_node(&store, hive).await,
Err(e) => Outcome::Failed(format!("logging in to the swarm secret store: {e}")),
}
};
tokio::time::timeout(PASS_DEADLINE, work)
.await
.unwrap_or_else(|_| {
Outcome::Failed(format!(
"copying the cache facts to {hive} timed out after {}s",
PASS_DEADLINE.as_secs()
))
})
}
/// One pass over `hives`.
///
/// # Errors
@ -305,6 +355,54 @@ mod tests {
);
}
/// The node and the pass copy through the same `copy_hive`: for one hive
/// they leave the store holding the same values.
#[tokio::test]
async fn the_node_and_the_pass_leave_the_same_copies() {
let by_pass = Memory::with(&SOURCES);
copy_pass(&by_pass, &["alpha".to_owned()])
.await
.expect("pass");
let by_node = Memory::with(&SOURCES);
assert_eq!(copy_node(&by_node, "alpha").await, Outcome::Done);
for leaf in Leaf::ALL {
let path = hive_path("alpha", leaf).expect("a hive path");
assert!(by_pass.get(&path).is_some(), "the pass wrote {path}");
assert_eq!(by_node.get(&path), by_pass.get(&path), "{path}");
}
}
/// The node copies for its hive only, and a hive already current is `Done`
/// without a write.
#[tokio::test]
async fn the_node_copies_for_its_own_hive_and_leaves_a_current_copy_alone() {
let store = Memory::with(&SOURCES);
assert_eq!(copy_node(&store, "alpha").await, Outcome::Done);
assert_eq!(
store.get("swarm/hives/alpha/attic/pull-token").as_deref(),
Some("tok")
);
assert!(store.get("swarm/hives/beta/attic/pull-token").is_none());
let writes = store.writes.lock().unwrap().len();
assert_eq!(copy_node(&store, "alpha").await, Outcome::Done);
assert_eq!(store.writes.lock().unwrap().len(), writes);
}
/// A copy that cannot be written fails the node, naming the hive and the
/// cause. Control: the same store copies for the hive it does not refuse.
#[tokio::test]
async fn a_copy_that_cannot_be_written_fails_the_node() {
let mut store = Memory::with(&SOURCES);
store.refuse = Some("swarm/hives/alpha/");
let Outcome::Failed(error) = copy_node(&store, "alpha").await else {
panic!("a refused write is not Done")
};
assert!(error.contains("alpha"), "{error}");
assert!(error.contains("denied"), "{error}");
assert_eq!(copy_node(&store, "beta").await, Outcome::Done);
}
#[tokio::test]
async fn one_hive_failing_does_not_stop_the_others() {
let mut store = Memory::with(&SOURCES);

8
swarm-controller/src/cache_miss.rs
View file

@ -12,10 +12,10 @@
//! refused token is "cannot tell" and rebuilds nothing.
//!
//! A record from the head with `cached` false (the builder's push failed, or the
//! cache or its token was not up yet) is not a hive's to use, and nothing else
//! pushes it before `internal/meta` moves. The pass sends its build without
//! asking the cache, and the record is rewritten with `cached` true once the
//! push lands.
//! cache or its token was not up yet) is not a hive's to use. The meta timer
//! builds its failed `BuildMeta` again, but only from the in-memory job graph, so
//! a record that outlives a restart has no retry. The pass sends its build
//! without asking the cache, and the push rewrites the record with `cached` true.
//!
//! A pass sends at most [`MAX_REBUILDS_PER_PASS`] builds, one after the other.
//! An agent whose rebuild failed, or built without pushing, is left alone for

3
swarm-controller/src/held_deploys.rs
View file

@ -6,7 +6,8 @@
//! under one of two `reason` labels:
//! - `failed`: `nix` failed, its timeout included.
//! - `unavailable`: the build got no verdict (no builder, a forge error, no
//! `internal/meta` head, a missing read grant).
//! `internal/meta` head, a missing read grant, a closure the builder built but
//! did not push to the cache).
//!
//! The ten-minute timer builds either again. A pending, running or finished
//! build does not count, so the value falls to zero when a build succeeds.

296
swarm-controller/src/jobq.rs
View file

@ -121,7 +121,8 @@ pub(crate) enum SwarmNodeKind {
BuildAgent { agent: String, rev: String, pr: u64 },
/// Build `agent` from `internal/meta`'s head on the swarm builder, and
/// record the closure in `agent-closures`. The `TriggerDeploy` behind it
/// runs only when the build is green. See `meta_build`.
/// runs only when the build is green and the builder pushed the closure to
/// the cache. See `meta_build`.
///
/// Carries no rev: it reads the head when it runs, which in most DAGs is
/// after the `UpdateMeta` that commits it.
@ -129,6 +130,12 @@ pub(crate) enum SwarmNodeKind {
/// `pr` is the merged config PR the build belongs to, if any: a red `nix`
/// build is reported on it once.
BuildMeta { agent: String, pr: Option<u64> },
/// Copy the swarm's cache facts to `hive`'s secret-store prefix, where the
/// hive reads them to pull a closure. The `TriggerDeploy` to the same hive
/// runs only when the copy is done. See `attic_copy`.
///
/// Carries no agent: the facts are the hive's, whatever it is told to deploy.
CopyAtticFacts { hive: String },
/// Re-mint `agent`'s queue secret if it is still stored and old enough.
/// See `agent_renewal`.
///
@ -189,6 +196,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
SwarmNodeKind::MintCiMetricsForgeToken => "mint_ci_metrics_forge_token".to_owned(),
SwarmNodeKind::BuildAgent { .. } => "build_agent".to_owned(),
SwarmNodeKind::BuildMeta { .. } => "build_meta".to_owned(),
SwarmNodeKind::CopyAtticFacts { .. } => "copy_attic_facts".to_owned(),
SwarmNodeKind::RenewAgentQueueCredential { .. } => {
"renew_agent_queue_credential".to_owned()
}
@ -227,7 +235,8 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
serde_json::json!({ "agent": agent, "archetype": archetype.name })
}
SwarmNodeKind::MintHiveSenderToken { hive }
| SwarmNodeKind::MintHiveForgeToken { hive } => serde_json::json!({ "hive": hive }),
| SwarmNodeKind::MintHiveForgeToken { hive }
| SwarmNodeKind::CopyAtticFacts { hive } => serde_json::json!({ "hive": hive }),
SwarmNodeKind::MintClassifierForgeToken
| SwarmNodeKind::MintBuilderForgeToken
| SwarmNodeKind::MintCiMetricsForgeToken
@ -440,17 +449,14 @@ async fn run_node_work(
build_agent(&deps, forge, agent, rev, pr).await
}
SwarmNodeKind::BuildMeta { agent, pr } => build_meta(&deps, forge, agent, pr).await,
SwarmNodeKind::CopyAtticFacts { hive } => crate::attic_copy::copy_node_now(&hive).await,
SwarmNodeKind::RenewAgentQueueCredential { agent } => renew_queue_credential(&agent).await,
SwarmNodeKind::SetAgentWanted { hive, agent } => {
set_agent_wanted(deps.wanted.as_deref(), &hive, &agent).await
}
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => match deps.queue {
None => Outcome::Failed(
"no swarm queue is configured on this host, so no hive can be told to deploy"
.to_owned(),
),
Some(client) => publish_deploy(&client, &hive, &agent, rev).await,
},
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => {
trigger_deploy(deps.queue.as_ref(), &hive, &agent, rev).await
}
SwarmNodeKind::UpdateMeta => finish_update_meta(update_meta(&deps, forge).await, &builder),
SwarmNodeKind::RecordFanout { pin } => record_fanout(deps.queue.as_ref(), &pin).await,
};
@ -849,11 +855,13 @@ async fn record_fanout(
}
}
/// The build and the deploy every path declares for one agent: a `BuildMeta`
/// of `agent`, then the `TriggerDeploy`, which runs only when the build is
/// green (`after_ok`), so a red build leaves it `Skipped`.
/// The build, the facts copy and the deploy every path declares for one agent:
/// a `BuildMeta` of `agent` and a `CopyAtticFacts` of `hive`, then the
/// `TriggerDeploy`, which runs only when both are done (`after_ok`), so a red or
/// unpushed build, or a failed copy, leaves it `Skipped`. The copy is not tied
/// to the build and waits on nothing.
///
/// `update` is the `UpdateMeta` the pair follows when it is not this pair's
/// `update` is the `UpdateMeta` these nodes follow when it is not their
/// parent. The build waits on it, since the build reads the head that update
/// commits. The deploy keeps its own edge to it too: `skipped_deploys` finds
/// the deploys a failed update skipped by that edge.
@ -874,25 +882,29 @@ fn gated_deploy<'a>(
if let Some(update) = update {
build = build.after_ok(update);
}
let copy = b.node(SwarmNodeKind::CopyAtticFacts {
hive: hive.to_owned(),
});
let mut deploy = b
.node(SwarmNodeKind::TriggerDeploy {
hive: hive.to_owned(),
agent: agent.to_owned(),
rev,
})
.after_ok(build);
.after_ok(build)
.after_ok(copy);
if let Some(update) = update {
deploy = deploy.after_ok(update);
}
deploy
}
/// Declare one build and rebuild of each `(hive, agent)` placement, which
/// nothing else tells to build again, and for a [`FanOut::Record`] the
/// `RecordFanout` that waits on every one of them. A deploy the build gate
/// skipped counts as sent: the pin is recorded, so one failing agent does
/// Declare one build, facts copy and deploy of each `(hive, agent)` placement,
/// which nothing else tells to build again, and for a [`FanOut::Record`] the
/// `RecordFanout` that waits on every one of them. A deploy the build or copy
/// gate skipped counts as sent: the pin is recorded, so one failing agent does
/// not resend every healthy one on each retry. A deploy skipped by a failed
/// build is re-declared by `queue_meta_update` on every round.
/// build or copy is re-declared by `queue_meta_update` on every round.
fn declare_fan_out(
grown: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
fan_out: &FanOut,
@ -1132,6 +1144,23 @@ async fn declare_new_agent(
}
}
/// The `TriggerDeploy` arm, lifted out for `clippy::too_many_lines`: fails when
/// no swarm queue is configured, else [`publish_deploy`].
async fn trigger_deploy(
queue: Option<&async_nats::Client>,
hive: &str,
agent: &str,
rev: Option<String>,
) -> hive_jobq::scheduler::Outcome {
match queue {
None => hive_jobq::scheduler::Outcome::Failed(
"no swarm queue is configured on this host, so no hive can be told to deploy"
.to_owned(),
),
Some(client) => publish_deploy(client, hive, agent, rev).await,
}
}
/// Publish one deploy request and report whether it left this process.
///
/// The flush is not belt-and-braces: `publish` hands the message to the
@ -1743,13 +1772,16 @@ pub(crate) fn spawn_meta_update_timer(
});
}
/// Why a deploy is `Skipped` and the timer should declare it again.
/// The node whose failure left a deploy `Skipped`, which the timer declares
/// again.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum SkipCause {
/// It waited on an `UpdateMeta` that `Failed`.
FailedUpdate,
/// It waited on a `BuildMeta` that `Failed`, whatever the reason.
FailedBuild,
/// The `UpdateMeta` it waited on.
Update,
/// The `BuildMeta` it waited on, whatever the reason.
Build,
/// The `CopyAtticFacts` it waited on.
Copy,
}
/// A deploy [`skipped_deploys`] found.
@ -1764,12 +1796,12 @@ struct SkippedDeploy {
}
/// The deploys the timer declares again: for each agent, its latest
/// `TriggerDeploy` when the scheduler skipped it because an `UpdateMeta` or a
/// `BuildMeta` it waited on `Failed`. A failed build is built again on every
/// round until one succeeds. A deploy skipped for another reason is left alone:
/// the pause declaration or the config seeding failing means the update that
/// would unblock the deploy was skipped too, and deploying past those is the
/// race those edges close.
/// `TriggerDeploy` when the scheduler skipped it because an `UpdateMeta`, a
/// `BuildMeta` or a `CopyAtticFacts` it waited on `Failed`. A failed build or
/// copy is declared again on every round until one succeeds. A deploy skipped
/// for another reason is left alone: the pause declaration or the config
/// seeding failing means the update that would unblock the deploy was skipped
/// too, and deploying past those is the race those edges close.
fn skipped_deploys(
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
) -> Vec<SkippedDeploy> {
@ -1801,9 +1833,11 @@ fn skipped_deploys(
dep_nodes().any(|dep| is(&dep.payload) && dep.state == hive_jobq::State::Failed)
};
let cause = if failed(|p| matches!(p, SwarmNodeKind::UpdateMeta)) {
SkipCause::FailedUpdate
SkipCause::Update
} else if failed(|p| matches!(p, SwarmNodeKind::BuildMeta { .. })) {
SkipCause::FailedBuild
SkipCause::Build
} else if failed(|p| matches!(p, SwarmNodeKind::CopyAtticFacts { .. })) {
SkipCause::Copy
} else {
return None;
};
@ -1829,9 +1863,9 @@ fn skipped_deploys(
/// update was failing is deployed by the run that succeeds.
///
/// When the latest `UpdateMeta` is `Done`, a deploy skipped by a failed build
/// still gets a fresh build and deploy, with no update, so a build that failed
/// is tried again on every round until it succeeds. That adds two nodes per
/// failing agent on every tick. Returns whether a job was inserted.
/// or copy still gets a fresh build, copy and deploy, with no update, so one
/// that failed is tried again on every round until it succeeds. That adds three
/// nodes per failing agent on every tick. Returns whether a job was inserted.
fn queue_meta_update(
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
) -> Result<bool> {
@ -1857,7 +1891,7 @@ fn queue_meta_update(
.is_none_or(|n| n.state != hive_jobq::State::Done);
let skipped: Vec<SkippedDeploy> = skipped_deploys(sched.graph())
.into_iter()
.filter(|s| update_due || s.cause == SkipCause::FailedBuild)
.filter(|s| update_due || s.cause != SkipCause::Update)
.collect();
if !update_due && skipped.is_empty() {
return Ok(false);
@ -2656,7 +2690,7 @@ mod tests {
}
/// Every `TriggerDeploy` in `graph` waits, strictly, on a `BuildMeta` of the
/// same agent that needs the builder. Panics, naming `path`, when one does
/// same agent that needs the builder, and on a `CopyAtticFacts` of its hive. Panics, naming `path`, when one does
/// not; returns how many deploys it checked.
fn assert_every_deploy_is_gated(
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
@ -2664,7 +2698,7 @@ mod tests {
) -> usize {
let mut checked = 0;
for deploy in graph.nodes() {
let SwarmNodeKind::TriggerDeploy { agent, .. } = &deploy.payload else {
let SwarmNodeKind::TriggerDeploy { hive, agent, .. } = &deploy.payload else {
continue;
};
let gate = deploy.deps.iter().find_map(|d| {
@ -2694,15 +2728,32 @@ mod tests {
)),
"{path}: {agent}'s build does not need the builder"
);
let copy = deploy.deps.iter().find_map(|d| {
match d {
hive_jobq::Dep::Node { id, when } => graph
.node(*id)
.filter(|n| {
matches!(&n.payload, SwarmNodeKind::CopyAtticFacts { hive: h } if h == hive)
})
.map(|_| *when),
hive_jobq::Dep::Resource { .. } => None,
}
});
let when = copy
.unwrap_or_else(|| panic!("{path}: {agent}'s deploy has no facts copy for {hive}"));
assert!(
!when.accepts(hive_jobq::TerminalState::Failed),
"{path}: {agent}'s deploy runs after a failed facts copy"
);
checked += 1;
}
checked
}
/// The four paths that send a deploy all declare the same build and deploy
/// pair, so a path that forgets the gate fails here.
/// The four paths that send a deploy all declare the same build, copy and
/// deploy, so a path that forgets a gate fails here.
#[tokio::test]
async fn all_four_deploy_paths_declare_the_gated_pair() {
async fn all_four_deploy_paths_declare_the_gated_deploy() {
let graph_of = |declare: fn(&TestBuilder)| {
let mut sched = hive_jobq::scheduler::Scheduler::new(
hive_jobq::Graph::new(),
@ -2908,20 +2959,28 @@ mod tests {
placed,
failing_agent,
crash_before_record,
None,
Failing::default(),
)
.await
}
/// [`pass`], with the build of one agent failing with the given error when
/// `failing_build` names it.
/// What [`pass_with`] makes fail besides the deploy and the record.
#[derive(Clone, Default)]
struct Failing {
/// The agent whose build fails, and with what.
build: Option<(&'static str, String)>,
/// The hive whose facts copy fails.
copy: Option<&'static str>,
}
/// [`pass`], with what `failing` names failing.
async fn pass_with(
pins: &Arc<FakePins>,
report: &meta_flake::Report,
placed: &[(&str, &str)],
failing_agent: Option<&'static str>,
crash_before_record: bool,
failing_build: Option<(&'static str, String)>,
failing: Failing,
) -> (FanOut, Arc<Mutex<TestScheduler>>) {
use hive_jobq::scheduler::{Outcome, Scheduler};
use meta_fanout::PinStore as _;
@ -2947,8 +3006,7 @@ mod tests {
.expect("the node just inserted is runnable");
update.await.1.expect("the grown nodes insert");
while let Some(runner) = Scheduler::claim_next(&sched, {
let (pins, sched, failing_build) =
(Arc::clone(pins), Arc::clone(&sched), failing_build.clone());
let (pins, sched, failing) = (Arc::clone(pins), Arc::clone(&sched), failing.clone());
move |_id, kind, builder| async move {
let outcome = match kind {
SwarmNodeKind::TriggerDeploy { agent, .. }
@ -2957,9 +3015,14 @@ mod tests {
Outcome::Failed("the hive did not take it".to_owned())
}
SwarmNodeKind::BuildMeta { agent, .. }
if failing_build.as_ref().is_some_and(|(a, _)| *a == agent) =>
if failing.build.as_ref().is_some_and(|(a, _)| *a == agent) =>
{
Outcome::Failed(failing_build.map(|(_, error)| error).expect("matched"))
Outcome::Failed(failing.build.map(|(_, error)| error).expect("matched"))
}
SwarmNodeKind::CopyAtticFacts { hive }
if failing.copy == Some(hive.as_str()) =>
{
Outcome::Failed("the store refused the write".to_owned())
}
SwarmNodeKind::RecordFanout { .. } if crash_before_record => {
Outcome::Failed("the controller died".to_owned())
@ -3021,6 +3084,11 @@ mod tests {
states_of(&sched, "trigger_deploy"),
[hive_jobq::State::Done; 3]
);
assert_eq!(
states_of(&sched, "copy_attic_facts"),
[hive_jobq::State::Done; 3],
"a copy is declared and run for every deploy"
);
assert_eq!(states_of(&sched, "record_fanout"), [hive_jobq::State::Done]);
assert_eq!(pins.held().as_deref(), Some("p1"));
}
@ -3164,6 +3232,10 @@ mod tests {
for error in [
"error: builder for argus failed".to_owned(),
"timed out: nix did not finish".to_owned(),
format!(
"{}built /nix/store/x-toplevel, but the builder did not push it to the cache",
meta_build::NO_VERDICT_PREFIX
),
format!(
"{}no swarm builder is listening",
meta_build::NO_VERDICT_PREFIX
@ -3176,7 +3248,10 @@ mod tests {
&PLACED,
None,
false,
Some(("argus", error.clone())),
Failing {
build: Some(("argus", error.clone())),
..Failing::default()
},
)
.await;
let mut builds = states_of(&sched, "build_meta");
@ -3214,6 +3289,68 @@ mod tests {
}
}
/// A failed facts copy to one hive skips the deploys to that hive only, and
/// the pin is still recorded. The timer declares the copy, a build and the
/// deploy again for the skipped deploy.
#[tokio::test]
async fn a_failed_copy_skips_its_deploy_and_the_timer_declares_it_again() {
use hive_jobq::State;
use hive_jobq_wire::WireNode as _;
let pins = Arc::new(FakePins::holding("p0"));
let (_, sched) = pass_with(
&pins,
&report("p1", true),
&PLACED,
None,
false,
Failing {
copy: Some("sec0nd"),
..Failing::default()
},
)
.await;
let mut copies = states_of(&sched, "copy_attic_facts");
copies.sort_by_key(|s| format!("{s:?}"));
assert_eq!(copies, [State::Done, State::Done, State::Failed]);
let deploys: Vec<_> = {
let guard = sched.lock().unwrap();
guard
.graph()
.nodes()
.filter_map(|n| match &n.payload {
SwarmNodeKind::TriggerDeploy { agent, .. } => Some((agent.clone(), n.state)),
_ => None,
})
.collect()
};
let state_of = |who: &str| deploys.iter().find(|(a, _)| a == who).map(|(_, s)| *s);
assert_eq!(state_of("iris"), Some(State::Skipped));
assert_eq!(state_of("atlas"), Some(State::Done));
assert_eq!(state_of("argus"), Some(State::Done));
assert_eq!(states_of(&sched, "record_fanout"), [State::Done]);
assert_eq!(pins.held().as_deref(), Some("p1"));
assert!(queue_meta_update(&sched).expect("tick"));
let guard = sched.lock().unwrap();
let pending = |label: &str| {
guard
.graph()
.nodes()
.filter(|n| n.payload.label() == label && n.state == State::Pending)
.map(|n| match &n.payload {
SwarmNodeKind::BuildMeta { agent, .. }
| SwarmNodeKind::TriggerDeploy { agent, .. } => agent.clone(),
SwarmNodeKind::CopyAtticFacts { hive } => hive.clone(),
other => panic!("not a build, copy or deploy: {other:?}"),
})
.collect::<Vec<_>>()
};
assert_eq!(pending("copy_attic_facts"), ["sec0nd"]);
assert_eq!(pending("build_meta"), ["iris"]);
assert_eq!(pending("trigger_deploy"), ["iris"]);
}
/// With nothing placed there is nothing to wait on, and the pin is
/// recorded all the same.
#[tokio::test]
@ -4297,6 +4434,65 @@ mod tests {
);
}
/// Run every runnable node; the facts copy to `hive` fails and everything
/// else succeeds.
async fn drain_failing_copy(sched: &SharedSched, hive: &'static str) {
use hive_jobq::scheduler::{Outcome, Scheduler};
while let Some(runner) =
Scheduler::claim_next(sched, move |_id, kind, builder| async move {
let outcome = match kind {
SwarmNodeKind::CopyAtticFacts { hive: h } if h == hive => {
Outcome::Failed("the store refused the write".to_owned())
}
_ => Outcome::Done,
};
(builder, outcome)
})
{
runner.await.1.expect("no growth declared");
}
}
/// A merged config whose facts copy fails, with the update `Done`: the
/// deploy is skipped, and the timer declares a copy, a build and the same
/// deploy again on every round, without an update, until a copy succeeds.
/// Then it stops.
#[tokio::test]
async fn a_failed_copy_is_declared_again_every_round_until_it_succeeds() {
use hive_jobq::State;
let sched = merged_atlas();
drain_failing_copy(&sched, "pr1ma").await;
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
assert_eq!(update_metas(&sched), 1);
assert!(queue_meta_update(&sched).expect("tick"));
assert_eq!(update_metas(&sched), 1, "the retry needs no update");
drain_failing_copy(&sched, "pr1ma").await;
assert_eq!(
deploys_of(&sched, "atlas"),
[State::Skipped, State::Skipped]
);
// A success on a later round deploys, once, with the merged rev.
assert!(queue_meta_update(&sched).expect("tick"));
drain(&sched, false, false).await;
assert_eq!(
deploys_of(&sched, "atlas"),
[State::Skipped, State::Skipped, State::Done]
);
assert!(
deploy_revs(&sched, "atlas")
.iter()
.all(|rev| rev.as_deref() == Some("abc"))
);
assert!(
!queue_meta_update(&sched).expect("tick"),
"the success ends the retries"
);
}
/// Only a merged config PR's build carries a PR: the creation DAG's does
/// not, so its failure has nobody to comment on.
#[test]

32
swarm-controller/src/meta_build.rs
View file

@ -8,14 +8,16 @@
//! key itself: the builder holds no grant on the bucket. A `Built` reply and
//! nothing else writes, so a red build leaves the previous record.
//!
//! The node ends one of two ways. `Done` is a green build. Every `Failed` skips
//! the deploy behind it, and the Jobs page shows why:
//! The node ends one of two ways. `Done` is a green build whose closure the
//! builder pushed to the cache. Every `Failed` skips the deploy behind it, and
//! the Jobs page shows why:
//! - `nix` failed, its timeout included: `Failed` with `nix`'s tail, or `nix
//! timed out`. The deploy stays held and counts as a failed build.
//! - The build got no verdict (no builder, a forge error, no `internal/meta`
//! head, a missing read grant, a failed closure write): `Failed` starting with
//! [`NO_VERDICT_PREFIX`], which the held-deploys gauge reads to label the
//! reason.
//! head, a missing read grant, a failed closure write, a closure the builder
//! built but did not push): `Failed` starting with [`NO_VERDICT_PREFIX`],
//! which the held-deploys gauge reads to label the reason. A built but
//! unpushed closure is still recorded, with `cached` false.
//!
//! Both are built again on every round of the meta timer
//! (`jobq::queue_meta_update`) until a build succeeds.
@ -384,6 +386,9 @@ pub(crate) async fn run(
nix_builds,
};
match queue.record(&agent, &closure).await {
Ok(()) if !cached => no_verdict(format!(
"built {out_path}, but the builder did not push it to the cache"
)),
Ok(()) => {
if let Some(pr) = pr {
clear_failure(head, pr, &agent, &built_rev).await;
@ -488,7 +493,7 @@ mod tests {
}
fn built() -> Self {
Self::built_cached(false)
Self::built_cached(true)
}
fn built_cached(cached: bool) -> Self {
@ -615,16 +620,25 @@ mod tests {
assert_eq!(closure.store_path, "/nix/store/x-toplevel");
assert_eq!(closure.system, "x86_64-linux");
assert!(closure.built_unix > 0);
assert!(!closure.cached);
assert!(closure.cached);
}
/// The record says what the builder said about the cache, both ways.
/// The record says what the builder said about the cache, both ways. Only a
/// pushed closure ends the node `Done`: the deploy waits on that.
#[tokio::test]
async fn the_record_carries_whether_the_builder_cached_the_closure() {
for cached in [true, false] {
let queue = FakeQueue::built_cached(cached);
let outcome = run_with(&FakeHead::reading(Ok(Some(REV))), &queue).await;
assert_eq!(outcome, Outcome::Done);
if cached {
assert_eq!(outcome, Outcome::Done);
} else {
let Outcome::Failed(error) = outcome else {
panic!("an uncached build is not Done: {outcome:?}")
};
assert!(is_no_verdict(&error), "{error}");
assert!(error.contains("did not push"), "{error}");
}
let recorded = queue.recorded.lock().unwrap();
let [(_, closure)] = recorded.as_slice() else {
panic!("one record: {recorded:?}")