mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

pin the declared closure once per rebuild and seed the marker from the running closure

Browse source
This commit is contained in:
atlas 2026-11-02 00:42:45 +01:00 • committed by mara
commit 4356f3840c
1 changed files with 456 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

176
hive-c0re/src/job_queue/exec/tests.rs
View file

@ -31,7 +31,9 @@ struct FakeHost {
running: bool,
/// What the swarm's builder recorded for the agent; `None` reads as no
/// record.
built: Option<AgentClosure>,
built: Mutex<Option<AgentClosure>>,
/// The store path pinned for the agent's rebuild.
pinned: Mutex<Option<String>>,
/// Store paths a download put in the store.
realised: Mutex<Vec<String>>,
/// Names of the calls that fail (`kill`, `realise`, `closure_record`, `swap_update`,
@ -48,7 +50,8 @@ impl FakeHost {
fn up() -> Self {
Self {
running: true,
built: Some(closure(true)),
built: Mutex::new(Some(closure(true))),
pinned: Mutex::new(Some(CLOSURE.to_owned())),
..Self::default()
}
}
@ -56,22 +59,39 @@ impl FakeHost {
fn down() -> Self {
Self {
running: false,
built: Some(closure(true)),
built: Mutex::new(Some(closure(true))),
pinned: Mutex::new(Some(CLOSURE.to_owned())),
..Self::default()
}
}
/// The builder recorded the closure for the agent, pushed or not.
fn built(mut self, cached: bool) -> Self {
self.built = Some(closure(cached));
fn built(self, cached: bool) -> Self {
*self.built.lock().unwrap() = Some(closure(cached));
self
}
fn without_record(mut self) -> Self {
self.built = None;
fn without_record(self) -> Self {
*self.built.lock().unwrap() = None;
self
}
/// The rebuild has no pinned closure, as when `MetaSync` has not run.
fn unpinned(self) -> Self {
*self.pinned.lock().unwrap() = None;
self
}
/// The builder records another closure, at `store_path`.
fn builder_records(&self, store_path: &str) {
self.built
.lock()
.unwrap()
.as_mut()
.expect("a record")
.store_path = store_path.to_owned();
}
fn failing(mut self, call: &'static str) -> Self {
self.failing.push(call);
self
@ -115,7 +135,21 @@ impl Host for FakeHost {
async fn closure_record(&self, name: &str) -> anyhow::Result<Option<AgentClosure>> {
self.step("closure_record", format!("closure_record {name}"))?;
Ok(self.built.clone())
Ok(self.built.lock().unwrap().clone())
}
fn pin_closure(&self, name: &str, store_path: &str) {
self.record(format!("pin_closure {name} {store_path}"));
*self.pinned.lock().unwrap() = Some(store_path.to_owned());
}
fn pinned_closure(&self, _name: &str) -> Option<String> {
self.pinned.lock().unwrap().clone()
}
fn unpin_closure(&self, name: &str) {
self.record(format!("unpin_closure {name}"));
*self.pinned.lock().unwrap() = None;
}
async fn realise_closure(&self, name: &str, store_path: &str) -> anyhow::Result<()> {
@ -326,15 +360,21 @@ async fn logging<T>(fut: impl std::future::Future<Output = T>) -> (T, Logged) {
/// Every way the swarm's closure for `alice` cannot be had, with the text the
/// error log must carry and the host calls that must not happen.
fn without_a_closure() -> [(&'static str, FakeHost, &'static str); 4] {
[
fn without_a_closure() -> Vec<(&'static str, FakeHost, &'static str)> {
let mut cases = without_a_record();
cases.push((
"download fails",
FakeHost::up().failing("realise"),
"Couldn't connect",
));
cases
}
/// The ways the swarm has no closure to pin for a rebuild.
fn without_a_record() -> Vec<(&'static str, FakeHost, &'static str)> {
vec![
("no record", FakeHost::up().without_record(), "no build"),
("not pushed", FakeHost::up().built(false), "did not push"),
(
"download fails",
FakeHost::up().failing("realise"),
"Couldn't connect",
),
(
"record unreadable",
FakeHost::up().failing("closure_record"),
@ -343,6 +383,18 @@ fn without_a_closure() -> [(&'static str, FakeHost, &'static str); 4] {
]
}
/// The ways a rebuild's `Prebuild` or `Swap` has no closure to download.
fn without_a_pinned_closure() -> Vec<(&'static str, FakeHost, &'static str)> {
vec![
("no pin", FakeHost::up().unpinned(), "no closure was pinned"),
(
"download fails",
FakeHost::up().failing("realise"),
"Couldn't connect",
),
]
}
#[tokio::test]
async fn prebuild_downloads_the_swarms_closure_when_the_container_is_up() {
let host = FakeHost::up();
@ -352,7 +404,6 @@ async fn prebuild_downloads_the_swarms_closure_when_the_container_is_up() {
host.calls(),
[
"is_running alice".to_owned(),
"closure_record alice".to_owned(),
format!("realise alice {CLOSURE}"),
]
);
@ -370,7 +421,7 @@ async fn prebuild_downloads_the_swarms_closure_when_the_container_is_up() {
/// container is untouched, and the log names the agent and why.
#[tokio::test]
async fn a_rebuild_without_a_closure_fails_the_prebuild_and_logs_an_error() {
for (case, host, why) in without_a_closure() {
for (case, host, why) in without_a_pinned_closure() {
let (result, log) = logging(run_prebuild(&host, "alice")).await;
let err = result.expect_err(case);
assert!(format!("{err:#}").starts_with("alice: "), "{case}: {err:#}");
@ -412,10 +463,10 @@ async fn swap_applies_the_swarms_closure() {
assert_eq!(
host.calls(),
[
"closure_record alice".to_owned(),
format!("realise alice {CLOSURE}"),
format!("swap_update alice {CLOSURE} node={}", id.get()),
format!("mark_deployed alice {CLOSURE}"),
"unpin_closure alice".to_owned(),
]
);
}
@ -424,7 +475,7 @@ async fn swap_applies_the_swarms_closure() {
/// was, and the log names the agent and why.
#[tokio::test]
async fn a_swap_without_a_closure_never_updates_and_logs_an_error() {
for (case, host, why) in without_a_closure() {
for (case, host, why) in without_a_pinned_closure() {
let (_dir, coord) = coordinator();
let (result, log) = logging(run_swap(&host, &coord, "alice", node_id())).await;
result.expect_err(case);
@ -450,8 +501,8 @@ async fn swap_marks_the_applied_closure_deployed_only_after_the_apply() {
let id = node_id();
run_swap(&host, &coord, "alice", id).await.expect("swap ok");
assert_eq!(
host.calls().last().map(String::as_str),
Some(format!("mark_deployed alice {CLOSURE}").as_str()),
host.calls()[2],
format!("mark_deployed alice {CLOSURE}"),
"{:?}",
host.calls()
);
@ -489,8 +540,8 @@ async fn a_failed_swap_reports_the_error_and_touches_no_container() {
.expect_err("update failure propagates");
assert!(format!("{err:#}").contains("update failed"), "{err:#}");
let calls = host.calls();
assert_eq!(calls.len(), 3, "{calls:?}");
assert!(calls[2].starts_with("swap_update alice"), "{calls:?}");
assert_eq!(calls.len(), 2, "{calls:?}");
assert!(calls[1].starts_with("swap_update alice"), "{calls:?}");
}
// --- Stop / StopForUpdate ---------------------------------------------------
@ -716,7 +767,7 @@ async fn pause_drain_times_out_ok_and_clears_pending() {
// --- MetaSync ---------------------------------------------------------------
#[tokio::test]
async fn meta_sync_only_prepares_the_rebuild() {
async fn meta_sync_prepares_the_rebuild_and_pins_the_declared_closure() {
let host = FakeHost::up();
run_meta_sync(&host, "alice").await.expect("ok");
assert_eq!(
@ -725,6 +776,8 @@ async fn meta_sync_only_prepares_the_rebuild() {
"refresh_archetype alice".to_owned(),
"prepare_rebuild_dirs alice".to_owned(),
"refresh_agent_set".to_owned(),
"closure_record alice".to_owned(),
format!("pin_closure alice {CLOSURE}"),
]
);
}
@ -733,7 +786,7 @@ async fn meta_sync_only_prepares_the_rebuild() {
async fn meta_sync_keeps_going_when_the_archetype_read_fails() {
let host = FakeHost::up().failing("refresh_archetype");
run_meta_sync(&host, "alice").await.expect("non-fatal");
assert_eq!(host.calls().len(), 3);
assert_eq!(host.calls().len(), 5);
}
#[tokio::test]
@ -747,6 +800,69 @@ async fn meta_sync_stops_at_the_first_fatal_step() {
}
}
/// A rebuild without a closure to deploy fails in `MetaSync`, before anything is
/// stopped, and pins nothing.
#[tokio::test]
async fn meta_sync_without_a_closure_fails_and_pins_nothing() {
for (case, host, why) in without_a_record() {
let host = host.unpinned();
let (result, log) = logging(run_meta_sync(&host, "alice")).await;
result.expect_err(case);
let errors = log.at(tracing::Level::ERROR);
assert_eq!(errors.len(), 1, "{case}: {errors:?}");
assert!(errors[0].contains(why), "{case}: {errors:?}");
assert!(
host.pinned_closure("alice").is_none(),
"{case}: {:?}",
host.calls()
);
}
}
/// The record is read once for a rebuild. A record that changes after `MetaSync`
/// (the builder finished another build) leaves `Prebuild` and `Swap` on the
/// closure that was pinned, and the marker names what was applied.
#[tokio::test]
async fn a_record_that_changes_mid_rebuild_does_not_change_what_swap_applies() {
const NEWER: &str = "/nix/store/zyxwvsrqpnmlkjihgfdcbadfhijklmn-nixos-system-alice";
let (_dir, coord) = coordinator();
let host = FakeHost::up().unpinned();
run_meta_sync(&host, "alice").await.expect("pinned");
run_prebuild(&host, "alice").await.expect("prebuilt");
host.builder_records(NEWER);
let id = node_id();
run_swap(&host, &coord, "alice", id).await.expect("swapped");
let calls = host.calls();
assert_eq!(
calls
.iter()
.filter(|c| c.starts_with("closure_record"))
.count(),
1,
"the record is read once: {calls:?}"
);
assert!(
calls.contains(&format!("swap_update alice {CLOSURE} node={}", id.get())),
"{calls:?}"
);
assert!(
calls.contains(&format!("mark_deployed alice {CLOSURE}")),
"{calls:?}"
);
assert!(
!calls.iter().any(|c| c.contains(NEWER)),
"control: the newer path is never touched: {calls:?}"
);
// Control: the changed record is what the next rebuild pins.
run_meta_sync(&host, "alice").await.expect("pinned again");
assert!(
host.calls().contains(&format!("pin_closure alice {NEWER}")),
"{:?}",
host.calls()
);
}
// --- Provision / Create / WriteDropin ----------------------------------------
#[tokio::test]
@ -1034,19 +1150,17 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() {
"refresh_archetype alice",
"prepare_rebuild_dirs alice",
"refresh_agent_set",
"closure_record alice",
"pin_closure alice",
],
),
(
alice(|agent| NodeKind::Prebuild { agent }),
&["is_running alice", "closure_record alice"],
&["is_running alice", "realise alice"],
),
(
alice(|agent| NodeKind::Swap { agent }),
&[
"closure_record alice",
"swap_update alice",
"mark_deployed alice",
],
&["realise alice", "swap_update alice", "mark_deployed alice"],
),
(alice(|agent| NodeKind::RebuildBookkeeping { agent }), &[]),
(