mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

deploy the closure the swarm declares, read no forge head, drop the update badge

Browse source
This commit is contained in:
atlas 2026-11-01 23:56:40 +01:00 • committed by mara
commit 47c2acb640
1 changed files with 647 additions and 2001 deletions
Download patch file Download diff file Expand all files Collapse all files

312
hive-c0re/src/job_queue/exec/tests.rs
View file

@ -19,7 +19,6 @@ use super::{
};
use crate::coordinator::{AgentPaths, Coordinator, HiveEnv};
use crate::job_queue::model::NodeKind;
use crate::meta_rev::MetaBuild;
use crate::power::Wanted;
use swarm_wire::agent_closures::AgentClosure;
@ -30,19 +29,16 @@ const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-
#[derive(Default)]
struct FakeHost {
running: bool,
rev: Option<String>,
/// What the swarm's builder recorded for the agent; `None` reads as no
/// record.
built: Option<AgentClosure>,
/// Store paths a download put in the store.
realised: Mutex<Vec<String>>,
/// Names of the calls that fail (`kill`, `realise`, `closure_record`, `swap_update`,
/// `set_paused`, ...). `recorded_build` fails by finding nothing.
/// `set_paused`, `mark_deployed`, ...).
failing: Vec<&'static str>,
/// The sweeps and pulls never finish, so only a deadline ends them.
hang: bool,
/// The agent has the same pins at every commit.
unmoved: bool,
runtime: PathBuf,
purge: Vec<PathBuf>,
calls: Mutex<Vec<String>>,
@ -52,8 +48,7 @@ impl FakeHost {
fn up() -> Self {
Self {
running: true,
rev: Some(REV.to_owned()),
built: Some(closure(REV, true)),
built: Some(closure(true)),
..Self::default()
}
}
@ -61,20 +56,14 @@ impl FakeHost {
fn down() -> Self {
Self {
running: false,
rev: Some(REV.to_owned()),
built: Some(closure(REV, true)),
built: Some(closure(true)),
..Self::default()
}
}
/// The builder recorded `meta_rev` for the agent, pushed or not.
fn built(mut self, meta_rev: &str, cached: bool) -> Self {
self.built = Some(closure(meta_rev, cached));
self
}
fn unmoved(mut self) -> Self {
self.unmoved = true;
/// The builder recorded the closure for the agent, pushed or not.
fn built(mut self, cached: bool) -> Self {
self.built = Some(closure(cached));
self
}
@ -124,31 +113,11 @@ impl Host for FakeHost {
Ok(())
}
fn recorded_rev(&self, name: &str) -> anyhow::Result<String> {
self.record(format!("recorded_rev {name}"));
self.rev
.clone()
.ok_or_else(|| anyhow::anyhow!("{name}: no internal/meta commit was recorded"))
}
async fn closure_record(&self, name: &str) -> anyhow::Result<Option<AgentClosure>> {
self.step("closure_record", format!("closure_record {name}"))?;
Ok(self.built.clone())
}
async fn pins_at(&self, name: &str, rev: &str) -> anyhow::Result<MetaBuild> {
self.step("pins_at", format!("pins_at {name} {rev}"))?;
Ok(MetaBuild {
rev: rev.to_owned(),
hyperhive: "dh".to_owned(),
config: Some(if self.unmoved {
"dc".to_owned()
} else {
format!("dc-{rev}")
}),
})
}
async fn realise_closure(&self, name: &str, store_path: &str) -> anyhow::Result<()> {
self.record(format!("realise {name} {store_path}"));
if self.fails("realise") {
@ -200,24 +169,10 @@ impl Host for FakeHost {
self.step("refresh_agent_set", "refresh_agent_set".to_owned())
}
async fn resolve_meta(&self, name: &str) -> anyhow::Result<MetaBuild> {
self.step("resolve_meta", format!("resolve_meta {name}"))?;
Ok(build())
}
fn record_meta(&self, name: &str, build: &MetaBuild) -> anyhow::Result<()> {
self.step("record_meta", format!("record_meta {name} {}", build.rev))
}
fn recorded_build(&self, name: &str) -> Option<MetaBuild> {
self.record(format!("recorded_build {name}"));
(!self.fails("recorded_build")).then(build)
}
fn mark_deployed(&self, name: &str, build: &MetaBuild) -> anyhow::Result<()> {
fn mark_deployed(&self, name: &str, store_path: &str) -> anyhow::Result<()> {
self.step(
"mark_deployed",
format!("mark_deployed {name} {}", build.rev),
format!("mark_deployed {name} {store_path}"),
)
}
@ -277,10 +232,10 @@ impl Host for FakeHost {
}
}
/// The builder's record of a closure built from `internal/meta` at `meta_rev`.
fn closure(meta_rev: &str, cached: bool) -> AgentClosure {
/// The builder's record of a closure.
fn closure(cached: bool) -> AgentClosure {
AgentClosure {
meta_rev: meta_rev.to_owned(),
meta_rev: REV.to_owned(),
store_path: CLOSURE.to_owned(),
system: "x86_64-linux".to_owned(),
built_unix: 1,
@ -289,14 +244,6 @@ fn closure(meta_rev: &str, cached: bool) -> AgentClosure {
}
}
fn build() -> MetaBuild {
MetaBuild {
rev: REV.to_owned(),
hyperhive: "h".to_owned(),
config: Some("c".to_owned()),
}
}
fn coordinator() -> (tempfile::TempDir, Arc<Coordinator>) {
let dir = tempfile::tempdir().expect("tempdir");
let coord = Coordinator::open(
@ -325,8 +272,6 @@ fn node_id() -> NodeId {
// --- Prebuild ---------------------------------------------------------------
const OTHER_REV: &str = "3333333333333333333333333333333333333333";
/// What a block logged, level and text: the message and each field.
#[derive(Clone, Default)]
struct Logged(Arc<Mutex<Vec<(tracing::Level, String)>>>);
@ -384,11 +329,7 @@ async fn logging<T>(fut: impl std::future::Future<Output = T>) -> (T, Logged) {
fn without_a_closure() -> [(&'static str, FakeHost, &'static str); 4] {
[
("no record", FakeHost::up().without_record(), "no build"),
(
"not pushed",
FakeHost::up().built(REV, false),
"did not push",
),
("not pushed", FakeHost::up().built(false), "did not push"),
(
"download fails",
FakeHost::up().failing("realise"),
@ -411,7 +352,6 @@ async fn prebuild_downloads_the_swarms_closure_when_the_container_is_up() {
host.calls(),
[
"is_running alice".to_owned(),
"recorded_rev alice".to_owned(),
"closure_record alice".to_owned(),
format!("realise alice {CLOSURE}"),
]
@ -454,39 +394,6 @@ async fn a_rebuild_without_a_closure_fails_the_prebuild_and_logs_an_error() {
}
}
/// Another commit that pins the agent the same is nothing to warn about.
#[tokio::test]
async fn a_closure_built_from_a_commit_with_the_same_pins_is_deployed_quietly() {
let host = FakeHost::up().built(OTHER_REV, true).unmoved();
let (result, log) = logging(run_prebuild(&host, "alice")).await;
result.expect("drift is not a refusal");
assert!(host.calls().contains(&format!("realise alice {CLOSURE}")));
assert!(log.at(tracing::Level::WARN).is_empty());
}
/// The builder's record is not for the commit this hive chose: its closure is
/// deployed and the difference is a warning.
#[tokio::test]
async fn a_closure_built_from_another_commit_is_deployed_with_a_warning() {
let host = FakeHost::up().built(OTHER_REV, true);
let (result, log) = logging(run_prebuild(&host, "alice")).await;
result.expect("drift is not a refusal");
assert!(
host.calls().contains(&format!("realise alice {CLOSURE}")),
"{:?}",
host.calls()
);
let warnings = log.at(tracing::Level::WARN);
assert_eq!(warnings.len(), 1, "{warnings:?}");
assert!(
[REV, OTHER_REV, CLOSURE]
.iter()
.all(|part| warnings[0].contains(part)),
"{warnings:?}"
);
assert!(log.at(tracing::Level::ERROR).is_empty());
}
#[tokio::test]
async fn a_stopped_container_looks_up_nothing() {
let host = FakeHost::down();
@ -494,22 +401,6 @@ async fn a_stopped_container_looks_up_nothing() {
assert_eq!(host.calls(), ["is_running alice"]);
}
#[tokio::test]
async fn prebuild_without_a_recorded_rev_fails_before_looking_up_a_closure() {
let host = FakeHost {
rev: None,
..FakeHost::up()
};
let err = run_prebuild(&host, "alice")
.await
.expect_err("nothing to deploy");
assert!(
format!("{err:#}").contains("no internal/meta commit"),
"{err:#}"
);
assert!(!host.calls().iter().any(|c| c.starts_with("closure_record")));
}
// --- Swap -------------------------------------------------------------------
#[tokio::test]
@ -521,10 +412,10 @@ async fn swap_applies_the_swarms_closure() {
assert_eq!(
host.calls(),
[
"recorded_rev alice".to_owned(),
"closure_record alice".to_owned(),
format!("realise alice {CLOSURE}"),
format!("swap_update alice {CLOSURE} node={}", id.get()),
format!("mark_deployed alice {CLOSURE}"),
]
);
}
@ -551,53 +442,40 @@ async fn a_swap_without_a_closure_never_updates_and_logs_an_error() {
}
}
/// The marker names the closure that was applied, and only an applied one.
#[tokio::test]
async fn swap_applies_a_closure_built_from_another_commit_with_a_warning() {
async fn swap_marks_the_applied_closure_deployed_only_after_the_apply() {
let (_dir, coord) = coordinator();
let host = FakeHost::down().built(OTHER_REV, true);
let host = FakeHost::down();
let id = node_id();
let (result, log) = logging(run_swap(&host, &coord, "alice", id)).await;
result.expect("drift is not a refusal");
// The pins of the commit the closure came from are read before the apply
// and recorded after it, in place of the ones this hive chose; the
// bookkeeping node then marks those.
run_swap(&host, &coord, "alice", id).await.expect("swap ok");
assert_eq!(
host.calls(),
[
"recorded_rev alice".to_owned(),
"closure_record alice".to_owned(),
format!("realise alice {CLOSURE}"),
format!("pins_at alice {OTHER_REV}"),
format!("pins_at alice {REV}"),
format!("pins_at alice {OTHER_REV}"),
format!("swap_update alice {CLOSURE} node={}", id.get()),
format!("record_meta alice {OTHER_REV}"),
]
host.calls().last().map(String::as_str),
Some(format!("mark_deployed alice {CLOSURE}").as_str()),
"{:?}",
host.calls()
);
let host = FakeHost::down().failing("swap_update");
run_swap(&host, &coord, "alice", node_id())
.await
.expect_err("apply fails");
assert!(
!host.calls().iter().any(|c| c.starts_with("mark_deployed")),
"{:?}",
host.calls()
);
assert_eq!(log.at(tracing::Level::WARN).len(), 1);
assert!(log.at(tracing::Level::ERROR).is_empty());
}
/// Pins that can't be read, or an apply that fails, leave the record alone.
/// The container runs the closure, so a marker that cannot be written is a
/// warning and not a failed swap.
#[tokio::test]
async fn a_drifted_swap_records_pins_only_for_an_apply_that_happened() {
for failing in ["pins_at", "swap_update"] {
let (_dir, coord) = coordinator();
let host = FakeHost::down().built(OTHER_REV, true).failing(failing);
run_swap(&host, &coord, "alice", node_id())
.await
.expect_err(failing);
let calls = host.calls();
assert!(
!calls.iter().any(|c| c.starts_with("record_meta")),
"{failing}: {calls:?}"
);
assert_eq!(
calls.iter().any(|c| c.starts_with("swap_update")),
failing == "swap_update",
"{failing}: {calls:?}"
);
}
async fn a_marker_that_cannot_be_written_does_not_fail_the_swap() {
let (_dir, coord) = coordinator();
let host = FakeHost::down().failing("mark_deployed");
let (result, log) = logging(run_swap(&host, &coord, "alice", node_id())).await;
result.expect("swap ok");
assert_eq!(log.at(tracing::Level::WARN).len(), 1);
}
/// A failed update fails the node and nothing else: the swap never restarts or
@ -611,21 +489,8 @@ async fn a_failed_swap_reports_the_error_and_touches_no_container() {
.expect_err("update failure propagates");
assert!(format!("{err:#}").contains("update failed"), "{err:#}");
let calls = host.calls();
assert_eq!(calls.len(), 4, "{calls:?}");
assert!(calls[3].starts_with("swap_update alice"), "{calls:?}");
}
#[tokio::test]
async fn swap_without_a_recorded_rev_never_updates() {
let (_dir, coord) = coordinator();
let host = FakeHost {
rev: None,
..FakeHost::down()
};
run_swap(&host, &coord, "alice", node_id())
.await
.expect_err("nothing to swap to");
assert_eq!(host.calls(), ["recorded_rev alice"]);
assert_eq!(calls.len(), 3, "{calls:?}");
assert!(calls[2].starts_with("swap_update alice"), "{calls:?}");
}
// --- Stop / StopForUpdate ---------------------------------------------------
@ -851,7 +716,7 @@ async fn pause_drain_times_out_ok_and_clears_pending() {
// --- MetaSync ---------------------------------------------------------------
#[tokio::test]
async fn meta_sync_prepares_then_resolves_and_records_last() {
async fn meta_sync_only_prepares_the_rebuild() {
let host = FakeHost::up();
run_meta_sync(&host, "alice").await.expect("ok");
assert_eq!(
@ -860,8 +725,6 @@ async fn meta_sync_prepares_then_resolves_and_records_last() {
"refresh_archetype alice".to_owned(),
"prepare_rebuild_dirs alice".to_owned(),
"refresh_agent_set".to_owned(),
"resolve_meta alice".to_owned(),
format!("record_meta alice {REV}"),
]
);
}
@ -870,17 +733,12 @@ async fn meta_sync_prepares_then_resolves_and_records_last() {
async fn meta_sync_keeps_going_when_the_archetype_read_fails() {
let host = FakeHost::up().failing("refresh_archetype");
run_meta_sync(&host, "alice").await.expect("non-fatal");
assert_eq!(host.calls().len(), 5);
assert_eq!(host.calls().len(), 3);
}
#[tokio::test]
async fn meta_sync_stops_at_the_first_fatal_step() {
for (fail, calls) in [
("prepare_rebuild_dirs", 2),
("refresh_agent_set", 3),
("resolve_meta", 4),
("record_meta", 5),
] {
for (fail, calls) in [("prepare_rebuild_dirs", 2), ("refresh_agent_set", 3)] {
let host = FakeHost::up().failing(fail);
run_meta_sync(&host, "alice")
.await
@ -913,11 +771,10 @@ async fn create_downloads_the_swarms_closure_and_creates_from_it() {
assert_eq!(
host.calls(),
[
"resolve_meta alice".to_owned(),
"closure_record alice".to_owned(),
format!("realise alice {CLOSURE}"),
format!("create_only alice {CLOSURE}"),
format!("mark_deployed alice {REV}"),
format!("mark_deployed alice {CLOSURE}"),
]
);
assert!(
@ -952,39 +809,11 @@ async fn a_create_without_a_closure_is_refused_and_logs_an_error() {
}
}
#[tokio::test]
async fn create_uses_a_closure_built_from_another_commit_with_a_warning() {
let host = FakeHost::up().built(OTHER_REV, true);
let (result, log) = logging(run_create(&host, "alice")).await;
result.expect("drift is not a refusal");
let calls = host.calls();
// Created from the swarm's closure, and marked as the commit it was built
// from, not the one this hive chose.
assert_eq!(
calls,
[
"resolve_meta alice".to_owned(),
"closure_record alice".to_owned(),
format!("realise alice {CLOSURE}"),
format!("pins_at alice {OTHER_REV}"),
format!("pins_at alice {REV}"),
format!("pins_at alice {OTHER_REV}"),
format!("create_only alice {CLOSURE}"),
format!("mark_deployed alice {OTHER_REV}"),
]
);
assert_eq!(log.at(tracing::Level::WARN).len(), 1);
assert!(log.at(tracing::Level::ERROR).is_empty());
}
#[tokio::test]
async fn a_failed_create_is_never_marked_deployed() {
let host = FakeHost::up().failing("create_only");
run_create(&host, "alice").await.expect_err("create fails");
assert_eq!(host.calls().len(), 4);
let host = FakeHost::up().failing("resolve_meta");
run_create(&host, "alice").await.expect_err("resolve fails");
assert_eq!(host.calls(), ["resolve_meta alice"]);
assert_eq!(host.calls().len(), 3);
}
// --- Start ------------------------------------------------------------------
@ -1102,38 +931,9 @@ async fn destroy_bookkeeping_continues_past_a_failed_refresh() {
// --- RebuildBookkeeping / EmitRebuilt ----------------------------------------
#[tokio::test]
async fn rebuild_bookkeeping_marks_the_recorded_build_deployed() {
async fn rebuild_bookkeeping_rescans_without_touching_the_host() {
let (_dir, coord) = coordinator();
let host = FakeHost::up();
run_rebuild_bookkeeping(&host, &coord, "alice")
.await
.expect("ok");
assert_eq!(
host.calls(),
[
"recorded_build alice".to_owned(),
format!("mark_deployed alice {REV}")
]
);
}
#[tokio::test]
async fn rebuild_bookkeeping_without_a_recording_marks_nothing() {
let (_dir, coord) = coordinator();
let host = FakeHost::up().failing("recorded_build");
run_rebuild_bookkeeping(&host, &coord, "alice")
.await
.expect("ok");
assert_eq!(host.calls(), ["recorded_build alice"]);
}
#[tokio::test]
async fn a_failed_marker_write_does_not_fail_the_bookkeeping_node() {
let (_dir, coord) = coordinator();
let host = FakeHost::up().failing("mark_deployed");
run_rebuild_bookkeeping(&host, &coord, "alice")
.await
.expect("warn only");
run_rebuild_bookkeeping(&coord).await.expect("ok");
}
#[tokio::test]
@ -1234,30 +1034,21 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() {
"refresh_archetype alice",
"prepare_rebuild_dirs alice",
"refresh_agent_set",
"resolve_meta alice",
"record_meta alice",
],
),
(
alice(|agent| NodeKind::Prebuild { agent }),
&[
"is_running alice",
"recorded_rev alice",
"closure_record alice",
],
&["is_running alice", "closure_record alice"],
),
(
alice(|agent| NodeKind::Swap { agent }),
&[
"recorded_rev alice",
"closure_record alice",
"swap_update alice",
"mark_deployed alice",
],
),
(
alice(|agent| NodeKind::RebuildBookkeeping { agent }),
&["recorded_build alice", "mark_deployed alice"],
),
(alice(|agent| NodeKind::RebuildBookkeeping { agent }), &[]),
(
alice(|agent| NodeKind::Provision { agent }),
&["provision alice"],
@ -1265,7 +1056,6 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() {
(
alice(|agent| NodeKind::Create { agent }),
&[
"resolve_meta alice",
"closure_record alice",
"create_only alice",
"mark_deployed alice",