mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

remove the hive's local build of agent closures

Browse source 
A rebuild or create downloads the closure the swarm built and applies it; with none to deploy it fails and changes nothing. hive-priv takes a store path instead of an internal/meta commit and builds nothing.
This commit is contained in:
atlas 2026-10-29 22:17:07 +01:00
commit 4590b04cac
1 changed files with 632 additions and 1351 deletions
Download patch file Download diff file Expand all files Collapse all files

521
hive-c0re/src/job_queue/exec/tests.rs
View file

@ -11,16 +11,17 @@ use hive_jobq::NodeId;
use hive_jobq::scheduler::Scheduler;
use super::{
Host, dispatch, drift, run_create, run_destroy_bookkeeping, run_destroy_container, run_drain,
Host, dispatch, run_create, run_destroy_bookkeeping, run_destroy_container, run_drain,
run_emit_rebuilt, run_forge_sweep, run_knowledge_pull, run_meta_sync, run_pause_drain,
run_pause_signal, run_prebuild, run_provision, run_purge_state, run_rebuild_bookkeeping,
run_reconcile, run_set_wanted, run_signal, run_start, run_stop, run_stop_for_update, run_swap,
run_wanted_pull, run_write_dropin, substitute,
run_wanted_pull, run_write_dropin,
};
use crate::coordinator::{AgentPaths, Coordinator, HiveEnv};
use crate::job_queue::model::NodeKind;
use crate::meta_rev::MetaBuild;
use crate::power::Wanted;
use swarm_wire::agent_closures::AgentClosure;
const REV: &str = "2222222222222222222222222222222222222222";
const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-alice";
@ -30,18 +31,12 @@ const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-
struct FakeHost {
running: bool,
rev: Option<String>,
/// The store path the swarm's record names, when it has a pushed build of
/// `REV`; `None` reads as no record.
closure: Option<String>,
/// A failing `realise` says nix refused the path for want of a signature.
refusal: bool,
/// The toplevel this hive's evaluation produces; `None` is `CLOSURE`.
evaluates: Option<String>,
/// What the swarm's builder recorded for the agent; `None` reads as no
/// record.
built: Option<AgentClosure>,
/// Store paths a download put in the store.
realised: Mutex<Vec<String>>,
/// Store paths a prebuild had to build because the store lacked them.
builds: Mutex<Vec<String>>,
/// Names of the calls that fail (`kill`, `realise`, `prebuild`, `swap_update`,
/// Names of the calls that fail (`kill`, `realise`, `closure_record`, `swap_update`,
/// `set_paused`, ...). `recorded_build` fails by finding nothing.
failing: Vec<&'static str>,
/// The sweeps and pulls never finish, so only a deadline ends them.
@ -56,6 +51,7 @@ impl FakeHost {
Self {
running: true,
rev: Some(REV.to_owned()),
built: Some(closure(REV, true)),
..Self::default()
}
}
@ -64,12 +60,19 @@ impl FakeHost {
Self {
running: false,
rev: Some(REV.to_owned()),
built: Some(closure(REV, true)),
..Self::default()
}
}
fn with_closure(mut self) -> Self {
self.closure = Some(CLOSURE.to_owned());
/// The builder recorded `meta_rev` for the agent, pushed or not.
fn built(mut self, meta_rev: &str, cached: bool) -> Self {
self.built = Some(closure(meta_rev, cached));
self
}
fn without_record(mut self) -> Self {
self.built = None;
self
}
@ -98,10 +101,6 @@ impl FakeHost {
fn calls(&self) -> Vec<String> {
self.calls.lock().unwrap().clone()
}
fn builds(&self) -> Vec<String> {
self.builds.lock().unwrap().clone()
}
}
impl Host for FakeHost {
@ -125,54 +124,29 @@ impl Host for FakeHost {
.ok_or_else(|| anyhow::anyhow!("{name}: no internal/meta commit was recorded"))
}
async fn cached_closure(&self, name: &str, rev: &str) -> anyhow::Result<String> {
self.record(format!("cached_closure {name} {rev}"));
self.closure
.clone()
.ok_or_else(|| anyhow::anyhow!("no-record: no build is recorded for the agent"))
async fn closure_record(&self, name: &str) -> anyhow::Result<Option<AgentClosure>> {
self.step("closure_record", format!("closure_record {name}"))?;
Ok(self.built.clone())
}
async fn realise_closure(&self, name: &str, store_path: &str) -> anyhow::Result<()> {
self.record(format!("realise {name} {store_path}"));
if self.fails("realise") {
if self.refusal {
anyhow::bail!(
"not available: {}: lacks a signature",
hive_priv_sock::SIGNATURE_REFUSED
);
}
anyhow::bail!("not available: Couldn't connect to server");
}
self.realised.lock().unwrap().push(store_path.to_owned());
Ok(())
}
async fn prebuild_toplevel(
&self,
name: &str,
rev: &str,
node_id: u64,
) -> anyhow::Result<Option<String>> {
self.record(format!("prebuild {name} {rev} node={node_id}"));
if self.fails("prebuild") {
anyhow::bail!("prebuild failed");
}
let path = self.evaluates.clone().unwrap_or_else(|| CLOSURE.to_owned());
if !self.realised.lock().unwrap().contains(&path) {
self.builds.lock().unwrap().push(path.clone());
}
Ok(Some(path))
}
async fn swap_update(
&self,
name: &str,
_hive: &HiveEnv,
_paths: &AgentPaths,
rev: &str,
toplevel: &str,
node_id: u64,
) -> anyhow::Result<()> {
self.record(format!("swap_update {name} {rev} node={node_id}"));
self.record(format!("swap_update {name} {toplevel} node={node_id}"));
if self.fails("swap_update") {
anyhow::bail!("nixos-container update failed");
}
@ -231,8 +205,8 @@ impl Host for FakeHost {
self.step("provision", format!("provision {name}"))
}
async fn create_only(&self, name: &str, rev: &str) -> anyhow::Result<()> {
self.step("create_only", format!("create_only {name} {rev}"))
async fn create_only(&self, name: &str, toplevel: &str) -> anyhow::Result<()> {
self.step("create_only", format!("create_only {name} {toplevel}"))
}
async fn write_dropins(
@ -287,6 +261,17 @@ impl Host for FakeHost {
}
}
/// The builder's record of a closure built from `internal/meta` at `meta_rev`.
fn closure(meta_rev: &str, cached: bool) -> AgentClosure {
AgentClosure {
meta_rev: meta_rev.to_owned(),
store_path: CLOSURE.to_owned(),
system: "x86_64-linux".to_owned(),
built_unix: 1,
cached,
}
}
fn build() -> MetaBuild {
MetaBuild {
rev: REV.to_owned(),
@ -323,188 +308,185 @@ fn node_id() -> NodeId {
// --- Prebuild ---------------------------------------------------------------
const OTHER_REV: &str = "3333333333333333333333333333333333333333";
/// What a block logged, level and text: the message and each field.
#[derive(Clone, Default)]
struct Logged(Arc<Mutex<Vec<(tracing::Level, String)>>>);
impl tracing::Subscriber for Logged {
fn enabled(&self, _: &tracing::Metadata<'_>) -> bool {
true
}
fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id {
tracing::span::Id::from_u64(1)
}
fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {}
fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {}
fn event(&self, event: &tracing::Event<'_>) {
use std::fmt::Write as _;
let mut text = String::new();
event.record(
&mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| {
let _ = write!(text, "{}={value:?} ", field.name());
},
);
self.0
.lock()
.unwrap()
.push((*event.metadata().level(), text));
}
fn enter(&self, _: &tracing::span::Id) {}
fn exit(&self, _: &tracing::span::Id) {}
}
impl Logged {
fn at(&self, level: tracing::Level) -> Vec<String> {
self.0
.lock()
.unwrap()
.iter()
.filter(|(l, _)| *l == level)
.map(|(_, text)| text.clone())
.collect()
}
}
/// `fut`'s output and what it logged. The test runtime is single-threaded, so
/// the subscriber set here sees every event the future emits.
async fn logging<T>(fut: impl std::future::Future<Output = T>) -> (T, Logged) {
let logged = Logged::default();
let _guard = tracing::subscriber::set_default(logged.clone());
let out = fut.await;
(out, logged)
}
/// Every way the swarm's closure for `alice` cannot be had, with the text the
/// error log must carry and the host calls that must not happen.
fn without_a_closure() -> [(&'static str, FakeHost, &'static str); 4] {
[
("no record", FakeHost::up().without_record(), "no build"),
(
"not pushed",
FakeHost::up().built(REV, false),
"did not push",
),
(
"download fails",
FakeHost::up().failing("realise"),
"Couldn't connect",
),
(
"record unreadable",
FakeHost::up().failing("closure_record"),
"closure_record failed",
),
]
}
#[tokio::test]
async fn prebuild_warms_the_recorded_rev_when_the_container_is_up() {
async fn prebuild_downloads_the_swarms_closure_when_the_container_is_up() {
let host = FakeHost::up();
let id = node_id();
run_prebuild(&host, "alice", id).await.expect("prebuild ok");
let (result, log) = logging(run_prebuild(&host, "alice")).await;
result.expect("prebuild ok");
assert_eq!(
host.calls(),
[
"is_running alice".to_owned(),
"recorded_rev alice".to_owned(),
format!("cached_closure alice {REV}"),
format!("prebuild alice {REV} node={}", id.get()),
]
);
}
/// The download is followed by the local build either way. When this hive
/// evaluates the path the swarm built, the build finds it in the store.
#[tokio::test]
async fn a_downloaded_closure_the_hive_also_evaluates_is_found_not_built() {
let host = FakeHost::up().with_closure();
let id = node_id();
run_prebuild(&host, "alice", id).await.expect("prebuild ok");
assert_eq!(
host.calls(),
[
"is_running alice".to_owned(),
"recorded_rev alice".to_owned(),
format!("cached_closure alice {REV}"),
"closure_record alice".to_owned(),
format!("realise alice {CLOSURE}"),
format!("prebuild alice {REV} node={}", id.get()),
]
);
assert_eq!(host.builds(), Vec::<String>::new());
}
/// When it evaluates another path, the build happens in `Prebuild`, after the
/// check that the agent is up and before `Swap` could stop it.
#[tokio::test]
async fn a_downloaded_closure_the_hive_does_not_evaluate_is_built_before_the_swap() {
let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-alice";
let host = FakeHost {
evaluates: Some(other.to_owned()),
..FakeHost::up().with_closure()
};
let id = node_id();
run_prebuild(&host, "alice", id).await.expect("prebuild ok");
let calls = host.calls();
assert_eq!(calls[0], "is_running alice", "{calls:?}");
assert_eq!(
calls.last().map(String::as_str),
Some(format!("prebuild alice {REV} node={}", id.get()).as_str()),
"{calls:?}"
);
assert_eq!(host.builds(), [other.to_owned()]);
}
#[test]
fn only_a_different_path_is_drift() {
assert_eq!(drift(CLOSURE, Some(CLOSURE)), None);
let note = drift(CLOSURE, Some("/nix/store/other")).expect("drift");
let info = log.at(tracing::Level::INFO);
assert!(
note.contains(CLOSURE) && note.contains("/nix/store/other"),
"{note}"
info.iter()
.any(|l| l.contains("prebuild: substituted") && l.contains(CLOSURE)),
"{info:?}"
);
assert!(drift(CLOSURE, None).expect("no path").contains(CLOSURE));
assert!(log.at(tracing::Level::WARN).is_empty());
assert!(log.at(tracing::Level::ERROR).is_empty());
}
/// No closure to deploy fails the node before `StopForUpdate`, so the running
/// container is untouched, and the log names the agent and why.
#[tokio::test]
async fn a_failed_download_builds_locally() {
let host = FakeHost::up().with_closure().failing("realise");
let id = node_id();
run_prebuild(&host, "alice", id)
.await
.expect("the build covers it");
let calls = host.calls();
assert_eq!(calls[3], format!("realise alice {CLOSURE}"), "{calls:?}");
assert_eq!(
calls[4],
format!("prebuild alice {REV} node={}", id.get()),
"{calls:?}"
);
assert_eq!(calls.len(), 5, "{calls:?}");
async fn a_rebuild_without_a_closure_fails_the_prebuild_and_logs_an_error() {
for (case, host, why) in without_a_closure() {
let (result, log) = logging(run_prebuild(&host, "alice")).await;
let err = result.expect_err(case);
assert!(format!("{err:#}").starts_with("alice: "), "{case}: {err:#}");
let errors = log.at(tracing::Level::ERROR);
assert_eq!(errors.len(), 1, "{case}: {errors:?}");
assert!(
errors[0].contains("alice") && errors[0].contains(why),
"{case}: {errors:?}"
);
let calls = host.calls();
assert!(
!calls.iter().any(|c| c.starts_with("kill")),
"{case}: {calls:?}"
);
assert!(
!calls
.iter()
.any(|c| c.starts_with("swap_update") || c.starts_with("create_only")),
"{case}: {calls:?}"
);
}
}
/// The fallback is the build, so a build that fails still fails the node.
/// The builder's record is not for the commit this hive chose: its closure is
/// deployed and the difference is a warning.
#[tokio::test]
async fn a_failed_download_and_a_failed_build_fail_the_node() {
let host = FakeHost::up()
.with_closure()
.failing("realise")
.failing("prebuild");
let err = run_prebuild(&host, "alice", node_id())
.await
.expect_err("nothing produced the closure");
assert!(format!("{err:#}").contains("prebuild failed"), "{err:#}");
async fn a_closure_built_from_another_commit_is_deployed_with_a_warning() {
let host = FakeHost::up().built(OTHER_REV, true);
let (result, log) = logging(run_prebuild(&host, "alice")).await;
result.expect("drift is not a refusal");
assert!(
host.calls().contains(&format!("realise alice {CLOSURE}")),
"{:?}",
host.calls()
);
let warnings = log.at(tracing::Level::WARN);
assert_eq!(warnings.len(), 1, "{warnings:?}");
assert!(
[REV, OTHER_REV, CLOSURE]
.iter()
.all(|part| warnings[0].contains(part)),
"{warnings:?}"
);
assert!(log.at(tracing::Level::ERROR).is_empty());
}
#[tokio::test]
async fn a_stopped_container_looks_up_nothing() {
let host = FakeHost::down().with_closure();
run_prebuild(&host, "alice", node_id())
.await
.expect("skipped");
assert_eq!(host.calls(), ["is_running alice"]);
}
/// The error log is for a path the cache served unsigned, not for a cache that
/// is down or a missing record.
#[tokio::test]
async fn only_a_signature_refusal_is_flagged() {
let down = substitute(
&FakeHost::up().with_closure().failing("realise"),
"alice",
REV,
)
.await
.expect_err("down");
assert!(!down.signature_refused, "{}", down.reason);
assert!(down.reason.contains("Couldn't connect"), "{}", down.reason);
let no_record = substitute(&FakeHost::up(), "alice", REV)
.await
.expect_err("no record");
assert!(!no_record.signature_refused);
assert!(
no_record.reason.starts_with("no-record"),
"{}",
no_record.reason
);
let refused = FakeHost {
refusal: true,
..FakeHost::up().with_closure().failing("realise")
};
let refused = substitute(&refused, "alice", REV)
.await
.expect_err("refused");
assert!(refused.signature_refused, "{}", refused.reason);
let ok = substitute(&FakeHost::up().with_closure(), "alice", REV).await;
assert_eq!(ok.ok().as_deref(), Some(CLOSURE));
}
#[tokio::test]
async fn prebuild_is_skipped_when_the_container_is_down() {
let host = FakeHost::down();
run_prebuild(&host, "alice", node_id())
.await
.expect("a skipped prebuild is ok");
run_prebuild(&host, "alice").await.expect("skipped");
assert_eq!(host.calls(), ["is_running alice"]);
}
#[tokio::test]
async fn prebuild_without_a_recorded_rev_fails_before_building() {
async fn prebuild_without_a_recorded_rev_fails_before_looking_up_a_closure() {
let host = FakeHost {
rev: None,
..FakeHost::up()
};
let err = run_prebuild(&host, "alice", node_id())
let err = run_prebuild(&host, "alice")
.await
.expect_err("nothing to build from");
.expect_err("nothing to deploy");
assert!(
format!("{err:#}").contains("no internal/meta commit"),
"{err:#}"
);
assert!(!host.calls().iter().any(|c| c.starts_with("prebuild")));
}
#[tokio::test]
async fn prebuild_failure_fails_the_node() {
let host = FakeHost::up().failing("prebuild");
let err = run_prebuild(&host, "alice", node_id())
.await
.expect_err("build failure propagates");
assert!(format!("{err:#}").contains("prebuild failed"), "{err:#}");
assert!(!host.calls().iter().any(|c| c.starts_with("closure_record")));
}
// --- Swap -------------------------------------------------------------------
#[tokio::test]
async fn swap_updates_at_the_recorded_rev() {
async fn swap_applies_the_swarms_closure() {
let (_dir, coord) = coordinator();
let host = FakeHost::down();
let id = node_id();
@ -513,11 +495,50 @@ async fn swap_updates_at_the_recorded_rev() {
host.calls(),
[
"recorded_rev alice".to_owned(),
format!("swap_update alice {REV} node={}", id.get()),
"closure_record alice".to_owned(),
format!("realise alice {CLOSURE}"),
format!("swap_update alice {CLOSURE} node={}", id.get()),
]
);
}
/// A swap with no closure to deploy never updates: the container stays as it
/// was, and the log names the agent and why.
#[tokio::test]
async fn a_swap_without_a_closure_never_updates_and_logs_an_error() {
for (case, host, why) in without_a_closure() {
let (_dir, coord) = coordinator();
let (result, log) = logging(run_swap(&host, &coord, "alice", node_id())).await;
result.expect_err(case);
let errors = log.at(tracing::Level::ERROR);
assert_eq!(errors.len(), 1, "{case}: {errors:?}");
assert!(
errors[0].contains("alice") && errors[0].contains(why),
"{case}: {errors:?}"
);
assert!(
!host.calls().iter().any(|c| c.starts_with("swap_update")),
"{case}: {:?}",
host.calls()
);
}
}
#[tokio::test]
async fn swap_applies_a_closure_built_from_another_commit_with_a_warning() {
let (_dir, coord) = coordinator();
let host = FakeHost::down().built(OTHER_REV, true);
let id = node_id();
let (result, log) = logging(run_swap(&host, &coord, "alice", id)).await;
result.expect("drift is not a refusal");
assert_eq!(
host.calls().last(),
Some(&format!("swap_update alice {CLOSURE} node={}", id.get()))
);
assert_eq!(log.at(tracing::Level::WARN).len(), 1);
assert!(log.at(tracing::Level::ERROR).is_empty());
}
/// A failed update fails the node and nothing else: the swap never restarts or
/// stops the container itself — recovery is the DAG's tail `Reconcile`.
#[tokio::test]
@ -529,8 +550,8 @@ async fn a_failed_swap_reports_the_error_and_touches_no_container() {
.expect_err("update failure propagates");
assert!(format!("{err:#}").contains("update failed"), "{err:#}");
let calls = host.calls();
assert_eq!(calls.len(), 2, "{calls:?}");
assert!(calls[1].starts_with("swap_update alice"), "{calls:?}");
assert_eq!(calls.len(), 4, "{calls:?}");
assert!(calls[3].starts_with("swap_update alice"), "{calls:?}");
}
#[tokio::test]
@ -824,84 +845,73 @@ async fn provision_and_write_dropin_forward_and_propagate_failure() {
}
#[tokio::test]
async fn create_without_a_record_builds_locally_at_the_resolved_rev() {
async fn create_downloads_the_swarms_closure_and_creates_from_it() {
let host = FakeHost::up();
run_create(&host, "alice", node_id()).await.expect("ok");
let (result, log) = logging(run_create(&host, "alice")).await;
result.expect("ok");
assert_eq!(
host.calls(),
[
"resolve_meta alice".to_owned(),
format!("cached_closure alice {REV}"),
format!("create_only alice {REV}"),
format!("mark_deployed alice {REV}"),
]
);
}
#[tokio::test]
async fn create_downloads_a_cached_closure_and_builds_nothing() {
let host = FakeHost::up().with_closure();
let id = node_id();
run_create(&host, "alice", id).await.expect("ok");
assert_eq!(
host.calls(),
[
"resolve_meta alice".to_owned(),
format!("cached_closure alice {REV}"),
"closure_record alice".to_owned(),
format!("realise alice {CLOSURE}"),
format!("prebuild alice {REV} node={}", id.get()),
format!("create_only alice {REV}"),
format!("create_only alice {CLOSURE}"),
format!("mark_deployed alice {REV}"),
]
);
assert_eq!(host.builds(), Vec::<String>::new());
}
#[tokio::test]
async fn create_builds_locally_when_the_download_fails() {
let host = FakeHost::up().with_closure().failing("realise");
run_create(&host, "alice", node_id()).await.expect("ok");
assert_eq!(
host.calls(),
[
"resolve_meta alice".to_owned(),
format!("cached_closure alice {REV}"),
format!("realise alice {CLOSURE}"),
format!("create_only alice {REV}"),
format!("mark_deployed alice {REV}"),
]
assert!(
log.at(tracing::Level::INFO)
.iter()
.any(|l| l.contains("create: substituted") && l.contains(CLOSURE))
);
assert!(log.at(tracing::Level::WARN).is_empty());
}
/// A create with no closure to deploy is refused: no container is made, none is
/// marked deployed, and the log names the agent and why.
#[tokio::test]
async fn a_create_without_a_closure_is_refused_and_logs_an_error() {
for (case, host, why) in without_a_closure() {
let (result, log) = logging(run_create(&host, "alice")).await;
let err = result.expect_err(case);
assert!(format!("{err:#}").starts_with("alice: "), "{case}: {err:#}");
let errors = log.at(tracing::Level::ERROR);
assert_eq!(errors.len(), 1, "{case}: {errors:?}");
assert!(
errors[0].contains("alice") && errors[0].contains(why),
"{case}: {errors:?}"
);
let calls = host.calls();
assert!(
!calls
.iter()
.any(|c| c.starts_with("create_only") || c.starts_with("mark_deployed")),
"{case}: {calls:?}"
);
}
}
#[tokio::test]
async fn create_builds_the_evaluated_path_when_it_differs_from_the_download() {
let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-alice";
let host = FakeHost {
evaluates: Some(other.to_owned()),
..FakeHost::up().with_closure()
};
run_create(&host, "alice", node_id()).await.expect("ok");
assert_eq!(host.builds(), [other.to_owned()]);
async fn create_uses_a_closure_built_from_another_commit_with_a_warning() {
let host = FakeHost::up().built(OTHER_REV, true);
let (result, log) = logging(run_create(&host, "alice")).await;
result.expect("drift is not a refusal");
let calls = host.calls();
assert!(
calls
.iter()
.any(|c| c == &format!("create_only alice {REV}")),
calls.contains(&format!("create_only alice {CLOSURE}")),
"{calls:?}"
);
assert_eq!(log.at(tracing::Level::WARN).len(), 1);
assert!(log.at(tracing::Level::ERROR).is_empty());
}
#[tokio::test]
async fn a_failed_create_is_never_marked_deployed() {
let host = FakeHost::up().failing("create_only");
run_create(&host, "alice", node_id())
.await
.expect_err("create fails");
assert_eq!(host.calls().len(), 3);
run_create(&host, "alice").await.expect_err("create fails");
assert_eq!(host.calls().len(), 4);
let host = FakeHost::up().failing("resolve_meta");
run_create(&host, "alice", node_id())
.await
.expect_err("resolve fails");
run_create(&host, "alice").await.expect_err("resolve fails");
assert_eq!(host.calls(), ["resolve_meta alice"]);
}
@ -1165,11 +1175,19 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() {
),
(
alice(|agent| NodeKind::Prebuild { agent }),
&["is_running alice", "recorded_rev alice", "prebuild alice"],
&[
"is_running alice",
"recorded_rev alice",
"closure_record alice",
],
),
(
alice(|agent| NodeKind::Swap { agent }),
&["recorded_rev alice", "swap_update alice"],
&[
"recorded_rev alice",
"closure_record alice",
"swap_update alice",
],
),
(
alice(|agent| NodeKind::RebuildBookkeeping { agent }),
@ -1183,6 +1201,7 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() {
alice(|agent| NodeKind::Create { agent }),
&[
"resolve_meta alice",
"closure_record alice",
"create_only alice",
"mark_deployed alice",
],