remove the hive's local build of agent closures
A rebuild or create downloads the closure the swarm built and applies it; with none to deploy it fails and changes nothing. hive-priv takes a store path instead of an internal/meta commit and builds nothing.
This commit is contained in:
atlas
parent
70538fb209
commit
4590b04cac
4 changed files with 632 additions and 1351 deletions
|
|
@ -9,6 +9,7 @@ use std::path::PathBuf;
|
|||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use swarm_wire::agent_closures::AgentClosure;
|
||||
|
||||
use hive_jobq::NodeId;
|
||||
|
||||
|
|
@ -34,7 +35,7 @@ pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from
|
|||
const PAUSE_ACK_TIMEOUT: std::time::Duration = GRACEFUL_STOP_TIMEOUT;
|
||||
|
||||
/// The container-state calls the executors below make on the host: systemd,
|
||||
/// `hive-priv`, the nix build, the `internal/meta` record. Production uses
|
||||
/// `hive-priv`, the swarm's closure record, the `internal/meta` record. Production uses
|
||||
/// [`RealHost`], which forwards to `lifecycle` / `meta_rev` / `Coordinator`
|
||||
/// unchanged; tests substitute a recording fake.
|
||||
pub(super) trait Host: Sync {
|
||||
|
|
@ -44,28 +45,19 @@ pub(super) trait Host: Sync {
|
|||
async fn kill(&self, name: &str) -> Result<()>;
|
||||
/// The `internal/meta` commit `MetaSync` recorded for the agent.
|
||||
fn recorded_rev(&self, name: &str) -> Result<String>;
|
||||
/// The store path of the closure the swarm built for the agent at the
|
||||
/// `internal/meta` commit `rev` and pushed to its cache. The error says why
|
||||
/// there is none.
|
||||
async fn cached_closure(&self, name: &str, rev: &str) -> Result<String>;
|
||||
/// The record the swarm's builder wrote for the agent; `None` when it wrote
|
||||
/// none.
|
||||
async fn closure_record(&self, name: &str) -> Result<Option<AgentClosure>>;
|
||||
/// Download `store_path` from the swarm's cache into the store, never
|
||||
/// building it.
|
||||
async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()>;
|
||||
/// Warm the agent's toplevel build from the `internal/meta` commit `rev`.
|
||||
/// The toplevel it printed, when it printed one.
|
||||
async fn prebuild_toplevel(
|
||||
&self,
|
||||
name: &str,
|
||||
rev: &str,
|
||||
node_id: u64,
|
||||
) -> Result<Option<String>>;
|
||||
/// Re-apply drop-ins, then `nixos-container update` at `rev`.
|
||||
/// Re-apply drop-ins, then point the container at the closure `toplevel`.
|
||||
async fn swap_update(
|
||||
&self,
|
||||
name: &str,
|
||||
hive: &HiveEnv,
|
||||
paths: &AgentPaths,
|
||||
rev: &str,
|
||||
toplevel: &str,
|
||||
node_id: u64,
|
||||
) -> Result<()>;
|
||||
/// Write or remove the agent's pause marker.
|
||||
|
|
@ -80,7 +72,8 @@ pub(super) trait Host: Sync {
|
|||
async fn refresh_agent_set(&self) -> Result<()>;
|
||||
/// Pin the `internal/meta` commit the agent builds from.
|
||||
async fn resolve_meta(&self, name: &str) -> Result<MetaBuild>;
|
||||
/// Record `build` as what `Prebuild` and `Swap` build from.
|
||||
/// Record `build` as the commit `Prebuild` and `Swap` expect the swarm's
|
||||
/// closure to be built from.
|
||||
fn record_meta(&self, name: &str, build: &MetaBuild) -> Result<()>;
|
||||
/// The build `MetaSync` recorded, if any.
|
||||
fn recorded_build(&self, name: &str) -> Option<MetaBuild>;
|
||||
|
|
@ -88,8 +81,8 @@ pub(super) trait Host: Sync {
|
|||
fn mark_deployed(&self, name: &str, build: &MetaBuild) -> Result<()>;
|
||||
/// First-spawn provisioning: repos, state subvolume, agent-set refresh.
|
||||
async fn provision(&self, name: &str, paths: &AgentPaths) -> Result<()>;
|
||||
/// `nixos-container create` at `rev`.
|
||||
async fn create_only(&self, name: &str, rev: &str) -> Result<()>;
|
||||
/// `nixos-container create` from the closure `toplevel`.
|
||||
async fn create_only(&self, name: &str, toplevel: &str) -> Result<()>;
|
||||
/// Write the nspawn and resource-limit drop-ins, then reload systemd.
|
||||
async fn write_dropins(&self, name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>;
|
||||
/// Start preamble, then start with the cold-start fallback.
|
||||
|
|
@ -128,33 +121,23 @@ impl Host for RealHost {
|
|||
Ok(crate::meta_rev::recorded_or_err(name)?.rev)
|
||||
}
|
||||
|
||||
async fn cached_closure(&self, name: &str, rev: &str) -> Result<String> {
|
||||
crate::attic::closure::path_for(name, rev).await
|
||||
async fn closure_record(&self, name: &str) -> Result<Option<AgentClosure>> {
|
||||
crate::attic::closure::record(name).await
|
||||
}
|
||||
|
||||
async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()> {
|
||||
crate::priv_client::realise_closure(name, store_path).await
|
||||
}
|
||||
|
||||
async fn prebuild_toplevel(
|
||||
&self,
|
||||
name: &str,
|
||||
rev: &str,
|
||||
node_id: u64,
|
||||
) -> Result<Option<String>> {
|
||||
let flake_ref = crate::meta_rev::flake_ref(name, rev);
|
||||
crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(node_id)).await
|
||||
}
|
||||
|
||||
async fn swap_update(
|
||||
&self,
|
||||
name: &str,
|
||||
hive: &HiveEnv,
|
||||
paths: &AgentPaths,
|
||||
rev: &str,
|
||||
toplevel: &str,
|
||||
node_id: u64,
|
||||
) -> Result<()> {
|
||||
crate::lifecycle::swap_update(name, hive, paths, rev, Some(node_id)).await
|
||||
crate::lifecycle::swap_update(name, hive, paths, toplevel, Some(node_id)).await
|
||||
}
|
||||
|
||||
async fn set_paused(&self, agent: &hive_types::Ident, paused: bool) -> Result<()> {
|
||||
|
|
@ -198,8 +181,8 @@ impl Host for RealHost {
|
|||
crate::lifecycle::provision_container(name, paths).await
|
||||
}
|
||||
|
||||
async fn create_only(&self, name: &str, rev: &str) -> Result<()> {
|
||||
crate::lifecycle::create_only(name, rev).await
|
||||
async fn create_only(&self, name: &str, toplevel: &str) -> Result<()> {
|
||||
crate::lifecycle::create_only(name, toplevel).await
|
||||
}
|
||||
|
||||
async fn write_dropins(&self, name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
||||
|
|
@ -300,11 +283,11 @@ async fn dispatch(
|
|||
// growth executors return what to grow rather than taking the builder.
|
||||
let result = match kind {
|
||||
NodeKind::MetaSync { .. } => run_meta_sync(host, agent).await,
|
||||
NodeKind::Prebuild { .. } => run_prebuild(host, agent, id).await,
|
||||
NodeKind::Prebuild { .. } => run_prebuild(host, agent).await,
|
||||
NodeKind::Swap { .. } => run_swap(host, coord, agent, id).await,
|
||||
NodeKind::RebuildBookkeeping { .. } => run_rebuild_bookkeeping(host, coord, agent).await,
|
||||
NodeKind::Provision { .. } => run_provision(host, agent).await,
|
||||
NodeKind::Create { .. } => run_create(host, agent, id).await,
|
||||
NodeKind::Create { .. } => run_create(host, agent).await,
|
||||
NodeKind::DestroyContainer { .. } => run_destroy_container(host, coord, agent).await,
|
||||
NodeKind::PurgeState { .. } => {
|
||||
run_purge_state(host, agent).await;
|
||||
|
|
@ -533,10 +516,11 @@ fn run_set_wanted(coord: &Arc<Coordinator>, agent: &str, up: bool) -> Result<()>
|
|||
}
|
||||
|
||||
/// The rebuild's preamble: runtime-dir prep, the agent-set refresh, and the
|
||||
/// `internal/meta` commit `Prebuild` and `Swap` build from.
|
||||
/// `internal/meta` commit `Prebuild` and `Swap` expect the swarm's closure to
|
||||
/// be built from.
|
||||
///
|
||||
/// Fails when `internal/meta` does not pin the agent's current config yet:
|
||||
/// there is no other flake to build it from.
|
||||
/// there is no commit to deploy.
|
||||
async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> {
|
||||
// Runs while the agent is still up — the runtime dir and MCP listener
|
||||
// already exist. Use the pure path accessor; no need to re-register the
|
||||
|
|
@ -550,98 +534,95 @@ async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> {
|
|||
}
|
||||
host.prepare_rebuild_dirs(name, &paths).await?;
|
||||
host.refresh_agent_set().await?;
|
||||
// Resolved once here so `Prebuild` and `Swap` build the same flake.
|
||||
// Resolved once here so `Prebuild` and `Swap` expect the same commit.
|
||||
let build = host.resolve_meta(name).await?;
|
||||
host.record_meta(name, &build)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Out-of-band toplevel build while the container keeps serving: warm
|
||||
/// `system.build.toplevel` so the later `Swap` hits cache and skips
|
||||
/// straight to the profile-swap, from the `internal/meta` commit the upstream
|
||||
/// `MetaSync` node recorded. The warm build is skipped when the
|
||||
/// container is already down: its only purpose is to shrink the swap's
|
||||
/// downtime window, so a stopped agent (no uptime to preserve) doesn't
|
||||
/// pay the double eval — `Swap` builds inline instead.
|
||||
///
|
||||
/// The swarm's cache is tried first: when the swarm built this agent at the
|
||||
/// recorded commit and pushed the closure, it is downloaded. The local build
|
||||
/// runs either way. After a download it is an evaluation and a no-op build when
|
||||
/// this hive evaluates the path the swarm built, and builds it, with the agent
|
||||
/// still up, when it doesn't; `Swap` never meets a build it didn't expect.
|
||||
async fn run_prebuild(host: &impl Host, name: &str, id: NodeId) -> Result<()> {
|
||||
// Warm the toplevel build only when the container is up — the whole
|
||||
// point of prebuild is to shrink the swap's downtime window. A
|
||||
// stopped agent has no uptime to preserve, so skip the (expensive)
|
||||
// eval and let the downstream `Swap` build inline.
|
||||
/// Download the swarm's closure for the agent while the container keeps
|
||||
/// serving, so a rebuild with no closure fails here, before `StopForUpdate`
|
||||
/// takes a healthy container down. `Swap` downloads the same closure again
|
||||
/// (a no-op once it is in the store) and applies it. Skipped when the container
|
||||
/// is already down: there is no uptime to protect, and `Swap` fails the same way.
|
||||
async fn run_prebuild(host: &impl Host, name: &str) -> Result<()> {
|
||||
if host.is_running(name).await {
|
||||
let rev = host.recorded_rev(name)?;
|
||||
let downloaded = match substitute(host, name, &rev).await {
|
||||
Ok(path) => Some(path),
|
||||
Err(fallback) => {
|
||||
let reason = &fallback.reason;
|
||||
if fallback.signature_refused {
|
||||
tracing::error!(%name, %rev, "prebuild: built locally: {reason}");
|
||||
} else {
|
||||
tracing::warn!(%name, %rev, "prebuild: built locally: {reason}");
|
||||
}
|
||||
None
|
||||
}
|
||||
};
|
||||
let evaluated = host.prebuild_toplevel(name, &rev, id.get()).await?;
|
||||
if let Some(path) = downloaded {
|
||||
match drift(&path, evaluated.as_deref()) {
|
||||
None => tracing::info!(%name, %rev, "prebuild: substituted {path}"),
|
||||
Some(note) => tracing::warn!(%name, %rev, "prebuild: {note}"),
|
||||
}
|
||||
}
|
||||
let path = closure_for(host, name, &rev, "prebuild").await?;
|
||||
tracing::info!(%name, %rev, "prebuild: substituted {path}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What to log when the closure downloaded for a rebuild isn't the one this
|
||||
/// hive's evaluation produced, which means the builder and the hive disagree
|
||||
/// about the agent's flake. `None` when they agree.
|
||||
fn drift(downloaded: &str, evaluated: Option<&str>) -> Option<String> {
|
||||
match evaluated {
|
||||
Some(path) if path == downloaded => None,
|
||||
Some(path) => Some(format!(
|
||||
"the downloaded closure {downloaded} is not the one this hive evaluates, {path}; built the latter locally"
|
||||
)),
|
||||
None => Some(format!(
|
||||
"downloaded {downloaded}, but the local build printed no path to compare it with"
|
||||
)),
|
||||
}
|
||||
/// Why the swarm's closure for an agent cannot be deployed.
|
||||
enum NoClosure {
|
||||
/// The builder has recorded no build of the agent.
|
||||
NoRecord,
|
||||
/// A build is recorded, but its closure cannot be had: the record could not
|
||||
/// be read, the builder did not push the closure, or the download failed.
|
||||
Unpullable(String),
|
||||
}
|
||||
|
||||
/// Why a rebuild or create builds instead of downloading the swarm's closure.
|
||||
struct LocalBuild {
|
||||
reason: String,
|
||||
/// The cache served a path no trusted key signed.
|
||||
signature_refused: bool,
|
||||
}
|
||||
|
||||
/// Download the closure the swarm built for `name` at `rev`; its store path.
|
||||
async fn substitute(host: &impl Host, name: &str, rev: &str) -> Result<String, LocalBuild> {
|
||||
let path = host
|
||||
.cached_closure(name, rev)
|
||||
/// The store path of the closure the swarm built for `name`, downloaded into
|
||||
/// the store. The deploy applies the swarm's closure whatever commit it was
|
||||
/// built from: when that is not the `rev` this hive chose, it is logged.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`without_closure`]'s refusal, after logging it.
|
||||
async fn closure_for(host: &impl Host, name: &str, rev: &str, step: &str) -> Result<String> {
|
||||
pull_closure(host, name, rev, step)
|
||||
.await
|
||||
.map_err(|e| LocalBuild {
|
||||
reason: format!("{e:#}"),
|
||||
signature_refused: false,
|
||||
})?;
|
||||
host.realise_closure(name, &path).await.map_err(|e| {
|
||||
let reason = format!("{e:#}");
|
||||
LocalBuild {
|
||||
signature_refused: reason.contains(hive_priv_sock::SIGNATURE_REFUSED),
|
||||
reason,
|
||||
}
|
||||
})?;
|
||||
Ok(path)
|
||||
.map_err(|why| without_closure(name, rev, step, &why))
|
||||
}
|
||||
|
||||
/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb),
|
||||
/// `nixos-container update`, then the post-rebuild bookkeeping tail
|
||||
async fn pull_closure(
|
||||
host: &impl Host,
|
||||
name: &str,
|
||||
rev: &str,
|
||||
step: &str,
|
||||
) -> Result<String, NoClosure> {
|
||||
let record = host
|
||||
.closure_record(name)
|
||||
.await
|
||||
.map_err(|e| NoClosure::Unpullable(format!("{e:#}")))?
|
||||
.ok_or(NoClosure::NoRecord)?;
|
||||
if !record.cached {
|
||||
return Err(NoClosure::Unpullable(
|
||||
"the builder did not push the closure".to_owned(),
|
||||
));
|
||||
}
|
||||
host.realise_closure(name, &record.store_path)
|
||||
.await
|
||||
.map_err(|e| NoClosure::Unpullable(format!("{e:#}")))?;
|
||||
if record.meta_rev != rev {
|
||||
tracing::warn!(
|
||||
%name, %rev, built_from = %record.meta_rev,
|
||||
"{step}: the swarm built {} from internal/meta {}, not the {rev} this hive chose; deploying the swarm's",
|
||||
record.store_path, record.meta_rev
|
||||
);
|
||||
}
|
||||
Ok(record.store_path)
|
||||
}
|
||||
|
||||
/// What a rebuild or create does when there is no closure to deploy: it
|
||||
/// fails, and nothing changes, so a running container keeps running as it is
|
||||
/// and a create makes none. The failure is logged at error level with the
|
||||
/// agent's name.
|
||||
///
|
||||
/// This is the one place that decides what an unpullable closure (the cache
|
||||
/// unreachable, a TLS error, a failed download) costs. A rebuild never builds
|
||||
/// locally on the hive.
|
||||
fn without_closure(name: &str, rev: &str, step: &str, why: &NoClosure) -> anyhow::Error {
|
||||
let reason = match why {
|
||||
NoClosure::NoRecord => "the swarm has recorded no build of this agent".to_owned(),
|
||||
NoClosure::Unpullable(e) => format!("the swarm's closure cannot be pulled: {e}"),
|
||||
};
|
||||
tracing::error!(%name, %rev, "{step}: {reason}; nothing was changed");
|
||||
anyhow::anyhow!("{name}: {reason}")
|
||||
}
|
||||
|
||||
/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), point the
|
||||
/// container at the swarm's closure, then the post-rebuild bookkeeping tail
|
||||
/// (rev marker, `Rebuilt` event, forge sync, kick, rescan).
|
||||
/// The recovery-start on failure is NOT here — the DAG's tail
|
||||
/// `Reconcile` runs after this node terminal ok *or* fail.
|
||||
|
|
@ -657,7 +638,11 @@ async fn run_swap(
|
|||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir)?;
|
||||
let rev = host.recorded_rev(name)?;
|
||||
let result = host.swap_update(name, &hive, &paths, &rev, id.get()).await;
|
||||
let toplevel = closure_for(host, name, &rev, "swap").await?;
|
||||
tracing::info!(%name, %rev, "swap: applying {toplevel}");
|
||||
let result = host
|
||||
.swap_update(name, &hive, &paths, &toplevel, id.get())
|
||||
.await;
|
||||
// On success the Ok-only bookkeeping tail (rev marker, forge
|
||||
// sync, kick, rescan, snapshot) runs in the sibling `RebuildBookkeeping` node,
|
||||
// which deps `AfterOk(Swap)`. On failure `RebuildBookkeeping` is cancel-cascaded
|
||||
|
|
@ -719,30 +704,13 @@ async fn run_provision(host: &impl Host, name: &str) -> Result<()> {
|
|||
/// node stays purely "create", not "create + start". Marks the agent as
|
||||
/// running the pins it was created from.
|
||||
///
|
||||
/// The swarm's cache is tried first, as in [`run_prebuild`]: when it holds the
|
||||
/// closure for the resolved commit it is downloaded, and the local evaluation
|
||||
/// that follows is a no-op build on a path match and builds the evaluated path
|
||||
/// on a mismatch. Without a cached closure the create builds it locally.
|
||||
async fn run_create(host: &impl Host, name: &str, id: NodeId) -> Result<()> {
|
||||
/// The container is created from the swarm's closure for the resolved commit;
|
||||
/// with none to deploy the create fails and makes no container.
|
||||
async fn run_create(host: &impl Host, name: &str) -> Result<()> {
|
||||
let build = host.resolve_meta(name).await?;
|
||||
match substitute(host, name, &build.rev).await {
|
||||
Ok(path) => {
|
||||
let evaluated = host.prebuild_toplevel(name, &build.rev, id.get()).await?;
|
||||
match drift(&path, evaluated.as_deref()) {
|
||||
None => tracing::info!(%name, rev = %build.rev, "create: substituted {path}"),
|
||||
Some(note) => tracing::warn!(%name, rev = %build.rev, "create: {note}"),
|
||||
}
|
||||
}
|
||||
Err(fallback) => {
|
||||
let reason = &fallback.reason;
|
||||
if fallback.signature_refused {
|
||||
tracing::error!(%name, rev = %build.rev, "create: built locally: {reason}");
|
||||
} else {
|
||||
tracing::warn!(%name, rev = %build.rev, "create: built locally: {reason}");
|
||||
}
|
||||
}
|
||||
}
|
||||
host.create_only(name, &build.rev).await?;
|
||||
let toplevel = closure_for(host, name, &build.rev, "create").await?;
|
||||
tracing::info!(%name, rev = %build.rev, "create: substituted {toplevel}");
|
||||
host.create_only(name, &toplevel).await?;
|
||||
host.mark_deployed(name, &build)?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,16 +11,17 @@ use hive_jobq::NodeId;
|
|||
use hive_jobq::scheduler::Scheduler;
|
||||
|
||||
use super::{
|
||||
Host, dispatch, drift, run_create, run_destroy_bookkeeping, run_destroy_container, run_drain,
|
||||
Host, dispatch, run_create, run_destroy_bookkeeping, run_destroy_container, run_drain,
|
||||
run_emit_rebuilt, run_forge_sweep, run_knowledge_pull, run_meta_sync, run_pause_drain,
|
||||
run_pause_signal, run_prebuild, run_provision, run_purge_state, run_rebuild_bookkeeping,
|
||||
run_reconcile, run_set_wanted, run_signal, run_start, run_stop, run_stop_for_update, run_swap,
|
||||
run_wanted_pull, run_write_dropin, substitute,
|
||||
run_wanted_pull, run_write_dropin,
|
||||
};
|
||||
use crate::coordinator::{AgentPaths, Coordinator, HiveEnv};
|
||||
use crate::job_queue::model::NodeKind;
|
||||
use crate::meta_rev::MetaBuild;
|
||||
use crate::power::Wanted;
|
||||
use swarm_wire::agent_closures::AgentClosure;
|
||||
|
||||
const REV: &str = "2222222222222222222222222222222222222222";
|
||||
const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-alice";
|
||||
|
|
@ -30,18 +31,12 @@ const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-
|
|||
struct FakeHost {
|
||||
running: bool,
|
||||
rev: Option<String>,
|
||||
/// The store path the swarm's record names, when it has a pushed build of
|
||||
/// `REV`; `None` reads as no record.
|
||||
closure: Option<String>,
|
||||
/// A failing `realise` says nix refused the path for want of a signature.
|
||||
refusal: bool,
|
||||
/// The toplevel this hive's evaluation produces; `None` is `CLOSURE`.
|
||||
evaluates: Option<String>,
|
||||
/// What the swarm's builder recorded for the agent; `None` reads as no
|
||||
/// record.
|
||||
built: Option<AgentClosure>,
|
||||
/// Store paths a download put in the store.
|
||||
realised: Mutex<Vec<String>>,
|
||||
/// Store paths a prebuild had to build because the store lacked them.
|
||||
builds: Mutex<Vec<String>>,
|
||||
/// Names of the calls that fail (`kill`, `realise`, `prebuild`, `swap_update`,
|
||||
/// Names of the calls that fail (`kill`, `realise`, `closure_record`, `swap_update`,
|
||||
/// `set_paused`, ...). `recorded_build` fails by finding nothing.
|
||||
failing: Vec<&'static str>,
|
||||
/// The sweeps and pulls never finish, so only a deadline ends them.
|
||||
|
|
@ -56,6 +51,7 @@ impl FakeHost {
|
|||
Self {
|
||||
running: true,
|
||||
rev: Some(REV.to_owned()),
|
||||
built: Some(closure(REV, true)),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
|
@ -64,12 +60,19 @@ impl FakeHost {
|
|||
Self {
|
||||
running: false,
|
||||
rev: Some(REV.to_owned()),
|
||||
built: Some(closure(REV, true)),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn with_closure(mut self) -> Self {
|
||||
self.closure = Some(CLOSURE.to_owned());
|
||||
/// The builder recorded `meta_rev` for the agent, pushed or not.
|
||||
fn built(mut self, meta_rev: &str, cached: bool) -> Self {
|
||||
self.built = Some(closure(meta_rev, cached));
|
||||
self
|
||||
}
|
||||
|
||||
fn without_record(mut self) -> Self {
|
||||
self.built = None;
|
||||
self
|
||||
}
|
||||
|
||||
|
|
@ -98,10 +101,6 @@ impl FakeHost {
|
|||
fn calls(&self) -> Vec<String> {
|
||||
self.calls.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
fn builds(&self) -> Vec<String> {
|
||||
self.builds.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Host for FakeHost {
|
||||
|
|
@ -125,54 +124,29 @@ impl Host for FakeHost {
|
|||
.ok_or_else(|| anyhow::anyhow!("{name}: no internal/meta commit was recorded"))
|
||||
}
|
||||
|
||||
async fn cached_closure(&self, name: &str, rev: &str) -> anyhow::Result<String> {
|
||||
self.record(format!("cached_closure {name} {rev}"));
|
||||
self.closure
|
||||
.clone()
|
||||
.ok_or_else(|| anyhow::anyhow!("no-record: no build is recorded for the agent"))
|
||||
async fn closure_record(&self, name: &str) -> anyhow::Result<Option<AgentClosure>> {
|
||||
self.step("closure_record", format!("closure_record {name}"))?;
|
||||
Ok(self.built.clone())
|
||||
}
|
||||
|
||||
async fn realise_closure(&self, name: &str, store_path: &str) -> anyhow::Result<()> {
|
||||
self.record(format!("realise {name} {store_path}"));
|
||||
if self.fails("realise") {
|
||||
if self.refusal {
|
||||
anyhow::bail!(
|
||||
"not available: {}: lacks a signature",
|
||||
hive_priv_sock::SIGNATURE_REFUSED
|
||||
);
|
||||
}
|
||||
anyhow::bail!("not available: Couldn't connect to server");
|
||||
}
|
||||
self.realised.lock().unwrap().push(store_path.to_owned());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn prebuild_toplevel(
|
||||
&self,
|
||||
name: &str,
|
||||
rev: &str,
|
||||
node_id: u64,
|
||||
) -> anyhow::Result<Option<String>> {
|
||||
self.record(format!("prebuild {name} {rev} node={node_id}"));
|
||||
if self.fails("prebuild") {
|
||||
anyhow::bail!("prebuild failed");
|
||||
}
|
||||
let path = self.evaluates.clone().unwrap_or_else(|| CLOSURE.to_owned());
|
||||
if !self.realised.lock().unwrap().contains(&path) {
|
||||
self.builds.lock().unwrap().push(path.clone());
|
||||
}
|
||||
Ok(Some(path))
|
||||
}
|
||||
|
||||
async fn swap_update(
|
||||
&self,
|
||||
name: &str,
|
||||
_hive: &HiveEnv,
|
||||
_paths: &AgentPaths,
|
||||
rev: &str,
|
||||
toplevel: &str,
|
||||
node_id: u64,
|
||||
) -> anyhow::Result<()> {
|
||||
self.record(format!("swap_update {name} {rev} node={node_id}"));
|
||||
self.record(format!("swap_update {name} {toplevel} node={node_id}"));
|
||||
if self.fails("swap_update") {
|
||||
anyhow::bail!("nixos-container update failed");
|
||||
}
|
||||
|
|
@ -231,8 +205,8 @@ impl Host for FakeHost {
|
|||
self.step("provision", format!("provision {name}"))
|
||||
}
|
||||
|
||||
async fn create_only(&self, name: &str, rev: &str) -> anyhow::Result<()> {
|
||||
self.step("create_only", format!("create_only {name} {rev}"))
|
||||
async fn create_only(&self, name: &str, toplevel: &str) -> anyhow::Result<()> {
|
||||
self.step("create_only", format!("create_only {name} {toplevel}"))
|
||||
}
|
||||
|
||||
async fn write_dropins(
|
||||
|
|
@ -287,6 +261,17 @@ impl Host for FakeHost {
|
|||
}
|
||||
}
|
||||
|
||||
/// The builder's record of a closure built from `internal/meta` at `meta_rev`.
|
||||
fn closure(meta_rev: &str, cached: bool) -> AgentClosure {
|
||||
AgentClosure {
|
||||
meta_rev: meta_rev.to_owned(),
|
||||
store_path: CLOSURE.to_owned(),
|
||||
system: "x86_64-linux".to_owned(),
|
||||
built_unix: 1,
|
||||
cached,
|
||||
}
|
||||
}
|
||||
|
||||
fn build() -> MetaBuild {
|
||||
MetaBuild {
|
||||
rev: REV.to_owned(),
|
||||
|
|
@ -323,188 +308,185 @@ fn node_id() -> NodeId {
|
|||
|
||||
// --- Prebuild ---------------------------------------------------------------
|
||||
|
||||
const OTHER_REV: &str = "3333333333333333333333333333333333333333";
|
||||
|
||||
/// What a block logged, level and text: the message and each field.
|
||||
#[derive(Clone, Default)]
|
||||
struct Logged(Arc<Mutex<Vec<(tracing::Level, String)>>>);
|
||||
|
||||
impl tracing::Subscriber for Logged {
|
||||
fn enabled(&self, _: &tracing::Metadata<'_>) -> bool {
|
||||
true
|
||||
}
|
||||
fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id {
|
||||
tracing::span::Id::from_u64(1)
|
||||
}
|
||||
fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {}
|
||||
fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {}
|
||||
fn event(&self, event: &tracing::Event<'_>) {
|
||||
use std::fmt::Write as _;
|
||||
|
||||
let mut text = String::new();
|
||||
event.record(
|
||||
&mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| {
|
||||
let _ = write!(text, "{}={value:?} ", field.name());
|
||||
},
|
||||
);
|
||||
self.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((*event.metadata().level(), text));
|
||||
}
|
||||
fn enter(&self, _: &tracing::span::Id) {}
|
||||
fn exit(&self, _: &tracing::span::Id) {}
|
||||
}
|
||||
|
||||
impl Logged {
|
||||
fn at(&self, level: tracing::Level) -> Vec<String> {
|
||||
self.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|(l, _)| *l == level)
|
||||
.map(|(_, text)| text.clone())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// `fut`'s output and what it logged. The test runtime is single-threaded, so
|
||||
/// the subscriber set here sees every event the future emits.
|
||||
async fn logging<T>(fut: impl std::future::Future<Output = T>) -> (T, Logged) {
|
||||
let logged = Logged::default();
|
||||
let _guard = tracing::subscriber::set_default(logged.clone());
|
||||
let out = fut.await;
|
||||
(out, logged)
|
||||
}
|
||||
|
||||
/// Every way the swarm's closure for `alice` cannot be had, with the text the
|
||||
/// error log must carry and the host calls that must not happen.
|
||||
fn without_a_closure() -> [(&'static str, FakeHost, &'static str); 4] {
|
||||
[
|
||||
("no record", FakeHost::up().without_record(), "no build"),
|
||||
(
|
||||
"not pushed",
|
||||
FakeHost::up().built(REV, false),
|
||||
"did not push",
|
||||
),
|
||||
(
|
||||
"download fails",
|
||||
FakeHost::up().failing("realise"),
|
||||
"Couldn't connect",
|
||||
),
|
||||
(
|
||||
"record unreadable",
|
||||
FakeHost::up().failing("closure_record"),
|
||||
"closure_record failed",
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prebuild_warms_the_recorded_rev_when_the_container_is_up() {
|
||||
async fn prebuild_downloads_the_swarms_closure_when_the_container_is_up() {
|
||||
let host = FakeHost::up();
|
||||
let id = node_id();
|
||||
run_prebuild(&host, "alice", id).await.expect("prebuild ok");
|
||||
let (result, log) = logging(run_prebuild(&host, "alice")).await;
|
||||
result.expect("prebuild ok");
|
||||
assert_eq!(
|
||||
host.calls(),
|
||||
[
|
||||
"is_running alice".to_owned(),
|
||||
"recorded_rev alice".to_owned(),
|
||||
format!("cached_closure alice {REV}"),
|
||||
format!("prebuild alice {REV} node={}", id.get()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// The download is followed by the local build either way. When this hive
|
||||
/// evaluates the path the swarm built, the build finds it in the store.
|
||||
#[tokio::test]
|
||||
async fn a_downloaded_closure_the_hive_also_evaluates_is_found_not_built() {
|
||||
let host = FakeHost::up().with_closure();
|
||||
let id = node_id();
|
||||
run_prebuild(&host, "alice", id).await.expect("prebuild ok");
|
||||
assert_eq!(
|
||||
host.calls(),
|
||||
[
|
||||
"is_running alice".to_owned(),
|
||||
"recorded_rev alice".to_owned(),
|
||||
format!("cached_closure alice {REV}"),
|
||||
"closure_record alice".to_owned(),
|
||||
format!("realise alice {CLOSURE}"),
|
||||
format!("prebuild alice {REV} node={}", id.get()),
|
||||
]
|
||||
);
|
||||
assert_eq!(host.builds(), Vec::<String>::new());
|
||||
}
|
||||
|
||||
/// When it evaluates another path, the build happens in `Prebuild`, after the
|
||||
/// check that the agent is up and before `Swap` could stop it.
|
||||
#[tokio::test]
|
||||
async fn a_downloaded_closure_the_hive_does_not_evaluate_is_built_before_the_swap() {
|
||||
let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-alice";
|
||||
let host = FakeHost {
|
||||
evaluates: Some(other.to_owned()),
|
||||
..FakeHost::up().with_closure()
|
||||
};
|
||||
let id = node_id();
|
||||
run_prebuild(&host, "alice", id).await.expect("prebuild ok");
|
||||
let calls = host.calls();
|
||||
assert_eq!(calls[0], "is_running alice", "{calls:?}");
|
||||
assert_eq!(
|
||||
calls.last().map(String::as_str),
|
||||
Some(format!("prebuild alice {REV} node={}", id.get()).as_str()),
|
||||
"{calls:?}"
|
||||
);
|
||||
assert_eq!(host.builds(), [other.to_owned()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_different_path_is_drift() {
|
||||
assert_eq!(drift(CLOSURE, Some(CLOSURE)), None);
|
||||
let note = drift(CLOSURE, Some("/nix/store/other")).expect("drift");
|
||||
let info = log.at(tracing::Level::INFO);
|
||||
assert!(
|
||||
note.contains(CLOSURE) && note.contains("/nix/store/other"),
|
||||
"{note}"
|
||||
info.iter()
|
||||
.any(|l| l.contains("prebuild: substituted") && l.contains(CLOSURE)),
|
||||
"{info:?}"
|
||||
);
|
||||
assert!(drift(CLOSURE, None).expect("no path").contains(CLOSURE));
|
||||
assert!(log.at(tracing::Level::WARN).is_empty());
|
||||
assert!(log.at(tracing::Level::ERROR).is_empty());
|
||||
}
|
||||
|
||||
/// No closure to deploy fails the node before `StopForUpdate`, so the running
|
||||
/// container is untouched, and the log names the agent and why.
|
||||
#[tokio::test]
|
||||
async fn a_failed_download_builds_locally() {
|
||||
let host = FakeHost::up().with_closure().failing("realise");
|
||||
let id = node_id();
|
||||
run_prebuild(&host, "alice", id)
|
||||
.await
|
||||
.expect("the build covers it");
|
||||
let calls = host.calls();
|
||||
assert_eq!(calls[3], format!("realise alice {CLOSURE}"), "{calls:?}");
|
||||
assert_eq!(
|
||||
calls[4],
|
||||
format!("prebuild alice {REV} node={}", id.get()),
|
||||
"{calls:?}"
|
||||
);
|
||||
assert_eq!(calls.len(), 5, "{calls:?}");
|
||||
async fn a_rebuild_without_a_closure_fails_the_prebuild_and_logs_an_error() {
|
||||
for (case, host, why) in without_a_closure() {
|
||||
let (result, log) = logging(run_prebuild(&host, "alice")).await;
|
||||
let err = result.expect_err(case);
|
||||
assert!(format!("{err:#}").starts_with("alice: "), "{case}: {err:#}");
|
||||
let errors = log.at(tracing::Level::ERROR);
|
||||
assert_eq!(errors.len(), 1, "{case}: {errors:?}");
|
||||
assert!(
|
||||
errors[0].contains("alice") && errors[0].contains(why),
|
||||
"{case}: {errors:?}"
|
||||
);
|
||||
let calls = host.calls();
|
||||
assert!(
|
||||
!calls.iter().any(|c| c.starts_with("kill")),
|
||||
"{case}: {calls:?}"
|
||||
);
|
||||
assert!(
|
||||
!calls
|
||||
.iter()
|
||||
.any(|c| c.starts_with("swap_update") || c.starts_with("create_only")),
|
||||
"{case}: {calls:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The fallback is the build, so a build that fails still fails the node.
|
||||
/// The builder's record is not for the commit this hive chose: its closure is
|
||||
/// deployed and the difference is a warning.
|
||||
#[tokio::test]
|
||||
async fn a_failed_download_and_a_failed_build_fail_the_node() {
|
||||
let host = FakeHost::up()
|
||||
.with_closure()
|
||||
.failing("realise")
|
||||
.failing("prebuild");
|
||||
let err = run_prebuild(&host, "alice", node_id())
|
||||
.await
|
||||
.expect_err("nothing produced the closure");
|
||||
assert!(format!("{err:#}").contains("prebuild failed"), "{err:#}");
|
||||
async fn a_closure_built_from_another_commit_is_deployed_with_a_warning() {
|
||||
let host = FakeHost::up().built(OTHER_REV, true);
|
||||
let (result, log) = logging(run_prebuild(&host, "alice")).await;
|
||||
result.expect("drift is not a refusal");
|
||||
assert!(
|
||||
host.calls().contains(&format!("realise alice {CLOSURE}")),
|
||||
"{:?}",
|
||||
host.calls()
|
||||
);
|
||||
let warnings = log.at(tracing::Level::WARN);
|
||||
assert_eq!(warnings.len(), 1, "{warnings:?}");
|
||||
assert!(
|
||||
[REV, OTHER_REV, CLOSURE]
|
||||
.iter()
|
||||
.all(|part| warnings[0].contains(part)),
|
||||
"{warnings:?}"
|
||||
);
|
||||
assert!(log.at(tracing::Level::ERROR).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_stopped_container_looks_up_nothing() {
|
||||
let host = FakeHost::down().with_closure();
|
||||
run_prebuild(&host, "alice", node_id())
|
||||
.await
|
||||
.expect("skipped");
|
||||
assert_eq!(host.calls(), ["is_running alice"]);
|
||||
}
|
||||
|
||||
/// The error log is for a path the cache served unsigned, not for a cache that
|
||||
/// is down or a missing record.
|
||||
#[tokio::test]
|
||||
async fn only_a_signature_refusal_is_flagged() {
|
||||
let down = substitute(
|
||||
&FakeHost::up().with_closure().failing("realise"),
|
||||
"alice",
|
||||
REV,
|
||||
)
|
||||
.await
|
||||
.expect_err("down");
|
||||
assert!(!down.signature_refused, "{}", down.reason);
|
||||
assert!(down.reason.contains("Couldn't connect"), "{}", down.reason);
|
||||
|
||||
let no_record = substitute(&FakeHost::up(), "alice", REV)
|
||||
.await
|
||||
.expect_err("no record");
|
||||
assert!(!no_record.signature_refused);
|
||||
assert!(
|
||||
no_record.reason.starts_with("no-record"),
|
||||
"{}",
|
||||
no_record.reason
|
||||
);
|
||||
|
||||
let refused = FakeHost {
|
||||
refusal: true,
|
||||
..FakeHost::up().with_closure().failing("realise")
|
||||
};
|
||||
let refused = substitute(&refused, "alice", REV)
|
||||
.await
|
||||
.expect_err("refused");
|
||||
assert!(refused.signature_refused, "{}", refused.reason);
|
||||
|
||||
let ok = substitute(&FakeHost::up().with_closure(), "alice", REV).await;
|
||||
assert_eq!(ok.ok().as_deref(), Some(CLOSURE));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prebuild_is_skipped_when_the_container_is_down() {
|
||||
let host = FakeHost::down();
|
||||
run_prebuild(&host, "alice", node_id())
|
||||
.await
|
||||
.expect("a skipped prebuild is ok");
|
||||
run_prebuild(&host, "alice").await.expect("skipped");
|
||||
assert_eq!(host.calls(), ["is_running alice"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prebuild_without_a_recorded_rev_fails_before_building() {
|
||||
async fn prebuild_without_a_recorded_rev_fails_before_looking_up_a_closure() {
|
||||
let host = FakeHost {
|
||||
rev: None,
|
||||
..FakeHost::up()
|
||||
};
|
||||
let err = run_prebuild(&host, "alice", node_id())
|
||||
let err = run_prebuild(&host, "alice")
|
||||
.await
|
||||
.expect_err("nothing to build from");
|
||||
.expect_err("nothing to deploy");
|
||||
assert!(
|
||||
format!("{err:#}").contains("no internal/meta commit"),
|
||||
"{err:#}"
|
||||
);
|
||||
assert!(!host.calls().iter().any(|c| c.starts_with("prebuild")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prebuild_failure_fails_the_node() {
|
||||
let host = FakeHost::up().failing("prebuild");
|
||||
let err = run_prebuild(&host, "alice", node_id())
|
||||
.await
|
||||
.expect_err("build failure propagates");
|
||||
assert!(format!("{err:#}").contains("prebuild failed"), "{err:#}");
|
||||
assert!(!host.calls().iter().any(|c| c.starts_with("closure_record")));
|
||||
}
|
||||
|
||||
// --- Swap -------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn swap_updates_at_the_recorded_rev() {
|
||||
async fn swap_applies_the_swarms_closure() {
|
||||
let (_dir, coord) = coordinator();
|
||||
let host = FakeHost::down();
|
||||
let id = node_id();
|
||||
|
|
@ -513,11 +495,50 @@ async fn swap_updates_at_the_recorded_rev() {
|
|||
host.calls(),
|
||||
[
|
||||
"recorded_rev alice".to_owned(),
|
||||
format!("swap_update alice {REV} node={}", id.get()),
|
||||
"closure_record alice".to_owned(),
|
||||
format!("realise alice {CLOSURE}"),
|
||||
format!("swap_update alice {CLOSURE} node={}", id.get()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// A swap with no closure to deploy never updates: the container stays as it
|
||||
/// was, and the log names the agent and why.
|
||||
#[tokio::test]
|
||||
async fn a_swap_without_a_closure_never_updates_and_logs_an_error() {
|
||||
for (case, host, why) in without_a_closure() {
|
||||
let (_dir, coord) = coordinator();
|
||||
let (result, log) = logging(run_swap(&host, &coord, "alice", node_id())).await;
|
||||
result.expect_err(case);
|
||||
let errors = log.at(tracing::Level::ERROR);
|
||||
assert_eq!(errors.len(), 1, "{case}: {errors:?}");
|
||||
assert!(
|
||||
errors[0].contains("alice") && errors[0].contains(why),
|
||||
"{case}: {errors:?}"
|
||||
);
|
||||
assert!(
|
||||
!host.calls().iter().any(|c| c.starts_with("swap_update")),
|
||||
"{case}: {:?}",
|
||||
host.calls()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn swap_applies_a_closure_built_from_another_commit_with_a_warning() {
|
||||
let (_dir, coord) = coordinator();
|
||||
let host = FakeHost::down().built(OTHER_REV, true);
|
||||
let id = node_id();
|
||||
let (result, log) = logging(run_swap(&host, &coord, "alice", id)).await;
|
||||
result.expect("drift is not a refusal");
|
||||
assert_eq!(
|
||||
host.calls().last(),
|
||||
Some(&format!("swap_update alice {CLOSURE} node={}", id.get()))
|
||||
);
|
||||
assert_eq!(log.at(tracing::Level::WARN).len(), 1);
|
||||
assert!(log.at(tracing::Level::ERROR).is_empty());
|
||||
}
|
||||
|
||||
/// A failed update fails the node and nothing else: the swap never restarts or
|
||||
/// stops the container itself — recovery is the DAG's tail `Reconcile`.
|
||||
#[tokio::test]
|
||||
|
|
@ -529,8 +550,8 @@ async fn a_failed_swap_reports_the_error_and_touches_no_container() {
|
|||
.expect_err("update failure propagates");
|
||||
assert!(format!("{err:#}").contains("update failed"), "{err:#}");
|
||||
let calls = host.calls();
|
||||
assert_eq!(calls.len(), 2, "{calls:?}");
|
||||
assert!(calls[1].starts_with("swap_update alice"), "{calls:?}");
|
||||
assert_eq!(calls.len(), 4, "{calls:?}");
|
||||
assert!(calls[3].starts_with("swap_update alice"), "{calls:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
|
@ -824,84 +845,73 @@ async fn provision_and_write_dropin_forward_and_propagate_failure() {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_without_a_record_builds_locally_at_the_resolved_rev() {
|
||||
async fn create_downloads_the_swarms_closure_and_creates_from_it() {
|
||||
let host = FakeHost::up();
|
||||
run_create(&host, "alice", node_id()).await.expect("ok");
|
||||
let (result, log) = logging(run_create(&host, "alice")).await;
|
||||
result.expect("ok");
|
||||
assert_eq!(
|
||||
host.calls(),
|
||||
[
|
||||
"resolve_meta alice".to_owned(),
|
||||
format!("cached_closure alice {REV}"),
|
||||
format!("create_only alice {REV}"),
|
||||
format!("mark_deployed alice {REV}"),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_downloads_a_cached_closure_and_builds_nothing() {
|
||||
let host = FakeHost::up().with_closure();
|
||||
let id = node_id();
|
||||
run_create(&host, "alice", id).await.expect("ok");
|
||||
assert_eq!(
|
||||
host.calls(),
|
||||
[
|
||||
"resolve_meta alice".to_owned(),
|
||||
format!("cached_closure alice {REV}"),
|
||||
"closure_record alice".to_owned(),
|
||||
format!("realise alice {CLOSURE}"),
|
||||
format!("prebuild alice {REV} node={}", id.get()),
|
||||
format!("create_only alice {REV}"),
|
||||
format!("create_only alice {CLOSURE}"),
|
||||
format!("mark_deployed alice {REV}"),
|
||||
]
|
||||
);
|
||||
assert_eq!(host.builds(), Vec::<String>::new());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_builds_locally_when_the_download_fails() {
|
||||
let host = FakeHost::up().with_closure().failing("realise");
|
||||
run_create(&host, "alice", node_id()).await.expect("ok");
|
||||
assert_eq!(
|
||||
host.calls(),
|
||||
[
|
||||
"resolve_meta alice".to_owned(),
|
||||
format!("cached_closure alice {REV}"),
|
||||
format!("realise alice {CLOSURE}"),
|
||||
format!("create_only alice {REV}"),
|
||||
format!("mark_deployed alice {REV}"),
|
||||
]
|
||||
assert!(
|
||||
log.at(tracing::Level::INFO)
|
||||
.iter()
|
||||
.any(|l| l.contains("create: substituted") && l.contains(CLOSURE))
|
||||
);
|
||||
assert!(log.at(tracing::Level::WARN).is_empty());
|
||||
}
|
||||
|
||||
/// A create with no closure to deploy is refused: no container is made, none is
|
||||
/// marked deployed, and the log names the agent and why.
|
||||
#[tokio::test]
|
||||
async fn a_create_without_a_closure_is_refused_and_logs_an_error() {
|
||||
for (case, host, why) in without_a_closure() {
|
||||
let (result, log) = logging(run_create(&host, "alice")).await;
|
||||
let err = result.expect_err(case);
|
||||
assert!(format!("{err:#}").starts_with("alice: "), "{case}: {err:#}");
|
||||
let errors = log.at(tracing::Level::ERROR);
|
||||
assert_eq!(errors.len(), 1, "{case}: {errors:?}");
|
||||
assert!(
|
||||
errors[0].contains("alice") && errors[0].contains(why),
|
||||
"{case}: {errors:?}"
|
||||
);
|
||||
let calls = host.calls();
|
||||
assert!(
|
||||
!calls
|
||||
.iter()
|
||||
.any(|c| c.starts_with("create_only") || c.starts_with("mark_deployed")),
|
||||
"{case}: {calls:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_builds_the_evaluated_path_when_it_differs_from_the_download() {
|
||||
let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-alice";
|
||||
let host = FakeHost {
|
||||
evaluates: Some(other.to_owned()),
|
||||
..FakeHost::up().with_closure()
|
||||
};
|
||||
run_create(&host, "alice", node_id()).await.expect("ok");
|
||||
assert_eq!(host.builds(), [other.to_owned()]);
|
||||
async fn create_uses_a_closure_built_from_another_commit_with_a_warning() {
|
||||
let host = FakeHost::up().built(OTHER_REV, true);
|
||||
let (result, log) = logging(run_create(&host, "alice")).await;
|
||||
result.expect("drift is not a refusal");
|
||||
let calls = host.calls();
|
||||
assert!(
|
||||
calls
|
||||
.iter()
|
||||
.any(|c| c == &format!("create_only alice {REV}")),
|
||||
calls.contains(&format!("create_only alice {CLOSURE}")),
|
||||
"{calls:?}"
|
||||
);
|
||||
assert_eq!(log.at(tracing::Level::WARN).len(), 1);
|
||||
assert!(log.at(tracing::Level::ERROR).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_create_is_never_marked_deployed() {
|
||||
let host = FakeHost::up().failing("create_only");
|
||||
run_create(&host, "alice", node_id())
|
||||
.await
|
||||
.expect_err("create fails");
|
||||
assert_eq!(host.calls().len(), 3);
|
||||
run_create(&host, "alice").await.expect_err("create fails");
|
||||
assert_eq!(host.calls().len(), 4);
|
||||
let host = FakeHost::up().failing("resolve_meta");
|
||||
run_create(&host, "alice", node_id())
|
||||
.await
|
||||
.expect_err("resolve fails");
|
||||
run_create(&host, "alice").await.expect_err("resolve fails");
|
||||
assert_eq!(host.calls(), ["resolve_meta alice"]);
|
||||
}
|
||||
|
||||
|
|
@ -1165,11 +1175,19 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() {
|
|||
),
|
||||
(
|
||||
alice(|agent| NodeKind::Prebuild { agent }),
|
||||
&["is_running alice", "recorded_rev alice", "prebuild alice"],
|
||||
&[
|
||||
"is_running alice",
|
||||
"recorded_rev alice",
|
||||
"closure_record alice",
|
||||
],
|
||||
),
|
||||
(
|
||||
alice(|agent| NodeKind::Swap { agent }),
|
||||
&["recorded_rev alice", "swap_update alice"],
|
||||
&[
|
||||
"recorded_rev alice",
|
||||
"closure_record alice",
|
||||
"swap_update alice",
|
||||
],
|
||||
),
|
||||
(
|
||||
alice(|agent| NodeKind::RebuildBookkeeping { agent }),
|
||||
|
|
@ -1183,6 +1201,7 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() {
|
|||
alice(|agent| NodeKind::Create { agent }),
|
||||
&[
|
||||
"resolve_meta alice",
|
||||
"closure_record alice",
|
||||
"create_only alice",
|
||||
"mark_deployed alice",
|
||||
],
|
||||
|
|
|
|||
|
|
@ -32,9 +32,9 @@ pub enum NodeKind {
|
|||
/// The rebuild's preamble: runtime-dir prep and the `internal/meta`
|
||||
/// commit the build reads.
|
||||
MetaSync { agent: String },
|
||||
/// Out-of-band toplevel build while the container keeps serving.
|
||||
/// Download of the swarm's closure while the container keeps serving.
|
||||
Prebuild { agent: String },
|
||||
/// `nixos-container update` profile-swap.
|
||||
/// Profile-swap onto the swarm's closure.
|
||||
Swap { agent: String },
|
||||
/// The post-`Swap` bookkeeping tail as a first-class node.
|
||||
RebuildBookkeeping { agent: String },
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ fn rebuild_subtree<'a>(
|
|||
.needs(Resource::Agent(a()))
|
||||
.after_ok(meta_sync);
|
||||
|
||||
// Siblings under the brace: the build and the quiesce chain run
|
||||
// Siblings under the brace: the download and the quiesce chain run
|
||||
// concurrently, borrowing the brace's grants rather than declaring their
|
||||
// own. Declaring the lease on both would make them mutually exclusive —
|
||||
// it is single-unit — which is exactly what this shape exists to avoid.
|
||||
|
|
@ -200,8 +200,8 @@ fn rebuild_subtree<'a>(
|
|||
|
||||
let drain = graceful.then(|| quiesce(builder, agent, agent_window));
|
||||
|
||||
// The container goes down here, not earlier: `AfterOk` the build so a
|
||||
// failed build never stops a healthy container, and `AfterOk` the drain so
|
||||
// The container goes down here, not earlier: `AfterOk` the download so a
|
||||
// missing closure never stops a healthy container, and `AfterOk` the drain so
|
||||
// the agent has checkpointed.
|
||||
let mut stop_for_update = builder
|
||||
.node(NodeKind::StopForUpdate { agent: a() })
|
||||
|
|
|
|||
Loading…
Reference in a new issue