mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

remove the hive's local build of agent closures

Browse source 
A rebuild or create downloads the closure the swarm built and applies it; with none to deploy it fails and changes nothing. hive-priv takes a store path instead of an internal/meta commit and builds nothing.
This commit is contained in:
atlas 2026-10-29 22:17:07 +01:00
commit 4590b04cac
1 changed files with 632 additions and 1351 deletions
Download patch file Download diff file Expand all files Collapse all files

244
hive-c0re/src/job_queue/exec.rs
View file

@ -9,6 +9,7 @@ use std::path::PathBuf;
use std::sync::{Arc, Mutex, OnceLock};
use anyhow::{Context as _, Result};
use swarm_wire::agent_closures::AgentClosure;
use hive_jobq::NodeId;
@ -34,7 +35,7 @@ pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from
const PAUSE_ACK_TIMEOUT: std::time::Duration = GRACEFUL_STOP_TIMEOUT;
/// The container-state calls the executors below make on the host: systemd,
/// `hive-priv`, the nix build, the `internal/meta` record. Production uses
/// `hive-priv`, the swarm's closure record, the `internal/meta` record. Production uses
/// [`RealHost`], which forwards to `lifecycle` / `meta_rev` / `Coordinator`
/// unchanged; tests substitute a recording fake.
pub(super) trait Host: Sync {
@ -44,28 +45,19 @@ pub(super) trait Host: Sync {
async fn kill(&self, name: &str) -> Result<()>;
/// The `internal/meta` commit `MetaSync` recorded for the agent.
fn recorded_rev(&self, name: &str) -> Result<String>;
/// The store path of the closure the swarm built for the agent at the
/// `internal/meta` commit `rev` and pushed to its cache. The error says why
/// there is none.
async fn cached_closure(&self, name: &str, rev: &str) -> Result<String>;
/// The record the swarm's builder wrote for the agent; `None` when it wrote
/// none.
async fn closure_record(&self, name: &str) -> Result<Option<AgentClosure>>;
/// Download `store_path` from the swarm's cache into the store, never
/// building it.
async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()>;
/// Warm the agent's toplevel build from the `internal/meta` commit `rev`.
/// The toplevel it printed, when it printed one.
async fn prebuild_toplevel(
&self,
name: &str,
rev: &str,
node_id: u64,
) -> Result<Option<String>>;
/// Re-apply drop-ins, then `nixos-container update` at `rev`.
/// Re-apply drop-ins, then point the container at the closure `toplevel`.
async fn swap_update(
&self,
name: &str,
hive: &HiveEnv,
paths: &AgentPaths,
rev: &str,
toplevel: &str,
node_id: u64,
) -> Result<()>;
/// Write or remove the agent's pause marker.
@ -80,7 +72,8 @@ pub(super) trait Host: Sync {
async fn refresh_agent_set(&self) -> Result<()>;
/// Pin the `internal/meta` commit the agent builds from.
async fn resolve_meta(&self, name: &str) -> Result<MetaBuild>;
/// Record `build` as what `Prebuild` and `Swap` build from.
/// Record `build` as the commit `Prebuild` and `Swap` expect the swarm's
/// closure to be built from.
fn record_meta(&self, name: &str, build: &MetaBuild) -> Result<()>;
/// The build `MetaSync` recorded, if any.
fn recorded_build(&self, name: &str) -> Option<MetaBuild>;
@ -88,8 +81,8 @@ pub(super) trait Host: Sync {
fn mark_deployed(&self, name: &str, build: &MetaBuild) -> Result<()>;
/// First-spawn provisioning: repos, state subvolume, agent-set refresh.
async fn provision(&self, name: &str, paths: &AgentPaths) -> Result<()>;
/// `nixos-container create` at `rev`.
async fn create_only(&self, name: &str, rev: &str) -> Result<()>;
/// `nixos-container create` from the closure `toplevel`.
async fn create_only(&self, name: &str, toplevel: &str) -> Result<()>;
/// Write the nspawn and resource-limit drop-ins, then reload systemd.
async fn write_dropins(&self, name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>;
/// Start preamble, then start with the cold-start fallback.
@ -128,33 +121,23 @@ impl Host for RealHost {
Ok(crate::meta_rev::recorded_or_err(name)?.rev)
}
async fn cached_closure(&self, name: &str, rev: &str) -> Result<String> {
crate::attic::closure::path_for(name, rev).await
async fn closure_record(&self, name: &str) -> Result<Option<AgentClosure>> {
crate::attic::closure::record(name).await
}
async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()> {
crate::priv_client::realise_closure(name, store_path).await
}
async fn prebuild_toplevel(
&self,
name: &str,
rev: &str,
node_id: u64,
) -> Result<Option<String>> {
let flake_ref = crate::meta_rev::flake_ref(name, rev);
crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(node_id)).await
}
async fn swap_update(
&self,
name: &str,
hive: &HiveEnv,
paths: &AgentPaths,
rev: &str,
toplevel: &str,
node_id: u64,
) -> Result<()> {
crate::lifecycle::swap_update(name, hive, paths, rev, Some(node_id)).await
crate::lifecycle::swap_update(name, hive, paths, toplevel, Some(node_id)).await
}
async fn set_paused(&self, agent: &hive_types::Ident, paused: bool) -> Result<()> {
@ -198,8 +181,8 @@ impl Host for RealHost {
crate::lifecycle::provision_container(name, paths).await
}
async fn create_only(&self, name: &str, rev: &str) -> Result<()> {
crate::lifecycle::create_only(name, rev).await
async fn create_only(&self, name: &str, toplevel: &str) -> Result<()> {
crate::lifecycle::create_only(name, toplevel).await
}
async fn write_dropins(&self, name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
@ -300,11 +283,11 @@ async fn dispatch(
// growth executors return what to grow rather than taking the builder.
let result = match kind {
NodeKind::MetaSync { .. } => run_meta_sync(host, agent).await,
NodeKind::Prebuild { .. } => run_prebuild(host, agent, id).await,
NodeKind::Prebuild { .. } => run_prebuild(host, agent).await,
NodeKind::Swap { .. } => run_swap(host, coord, agent, id).await,
NodeKind::RebuildBookkeeping { .. } => run_rebuild_bookkeeping(host, coord, agent).await,
NodeKind::Provision { .. } => run_provision(host, agent).await,
NodeKind::Create { .. } => run_create(host, agent, id).await,
NodeKind::Create { .. } => run_create(host, agent).await,
NodeKind::DestroyContainer { .. } => run_destroy_container(host, coord, agent).await,
NodeKind::PurgeState { .. } => {
run_purge_state(host, agent).await;
@ -533,10 +516,11 @@ fn run_set_wanted(coord: &Arc<Coordinator>, agent: &str, up: bool) -> Result<()>
}
/// The rebuild's preamble: runtime-dir prep, the agent-set refresh, and the
/// `internal/meta` commit `Prebuild` and `Swap` build from.
/// `internal/meta` commit `Prebuild` and `Swap` expect the swarm's closure to
/// be built from.
///
/// Fails when `internal/meta` does not pin the agent's current config yet:
/// there is no other flake to build it from.
/// there is no commit to deploy.
async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> {
// Runs while the agent is still up — the runtime dir and MCP listener
// already exist. Use the pure path accessor; no need to re-register the
@ -550,98 +534,95 @@ async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> {
}
host.prepare_rebuild_dirs(name, &paths).await?;
host.refresh_agent_set().await?;
// Resolved once here so `Prebuild` and `Swap` build the same flake.
// Resolved once here so `Prebuild` and `Swap` expect the same commit.
let build = host.resolve_meta(name).await?;
host.record_meta(name, &build)?;
Ok(())
}
/// Out-of-band toplevel build while the container keeps serving: warm
/// `system.build.toplevel` so the later `Swap` hits cache and skips
/// straight to the profile-swap, from the `internal/meta` commit the upstream
/// `MetaSync` node recorded. The warm build is skipped when the
/// container is already down: its only purpose is to shrink the swap's
/// downtime window, so a stopped agent (no uptime to preserve) doesn't
/// pay the double eval — `Swap` builds inline instead.
///
/// The swarm's cache is tried first: when the swarm built this agent at the
/// recorded commit and pushed the closure, it is downloaded. The local build
/// runs either way. After a download it is an evaluation and a no-op build when
/// this hive evaluates the path the swarm built, and builds it, with the agent
/// still up, when it doesn't; `Swap` never meets a build it didn't expect.
async fn run_prebuild(host: &impl Host, name: &str, id: NodeId) -> Result<()> {
// Warm the toplevel build only when the container is up — the whole
// point of prebuild is to shrink the swap's downtime window. A
// stopped agent has no uptime to preserve, so skip the (expensive)
// eval and let the downstream `Swap` build inline.
/// Download the swarm's closure for the agent while the container keeps
/// serving, so a rebuild with no closure fails here, before `StopForUpdate`
/// takes a healthy container down. `Swap` downloads the same closure again
/// (a no-op once it is in the store) and applies it. Skipped when the container
/// is already down: there is no uptime to protect, and `Swap` fails the same way.
async fn run_prebuild(host: &impl Host, name: &str) -> Result<()> {
if host.is_running(name).await {
let rev = host.recorded_rev(name)?;
let downloaded = match substitute(host, name, &rev).await {
Ok(path) => Some(path),
Err(fallback) => {
let reason = &fallback.reason;
if fallback.signature_refused {
tracing::error!(%name, %rev, "prebuild: built locally: {reason}");
} else {
tracing::warn!(%name, %rev, "prebuild: built locally: {reason}");
}
None
}
};
let evaluated = host.prebuild_toplevel(name, &rev, id.get()).await?;
if let Some(path) = downloaded {
match drift(&path, evaluated.as_deref()) {
None => tracing::info!(%name, %rev, "prebuild: substituted {path}"),
Some(note) => tracing::warn!(%name, %rev, "prebuild: {note}"),
}
}
let path = closure_for(host, name, &rev, "prebuild").await?;
tracing::info!(%name, %rev, "prebuild: substituted {path}");
}
Ok(())
}
/// What to log when the closure downloaded for a rebuild isn't the one this
/// hive's evaluation produced, which means the builder and the hive disagree
/// about the agent's flake. `None` when they agree.
fn drift(downloaded: &str, evaluated: Option<&str>) -> Option<String> {
match evaluated {
Some(path) if path == downloaded => None,
Some(path) => Some(format!(
"the downloaded closure {downloaded} is not the one this hive evaluates, {path}; built the latter locally"
)),
None => Some(format!(
"downloaded {downloaded}, but the local build printed no path to compare it with"
)),
}
/// Why the swarm's closure for an agent cannot be deployed.
enum NoClosure {
/// The builder has recorded no build of the agent.
NoRecord,
/// A build is recorded, but its closure cannot be had: the record could not
/// be read, the builder did not push the closure, or the download failed.
Unpullable(String),
}
/// Why a rebuild or create builds instead of downloading the swarm's closure.
struct LocalBuild {
reason: String,
/// The cache served a path no trusted key signed.
signature_refused: bool,
}
/// Download the closure the swarm built for `name` at `rev`; its store path.
async fn substitute(host: &impl Host, name: &str, rev: &str) -> Result<String, LocalBuild> {
let path = host
.cached_closure(name, rev)
/// The store path of the closure the swarm built for `name`, downloaded into
/// the store. The deploy applies the swarm's closure whatever commit it was
/// built from: when that is not the `rev` this hive chose, it is logged.
///
/// # Errors
/// [`without_closure`]'s refusal, after logging it.
async fn closure_for(host: &impl Host, name: &str, rev: &str, step: &str) -> Result<String> {
pull_closure(host, name, rev, step)
.await
.map_err(|e| LocalBuild {
reason: format!("{e:#}"),
signature_refused: false,
})?;
host.realise_closure(name, &path).await.map_err(|e| {
let reason = format!("{e:#}");
LocalBuild {
signature_refused: reason.contains(hive_priv_sock::SIGNATURE_REFUSED),
reason,
}
})?;
Ok(path)
.map_err(|why| without_closure(name, rev, step, &why))
}
/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb),
/// `nixos-container update`, then the post-rebuild bookkeeping tail
async fn pull_closure(
host: &impl Host,
name: &str,
rev: &str,
step: &str,
) -> Result<String, NoClosure> {
let record = host
.closure_record(name)
.await
.map_err(|e| NoClosure::Unpullable(format!("{e:#}")))?
.ok_or(NoClosure::NoRecord)?;
if !record.cached {
return Err(NoClosure::Unpullable(
"the builder did not push the closure".to_owned(),
));
}
host.realise_closure(name, &record.store_path)
.await
.map_err(|e| NoClosure::Unpullable(format!("{e:#}")))?;
if record.meta_rev != rev {
tracing::warn!(
%name, %rev, built_from = %record.meta_rev,
"{step}: the swarm built {} from internal/meta {}, not the {rev} this hive chose; deploying the swarm's",
record.store_path, record.meta_rev
);
}
Ok(record.store_path)
}
/// What a rebuild or create does when there is no closure to deploy: it
/// fails, and nothing changes, so a running container keeps running as it is
/// and a create makes none. The failure is logged at error level with the
/// agent's name.
///
/// This is the one place that decides what an unpullable closure (the cache
/// unreachable, a TLS error, a failed download) costs. A rebuild never builds
/// locally on the hive.
fn without_closure(name: &str, rev: &str, step: &str, why: &NoClosure) -> anyhow::Error {
let reason = match why {
NoClosure::NoRecord => "the swarm has recorded no build of this agent".to_owned(),
NoClosure::Unpullable(e) => format!("the swarm's closure cannot be pulled: {e}"),
};
tracing::error!(%name, %rev, "{step}: {reason}; nothing was changed");
anyhow::anyhow!("{name}: {reason}")
}
/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), point the
/// container at the swarm's closure, then the post-rebuild bookkeeping tail
/// (rev marker, `Rebuilt` event, forge sync, kick, rescan).
/// The recovery-start on failure is NOT here — the DAG's tail
/// `Reconcile` runs after this node terminal ok *or* fail.
@ -657,7 +638,11 @@ async fn run_swap(
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir)?;
let rev = host.recorded_rev(name)?;
let result = host.swap_update(name, &hive, &paths, &rev, id.get()).await;
let toplevel = closure_for(host, name, &rev, "swap").await?;
tracing::info!(%name, %rev, "swap: applying {toplevel}");
let result = host
.swap_update(name, &hive, &paths, &toplevel, id.get())
.await;
// On success the Ok-only bookkeeping tail (rev marker, forge
// sync, kick, rescan, snapshot) runs in the sibling `RebuildBookkeeping` node,
// which deps `AfterOk(Swap)`. On failure `RebuildBookkeeping` is cancel-cascaded
@ -719,30 +704,13 @@ async fn run_provision(host: &impl Host, name: &str) -> Result<()> {
/// node stays purely "create", not "create + start". Marks the agent as
/// running the pins it was created from.
///
/// The swarm's cache is tried first, as in [`run_prebuild`]: when it holds the
/// closure for the resolved commit it is downloaded, and the local evaluation
/// that follows is a no-op build on a path match and builds the evaluated path
/// on a mismatch. Without a cached closure the create builds it locally.
async fn run_create(host: &impl Host, name: &str, id: NodeId) -> Result<()> {
/// The container is created from the swarm's closure for the resolved commit;
/// with none to deploy the create fails and makes no container.
async fn run_create(host: &impl Host, name: &str) -> Result<()> {
let build = host.resolve_meta(name).await?;
match substitute(host, name, &build.rev).await {
Ok(path) => {
let evaluated = host.prebuild_toplevel(name, &build.rev, id.get()).await?;
match drift(&path, evaluated.as_deref()) {
None => tracing::info!(%name, rev = %build.rev, "create: substituted {path}"),
Some(note) => tracing::warn!(%name, rev = %build.rev, "create: {note}"),
}
}
Err(fallback) => {
let reason = &fallback.reason;
if fallback.signature_refused {
tracing::error!(%name, rev = %build.rev, "create: built locally: {reason}");
} else {
tracing::warn!(%name, rev = %build.rev, "create: built locally: {reason}");
}
}
}
host.create_only(name, &build.rev).await?;
let toplevel = closure_for(host, name, &build.rev, "create").await?;
tracing::info!(%name, rev = %build.rev, "create: substituted {toplevel}");
host.create_only(name, &toplevel).await?;
host.mark_deployed(name, &build)?;
Ok(())
}