deploy the closure the swarm declares, read no forge head, drop the update badge
This commit is contained in:
atlas
mara
parent
279428a66f
commit
47c2acb640
1 changed files with 647 additions and 2001 deletions
|
|
@ -1,5 +1,5 @@
|
|||
//! Node executors — one async fn per [`NodeKind`], each a thin wrapper
|
||||
//! over existing `lifecycle.rs` / `meta_rev.rs` / `actions.rs` code. Node
|
||||
//! over existing `lifecycle.rs` / `actions.rs` code. Node
|
||||
//! executors keep their own internal error handling where it exists
|
||||
//! today (cold-start fallback inside `Reconcile`, warn-only forge sync in
|
||||
//! the `Swap` tail); DAG-level failure handling is cancel-downstream in
|
||||
|
|
@ -15,7 +15,6 @@ use hive_jobq::NodeId;
|
|||
|
||||
use super::model::NodeKind;
|
||||
use crate::coordinator::{AgentPaths, Coordinator, HiveEnv};
|
||||
use crate::meta_rev::MetaBuild;
|
||||
use crate::power::{ReconcileAction, reconcile_action};
|
||||
use crate::stats::sweep_health::{self, SweepHealth};
|
||||
|
||||
|
|
@ -35,21 +34,17 @@ pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from
|
|||
const PAUSE_ACK_TIMEOUT: std::time::Duration = GRACEFUL_STOP_TIMEOUT;
|
||||
|
||||
/// The container-state calls the executors below make on the host: systemd,
|
||||
/// `hive-priv`, the swarm's closure record, the `internal/meta` record. Production uses
|
||||
/// [`RealHost`], which forwards to `lifecycle` / `meta_rev` / `Coordinator`
|
||||
/// `hive-priv`, the swarm's closure record and the marker of the closure applied.
|
||||
/// Production uses [`RealHost`], which forwards to `lifecycle` / `Coordinator`
|
||||
/// unchanged; tests substitute a recording fake.
|
||||
pub(super) trait Host: Sync {
|
||||
/// Whether the agent's container unit is active.
|
||||
async fn is_running(&self, name: &str) -> bool;
|
||||
/// Stop the agent's container.
|
||||
async fn kill(&self, name: &str) -> Result<()>;
|
||||
/// The `internal/meta` commit `MetaSync` recorded for the agent.
|
||||
fn recorded_rev(&self, name: &str) -> Result<String>;
|
||||
/// The record the swarm's builder wrote for the agent; `None` when it wrote
|
||||
/// none.
|
||||
async fn closure_record(&self, name: &str) -> Result<Option<AgentClosure>>;
|
||||
/// The pins `internal/meta` at the commit `rev` holds for the agent.
|
||||
async fn pins_at(&self, name: &str, rev: &str) -> Result<MetaBuild>;
|
||||
/// Download `store_path` from the swarm's cache into the store, never
|
||||
/// building it.
|
||||
async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()>;
|
||||
|
|
@ -72,15 +67,8 @@ pub(super) trait Host: Sync {
|
|||
async fn prepare_rebuild_dirs(&self, name: &str, paths: &AgentPaths) -> Result<()>;
|
||||
/// Rewrite the host-side files that follow the agent set.
|
||||
async fn refresh_agent_set(&self) -> Result<()>;
|
||||
/// Pin the `internal/meta` commit the agent builds from.
|
||||
async fn resolve_meta(&self, name: &str) -> Result<MetaBuild>;
|
||||
/// Record `build` as the commit `Prebuild` and `Swap` expect the swarm's
|
||||
/// closure to be built from.
|
||||
fn record_meta(&self, name: &str, build: &MetaBuild) -> Result<()>;
|
||||
/// The build `MetaSync` recorded, if any.
|
||||
fn recorded_build(&self, name: &str) -> Option<MetaBuild>;
|
||||
/// Mark the agent as running the pins of `build`.
|
||||
fn mark_deployed(&self, name: &str, build: &MetaBuild) -> Result<()>;
|
||||
/// Mark the agent as running the closure at `store_path`.
|
||||
fn mark_deployed(&self, name: &str, store_path: &str) -> Result<()>;
|
||||
/// First-spawn provisioning: repos, state subvolume, agent-set refresh.
|
||||
async fn provision(&self, name: &str, paths: &AgentPaths) -> Result<()>;
|
||||
/// `nixos-container create` from the closure `toplevel`.
|
||||
|
|
@ -117,18 +105,10 @@ impl Host for RealHost {
|
|||
crate::lifecycle::kill(name).await
|
||||
}
|
||||
|
||||
fn recorded_rev(&self, name: &str) -> Result<String> {
|
||||
Ok(crate::meta_rev::recorded_or_err(name)?.rev)
|
||||
}
|
||||
|
||||
async fn closure_record(&self, name: &str) -> Result<Option<AgentClosure>> {
|
||||
crate::attic::closure::record(name).await
|
||||
}
|
||||
|
||||
async fn pins_at(&self, name: &str, rev: &str) -> Result<MetaBuild> {
|
||||
crate::meta_rev::resolve_at(name, rev).await
|
||||
}
|
||||
|
||||
async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()> {
|
||||
crate::priv_client::realise_closure(name, store_path).await
|
||||
}
|
||||
|
|
@ -165,20 +145,8 @@ impl Host for RealHost {
|
|||
crate::agent_set::refresh(&agents).await
|
||||
}
|
||||
|
||||
async fn resolve_meta(&self, name: &str) -> Result<MetaBuild> {
|
||||
crate::meta_rev::resolve(name).await
|
||||
}
|
||||
|
||||
fn record_meta(&self, name: &str, build: &MetaBuild) -> Result<()> {
|
||||
crate::meta_rev::record(name, build)
|
||||
}
|
||||
|
||||
fn recorded_build(&self, name: &str) -> Option<MetaBuild> {
|
||||
crate::meta_rev::recorded(name)
|
||||
}
|
||||
|
||||
fn mark_deployed(&self, name: &str, build: &MetaBuild) -> Result<()> {
|
||||
crate::meta_rev::mark_deployed(name, build)
|
||||
fn mark_deployed(&self, name: &str, store_path: &str) -> Result<()> {
|
||||
crate::attic::closure::mark_deployed(name, store_path)
|
||||
}
|
||||
|
||||
async fn provision(&self, name: &str, paths: &AgentPaths) -> Result<()> {
|
||||
|
|
@ -283,7 +251,7 @@ async fn dispatch(
|
|||
NodeKind::MetaSync { .. } => run_meta_sync(host, agent).await,
|
||||
NodeKind::Prebuild { .. } => run_prebuild(host, agent).await,
|
||||
NodeKind::Swap { .. } => run_swap(host, coord, agent, id).await,
|
||||
NodeKind::RebuildBookkeeping { .. } => run_rebuild_bookkeeping(host, coord, agent).await,
|
||||
NodeKind::RebuildBookkeeping { .. } => run_rebuild_bookkeeping(coord).await,
|
||||
NodeKind::Provision { .. } => run_provision(host, agent).await,
|
||||
NodeKind::Create { .. } => run_create(host, agent).await,
|
||||
NodeKind::DestroyContainer { .. } => run_destroy_container(host, coord, agent).await,
|
||||
|
|
@ -502,12 +470,9 @@ fn run_set_wanted(coord: &Arc<Coordinator>, agent: &str, up: bool) -> Result<()>
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// The rebuild's preamble: runtime-dir prep, the agent-set refresh, and the
|
||||
/// `internal/meta` commit `Prebuild` and `Swap` expect the swarm's closure to
|
||||
/// be built from.
|
||||
///
|
||||
/// Fails when `internal/meta` does not pin the agent's current config yet:
|
||||
/// there is no commit to deploy.
|
||||
/// The rebuild's preamble: runtime-dir prep, the archetype and agent-set
|
||||
/// refresh. Reads nothing from the forge: what a rebuild deploys is the swarm's
|
||||
/// declared closure, read by `Prebuild` and `Swap`.
|
||||
async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> {
|
||||
// Runs while the agent is still up — the runtime dir and MCP listener
|
||||
// already exist. Use the pure path accessor; no need to re-register the
|
||||
|
|
@ -521,9 +486,6 @@ async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> {
|
|||
}
|
||||
host.prepare_rebuild_dirs(name, &paths).await?;
|
||||
host.refresh_agent_set().await?;
|
||||
// Resolved once here so `Prebuild` and `Swap` expect the same commit.
|
||||
let build = host.resolve_meta(name).await?;
|
||||
host.record_meta(name, &build)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -534,9 +496,8 @@ async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> {
|
|||
/// is already down: there is no uptime to protect, and `Swap` fails the same way.
|
||||
async fn run_prebuild(host: &impl Host, name: &str) -> Result<()> {
|
||||
if host.is_running(name).await {
|
||||
let rev = host.recorded_rev(name)?;
|
||||
let closure = closure_for(host, name, &rev, "prebuild").await?;
|
||||
tracing::info!(%name, %rev, "prebuild: substituted {}", closure.path);
|
||||
let closure = closure_for(host, name, "prebuild").await?;
|
||||
tracing::info!(%name, "prebuild: substituted {closure}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -550,32 +511,19 @@ enum NoClosure {
|
|||
Unpullable(String),
|
||||
}
|
||||
|
||||
/// The closure the swarm built for an agent, in the store.
|
||||
struct Closure {
|
||||
path: String,
|
||||
/// The `internal/meta` commit the builder built it from.
|
||||
meta_rev: String,
|
||||
}
|
||||
|
||||
/// The closure the swarm built for `name`, downloaded into the store. The
|
||||
/// deploy applies the swarm's closure whatever commit it was built from: when
|
||||
/// that is not the `rev` this hive chose, it is logged, and the caller records
|
||||
/// the pins of the commit actually deployed ([`deployed_pins`]).
|
||||
/// The store path of the closure the swarm declares for `name`, downloaded into
|
||||
/// the store. Whatever `internal/meta` commit it was built from, it is the one
|
||||
/// deployed.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`without_closure`]'s refusal, after logging it.
|
||||
async fn closure_for(host: &impl Host, name: &str, rev: &str, step: &str) -> Result<Closure> {
|
||||
pull_closure(host, name, rev, step)
|
||||
async fn closure_for(host: &impl Host, name: &str, step: &str) -> Result<String> {
|
||||
pull_closure(host, name)
|
||||
.await
|
||||
.map_err(|why| without_closure(name, rev, step, &why))
|
||||
.map_err(|why| without_closure(name, step, &why))
|
||||
}
|
||||
|
||||
async fn pull_closure(
|
||||
host: &impl Host,
|
||||
name: &str,
|
||||
rev: &str,
|
||||
step: &str,
|
||||
) -> Result<Closure, NoClosure> {
|
||||
async fn pull_closure(host: &impl Host, name: &str) -> Result<String, NoClosure> {
|
||||
let record = host
|
||||
.closure_record(name)
|
||||
.await
|
||||
|
|
@ -589,52 +537,7 @@ async fn pull_closure(
|
|||
host.realise_closure(name, &record.store_path)
|
||||
.await
|
||||
.map_err(|e| NoClosure::Unpullable(format!("{e:#}")))?;
|
||||
if record.meta_rev != rev && !same_pins(host, name, &record.meta_rev, rev).await {
|
||||
tracing::warn!(
|
||||
%name, %rev, built_from = %record.meta_rev,
|
||||
"{step}: the swarm built {} from internal/meta {}, not the {rev} this hive chose; deploying the swarm's",
|
||||
record.store_path, record.meta_rev
|
||||
);
|
||||
}
|
||||
Ok(Closure {
|
||||
path: record.store_path,
|
||||
meta_rev: record.meta_rev,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the agent has the same pins at the commits `built` and `chosen`. A
|
||||
/// commit whose pins cannot be read counts as different.
|
||||
async fn same_pins(host: &impl Host, name: &str, built: &str, chosen: &str) -> bool {
|
||||
match (
|
||||
host.pins_at(name, built).await,
|
||||
host.pins_at(name, chosen).await,
|
||||
) {
|
||||
(Ok(built), Ok(chosen)) => built.same_pins(&chosen),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The pins to record for a deploy of `closure` when this hive chose the commit
|
||||
/// `rev`: those of the commit the closure was built from, so the markers say
|
||||
/// what runs. `None` when they are the same commit.
|
||||
///
|
||||
/// # Errors
|
||||
/// The pins of the closure's commit could not be read. The caller fails before
|
||||
/// applying anything.
|
||||
async fn deployed_pins(
|
||||
host: &impl Host,
|
||||
name: &str,
|
||||
closure: &Closure,
|
||||
rev: &str,
|
||||
) -> Result<Option<MetaBuild>> {
|
||||
if closure.meta_rev == rev {
|
||||
return Ok(None);
|
||||
}
|
||||
let pins = host
|
||||
.pins_at(name, &closure.meta_rev)
|
||||
.await
|
||||
.with_context(|| format!("{name}: reading the pins of the commit the swarm built from"))?;
|
||||
Ok(Some(pins))
|
||||
Ok(record.store_path)
|
||||
}
|
||||
|
||||
/// What a rebuild or create does when there is no closure to deploy: it
|
||||
|
|
@ -645,18 +548,19 @@ async fn deployed_pins(
|
|||
/// This is the one place that decides what an unpullable closure (the cache
|
||||
/// unreachable, a TLS error, a failed download) costs. A rebuild never builds
|
||||
/// locally on the hive.
|
||||
fn without_closure(name: &str, rev: &str, step: &str, why: &NoClosure) -> anyhow::Error {
|
||||
fn without_closure(name: &str, step: &str, why: &NoClosure) -> anyhow::Error {
|
||||
let reason = match why {
|
||||
NoClosure::NoRecord => "the swarm has recorded no build of this agent".to_owned(),
|
||||
NoClosure::Unpullable(e) => format!("the swarm's closure cannot be pulled: {e}"),
|
||||
};
|
||||
tracing::error!(%name, %rev, "{step}: {reason}; nothing was changed");
|
||||
tracing::error!(%name, "{step}: {reason}; nothing was changed");
|
||||
anyhow::anyhow!("{name}: {reason}")
|
||||
}
|
||||
|
||||
/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), point the
|
||||
/// container at the swarm's closure, then the post-rebuild bookkeeping tail
|
||||
/// (rev marker, `Rebuilt` event, forge sync, kick, rescan).
|
||||
/// container at the swarm's closure and, once that holds, mark the agent as
|
||||
/// running it. The post-rebuild bookkeeping tail (`Rebuilt` event, forge sync,
|
||||
/// kick, rescan) is the next node.
|
||||
/// The recovery-start on failure is NOT here — the DAG's tail
|
||||
/// `Reconcile` runs after this node terminal ok *or* fail.
|
||||
async fn run_swap(
|
||||
|
|
@ -670,21 +574,20 @@ async fn run_swap(
|
|||
let agent_dir = crate::paths::agent_runtime_dir(name);
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir)?;
|
||||
let rev = host.recorded_rev(name)?;
|
||||
let closure = closure_for(host, name, &rev, "swap").await?;
|
||||
let deployed = deployed_pins(host, name, &closure, &rev).await?;
|
||||
tracing::info!(%name, %rev, "swap: applying {}", closure.path);
|
||||
let closure = closure_for(host, name, "swap").await?;
|
||||
tracing::info!(%name, "swap: applying {closure}");
|
||||
let result = host
|
||||
.swap_update(name, &hive, &paths, &closure.path, id.get())
|
||||
.swap_update(name, &hive, &paths, &closure, id.get())
|
||||
.await;
|
||||
// The bookkeeping node marks what the record holds, so it holds what was
|
||||
// applied.
|
||||
// The marker names what was applied; a failed write is not a failed swap
|
||||
// (the container runs the closure), it only means the next comparison
|
||||
// sees the declaration as new.
|
||||
if result.is_ok()
|
||||
&& let Some(deployed) = &deployed
|
||||
&& let Err(e) = host.mark_deployed(name, &closure)
|
||||
{
|
||||
host.record_meta(name, deployed)?;
|
||||
tracing::warn!(%name, error = ?e, "write the deployed-closure marker failed");
|
||||
}
|
||||
// On success the Ok-only bookkeeping tail (rev marker, forge
|
||||
// On success the Ok-only bookkeeping tail (forge
|
||||
// sync, kick, rescan, snapshot) runs in the sibling `RebuildBookkeeping` node,
|
||||
// which deps `AfterOk(Swap)`. On failure `RebuildBookkeeping` is cancel-cascaded
|
||||
// and the tail `Reconcile` (`AfterAny(RebuildBookkeeping)`) handles recovery; here
|
||||
|
|
@ -702,16 +605,7 @@ async fn run_swap(
|
|||
/// means the profile swap succeeded. Store/forge work only — no nix
|
||||
/// build (build-slot-exempt); the agent lease taken at `Swap` is still held
|
||||
/// (the whole chain up to `Reconcile` is one agent's subgraph).
|
||||
async fn run_rebuild_bookkeeping(
|
||||
host: &impl Host,
|
||||
coord: &Arc<Coordinator>,
|
||||
name: &str,
|
||||
) -> Result<()> {
|
||||
if let Some(build) = host.recorded_build(name)
|
||||
&& let Err(e) = host.mark_deployed(name, &build)
|
||||
{
|
||||
tracing::warn!(%name, error = ?e, "write rev markers failed");
|
||||
}
|
||||
async fn run_rebuild_bookkeeping(coord: &Arc<Coordinator>) -> Result<()> {
|
||||
// The `Rebuilt` manager event is emitted exactly once per agent by the DAG's
|
||||
// `EmitRebuilt` tail — emitting ok here and letting a failed tail `Reconcile`
|
||||
// add a contradictory !ok would double-report the same rebuild.
|
||||
|
|
@ -724,7 +618,7 @@ async fn run_rebuild_bookkeeping(
|
|||
// alongside `run_start`'s own kick for the same boot. `hive-agent` now
|
||||
// self-announces once on its own startup instead — one message per
|
||||
// actual boot, not one per DAG node that happens to run during it.
|
||||
// Rescan so dashboards drop the "needs update" chip.
|
||||
// Rescan so dashboards see the rebuilt container.
|
||||
coord.rescan_containers_and_emit().await;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -743,19 +637,15 @@ async fn run_provision(host: &impl Host, name: &str) -> Result<()> {
|
|||
/// dir creation and MCP listener registration are deferred to the tail
|
||||
/// `Reconcile` (`converge_start_preamble` + `register_agent`) so this
|
||||
/// node stays purely "create", not "create + start". Marks the agent as
|
||||
/// running the pins it was created from.
|
||||
/// running the closure it was created from.
|
||||
///
|
||||
/// The container is created from the swarm's closure for the resolved commit;
|
||||
/// with none to deploy the create fails and makes no container.
|
||||
/// The container is created from the swarm's declared closure; with none to
|
||||
/// deploy the create fails and makes no container.
|
||||
async fn run_create(host: &impl Host, name: &str) -> Result<()> {
|
||||
let build = host.resolve_meta(name).await?;
|
||||
let closure = closure_for(host, name, &build.rev, "create").await?;
|
||||
let build = deployed_pins(host, name, &closure, &build.rev)
|
||||
.await?
|
||||
.unwrap_or(build);
|
||||
tracing::info!(%name, rev = %build.rev, "create: substituted {}", closure.path);
|
||||
host.create_only(name, &closure.path).await?;
|
||||
host.mark_deployed(name, &build)?;
|
||||
let closure = closure_for(host, name, "create").await?;
|
||||
tracing::info!(%name, "create: substituted {closure}");
|
||||
host.create_only(name, &closure).await?;
|
||||
host.mark_deployed(name, &closure)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue