//! Auto-update: rebuild the agents whose swarm-declared closure the hive has //! not applied. An agent's `agent-closures` record names the store path the //! swarm wants it to run; the agent's `applied/..closure` marker holds //! the one the hive last swapped to ([`crate::attic::closure::pending`]). //! //! A pass runs on `hive-c0re serve` boot, then every [`PASS_INTERVAL`]. A boot //! pass is how a hive that missed the swarm's deploy request catches up; the //! periodic one covers a request lost while the hive was up. Each pass //! rebuilds each stale agent whose `wanted` power intent is `Up` as one `Boot` //! DAG, and the boot pass also converges every other drifted agent to its //! persisted `wanted` via `Reconcile` DAGs. Rules that keep nix work minimal: //! //! 1. **Stale but wanted-offline agents** get no rebuild — their //! rebuild happens the first time they're started (the start //! submit path upgrades a stale start to rebuild+start). //! 2. **Agents running the declared closure** are skipped. //! 3. **A declaration the hive cannot read or act on** (no connection to the //! swarm queue, no record, the closure not in the cache, no readable running //! closure to compare with) rebuilds nothing. //! 4. **A declaration is attempted once per process** by the periodic pass, and //! never while a rebuild of the agent is queued or running, so a closure //! that fails to deploy does not restart its agent every interval. //! //! See `docs/scheduler/coordinator.md::Boot reconcile`. use std::collections::HashMap; use std::path::Path; use std::sync::Arc; use std::time::Duration; use anyhow::Result; use crate::coordinator::Coordinator; use crate::lifecycle::{self, AGENT_PREFIX}; /// How often the swarm's declarations are compared with what the hive runs /// after the boot pass. const PASS_INTERVAL: Duration = Duration::from_mins(5); /// Resolve the current rev of `hyperhive_flake`. For a path on disk we /// canonicalize (following symlinks) so a /etc/hyperhive → /nix/store/... /// update yields a different string. For anything else we return None. #[must_use] pub fn current_flake_rev(hyperhive_flake: &str) -> Option { let path = Path::new(hyperhive_flake); if !path.exists() { return None; } std::fs::canonicalize(path) .ok() .map(|p| p.display().to_string()) } /// What one pass decided. struct Plan { /// Stale ∧ wanted up, running agents first. rebuilds: Vec, /// Agents whose observed power state drifted from `wanted`; empty after the /// boot pass. drifted: Vec, deferred: usize, skipped: usize, } /// Whether an agent needs no rebuild this pass: nothing is pending, a rebuild /// is already queued, or this process already attempted the declaration /// (`attempted`) and the periodic pass does not retry it. fn is_fresh(pending: Option<&str>, attempted: Option<&str>, rebuild_live: bool) -> bool { pending.is_none() || rebuild_live || pending == attempted } /// Classify every agent by what the swarm declares + persisted `wanted` /// intent. `attempted` holds the declaration this process last rebuilt each /// agent for; the agents it adds to `Plan::rebuilds` are recorded in it. async fn classify( coord: &Arc, attempted: &mut HashMap, ) -> Option { let containers = match lifecycle::list().await { Ok(c) => c, Err(e) => { tracing::warn!(error = ?e, "auto-update: nixos-container list failed"); return None; } }; // Resolve container names to logical agent names, then sort. The // parent field this used to depth-sort by is gone; with no // hierarchy left to respect, alphabetical is the whole order — and it // is exactly what the depth sort already produced once every agent // was a root. let mut logical_names: Vec = containers .iter() .filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned)) .collect(); logical_names.sort(); // `get_or_seed` doubles as the one-time migration: an // agent without an `agent_power` row is seeded from its observed // state (running ⇒ Up), after which the DB is authoritative. // stale ∧ wanted=Up → rebuild. The bool is the agent's observed // running state, used to order running agents first before submit. let mut rebuilds: Vec<(String, bool)> = Vec::new(); let mut drifted: Vec = Vec::new(); // fresh ∧ wanted≠observed → reconcile let mut deferred = 0usize; let mut skipped = 0usize; for name in &logical_names { let running = lifecycle::is_running(name).await; let wanted = match coord.power.get_or_seed(name, running) { Ok(w) => Some(w), Err(e) => { tracing::warn!(%name, error = ?e, "agent_power read failed — reconciling so it surfaces"); None } }; let pending = crate::attic::closure::pending_path(name).await; let fresh = is_fresh( pending.as_deref(), attempted.get(name).map(String::as_str), coord.job_queue.rebuild_live(name), ); match boot_action(wanted, fresh, running) { BootAction::Rebuild => { if let Some(path) = pending { attempted.insert(name.clone(), path); } rebuilds.push((name.clone(), running)); } BootAction::Defer { reconcile } => { deferred += 1; tracing::debug!(%name, "auto-update: stale but offline — deferring rebuild to on-start"); if reconcile { drifted.push(name.clone()); } } BootAction::Skip { reconcile } => { skipped += 1; if reconcile { drifted.push(name.clone()); } } BootAction::Unreadable => drifted.push(name.clone()), } } // Rebuild running agents first. All entries are wanted=Up; // among them, warm the live/serving agents onto the declared closure before the // stopped-but-wanted-up ones so the scarce build slots hit uptime-critical // agents first. Stable sort keeps the alphabetical order within each // running/stopped group. The `drifted` reconciles aren't sorted // — they hold no build slot and run concurrently, so their order is moot. rebuilds.sort_by_key(|(_, running)| !running); Some(Plan { rebuilds: rebuilds.into_iter().map(|(name, _)| name).collect(), drifted, deferred, skipped, }) } /// Auto-update (see the module doc): the boot pass, then a rebuild-only pass /// every [`PASS_INTERVAL`] until shutdown. Returns Ok even if some submissions /// failed. pub async fn run(coord: Arc) -> Result<()> { let mut shutdown = coord.shutdown_rx(); let mut attempted = HashMap::new(); if let Some(plan) = classify(&coord, &mut attempted).await { tracing::info!( total = plan.rebuilds.len() + plan.deferred + plan.skipped, rebuilds = plan.rebuilds.len(), reconciles = plan.drifted.len(), deferred = plan.deferred, up_to_date = plan.skipped, "boot reconcile" ); submit_boot_tree( &coord, &plan.rebuilds, plan.drifted, plan.deferred, plan.skipped, ); } submit_startup_sweep_nodes(&coord); loop { tokio::select! { () = tokio::time::sleep(PASS_INTERVAL) => {} _ = shutdown.changed() => return Ok(()), } if let Some(plan) = classify(&coord, &mut attempted).await && !plan.rebuilds.is_empty() { tracing::info!( rebuilds = plan.rebuilds.len(), "auto-update: the swarm declared new closures" ); submit_boot_tree( &coord, &plan.rebuilds, Vec::new(), plan.deferred, plan.skipped, ); } } } /// What the boot sweep does with one agent. #[derive(Debug, PartialEq, Eq)] enum BootAction { /// Stale ∧ wanted up: rebuild to the swarm-declared closure. The DAG's /// tail `Reconcile` brings the agent (back) up, covering both the /// running-stale and stopped-but-wanted-up cases. Rebuild, /// Stale ∧ wanted offline: no boot-time nix work — the start submit /// path upgrades a stale start to a rebuild. Defer { reconcile: bool }, /// Already running the declared closure, or nothing to deploy. Skip { reconcile: bool }, /// The intent could not be read. Unreadable, } /// `wanted` is `None` when the store could not answer for this agent. /// /// That case has no neutral stand-in, which is why it is a variant rather /// than a fallback value: `Wanted::from_running(running)` is precisely the /// value for which `reconcile_action` returns `Noop`, so guessing it here /// puts the agent beyond every check below — a corrupt row then leaves one /// `warn!` per boot and no other trace. The queued reconcile's own /// `get_or_seed` fails as a per-agent node instead. fn boot_action(wanted: Option, fresh: bool, running: bool) -> BootAction { let Some(wanted) = wanted else { return BootAction::Unreadable; }; let reconcile = crate::power::reconcile_action(wanted, running) != crate::power::ReconcileAction::Noop; if fresh { BootAction::Skip { reconcile } } else if wanted == crate::power::Wanted::Up { BootAction::Rebuild } else { BootAction::Defer { reconcile } } } /// Submit the boot-time knowledge and wanted-state sweeps as DAG /// nodes — `KnowledgePull`, `WantedPull`. Unlike /// [`submit_boot_tree`] this runs on **every** boot, quiet or not: these /// aren't config-drift work, they're startup housekeeping that always needs /// to happen, and the point of moving them here is exactly so they show up /// as real work on the dashboard instead of an invisible `tokio::spawn` that /// only surfaces on failure. Independent, build-slot- and lease-exempt /// roots — no dependency edges between them, matching the existing /// `Reconcile`-root pattern in [`boot_nodes`]. `KnowledgePull` holds its /// sweep's own resource, so it queues behind any pass of the same sweep /// already running. fn submit_startup_sweep_nodes(coord: &Arc) { use crate::job_queue::NodeKind; if let Err(e) = coord.job_queue.insert_job(|b| { let _ = crate::job_queue::templates::knowledge_pull(b); let _ = b.node(NodeKind::WantedPull); Vec::new() }) { tracing::warn!(error = ?e, "boot: startup sweep DAG insert failed"); } } /// The boot DAG's node declarations, split out of [`submit_boot_tree`] so they /// can be exercised without a live [`Coordinator`]. /// /// That split is not cosmetic: this path constructs nodes outside /// `job_queue/`, so it is the one place a resource declaration can be forgotten /// without any in-module test noticing. It has happened once already — the /// boot `Reconcile`s silently declared nothing when kind-derived resources /// were removed, which drops the agent lease a boot reconcile needs to not /// race another DAG's container ops. pub(crate) fn boot_nodes( builder: &crate::job_queue::JobBuilder, fanout: &[String], drifted: Vec, ) { use crate::job_queue::NodeKind; use crate::job_queue::resource::Resource; // One graceful rebuild subgraph per stale agent, each an independent // group root: agents that were mid-turn when the host came up drain rather // than being cut off, and the rebuilds overlap on their own leases. for agent in fanout { let _ = crate::job_queue::templates::graceful_rebuild_nodes(builder, agent, None); } // One boot Reconcile per drifted agent — independent roots. for name in drifted { // Name the lease before the agent string moves into the kind. let lease = Resource::Agent(name.clone()); let _ = builder .node(NodeKind::Reconcile { agent: name }) .needs(lease); } } /// Submit this boot's work as **one DAG** (no anchor node, no per-agent /// child DAGs). It holds one graceful rebuild subgraph per stale agent, and /// every drifted agent gets a boot `Reconcile` as an independent root, so they /// converge concurrently with the rebuilds. No-op when there's nothing to do. fn submit_boot_tree( coord: &Arc, fanout: &[String], drifted: Vec, n_deferred: usize, n_skipped: usize, ) { // Fully-quiet boot (nothing to rebuild, nothing drifted) inserts nothing. if fanout.is_empty() && drifted.is_empty() { return; } // The summary the sweep used to hand the container as its `reason` is a log // line now: it was only ever stored on a node nobody read, and the counts // are worth having where they can actually be seen. tracing::info!( rebuilds = fanout.len(), reconciles = drifted.len(), deferred = n_deferred, up_to_date = n_skipped, "boot: sweep" ); // The sweep's own rebuild subgraphs emit their `Rebuilt` events as they // land; the boot DAG as a whole has no terminal side effect, so no tail. // The subgraphs also carry their own per-agent crash-watch suppression // during their `Swap` (applied at claim time); a reconcile-only boot needs // no transient. if let Err(e) = coord.job_queue.insert_job(|b| { boot_nodes(b, fanout, drifted); Vec::new() }) { tracing::warn!(error = ?e, "boot: sweep DAG insert failed"); } coord.emit_rebuild_queue_snapshot(); } #[cfg(test)] mod tests { use super::{BootAction, boot_action, is_fresh}; use crate::power::Wanted; // The regression the `Unreadable` variant exists for: the arm used to // substitute `from_running(running)`, which is `Noop` against BOTH // observations — so no combination of inputs could reach a reconcile. #[test] fn an_unreadable_intent_is_never_settled_whatever_the_agent_is_doing() { for fresh in [true, false] { for running in [true, false] { assert_eq!( boot_action(None, fresh, running), BootAction::Unreadable, "fresh={fresh} running={running}" ); } } } #[test] fn a_fresh_agent_doing_what_it_should_is_skipped_without_a_reconcile() { assert_eq!( boot_action(Some(Wanted::Up), true, true), BootAction::Skip { reconcile: false } ); assert_eq!( boot_action(Some(Wanted::Offline), true, false), BootAction::Skip { reconcile: false } ); } #[test] fn a_fresh_agent_that_drifted_from_its_intent_is_reconciled() { assert_eq!( boot_action(Some(Wanted::Up), true, false), BootAction::Skip { reconcile: true } ); assert_eq!( boot_action(Some(Wanted::Offline), true, true), BootAction::Skip { reconcile: true } ); } #[test] fn a_stale_agent_wanted_up_is_rebuilt_running_or_not() { assert_eq!( boot_action(Some(Wanted::Up), false, true), BootAction::Rebuild ); assert_eq!( boot_action(Some(Wanted::Up), false, false), BootAction::Rebuild ); } #[test] fn a_stale_agent_wanted_offline_defers_and_still_reports_drift() { assert_eq!( boot_action(Some(Wanted::Offline), false, false), BootAction::Defer { reconcile: false } ); assert_eq!( boot_action(Some(Wanted::Offline), false, true), BootAction::Defer { reconcile: true } ); } /// A changed declaration is rebuilt with no operator step, and with no read /// of the forge: the boot pass reaches `Rebuild` from the record and marker /// alone. #[test] fn a_changed_declaration_for_a_wanted_up_agent_is_rebuilt_unprompted() { let fresh = is_fresh(Some("/nix/store/new"), None, false); assert_eq!( boot_action(Some(Wanted::Up), fresh, true), BootAction::Rebuild ); let fresh = is_fresh(None, None, false); assert_eq!( boot_action(Some(Wanted::Up), fresh, true), BootAction::Skip { reconcile: false }, "control: nothing declared, nothing rebuilt" ); } #[test] fn a_declaration_is_attempted_once_and_never_beside_a_live_rebuild() { assert!(is_fresh(None, None, false), "nothing pending"); assert!( is_fresh(None, Some("/nix/store/b"), false), "nothing pending now, whatever was attempted" ); assert!( !is_fresh(Some("/nix/store/b"), None, false), "control: a new declaration" ); assert!( is_fresh(Some("/nix/store/b"), None, true), "a rebuild is queued" ); assert!( is_fresh(Some("/nix/store/b"), Some("/nix/store/b"), false), "already attempted" ); assert!( !is_fresh(Some("/nix/store/c"), Some("/nix/store/b"), false), "a newer declaration is attempted again" ); } }