//! Startup auto-update: on `hive-c0re serve` boot, rebuild every known //! container unconditionally. `nixos-container update` is a no-op at the //! nix level when nothing changed (same store path), so the cost is low //! and avoids rev-marker staleness (all agents always need an update pass //! when any meta commit lands). See `docs/coordinator.md::Auto-update sweep`. use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::{Context, Result}; use crate::coordinator::Coordinator; use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME}; /// Marker file recording the hyperhive rev a sub-agent's container was last /// built against. Sibling of `applied//` (rather than inside it) to /// keep it out of the applied repo's git history. Uses a leading dot so a /// glob over `applied/*` doesn't include it. pub fn rev_marker_path(name: &str) -> PathBuf { PathBuf::from(format!("/var/lib/hyperhive/applied/.{name}.hyperhive-rev")) } /// Resolve the current rev of `hyperhive_flake`. For a path on disk we /// canonicalize (following symlinks) so a /etc/hyperhive → /nix/store/... /// update yields a different string. For anything else we return None. #[must_use] pub fn current_flake_rev(hyperhive_flake: &str) -> Option { let path = Path::new(hyperhive_flake); if !path.exists() { return None; } std::fs::canonicalize(path) .ok() .map(|p| p.display().to_string()) } /// Returns true when the applied repo has commits that have not yet been /// deployed (i.e. the applied HEAD differs from the sha currently locked in /// meta's flake.lock). This is the semantic the dashboard `needs_update` chip /// conveys: "there is a config change ready to apply via rebuild." /// /// Async on purpose: this runs per agent inside `container_view::build_all`, /// which fires on the ~10s dashboard sweep, every `AgentStatus` request, and /// every `rescan_containers_and_emit` after a lifecycle step. A synchronous /// `git` fork here blocks a tokio worker for the whole exec — under /// nix-build disk saturation that's long enough that concurrent sweeps /// starved the runtime and stalled the per-agent sockets. pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool { let applied_head = tokio::process::Command::new("git") .args([ "-C", &format!("/var/lib/hyperhive/applied/{name}"), "rev-parse", "HEAD", ]) .output() .await .ok() .filter(|o| o.status.success()) .and_then(|o| String::from_utf8(o.stdout).ok()) .map(|s| s.trim().to_owned()); match (applied_head.as_deref(), deployed_sha) { (Some(head), Some(sha)) => !head.starts_with(sha) && !sha.starts_with(head), _ => false, } } /// Rebuild one sub-agent and refresh its marker. Used by both the startup /// scanner and the dashboard's manual "update" button so the two paths /// can't diverge. /// /// `queue_entry_id` is `Some(id)` when the rebuild was dispatched from /// the `rebuild_queue` worker (lets the function annotate its phase via /// `coord.set_queue_step`) and `None` when called directly (e.g. the /// root-agent migration nudge in `ensure_root_agent`). /// /// `relock` bumps the agent's meta input to `applied//main` before /// the container rebuild. Pass `false` only for meta-update cascade /// rebuilds, where re-locking would revert the bump the cascade just /// committed (see `lifecycle::rebuild`). /// /// `defer_start_source` is `Some(source)` for queue-dispatched rebuilds: /// instead of holding the serialized build lane through the container /// boot, the start-after-rebuild is enqueued as a fast-lane `Start` /// entry (grouped under this rebuild via `parent_id`, same split as the /// graceful-stop follow-up). Pass `None` for direct callers to keep the /// start inline. /// /// # Errors /// /// Propagates errors from `coord.ensure_runtime` and `lifecycle::rebuild`. pub async fn rebuild_agent( coord: &Arc, name: &str, current_rev: &str, queue_entry_id: Option, relock: bool, defer_start_source: Option, ) -> Result<()> { tracing::info!(%name, rev = %current_rev, "rebuild agent"); let agent_dir = coord .ensure_runtime(name) .with_context(|| format!("ensure_runtime {name}"))?; let hive = coord.hive_env(); let paths = Coordinator::agent_paths(name, agent_dir); // Suppress crash_watch during the stop+start window inside // lifecycle::rebuild. Dashboard rebuilds already do this via // lifecycle_action; this catches the auto-update scan + any // other direct caller. let guard = coord.transient_guard(name, crate::coordinator::TransientKind::Rebuilding); let result = lifecycle::rebuild( name, &hive, &paths, relock, defer_start_source.is_some(), &|step| coord.set_queue_step(queue_entry_id, step), &|log_id| { if let Some(qid) = queue_entry_id && coord.rebuild_queue.set_build_log_id(qid, log_id) { coord.emit_rebuild_queue_snapshot(); } }, ) .await; drop(guard); match &result { Ok(needs_start) => { if let Err(e) = std::fs::write(rev_marker_path(name), current_rev) { tracing::warn!(%name, error = ?e, "write rev marker failed"); } // Deferred start: hand the container boot to the fast lane so // this build-lane entry completes now and the next queued // rebuild's nix build overlaps with the boot. `parent_id` // groups the follow-up under this rebuild on the dashboard — // same split the graceful-stop path uses for its container // stop. A start failure surfaces on the Start entry (with // the cold-start fallback) instead of failing the rebuild. if *needs_start && let Some(source) = defer_start_source { coord .rebuild_queue .enqueue_full(crate::rebuild_queue::FullEnqueue { kind: crate::rebuild_queue::QueueKind::Start, agent: name.to_owned(), source, reason: format!("start after rebuild of {name}"), parent_id: queue_entry_id, inputs: Vec::new(), approval_id: None, perm_payload: None, depends_on: Vec::new(), }); coord.emit_rebuild_queue_snapshot(); } coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { agent: name.to_owned(), ok: true, note: None, sha: None, tag: None, }); coord.set_queue_step(queue_entry_id, "forge sync"); // Run the full forge sync on every successful rebuild so // the rebuild path is equivalent to the hive-c0re startup // sweep: token, config-repo mirror, meta read access, and // meta remote are all kept in sync. Recovers missing tokens // (e.g. first-spawn seeding failed transiently) without // requiring a full hive-c0re restart. crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await; // Mirror the matrix side of the startup sweep: if hive-matrix // is present, ensure this agent has a registered user + token. // Idempotent (skips if token file already exists). Keeps the // rebuild path equivalent to the startup sweep for newly-spawned // agents that missed ensure_all(). crate::matrix::sync_agent_standalone(name).await; // Wake the agent on its next turn so claude sees a // "you were rebuilt — check /state/ for notes, --continue // session intact" hint. Covers dashboard rebuild, admin // CLI rebuild, auto-update startup scan, and the // dashboard's meta-input update path — all of which // route through rebuild_agent. coord.kick_agent(name, "container rebuilt"); // Container state (needs_update, deployed_sha) may have // shifted — rescan so dashboards drop the "needs update" // chip without waiting for the next /api/state poll. coord.rescan_containers_and_emit().await; // Lock bump → meta-inputs panel needs to re-render. crate::dashboard::emit_meta_inputs_snapshot(coord); } Err(e) => { coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt { agent: name.to_owned(), ok: false, note: Some(format!("{e:#}")), sha: None, tag: None, }); coord.rescan_containers_and_emit().await; } } result.map(|_| ()) } /// Whether this hive is "ruthless" — running with no root/manager agent at /// all (no ruth). When true, hive-c0re skips the root-agent create/start /// sweep entirely. Controlled by the host option /// `services.hyperhive.ruthless`, threaded in via the `HYPERHIVE_RUTHLESS` /// env var. Defaults to `false` when the var is unset (back-compat: the /// root agent was always auto-managed before this opt-out existed); only /// an explicit `true` / `1` / `yes` enables ruthless mode. fn ruthless() -> bool { match std::env::var("HYPERHIVE_RUTHLESS") { Ok(v) => matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes"), Err(_) => false, } } /// Auto-create the manager container on startup if it isn't already there. /// hive-c0re manages the manager end-to-end: operators no longer declare /// `containers.h-ruth` in their host NixOS config. Bypasses the approval /// queue — the root/manager is auto-managed by default. Operators who /// don't want a root agent at all set `services.hyperhive.ruthless = true`, /// which short-circuits this whole function. Idempotent. pub async fn ensure_root_agent(coord: &Arc) -> Result<()> { if ruthless() { tracing::info!( "ruthless mode (services.hyperhive.ruthless = true) - skipping root agent create/start" ); return Ok(()); } let existing = lifecycle::list().await.unwrap_or_default(); let current_rev = current_flake_rev(&coord.hyperhive_flake); if existing .iter() .any(|c| c.strip_prefix(AGENT_PREFIX) == Some(MANAGER_NAME)) { // Container exists already. If it predates the unified lifecycle // (no applied flake on disk) we must rebuild — otherwise it's // running whatever the host-declarative config was at create // time, with a wrong systemd unit and port. let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix"); if !applied_flake.exists() && let Some(rev) = current_rev.as_ref() { tracing::warn!( "manager container exists but no applied flake — forcing rebuild to migrate" ); let coord_clone = coord.clone(); if let Err(e) = rebuild_agent(&coord_clone, MANAGER_NAME, rev.as_str(), None, true, None).await { tracing::warn!(error = ?e, "manager migration rebuild failed"); } } else { tracing::debug!("manager container already present"); } // hive-c0re auto-manages the root/manager by default, so a // present-but-stopped root (e.g. a first-start failure on a fresh // install) is brought back up here: the startup sweep's rebuild only // restarts a container that was already running, so without this it // stays down until a manual `nixos-container start`. The sub-agent // `was_running` guard is intentionally left untouched. (Operators // opt out of this whole auto-management with // `services.hyperhive.ruthless = true`, gated at the top of // this function.) if !lifecycle::is_running(MANAGER_NAME).await { tracing::info!("manager container present but not running — starting"); if let Err(e) = lifecycle::start(MANAGER_NAME).await { tracing::warn!(error = ?e, "manager start failed"); } } return Ok(()); } tracing::info!("manager container missing — spawning"); let runtime = coord.ensure_runtime(MANAGER_NAME)?; let hive = coord.hive_env(); let paths = Coordinator::agent_paths(MANAGER_NAME, runtime); lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?; if let Some(rev) = current_rev { let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev); } Ok(()) } /// Sort `names` in-place so parents precede their children in the topology. /// Uses BFS from root agents (depth 0). Agents absent from `topo` sort last, /// alphabetically within their tier. Stable within each depth tier. pub fn topology_sort( names: &mut [String], topo: &std::collections::BTreeMap>, ) { use std::collections::{HashMap, VecDeque}; // Build depth map using owned clones so the borrow on `names` is released // before the sort_by mutable borrow. let name_set: Vec = names.to_vec(); let mut depth: HashMap = HashMap::new(); let mut queue: VecDeque = VecDeque::new(); // Seed roots: entries with no parent, or names not present in topo at all. for name in &name_set { if topo.get(name).is_none_or(Option::is_none) { depth.insert(name.clone(), 0); queue.push_back(name.clone()); } } // BFS to assign depths to children. while let Some(parent) = queue.pop_front() { let d = depth[&parent] + 1; for name in &name_set { let is_child = topo.get(name).and_then(|p| p.as_deref()) == Some(parent.as_str()); if is_child && !depth.contains_key(name) { depth.insert(name.clone(), d); queue.push_back(name.clone()); } } } names.sort_by(|a, b| { let da = depth.get(a).copied().unwrap_or(usize::MAX); let db = depth.get(b).copied().unwrap_or(usize::MAX); da.cmp(&db).then(a.cmp(b)) }); } /// Rebuild every container on startup. Enqueues a `StartupSweep` parent /// entry (agent = `"hyperhive"`) followed by per-agent `Rebuild` children /// linked via `parent_id`. The dashboard renders them nested so the operator /// can see at a glance "boot N agents, here is each rebuild's status". /// Returns Ok even if some rebuilds failed. pub async fn run(coord: Arc) -> Result<()> { let containers = match lifecycle::list().await { Ok(c) => c, Err(e) => { tracing::warn!(error = ?e, "auto-update: nixos-container list failed"); return Ok(()); } }; // Enqueue the parent sweep entry. The worker processes it trivially // (no-op dispatch) so it completes quickly; its purpose is to give the // dashboard a "why" header for the per-agent child rebuilds below. let sweep_id = coord.rebuild_queue.enqueue( crate::rebuild_queue::QueueKind::StartupSweep, "hyperhive".to_owned(), crate::rebuild_queue::QueueSource::AutoUpdate, format!("startup sweep ({} containers)", containers.len()), None, ); tracing::info!( agents = containers.len(), sweep_id, "auto-update: queueing all on startup" ); // Resolve container names to logical agent names, then sort by // topology depth so parents are always rebuilt before their // children. Root agents (depth 0) go first; agents absent from // the topology file sort last (stable, alphabetical within tier). let mut logical_names: Vec = containers .iter() .filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned)) .collect(); let topo = crate::topology::read(); topology_sort(&mut logical_names, &topo); for name in logical_names { coord.rebuild_queue.enqueue( crate::rebuild_queue::QueueKind::Rebuild, name, crate::rebuild_queue::QueueSource::StartupSweep, "startup sweep".to_owned(), Some(sweep_id), ); } coord.emit_rebuild_queue_snapshot(); Ok(()) }