hyperhive/hive-c0re/src/workers/auto_update.rs

//! Boot reconcile: on `hive-c0re serve` boot, (a) run the config path
//! for agents whose per-agent rev marker is stale — a `StartupSweep`
//! DAG (meta hyperhive lock bump) fanning out `Rebuild` children for
//! the stale agents whose `wanted` power intent is `Up` — and (b)
//! converge every other drifted agent to its persisted `wanted` via
//! `Reconcile` DAGs. Two rules keep boot-time nix work minimal:
//!
//! 1. **Stale but wanted-offline agents** get no rebuild — their
//!    rebuild happens the first time they're started (the start
//!    submit path upgrades a stale start to rebuild+start). The sweep
//!    parent still runs whenever *any* marker is stale so the meta
//!    hyperhive lock is bumped for those later start-upgrades.
//! 2. **Agents whose rev marker matches** the current hyperhive flake
//!    path are skipped — nothing changed, no nix work to do.
//!
//! Booting with no config change performs no meta commit — only
//! reconciles. See `docs/coordinator.md::Boot reconcile`.

use std::path::{Path, PathBuf};
use std::sync::Arc;

use anyhow::Result;

use crate::coordinator::Coordinator;
use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME};

/// Marker file recording the hyperhive rev a sub-agent's container was last
/// built against. Sibling of `applied/<name>/` (rather than inside it) to
/// keep it out of the applied repo's git history. Uses a leading dot so a
/// glob over `applied/*` doesn't include it.
pub fn rev_marker_path(name: &str) -> PathBuf {
    PathBuf::from(format!("/var/lib/hyperhive/applied/.{name}.hyperhive-rev"))
}

/// Resolve the current rev of `hyperhive_flake`. For a path on disk we
/// canonicalize (following symlinks) so a /etc/hyperhive → /nix/store/...
/// update yields a different string. For anything else we return None.
#[must_use]
pub fn current_flake_rev(hyperhive_flake: &str) -> Option<String> {
    let path = Path::new(hyperhive_flake);
    if !path.exists() {
        return None;
    }
    std::fs::canonicalize(path)
        .ok()
        .map(|p| p.display().to_string())
}

/// Returns true when the applied repo has commits that have not yet been
/// deployed (i.e. the applied HEAD differs from the sha currently locked in
/// meta's flake.lock). This is the semantic the dashboard `needs_update` chip
/// conveys: "there is a config change ready to apply via rebuild."
///
/// Async on purpose: this runs per agent inside `container_view::build_all`,
/// which fires on the ~10s dashboard sweep, every `AgentStatus` request, and
/// every `rescan_containers_and_emit` after a lifecycle step. A synchronous
/// `git` fork here blocks a tokio worker for the whole exec — under
/// nix-build disk saturation that's long enough that concurrent sweeps
/// starved the runtime and stalled the per-agent sockets.
pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool {
    let applied_head = tokio::process::Command::new("git")
        .args([
            "-C",
            &format!("/var/lib/hyperhive/applied/{name}"),
            "rev-parse",
            "HEAD",
        ])
        .output()
        .await
        .ok()
        .filter(|o| o.status.success())
        .and_then(|o| String::from_utf8(o.stdout).ok())
        .map(|s| s.trim().to_owned());

    match (applied_head.as_deref(), deployed_sha) {
        (Some(head), Some(sha)) => !head.starts_with(sha) && !sha.starts_with(head),
        _ => false,
    }
}

/// Whether this hive is "ruthless" — running with no root/manager agent at
/// all (no ruth). When true, hive-c0re skips the root-agent create/start
/// sweep entirely. Controlled by the host option
/// `services.hyperhive.ruthless`, threaded in via the `HYPERHIVE_RUTHLESS`
/// env var. Defaults to `false` when the var is unset (back-compat: the
/// root agent was always auto-managed before this opt-out existed); only
/// an explicit `true` / `1` / `yes` enables ruthless mode.
fn ruthless() -> bool {
    match std::env::var("HYPERHIVE_RUTHLESS") {
        Ok(v) => matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes"),
        Err(_) => false,
    }
}

/// Auto-create the manager container on startup if it isn't already there.
/// hive-c0re manages the manager end-to-end: operators no longer declare
/// `containers.h-ruth` in their host NixOS config. Bypasses the approval
/// queue — the root/manager is auto-managed by default. Operators who
/// don't want a root agent at all set `services.hyperhive.ruthless = true`,
/// which short-circuits this whole function. Idempotent.
pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
    if ruthless() {
        tracing::info!(
            "ruthless mode (services.hyperhive.ruthless = true) - skipping root agent create/start"
        );
        return Ok(());
    }
    let existing = lifecycle::list().await.unwrap_or_default();
    let current_rev = current_flake_rev(&coord.hyperhive_flake);
    if existing
        .iter()
        .any(|c| c.strip_prefix(AGENT_PREFIX) == Some(MANAGER_NAME))
    {
        // Container exists already. If it predates the unified lifecycle
        // (no applied flake on disk) we must rebuild — otherwise it's
        // running whatever the host-declarative config was at create
        // time, with a wrong systemd unit and port.
        let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix");
        if !applied_flake.exists() && current_rev.is_some() {
            tracing::warn!(
                "manager container exists but no applied flake — forcing rebuild to migrate"
            );
            if let Err(e) = coord.job_queue.submit(crate::job_queue::templates::rebuild(
                MANAGER_NAME,
                crate::job_queue::Source::AutoUpdate,
                "manager migration: no applied flake".to_owned(),
                None,
                true,
            )) {
                tracing::warn!(error = ?e, "manager migration rebuild submit failed");
            }
        } else {
            tracing::debug!("manager container already present");
        }
        // hive-c0re auto-manages the root/manager by default, so a
        // present-but-stopped root (e.g. a first-start failure on a fresh
        // install) is brought back up here: the startup sweep's rebuild only
        // restarts a container that was already running, so without this it
        // stays down until a manual `nixos-container start`. The sub-agent
        // `was_running` guard is intentionally left untouched. (Operators
        // opt out of this whole auto-management with
        // `services.hyperhive.ruthless = true`, gated at the top of
        // this function.)
        if !lifecycle::is_running(MANAGER_NAME).await {
            tracing::info!("manager container present but not running — starting");
            if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
                tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
            }
            if let Err(e) = lifecycle::start(MANAGER_NAME).await {
                tracing::warn!(error = ?e, "manager start failed");
            }
        }
        return Ok(());
    }
    tracing::info!("manager container missing — spawning");
    let runtime = coord.ensure_runtime(MANAGER_NAME)?;
    let hive = coord.hive_env();
    let paths = Coordinator::agent_paths(MANAGER_NAME, runtime);
    lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?;
    if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
        tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
    }
    if let Some(rev) = current_rev {
        let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev);
    }
    Ok(())
}

/// Sort `names` in-place so parents precede their children in the topology.
/// Uses BFS from root agents (depth 0). Agents absent from `topo` sort last,
/// alphabetically within their tier. Stable within each depth tier.
pub fn topology_sort(
    names: &mut [String],
    topo: &std::collections::BTreeMap<String, Option<String>>,
) {
    use std::collections::{HashMap, VecDeque};
    // Build depth map using owned clones so the borrow on `names` is released
    // before the sort_by mutable borrow.
    let name_set: Vec<String> = names.to_vec();
    let mut depth: HashMap<String, usize> = HashMap::new();
    let mut queue: VecDeque<String> = VecDeque::new();
    // Seed roots: entries with no parent, or names not present in topo at all.
    for name in &name_set {
        if topo.get(name).is_none_or(Option::is_none) {
            depth.insert(name.clone(), 0);
            queue.push_back(name.clone());
        }
    }
    // BFS to assign depths to children.
    while let Some(parent) = queue.pop_front() {
        let d = depth[&parent] + 1;
        for name in &name_set {
            let is_child = topo.get(name).and_then(|p| p.as_deref()) == Some(parent.as_str());
            if is_child && !depth.contains_key(name) {
                depth.insert(name.clone(), d);
                queue.push_back(name.clone());
            }
        }
    }
    names.sort_by(|a, b| {
        let da = depth.get(a).copied().unwrap_or(usize::MAX);
        let db = depth.get(b).copied().unwrap_or(usize::MAX);
        da.cmp(&db).then(a.cmp(b))
    });
}

/// Boot reconcile (see the module doc): classify every agent by rev
/// freshness + persisted `wanted` intent, submit one `StartupSweep`
/// DAG (hyperhive lock bump → fan-out rebuilds for stale wanted-up
/// agents) when anything is stale, and `Reconcile` DAGs for agents
/// whose observed power state drifted from `wanted`. Returns Ok even
/// if some submissions failed.
pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
    let containers = match lifecycle::list().await {
        Ok(c) => c,
        Err(e) => {
            tracing::warn!(error = ?e, "boot reconcile: nixos-container list failed");
            return Ok(());
        }
    };

    let current_rev = current_flake_rev(&coord.hyperhive_flake);

    // Resolve container names to logical agent names, then sort by
    // topology depth so parents are always rebuilt before their
    // children. Root agents (depth 0) go first; agents absent from
    // the topology file sort last (stable, alphabetical within tier).
    let mut logical_names: Vec<String> = containers
        .iter()
        .filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned))
        .collect();
    let topo = crate::topology::read();
    topology_sort(&mut logical_names, &topo);

    // Classify. `get_or_seed` doubles as the one-time migration: an
    // agent without an `agent_power` row is seeded from its observed
    // state (running ⇒ Up), after which the DB is authoritative.
    let mut any_stale = false;
    let mut fanout: Vec<String> = Vec::new(); // stale ∧ wanted=Up → sweep rebuild
    let mut drifted: Vec<String> = Vec::new(); // fresh ∧ wanted≠observed → reconcile
    let mut n_deferred = 0usize;
    let mut n_skipped = 0usize;
    for name in &logical_names {
        let running = lifecycle::is_running(name).await;
        let wanted = match coord.power.get_or_seed(name, running) {
            Ok(w) => w,
            Err(e) => {
                tracing::warn!(%name, error = ?e, "agent_power read failed — assuming observed");
                crate::power::Wanted::from_running(running)
            }
        };
        let fresh = current_rev.as_ref().is_some_and(|rev| {
            std::fs::read_to_string(rev_marker_path(name))
                .is_ok_and(|stored| stored == rev.as_str())
        });
        if fresh {
            n_skipped += 1;
        } else {
            any_stale = true;
            if wanted == crate::power::Wanted::Up {
                // Rebuild against the post-bump lock; the DAG's tail
                // Reconcile brings the agent (back) up — covering both
                // the running-stale and stopped-but-wanted-up cases.
                fanout.push(name.clone());
                continue;
            }
            // Stale but wanted offline: no boot-time nix work — the
            // start submit path upgrades a stale start to a rebuild.
            n_deferred += 1;
            tracing::debug!(%name, "boot reconcile: stale but offline — deferring rebuild to on-start");
        }
        if crate::power::reconcile_action(wanted, running) != crate::power::ReconcileAction::Noop {
            drifted.push(name.clone());
        }
    }

    tracing::info!(
        total = containers.len(),
        rebuilds = fanout.len(),
        reconciles = drifted.len(),
        deferred = n_deferred,
        up_to_date = n_skipped,
        "boot reconcile"
    );

    // Sweep parent whenever ANY marker is stale — even when every
    // stale agent is wanted-offline: the hyperhive lock bump must land
    // now so their later start-upgrade rebuilds build against it.
    // No stale agents ⇒ no sweep ⇒ no meta commit on a no-change boot.
    if any_stale {
        let reason = format!(
            "startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date",
            fanout.len(),
            n_deferred,
            n_skipped,
        );
        if let Err(e) = coord
            .job_queue
            .submit(crate::job_queue::templates::startup_sweep(reason, fanout))
        {
            tracing::warn!(error = ?e, "boot reconcile: sweep submit failed");
        }
    }
    for name in drifted {
        if let Err(e) = coord
            .job_queue
            .submit(crate::job_queue::templates::reconcile_only(
                crate::job_queue::Template::Reconcile,
                &name,
                crate::job_queue::Source::AutoUpdate,
                "boot reconcile".to_owned(),
                None,
            ))
        {
            tracing::warn!(%name, error = ?e, "boot reconcile: submit failed");
        }
    }
    coord.emit_rebuild_queue_snapshot();
    Ok(())
}