377 lines
16 KiB
Rust
377 lines
16 KiB
Rust
//! Startup auto-update: on `hive-c0re serve` boot, rebuild every known
|
|
//! container unconditionally. `nixos-container update` is a no-op at the
|
|
//! nix level when nothing changed (same store path), so the cost is low
|
|
//! and avoids rev-marker staleness (all agents always need an update pass
|
|
//! when any meta commit lands). See `docs/coordinator.md::Auto-update sweep`.
|
|
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::Arc;
|
|
|
|
use anyhow::{Context, Result};
|
|
|
|
use crate::coordinator::Coordinator;
|
|
use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME};
|
|
|
|
/// Marker file recording the hyperhive rev a sub-agent's container was last
|
|
/// built against. Sibling of `applied/<name>/` (rather than inside it) to
|
|
/// keep it out of the applied repo's git history. Uses a leading dot so a
|
|
/// glob over `applied/*` doesn't include it.
|
|
pub fn rev_marker_path(name: &str) -> PathBuf {
|
|
PathBuf::from(format!("/var/lib/hyperhive/applied/.{name}.hyperhive-rev"))
|
|
}
|
|
|
|
/// Resolve the current rev of `hyperhive_flake`. For a path on disk we
|
|
/// canonicalize (following symlinks) so a /etc/hyperhive → /nix/store/...
|
|
/// update yields a different string. For anything else we return None.
|
|
#[must_use]
|
|
pub fn current_flake_rev(hyperhive_flake: &str) -> Option<String> {
|
|
let path = Path::new(hyperhive_flake);
|
|
if !path.exists() {
|
|
return None;
|
|
}
|
|
std::fs::canonicalize(path)
|
|
.ok()
|
|
.map(|p| p.display().to_string())
|
|
}
|
|
|
|
/// Returns true when the applied repo has commits that have not yet been
|
|
/// deployed (i.e. the applied HEAD differs from the sha currently locked in
|
|
/// meta's flake.lock). This is the semantic the dashboard `needs_update` chip
|
|
/// conveys: "there is a config change ready to apply via rebuild."
|
|
///
|
|
/// Async on purpose: this runs per agent inside `container_view::build_all`,
|
|
/// which fires on the ~10s dashboard sweep, every `AgentStatus` request, and
|
|
/// every `rescan_containers_and_emit` after a lifecycle step. A synchronous
|
|
/// `git` fork here blocks a tokio worker for the whole exec — under
|
|
/// nix-build disk saturation that's long enough that concurrent sweeps
|
|
/// starved the runtime and stalled the per-agent sockets.
|
|
pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool {
|
|
let applied_head = tokio::process::Command::new("git")
|
|
.args([
|
|
"-C",
|
|
&format!("/var/lib/hyperhive/applied/{name}"),
|
|
"rev-parse",
|
|
"HEAD",
|
|
])
|
|
.output()
|
|
.await
|
|
.ok()
|
|
.filter(|o| o.status.success())
|
|
.and_then(|o| String::from_utf8(o.stdout).ok())
|
|
.map(|s| s.trim().to_owned());
|
|
|
|
match (applied_head.as_deref(), deployed_sha) {
|
|
(Some(head), Some(sha)) => !head.starts_with(sha) && !sha.starts_with(head),
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Rebuild one sub-agent and refresh its marker. Used by both the startup
|
|
/// scanner and the dashboard's manual "update" button so the two paths
|
|
/// can't diverge.
|
|
///
|
|
/// `queue_entry_id` is `Some(id)` when the rebuild was dispatched from
|
|
/// the `rebuild_queue` worker (lets the function annotate its phase via
|
|
/// `coord.set_queue_step`) and `None` when called directly (e.g. the
|
|
/// root-agent migration nudge in `ensure_root_agent`).
|
|
///
|
|
/// `relock` bumps the agent's meta input to `applied/<n>/main` before
|
|
/// the container rebuild. Pass `false` only for meta-update cascade
|
|
/// rebuilds, where re-locking would revert the bump the cascade just
|
|
/// committed (see `lifecycle::rebuild`).
|
|
///
|
|
/// `defer_start_source` is `Some(source)` for queue-dispatched rebuilds:
|
|
/// instead of holding the serialized build lane through the container
|
|
/// boot, the start-after-rebuild is enqueued as a fast-lane `Start`
|
|
/// entry (grouped under this rebuild via `parent_id`, same split as the
|
|
/// graceful-stop follow-up). Pass `None` for direct callers to keep the
|
|
/// start inline.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Propagates errors from `coord.ensure_runtime` and `lifecycle::rebuild`.
|
|
pub async fn rebuild_agent(
|
|
coord: &Arc<Coordinator>,
|
|
name: &str,
|
|
current_rev: &str,
|
|
queue_entry_id: Option<u64>,
|
|
relock: bool,
|
|
defer_start_source: Option<crate::rebuild_queue::QueueSource>,
|
|
) -> Result<()> {
|
|
tracing::info!(%name, rev = %current_rev, "rebuild agent");
|
|
let agent_dir = coord
|
|
.ensure_runtime(name)
|
|
.with_context(|| format!("ensure_runtime {name}"))?;
|
|
let hive = coord.hive_env();
|
|
let paths = Coordinator::agent_paths(name, agent_dir);
|
|
// Suppress crash_watch during the stop+start window inside
|
|
// lifecycle::rebuild. Dashboard rebuilds already do this via
|
|
// lifecycle_action; this catches the auto-update scan + any
|
|
// other direct caller.
|
|
let guard = coord.transient_guard(name, crate::coordinator::TransientKind::Rebuilding);
|
|
let result = lifecycle::rebuild(
|
|
name,
|
|
&hive,
|
|
&paths,
|
|
relock,
|
|
defer_start_source.is_some(),
|
|
&|step| coord.set_queue_step(queue_entry_id, step),
|
|
&|log_id| {
|
|
if let Some(qid) = queue_entry_id
|
|
&& coord.rebuild_queue.set_build_log_id(qid, log_id)
|
|
{
|
|
coord.emit_rebuild_queue_snapshot();
|
|
}
|
|
},
|
|
)
|
|
.await;
|
|
drop(guard);
|
|
match &result {
|
|
Ok(needs_start) => {
|
|
if let Err(e) = std::fs::write(rev_marker_path(name), current_rev) {
|
|
tracing::warn!(%name, error = ?e, "write rev marker failed");
|
|
}
|
|
// Deferred start: hand the container boot to the fast lane so
|
|
// this build-lane entry completes now and the next queued
|
|
// rebuild's nix build overlaps with the boot. `parent_id`
|
|
// groups the follow-up under this rebuild on the dashboard —
|
|
// same split the graceful-stop path uses for its container
|
|
// stop. A start failure surfaces on the Start entry (with
|
|
// the cold-start fallback) instead of failing the rebuild.
|
|
if *needs_start && let Some(source) = defer_start_source {
|
|
coord
|
|
.rebuild_queue
|
|
.enqueue_full(crate::rebuild_queue::FullEnqueue {
|
|
kind: crate::rebuild_queue::QueueKind::Start,
|
|
agent: name.to_owned(),
|
|
source,
|
|
reason: format!("start after rebuild of {name}"),
|
|
parent_id: queue_entry_id,
|
|
inputs: Vec::new(),
|
|
approval_id: None,
|
|
perm_payload: None,
|
|
depends_on: Vec::new(),
|
|
});
|
|
coord.emit_rebuild_queue_snapshot();
|
|
}
|
|
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
|
|
agent: name.to_owned(),
|
|
ok: true,
|
|
note: None,
|
|
sha: None,
|
|
tag: None,
|
|
});
|
|
coord.set_queue_step(queue_entry_id, "forge sync");
|
|
// Run the full forge sync on every successful rebuild so
|
|
// the rebuild path is equivalent to the hive-c0re startup
|
|
// sweep: token, config-repo mirror, meta read access, and
|
|
// meta remote are all kept in sync. Recovers missing tokens
|
|
// (e.g. first-spawn seeding failed transiently) without
|
|
// requiring a full hive-c0re restart.
|
|
crate::forge::sync_agent(name, crate::forge::core_token().as_deref()).await;
|
|
// Mirror the matrix side of the startup sweep: if hive-matrix
|
|
// is present, ensure this agent has a registered user + token.
|
|
// Idempotent (skips if token file already exists). Keeps the
|
|
// rebuild path equivalent to the startup sweep for newly-spawned
|
|
// agents that missed ensure_all().
|
|
crate::matrix::sync_agent_standalone(name).await;
|
|
// Wake the agent on its next turn so claude sees a
|
|
// "you were rebuilt — check /state/ for notes, --continue
|
|
// session intact" hint. Covers dashboard rebuild, admin
|
|
// CLI rebuild, auto-update startup scan, and the
|
|
// dashboard's meta-input update path — all of which
|
|
// route through rebuild_agent.
|
|
coord.kick_agent(name, "container rebuilt");
|
|
// Container state (needs_update, deployed_sha) may have
|
|
// shifted — rescan so dashboards drop the "needs update"
|
|
// chip without waiting for the next /api/state poll.
|
|
coord.rescan_containers_and_emit().await;
|
|
// Lock bump → meta-inputs panel needs to re-render.
|
|
crate::dashboard::emit_meta_inputs_snapshot(coord);
|
|
}
|
|
Err(e) => {
|
|
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
|
|
agent: name.to_owned(),
|
|
ok: false,
|
|
note: Some(format!("{e:#}")),
|
|
sha: None,
|
|
tag: None,
|
|
});
|
|
coord.rescan_containers_and_emit().await;
|
|
}
|
|
}
|
|
result.map(|_| ())
|
|
}
|
|
|
|
/// Whether this hive is "ruthless" — running with no root/manager agent at
|
|
/// all (no ruth). When true, hive-c0re skips the root-agent create/start
|
|
/// sweep entirely. Controlled by the host option
|
|
/// `services.hyperhive.ruthless`, threaded in via the `HYPERHIVE_RUTHLESS`
|
|
/// env var. Defaults to `false` when the var is unset (back-compat: the
|
|
/// root agent was always auto-managed before this opt-out existed); only
|
|
/// an explicit `true` / `1` / `yes` enables ruthless mode.
|
|
fn ruthless() -> bool {
|
|
match std::env::var("HYPERHIVE_RUTHLESS") {
|
|
Ok(v) => matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes"),
|
|
Err(_) => false,
|
|
}
|
|
}
|
|
|
|
/// Auto-create the manager container on startup if it isn't already there.
|
|
/// hive-c0re manages the manager end-to-end: operators no longer declare
|
|
/// `containers.h-ruth` in their host NixOS config. Bypasses the approval
|
|
/// queue — the root/manager is auto-managed by default. Operators who
|
|
/// don't want a root agent at all set `services.hyperhive.ruthless = true`,
|
|
/// which short-circuits this whole function. Idempotent.
|
|
pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
|
|
if ruthless() {
|
|
tracing::info!(
|
|
"ruthless mode (services.hyperhive.ruthless = true) - skipping root agent create/start"
|
|
);
|
|
return Ok(());
|
|
}
|
|
let existing = lifecycle::list().await.unwrap_or_default();
|
|
let current_rev = current_flake_rev(&coord.hyperhive_flake);
|
|
if existing
|
|
.iter()
|
|
.any(|c| c.strip_prefix(AGENT_PREFIX) == Some(MANAGER_NAME))
|
|
{
|
|
// Container exists already. If it predates the unified lifecycle
|
|
// (no applied flake on disk) we must rebuild — otherwise it's
|
|
// running whatever the host-declarative config was at create
|
|
// time, with a wrong systemd unit and port.
|
|
let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix");
|
|
if !applied_flake.exists()
|
|
&& let Some(rev) = current_rev.as_ref()
|
|
{
|
|
tracing::warn!(
|
|
"manager container exists but no applied flake — forcing rebuild to migrate"
|
|
);
|
|
let coord_clone = coord.clone();
|
|
if let Err(e) =
|
|
rebuild_agent(&coord_clone, MANAGER_NAME, rev.as_str(), None, true, None).await
|
|
{
|
|
tracing::warn!(error = ?e, "manager migration rebuild failed");
|
|
}
|
|
} else {
|
|
tracing::debug!("manager container already present");
|
|
}
|
|
// hive-c0re auto-manages the root/manager by default, so a
|
|
// present-but-stopped root (e.g. a first-start failure on a fresh
|
|
// install) is brought back up here: the startup sweep's rebuild only
|
|
// restarts a container that was already running, so without this it
|
|
// stays down until a manual `nixos-container start`. The sub-agent
|
|
// `was_running` guard is intentionally left untouched. (Operators
|
|
// opt out of this whole auto-management with
|
|
// `services.hyperhive.ruthless = true`, gated at the top of
|
|
// this function.)
|
|
if !lifecycle::is_running(MANAGER_NAME).await {
|
|
tracing::info!("manager container present but not running — starting");
|
|
if let Err(e) = lifecycle::start(MANAGER_NAME).await {
|
|
tracing::warn!(error = ?e, "manager start failed");
|
|
}
|
|
}
|
|
return Ok(());
|
|
}
|
|
tracing::info!("manager container missing — spawning");
|
|
let runtime = coord.ensure_runtime(MANAGER_NAME)?;
|
|
let hive = coord.hive_env();
|
|
let paths = Coordinator::agent_paths(MANAGER_NAME, runtime);
|
|
lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?;
|
|
if let Some(rev) = current_rev {
|
|
let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Sort `names` in-place so parents precede their children in the topology.
|
|
/// Uses BFS from root agents (depth 0). Agents absent from `topo` sort last,
|
|
/// alphabetically within their tier. Stable within each depth tier.
|
|
pub fn topology_sort(
|
|
names: &mut [String],
|
|
topo: &std::collections::BTreeMap<String, Option<String>>,
|
|
) {
|
|
use std::collections::{HashMap, VecDeque};
|
|
// Build depth map using owned clones so the borrow on `names` is released
|
|
// before the sort_by mutable borrow.
|
|
let name_set: Vec<String> = names.to_vec();
|
|
let mut depth: HashMap<String, usize> = HashMap::new();
|
|
let mut queue: VecDeque<String> = VecDeque::new();
|
|
// Seed roots: entries with no parent, or names not present in topo at all.
|
|
for name in &name_set {
|
|
if topo.get(name).is_none_or(Option::is_none) {
|
|
depth.insert(name.clone(), 0);
|
|
queue.push_back(name.clone());
|
|
}
|
|
}
|
|
// BFS to assign depths to children.
|
|
while let Some(parent) = queue.pop_front() {
|
|
let d = depth[&parent] + 1;
|
|
for name in &name_set {
|
|
let is_child = topo.get(name).and_then(|p| p.as_deref()) == Some(parent.as_str());
|
|
if is_child && !depth.contains_key(name) {
|
|
depth.insert(name.clone(), d);
|
|
queue.push_back(name.clone());
|
|
}
|
|
}
|
|
}
|
|
names.sort_by(|a, b| {
|
|
let da = depth.get(a).copied().unwrap_or(usize::MAX);
|
|
let db = depth.get(b).copied().unwrap_or(usize::MAX);
|
|
da.cmp(&db).then(a.cmp(b))
|
|
});
|
|
}
|
|
|
|
/// Rebuild every container on startup. Enqueues a `StartupSweep` parent
|
|
/// entry (agent = `"hyperhive"`) followed by per-agent `Rebuild` children
|
|
/// linked via `parent_id`. The dashboard renders them nested so the operator
|
|
/// can see at a glance "boot N agents, here is each rebuild's status".
|
|
/// Returns Ok even if some rebuilds failed.
|
|
pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
|
|
let containers = match lifecycle::list().await {
|
|
Ok(c) => c,
|
|
Err(e) => {
|
|
tracing::warn!(error = ?e, "auto-update: nixos-container list failed");
|
|
return Ok(());
|
|
}
|
|
};
|
|
|
|
// Enqueue the parent sweep entry. The worker processes it trivially
|
|
// (no-op dispatch) so it completes quickly; its purpose is to give the
|
|
// dashboard a "why" header for the per-agent child rebuilds below.
|
|
let sweep_id = coord.rebuild_queue.enqueue(
|
|
crate::rebuild_queue::QueueKind::StartupSweep,
|
|
"hyperhive".to_owned(),
|
|
crate::rebuild_queue::QueueSource::AutoUpdate,
|
|
format!("startup sweep ({} containers)", containers.len()),
|
|
None,
|
|
);
|
|
|
|
tracing::info!(
|
|
agents = containers.len(),
|
|
sweep_id,
|
|
"auto-update: queueing all on startup"
|
|
);
|
|
|
|
// Resolve container names to logical agent names, then sort by
|
|
// topology depth so parents are always rebuilt before their
|
|
// children. Root agents (depth 0) go first; agents absent from
|
|
// the topology file sort last (stable, alphabetical within tier).
|
|
let mut logical_names: Vec<String> = containers
|
|
.iter()
|
|
.filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned))
|
|
.collect();
|
|
let topo = crate::topology::read();
|
|
topology_sort(&mut logical_names, &topo);
|
|
for name in logical_names {
|
|
coord.rebuild_queue.enqueue(
|
|
crate::rebuild_queue::QueueKind::Rebuild,
|
|
name,
|
|
crate::rebuild_queue::QueueSource::StartupSweep,
|
|
"startup sweep".to_owned(),
|
|
Some(sweep_id),
|
|
);
|
|
}
|
|
coord.emit_rebuild_queue_snapshot();
|
|
Ok(())
|
|
}
|