vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

refactor(hive-c0re): group src-root files into submodules

Browse source 
stores/ (sqlite-backed host stores + db helper), stats/, agent_config/,
workers/ — pure git-mv moves; crate-root re-exports keep every
crate::<module> path compiling. flake_check stays at root (synchronous
approval-flow validation, not a background worker)
This commit is contained in:
müde 2026-07-06 22:38:47 +02:00
commit 0e4b5a1120
29 changed files with 68 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

320
hive-c0re/src/auto_update.rs
View file

@ -1,320 +0,0 @@
//! Boot reconcile: on `hive-c0re serve` boot, (a) run the config path
//! for agents whose per-agent rev marker is stale — a `StartupSweep`
//! DAG (meta hyperhive lock bump) fanning out `Rebuild` children for
//! the stale agents whose `wanted` power intent is `Up` — and (b)
//! converge every other drifted agent to its persisted `wanted` via
//! `Reconcile` DAGs. Two rules keep boot-time nix work minimal:
//!
//! 1. **Stale but wanted-offline agents** get no rebuild — their
//! rebuild happens the first time they're started (the start
//! submit path upgrades a stale start to rebuild+start). The sweep
//! parent still runs whenever *any* marker is stale so the meta
//! hyperhive lock is bumped for those later start-upgrades.
//! 2. **Agents whose rev marker matches** the current hyperhive flake
//! path are skipped — nothing changed, no nix work to do.
//!
//! Booting with no config change performs no meta commit — only
//! reconciles. See `docs/coordinator.md::Boot reconcile`.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::Result;
use crate::coordinator::Coordinator;
use crate::lifecycle::{self, AGENT_PREFIX, MANAGER_NAME};
/// Marker file recording the hyperhive rev a sub-agent's container was last
/// built against. Sibling of `applied/<name>/` (rather than inside it) to
/// keep it out of the applied repo's git history. Uses a leading dot so a
/// glob over `applied/*` doesn't include it.
pub fn rev_marker_path(name: &str) -> PathBuf {
PathBuf::from(format!("/var/lib/hyperhive/applied/.{name}.hyperhive-rev"))
}
/// Resolve the current rev of `hyperhive_flake`. For a path on disk we
/// canonicalize (following symlinks) so a /etc/hyperhive → /nix/store/...
/// update yields a different string. For anything else we return None.
#[must_use]
pub fn current_flake_rev(hyperhive_flake: &str) -> Option<String> {
let path = Path::new(hyperhive_flake);
if !path.exists() {
return None;
}
std::fs::canonicalize(path)
.ok()
.map(|p| p.display().to_string())
}
/// Returns true when the applied repo has commits that have not yet been
/// deployed (i.e. the applied HEAD differs from the sha currently locked in
/// meta's flake.lock). This is the semantic the dashboard `needs_update` chip
/// conveys: "there is a config change ready to apply via rebuild."
///
/// Async on purpose: this runs per agent inside `container_view::build_all`,
/// which fires on the ~10s dashboard sweep, every `AgentStatus` request, and
/// every `rescan_containers_and_emit` after a lifecycle step. A synchronous
/// `git` fork here blocks a tokio worker for the whole exec — under
/// nix-build disk saturation that's long enough that concurrent sweeps
/// starved the runtime and stalled the per-agent sockets.
pub async fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool {
let applied_head = tokio::process::Command::new("git")
.args([
"-C",
&format!("/var/lib/hyperhive/applied/{name}"),
"rev-parse",
"HEAD",
])
.output()
.await
.ok()
.filter(|o| o.status.success())
.and_then(|o| String::from_utf8(o.stdout).ok())
.map(|s| s.trim().to_owned());
match (applied_head.as_deref(), deployed_sha) {
(Some(head), Some(sha)) => !head.starts_with(sha) && !sha.starts_with(head),
_ => false,
}
}
/// Whether this hive is "ruthless" — running with no root/manager agent at
/// all (no ruth). When true, hive-c0re skips the root-agent create/start
/// sweep entirely. Controlled by the host option
/// `services.hyperhive.ruthless`, threaded in via the `HYPERHIVE_RUTHLESS`
/// env var. Defaults to `false` when the var is unset (back-compat: the
/// root agent was always auto-managed before this opt-out existed); only
/// an explicit `true` / `1` / `yes` enables ruthless mode.
fn ruthless() -> bool {
match std::env::var("HYPERHIVE_RUTHLESS") {
Ok(v) => matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes"),
Err(_) => false,
}
}
/// Auto-create the manager container on startup if it isn't already there.
/// hive-c0re manages the manager end-to-end: operators no longer declare
/// `containers.h-ruth` in their host NixOS config. Bypasses the approval
/// queue — the root/manager is auto-managed by default. Operators who
/// don't want a root agent at all set `services.hyperhive.ruthless = true`,
/// which short-circuits this whole function. Idempotent.
pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
if ruthless() {
tracing::info!(
"ruthless mode (services.hyperhive.ruthless = true) - skipping root agent create/start"
);
return Ok(());
}
let existing = lifecycle::list().await.unwrap_or_default();
let current_rev = current_flake_rev(&coord.hyperhive_flake);
if existing
.iter()
.any(|c| c.strip_prefix(AGENT_PREFIX) == Some(MANAGER_NAME))
{
// Container exists already. If it predates the unified lifecycle
// (no applied flake on disk) we must rebuild — otherwise it's
// running whatever the host-declarative config was at create
// time, with a wrong systemd unit and port.
let applied_flake = Coordinator::agent_applied_dir(MANAGER_NAME).join("flake.nix");
if !applied_flake.exists() && current_rev.is_some() {
tracing::warn!(
"manager container exists but no applied flake — forcing rebuild to migrate"
);
if let Err(e) = coord.job_queue.submit(crate::job_queue::templates::rebuild(
MANAGER_NAME,
crate::job_queue::Source::AutoUpdate,
"manager migration: no applied flake".to_owned(),
None,
true,
)) {
tracing::warn!(error = ?e, "manager migration rebuild submit failed");
}
} else {
tracing::debug!("manager container already present");
}
// hive-c0re auto-manages the root/manager by default, so a
// present-but-stopped root (e.g. a first-start failure on a fresh
// install) is brought back up here: the startup sweep's rebuild only
// restarts a container that was already running, so without this it
// stays down until a manual `nixos-container start`. The sub-agent
// `was_running` guard is intentionally left untouched. (Operators
// opt out of this whole auto-management with
// `services.hyperhive.ruthless = true`, gated at the top of
// this function.)
if !lifecycle::is_running(MANAGER_NAME).await {
tracing::info!("manager container present but not running — starting");
if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
}
if let Err(e) = lifecycle::start(MANAGER_NAME).await {
tracing::warn!(error = ?e, "manager start failed");
}
}
return Ok(());
}
tracing::info!("manager container missing — spawning");
let runtime = coord.ensure_runtime(MANAGER_NAME)?;
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(MANAGER_NAME, runtime);
lifecycle::spawn(MANAGER_NAME, &hive, &paths).await?;
if let Err(e) = coord.power.set(MANAGER_NAME, crate::power::Wanted::Up) {
tracing::warn!(error = ?e, "agent_power: set manager wanted=up failed");
}
if let Some(rev) = current_rev {
let _ = std::fs::write(rev_marker_path(MANAGER_NAME), &rev);
}
Ok(())
}
/// Sort `names` in-place so parents precede their children in the topology.
/// Uses BFS from root agents (depth 0). Agents absent from `topo` sort last,
/// alphabetically within their tier. Stable within each depth tier.
pub fn topology_sort(
names: &mut [String],
topo: &std::collections::BTreeMap<String, Option<String>>,
) {
use std::collections::{HashMap, VecDeque};
// Build depth map using owned clones so the borrow on `names` is released
// before the sort_by mutable borrow.
let name_set: Vec<String> = names.to_vec();
let mut depth: HashMap<String, usize> = HashMap::new();
let mut queue: VecDeque<String> = VecDeque::new();
// Seed roots: entries with no parent, or names not present in topo at all.
for name in &name_set {
if topo.get(name).is_none_or(Option::is_none) {
depth.insert(name.clone(), 0);
queue.push_back(name.clone());
}
}
// BFS to assign depths to children.
while let Some(parent) = queue.pop_front() {
let d = depth[&parent] + 1;
for name in &name_set {
let is_child = topo.get(name).and_then(|p| p.as_deref()) == Some(parent.as_str());
if is_child && !depth.contains_key(name) {
depth.insert(name.clone(), d);
queue.push_back(name.clone());
}
}
}
names.sort_by(|a, b| {
let da = depth.get(a).copied().unwrap_or(usize::MAX);
let db = depth.get(b).copied().unwrap_or(usize::MAX);
da.cmp(&db).then(a.cmp(b))
});
}
/// Boot reconcile (see the module doc): classify every agent by rev
/// freshness + persisted `wanted` intent, submit one `StartupSweep`
/// DAG (hyperhive lock bump → fan-out rebuilds for stale wanted-up
/// agents) when anything is stale, and `Reconcile` DAGs for agents
/// whose observed power state drifted from `wanted`. Returns Ok even
/// if some submissions failed.
pub async fn run(coord: Arc<Coordinator>) -> Result<()> {
let containers = match lifecycle::list().await {
Ok(c) => c,
Err(e) => {
tracing::warn!(error = ?e, "boot reconcile: nixos-container list failed");
return Ok(());
}
};
let current_rev = current_flake_rev(&coord.hyperhive_flake);
// Resolve container names to logical agent names, then sort by
// topology depth so parents are always rebuilt before their
// children. Root agents (depth 0) go first; agents absent from
// the topology file sort last (stable, alphabetical within tier).
let mut logical_names: Vec<String> = containers
.iter()
.filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned))
.collect();
let topo = crate::topology::read();
topology_sort(&mut logical_names, &topo);
// Classify. `get_or_seed` doubles as the one-time migration: an
// agent without an `agent_power` row is seeded from its observed
// state (running ⇒ Up), after which the DB is authoritative.
let mut any_stale = false;
let mut fanout: Vec<String> = Vec::new(); // stale ∧ wanted=Up → sweep rebuild
let mut drifted: Vec<String> = Vec::new(); // fresh ∧ wanted≠observed → reconcile
let mut n_deferred = 0usize;
let mut n_skipped = 0usize;
for name in &logical_names {
let running = lifecycle::is_running(name).await;
let wanted = match coord.power.get_or_seed(name, running) {
Ok(w) => w,
Err(e) => {
tracing::warn!(%name, error = ?e, "agent_power read failed — assuming observed");
crate::power::Wanted::from_running(running)
}
};
let fresh = current_rev.as_ref().is_some_and(|rev| {
std::fs::read_to_string(rev_marker_path(name))
.is_ok_and(|stored| stored == rev.as_str())
});
if fresh {
n_skipped += 1;
} else {
any_stale = true;
if wanted == crate::power::Wanted::Up {
// Rebuild against the post-bump lock; the DAG's tail
// Reconcile brings the agent (back) up — covering both
// the running-stale and stopped-but-wanted-up cases.
fanout.push(name.clone());
continue;
}
// Stale but wanted offline: no boot-time nix work — the
// start submit path upgrades a stale start to a rebuild.
n_deferred += 1;
tracing::debug!(%name, "boot reconcile: stale but offline — deferring rebuild to on-start");
}
if crate::power::reconcile_action(wanted, running) != crate::power::ReconcileAction::Noop {
drifted.push(name.clone());
}
}
tracing::info!(
total = containers.len(),
rebuilds = fanout.len(),
reconciles = drifted.len(),
deferred = n_deferred,
up_to_date = n_skipped,
"boot reconcile"
);
// Sweep parent whenever ANY marker is stale — even when every
// stale agent is wanted-offline: the hyperhive lock bump must land
// now so their later start-upgrade rebuilds build against it.
// No stale agents ⇒ no sweep ⇒ no meta commit on a no-change boot.
if any_stale {
let reason = format!(
"startup sweep: {} rebuild(s), {} deferred (offline), {} up-to-date",
fanout.len(),
n_deferred,
n_skipped,
);
if let Err(e) = coord
.job_queue
.submit(crate::job_queue::templates::startup_sweep(reason, fanout))
{
tracing::warn!(error = ?e, "boot reconcile: sweep submit failed");
}
}
for name in drifted {
if let Err(e) = coord
.job_queue
.submit(crate::job_queue::templates::reconcile_only(
crate::job_queue::Template::Reconcile,
&name,
crate::job_queue::Source::AutoUpdate,
"boot reconcile".to_owned(),
None,
))
{
tracing::warn!(%name, error = ?e, "boot reconcile: submit failed");
}
}
coord.emit_rebuild_queue_snapshot();
Ok(())
}