vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

add a host seam to the job-queue executors and test them

Browse source
This commit is contained in:
atlas 2026-10-27 19:03:54 +01:00 • committed by mara
commit 61292e865e
1 changed files with 568 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

143
hive-c0re/src/job_queue/exec.rs
View file

@ -12,7 +12,7 @@ use anyhow::{Context as _, Result};
use hive_jobq::NodeId;
use super::model::NodeKind;
use crate::coordinator::Coordinator;
use crate::coordinator::{AgentPaths, Coordinator, HiveEnv};
use crate::power::{ReconcileAction, reconcile_action};
use crate::stats::sweep_health::{self, SweepHealth};
@ -31,6 +31,69 @@ pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from
/// change to one doesn't silently retune the other.
const PAUSE_ACK_TIMEOUT: std::time::Duration = GRACEFUL_STOP_TIMEOUT;
/// The container-state calls the executors below make on the host: systemd,
/// `hive-priv`, the nix build, the `internal/meta` record. Production uses
/// [`RealHost`], which forwards to `lifecycle` / `meta_rev` / `Coordinator`
/// unchanged; tests substitute a recording fake.
pub(super) trait Host: Sync {
/// Whether the agent's container unit is active.
async fn is_running(&self, name: &str) -> bool;
/// Stop the agent's container.
async fn kill(&self, name: &str) -> Result<()>;
/// The `internal/meta` commit `MetaSync` recorded for the agent.
fn recorded_rev(&self, name: &str) -> Result<String>;
/// Warm the agent's toplevel build from the `internal/meta` commit `rev`.
async fn prebuild_toplevel(&self, name: &str, rev: &str, node_id: u64) -> Result<()>;
/// Re-apply drop-ins, then `nixos-container update` at `rev`.
async fn swap_update(
&self,
name: &str,
hive: &HiveEnv,
paths: &AgentPaths,
rev: &str,
node_id: u64,
) -> Result<()>;
/// Write or remove the agent's pause marker.
async fn set_paused(&self, agent: &hive_types::Ident, paused: bool) -> Result<()>;
}
/// [`Host`] over the real system.
pub(super) struct RealHost;
impl Host for RealHost {
async fn is_running(&self, name: &str) -> bool {
crate::lifecycle::is_running(name).await
}
async fn kill(&self, name: &str) -> Result<()> {
crate::lifecycle::kill(name).await
}
fn recorded_rev(&self, name: &str) -> Result<String> {
Ok(crate::meta_rev::recorded_or_err(name)?.rev)
}
async fn prebuild_toplevel(&self, name: &str, rev: &str, node_id: u64) -> Result<()> {
let flake_ref = crate::meta_rev::flake_ref(name, rev);
crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(node_id)).await
}
async fn swap_update(
&self,
name: &str,
hive: &HiveEnv,
paths: &AgentPaths,
rev: &str,
node_id: u64,
) -> Result<()> {
crate::lifecycle::swap_update(name, hive, paths, rev, Some(node_id)).await
}
async fn set_paused(&self, agent: &hive_types::Ident, paused: bool) -> Result<()> {
Coordinator::set_paused(agent, paused).await
}
}
/// Run one claimed node to completion. Called from a task the
/// scheduler spawns per claim; the `Result` (stringified) becomes the
/// node's terminal state.
@ -68,8 +131,8 @@ pub(super) async fn run_node(
// growth executors return what to grow rather than taking the builder.
let result = match kind {
NodeKind::MetaSync { .. } => run_meta_sync(agent).await,
NodeKind::Prebuild { .. } => run_prebuild(agent, id).await,
NodeKind::Swap { .. } => run_swap(coord, agent, id).await,
NodeKind::Prebuild { .. } => run_prebuild(&RealHost, agent, id).await,
NodeKind::Swap { .. } => run_swap(&RealHost, coord, agent, id).await,
NodeKind::RebuildBookkeeping { .. } => run_rebuild_bookkeeping(coord, agent).await,
NodeKind::Provision { .. } => run_provision(agent).await,
NodeKind::Create { .. } => run_create(agent).await,
@ -83,21 +146,21 @@ pub(super) async fn run_node(
Ok(())
}
NodeKind::Reconcile { .. } => run_reconcile(coord, agent).await.map(|sub| {
NodeKind::Reconcile { .. } => run_reconcile(&RealHost, coord, agent).await.map(|sub| {
if let Some(kind) = sub {
super::templates::fanned_out_mechanical(&builder, kind);
}
}),
NodeKind::Start { .. } => run_start(coord, agent).await,
NodeKind::Stop { .. } => run_stop(coord, agent).await,
NodeKind::StopForUpdate { .. } => run_stop_for_update(coord, agent).await,
NodeKind::Stop { .. } => run_stop(&RealHost, coord, agent).await,
NodeKind::StopForUpdate { .. } => run_stop_for_update(&RealHost, coord, agent).await,
NodeKind::Signal { .. } => {
run_signal(coord, agent);
Ok(())
}
NodeKind::Drain { .. } => run_drain(coord, agent).await,
NodeKind::PauseSignal { .. } => run_pause_signal(coord, agent).await,
NodeKind::PauseDrain { .. } => run_pause_drain(coord, agent).await,
NodeKind::Drain { .. } => run_drain(coord, agent, GRACEFUL_STOP_TIMEOUT).await,
NodeKind::PauseSignal { .. } => run_pause_signal(&RealHost, coord, agent).await,
NodeKind::PauseDrain { .. } => run_pause_drain(coord, agent, PAUSE_ACK_TIMEOUT).await,
NodeKind::WriteDropin { .. } => run_write_dropin(coord, agent).await,
// The payload rides the node and is destructured here, so the executor
// takes it directly instead of re-matching the kind behind a `bail!`
@ -392,15 +455,14 @@ async fn run_meta_sync(name: &str) -> Result<()> {
/// container is already down: its only purpose is to shrink the swap's
/// downtime window, so a stopped agent (no uptime to preserve) doesn't
/// pay the double eval — `Swap` builds inline instead.
async fn run_prebuild(name: &str, id: NodeId) -> Result<()> {
async fn run_prebuild(host: &impl Host, name: &str, id: NodeId) -> Result<()> {
// Warm the toplevel build only when the container is up — the whole
// point of prebuild is to shrink the swap's downtime window. A
// stopped agent has no uptime to preserve, so skip the (expensive)
// eval and let the downstream `Swap` build inline.
if crate::lifecycle::is_running(name).await {
let build = crate::meta_rev::recorded_or_err(name)?;
let flake_ref = crate::meta_rev::flake_ref(name, &build.rev);
crate::lifecycle::prebuild_toplevel(name, &flake_ref, Some(id.get())).await?;
if host.is_running(name).await {
let rev = host.recorded_rev(name)?;
host.prebuild_toplevel(name, &rev, id.get()).await?;
}
Ok(())
}
@ -410,15 +472,19 @@ async fn run_prebuild(name: &str, id: NodeId) -> Result<()> {
/// (rev marker, `Rebuilt` event, forge/matrix sync, kick, rescan).
/// The recovery-start on failure is NOT here — the DAG's tail
/// `Reconcile` runs after this node terminal ok *or* fail.
async fn run_swap(coord: &Arc<Coordinator>, name: &str, id: NodeId) -> Result<()> {
async fn run_swap(
host: &impl Host,
coord: &Arc<Coordinator>,
name: &str,
id: NodeId,
) -> Result<()> {
// Swap runs on an already-existing (stopped) container — runtime dir
// and listener were created earlier. Pure path accessor suffices.
let agent_dir = crate::paths::agent_runtime_dir(name);
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir)?;
let build = crate::meta_rev::recorded_or_err(name)?;
let result =
crate::lifecycle::swap_update(name, &hive, &paths, &build.rev, Some(id.get())).await;
let rev = host.recorded_rev(name)?;
let result = host.swap_update(name, &hive, &paths, &rev, id.get()).await;
// On success the Ok-only bookkeeping tail (rev marker, forge/matrix
// sync, kick, rescan, snapshot) runs in the sibling `RebuildBookkeeping` node,
// which deps `AfterOk(Swap)`. On failure `RebuildBookkeeping` is cancel-cascaded
@ -493,8 +559,12 @@ async fn run_create(name: &str) -> Result<()> {
/// declaring it — the declaration has to happen outside any `.await`, see
/// [`run_node`]. `NodeKind` carries the agent it targets, so this node's agent
/// is stamped into the fanned-out kind here.
async fn run_reconcile(coord: &Arc<Coordinator>, name: &str) -> Result<Option<NodeKind>> {
let running = crate::lifecycle::is_running(name).await;
async fn run_reconcile(
host: &impl Host,
coord: &Arc<Coordinator>,
name: &str,
) -> Result<Option<NodeKind>> {
let running = host.is_running(name).await;
let wanted = coord.power.get_or_seed(name, running)?;
Ok(match reconcile_action(wanted, running) {
ReconcileAction::Start => Some(NodeKind::Start {
@ -542,10 +612,10 @@ async fn run_start(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
/// Mechanical container stop — the sub-step a `Reconcile` planner fans
/// out when it observes `wanted = Offline` and the container up.
async fn run_stop(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
async fn run_stop(host: &impl Host, coord: &Arc<Coordinator>, name: &str) -> Result<()> {
// See `run_start`: no node-local guard — `Stop` reports `Stopping` from its
// own kind now.
crate::lifecycle::kill(name).await?;
host.kill(name).await?;
coord.unregister_agent(name);
crate::swarm_notices::notify(
"core",
@ -560,8 +630,8 @@ async fn run_stop(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
/// Mechanical stop for the profile swap. Never *changes* `wanted`;
/// noop when already stopped.
async fn run_stop_for_update(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
if crate::lifecycle::is_running(name).await {
async fn run_stop_for_update(host: &impl Host, coord: &Arc<Coordinator>, name: &str) -> Result<()> {
if host.is_running(name).await {
// Seed a missing agent_power row from the PRE-stop observation
// — the DAG's tail `Reconcile` observes only the mechanically
// stopped state and would otherwise seed a running-but-unknown
@ -569,7 +639,7 @@ async fn run_stop_for_update(coord: &Arc<Coordinator>, name: &str) -> Result<()>
if let Err(e) = coord.power.get_or_seed(name, true) {
tracing::warn!(%name, error = ?e, "agent_power: pre-stop seed failed");
}
crate::lifecycle::kill(name).await?;
host.kill(name).await?;
coord.rescan_containers_and_emit().await;
}
Ok(())
@ -595,8 +665,12 @@ fn run_signal(coord: &Arc<Coordinator>, name: &str) {
/// Await the harness clearing the fence (`GracefulStopComplete`) or
/// the timeout — either way the downstream `Reconcile` proceeds with
/// the actual stop.
async fn run_drain(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT;
async fn run_drain(
coord: &Arc<Coordinator>,
name: &str,
timeout: std::time::Duration,
) -> Result<()> {
let deadline = std::time::Instant::now() + timeout;
while coord.is_graceful_stop_pending(name) {
if std::time::Instant::now() >= deadline {
tracing::warn!(agent = %name, "graceful stop: drain timed out — hard-stopping");
@ -621,11 +695,11 @@ async fn run_drain(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
/// no edge for `run_pause_drain` to observe — marking pending here
/// would just burn its timeout every time an operator re-confirms a
/// pause that already took.
async fn run_pause_signal(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
async fn run_pause_signal(host: &impl Host, coord: &Arc<Coordinator>, name: &str) -> Result<()> {
let agent = hive_types::Ident::parse(name)
.map_err(|e| anyhow::anyhow!("invalid agent name for pause {name:?}: {e}"))?;
let already_paused = Coordinator::is_paused(&agent);
Coordinator::set_paused(&agent, true).await?;
host.set_paused(&agent, true).await?;
if !already_paused {
coord.mark_pause_pending(name);
}
@ -643,8 +717,12 @@ async fn run_pause_signal(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
/// a timed-out wait doesn't leave the agent un-paused, just leaves the
/// dashboard's "pausing…" badge running a little longer than it needed
/// to.
async fn run_pause_drain(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
let deadline = std::time::Instant::now() + PAUSE_ACK_TIMEOUT;
async fn run_pause_drain(
coord: &Arc<Coordinator>,
name: &str,
timeout: std::time::Duration,
) -> Result<()> {
let deadline = std::time::Instant::now() + timeout;
while coord.is_pause_pending(name) {
if std::time::Instant::now() >= deadline {
tracing::warn!(agent = %name, "pause: ack wait timed out — marker is set regardless");
@ -667,3 +745,6 @@ async fn run_write_dropin(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
crate::lifecycle::write_dropins(name, &hive, &paths).await?;
Ok(())
}
#[cfg(test)]
mod tests;