mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

c0re: keep power intent in memory, drop agent_power/kv tables and the stop snapshot

Browse source
This commit is contained in:
damocles 2026-11-07 01:22:18 +01:00 • committed by mara
commit b77fa9edc3
2 changed files with 222 additions and 553 deletions
Download patch file Download diff file Expand all files Collapse all files

42
hive-c0re/src/job_queue/exec.rs
View file

@ -8,7 +8,7 @@
use std::path::PathBuf;
use std::sync::{Arc, Mutex, OnceLock};
use anyhow::{Context as _, Result};
use anyhow::Result;
use swarm_wire::agent_closures::AgentClosure;
use hive_jobq::NodeId;
@ -305,7 +305,10 @@ async fn dispatch(
run_emit_rebuilt(host, coord, agent, coord.job_queue.root_of(id), *ok).await;
Ok(())
}
NodeKind::SetWanted { up, .. } => run_set_wanted(coord, agent, *up),
NodeKind::SetWanted { up, .. } => {
run_set_wanted(coord, agent, *up);
Ok(())
}
// Braces carry no work of their own; completing one lets it reach
// `Finishing` so the nodes under it start. What they declare stays held
// until their whole subtree settles.
@ -425,7 +428,7 @@ async fn run_purge_state(host: &impl Host, agent: &str) {
/// Post-destroy bookkeeping. Infallible by construction: every step is
/// warn-and-continue, because the destroy it follows has already succeeded and
/// none of this is undoable — a failed agent-set refresh or power-store write is a
/// none of this is undoable — a failed agent-set refresh is a
/// bookkeeping drift to log, not a reason to red a DAG whose container is
/// already gone.
async fn run_destroy_bookkeeping(host: &impl Host, coord: &Arc<Coordinator>, agent: &str) {
@ -433,11 +436,9 @@ async fn run_destroy_bookkeeping(host: &impl Host, coord: &Arc<Coordinator>, age
if let Err(e) = host.refresh_agent_set().await {
tracing::warn!(error = ?e, %agent, "agent_set refresh after destroy failed");
}
// Drop the durable power intent — a future agent of the same name seeds
// fresh from its observed state.
if let Err(e) = coord.power.remove(agent) {
tracing::warn!(%agent, error = ?e, "agent_power: remove on destroy failed");
}
// Drop the power intent — a future agent of the same name seeds fresh from
// its observed state.
coord.power.remove(agent);
host.notify(
format!("destroyed:{agent}"),
format!("agent '{agent}' destroyed"),
@ -467,25 +468,18 @@ async fn run_emit_rebuilt(
host.notify(format!("rebuilt:{agent}"), summary).await;
}
/// Write the agent's durable power intent — the DAG-node form of the old
/// pre-submit `set_wanted` side effect. Store-only (no container touch), so
/// Record the agent's power intent — the DAG-node form of the old pre-submit
/// `set_wanted` side effect. Memory-only (no container touch), so
/// build-slot-exempt; but it declares the agent's lifecycle lease
/// (`Resource::Agent`) so the whole power-op DAG is atomic per-agent.
/// The downstream `Reconcile` reads the intent this writes. Unlike the old
/// warn-and-continue write, a failed write fails the node (cancel-downstream
/// cancels the `Reconcile`) rather than letting it converge to a stale
/// intent — that atomicity is the point of moving it into the DAG.
fn run_set_wanted(coord: &Arc<Coordinator>, agent: &str, up: bool) -> Result<()> {
/// The downstream `Reconcile` reads the intent this writes.
fn run_set_wanted(coord: &Arc<Coordinator>, agent: &str, up: bool) {
let wanted = if up {
crate::power::Wanted::Up
} else {
crate::power::Wanted::Offline
};
coord
.power
.set(agent, wanted)
.with_context(|| format!("set wanted={} for agent {agent}", <&str>::from(wanted)))?;
Ok(())
coord.power.set(agent, wanted);
}
/// The rebuild's preamble: runtime-dir prep, the archetype and agent-set
@ -719,7 +713,7 @@ async fn run_reconcile(
name: &str,
) -> Result<Option<NodeKind>> {
let running = host.is_running(name).await;
let wanted = coord.power.get_or_seed(name, running)?;
let wanted = coord.power.get_or_seed(name, running);
Ok(match reconcile_action(wanted, running) {
ReconcileAction::Start => Some(NodeKind::Start {
agent: name.to_owned(),
@ -780,13 +774,11 @@ async fn run_stop(host: &impl Host, coord: &Arc<Coordinator>, name: &str) -> Res
/// noop when already stopped.
async fn run_stop_for_update(host: &impl Host, coord: &Arc<Coordinator>, name: &str) -> Result<()> {
if host.is_running(name).await {
// Seed a missing agent_power row from the PRE-stop observation
// Seed a missing intent from the PRE-stop observation
// — the DAG's tail `Reconcile` observes only the mechanically
// stopped state and would otherwise seed a running-but-unknown
// agent as `Offline`, stranding it down after its own rebuild.
if let Err(e) = coord.power.get_or_seed(name, true) {
tracing::warn!(%name, error = ?e, "agent_power: pre-stop seed failed");
}
coord.power.get_or_seed(name, true);
host.kill(name).await?;
coord.rescan_containers_and_emit().await;
}

48
hive-c0re/src/job_queue/exec/tests.rs
View file

@ -572,21 +572,18 @@ async fn stop_for_update_kills_a_running_container_and_seeds_its_intent_up() {
.expect("stop ok");
assert_eq!(host.calls(), ["is_running alice", "kill alice"]);
// Seeded from the pre-stop observation, so the tail Reconcile restarts it.
assert_eq!(coord.power.get("alice").expect("read"), Some(Wanted::Up));
assert_eq!(coord.power.get("alice"), Some(Wanted::Up));
}
#[tokio::test]
async fn stop_for_update_keeps_an_existing_offline_intent() {
let (_dir, coord) = coordinator();
coord.power.set("alice", Wanted::Offline).expect("seed");
coord.power.set("alice", Wanted::Offline);
let host = FakeHost::up();
run_stop_for_update(&host, &coord, "alice")
.await
.expect("stop ok");
assert_eq!(
coord.power.get("alice").expect("read"),
Some(Wanted::Offline)
);
assert_eq!(coord.power.get("alice"), Some(Wanted::Offline));
}
#[tokio::test]
@ -597,7 +594,7 @@ async fn stop_for_update_is_a_noop_on_a_stopped_container() {
.await
.expect("noop ok");
assert_eq!(host.calls(), ["is_running alice"]);
assert_eq!(coord.power.get("alice").expect("read"), None);
assert_eq!(coord.power.get("alice"), None);
}
#[tokio::test]
@ -607,7 +604,7 @@ async fn stop_for_update_failure_fails_the_node_after_seeding() {
run_stop_for_update(&host, &coord, "alice")
.await
.expect_err("a failed stop fails the node");
assert_eq!(coord.power.get("alice").expect("read"), Some(Wanted::Up));
assert_eq!(coord.power.get("alice"), Some(Wanted::Up));
}
// --- Reconcile --------------------------------------------------------------
@ -615,7 +612,7 @@ async fn stop_for_update_failure_fails_the_node_after_seeding() {
#[tokio::test]
async fn reconcile_fans_out_start_for_wanted_up_and_down() {
let (_dir, coord) = coordinator();
coord.power.set("alice", Wanted::Up).expect("set");
coord.power.set("alice", Wanted::Up);
let sub = run_reconcile(&FakeHost::down(), &coord, "alice")
.await
.expect("ok");
@ -625,7 +622,7 @@ async fn reconcile_fans_out_start_for_wanted_up_and_down() {
#[tokio::test]
async fn reconcile_fans_out_stop_for_wanted_offline_and_up() {
let (_dir, coord) = coordinator();
coord.power.set("alice", Wanted::Offline).expect("set");
coord.power.set("alice", Wanted::Offline);
let sub = run_reconcile(&FakeHost::up(), &coord, "alice")
.await
.expect("ok");
@ -635,7 +632,7 @@ async fn reconcile_fans_out_stop_for_wanted_offline_and_up() {
#[tokio::test]
async fn reconcile_is_a_noop_when_state_matches_intent() {
let (_dir, coord) = coordinator();
coord.power.set("alice", Wanted::Up).expect("set");
coord.power.set("alice", Wanted::Up);
let sub = run_reconcile(&FakeHost::up(), &coord, "alice")
.await
.expect("ok");
@ -649,7 +646,7 @@ async fn reconcile_seeds_a_missing_intent_from_the_observed_state() {
.await
.expect("ok");
assert!(sub.is_none());
assert_eq!(coord.power.get("alice").expect("read"), Some(Wanted::Up));
assert_eq!(coord.power.get("alice"), Some(Wanted::Up));
}
// --- Set wanted -------------------------------------------------------------
@ -657,13 +654,10 @@ async fn reconcile_seeds_a_missing_intent_from_the_observed_state() {
#[tokio::test]
async fn set_wanted_writes_the_intent() {
let (_dir, coord) = coordinator();
run_set_wanted(&coord, "alice", false).expect("ok");
assert_eq!(
coord.power.get("alice").expect("read"),
Some(Wanted::Offline)
);
run_set_wanted(&coord, "alice", true).expect("ok");
assert_eq!(coord.power.get("alice").expect("read"), Some(Wanted::Up));
run_set_wanted(&coord, "alice", false);
assert_eq!(coord.power.get("alice"), Some(Wanted::Offline));
run_set_wanted(&coord, "alice", true);
assert_eq!(coord.power.get("alice"), Some(Wanted::Up));
}
// --- Signal / Drain ---------------------------------------------------------
@ -1015,7 +1009,7 @@ async fn purge_removes_every_state_dir_even_when_the_subvolume_delete_fails() {
#[tokio::test]
async fn destroy_bookkeeping_drops_the_power_row_and_announces() {
let (_dir, coord) = coordinator();
coord.power.set("alice", Wanted::Offline).expect("set");
coord.power.set("alice", Wanted::Offline);
let host = FakeHost::up();
run_destroy_bookkeeping(&host, &coord, "alice").await;
assert_eq!(
@ -1026,10 +1020,7 @@ async fn destroy_bookkeeping_drops_the_power_row_and_announces() {
]
);
// The Offline row is gone, so a fresh seed from "running" gives Up.
assert_eq!(
coord.power.get_or_seed("alice", true).expect("seed"),
Wanted::Up
);
assert_eq!(coord.power.get_or_seed("alice", true), Wanted::Up);
}
#[tokio::test]
@ -1257,10 +1248,7 @@ async fn dispatch_set_wanted_writes_the_intent_and_emit_rebuilt_announces() {
},
)
.await;
assert_eq!(
coord.power.get("alice").expect("read"),
Some(Wanted::Offline)
);
assert_eq!(coord.power.get("alice"), Some(Wanted::Offline));
let settled = run_dag(
&host,
&coord,
@ -1325,7 +1313,7 @@ async fn dispatch_reports_a_failing_executor_as_a_failed_node() {
#[tokio::test]
async fn reconcile_whose_start_fails_fails_the_dag_without_registering() {
let (_dir, coord) = coordinator();
coord.power.set("alice", Wanted::Up).expect("set");
coord.power.set("alice", Wanted::Up);
let host = FakeHost::down().failing("start");
let settled = run_dag(&host, &coord, alice(|agent| NodeKind::Reconcile { agent })).await;
let by_kind = |k: &str| settled.iter().find(|s| s.0 == k).cloned();
@ -1346,7 +1334,7 @@ async fn reconcile_whose_start_fails_fails_the_dag_without_registering() {
#[tokio::test]
async fn reconcile_fans_out_a_stop_that_kills_and_announces() {
let (_dir, coord) = coordinator();
coord.power.set("alice", Wanted::Offline).expect("set");
coord.power.set("alice", Wanted::Offline);
let host = FakeHost::up();
let settled = run_dag(&host, &coord, alice(|agent| NodeKind::Reconcile { agent })).await;
assert!(settled.iter().any(|s| s.0 == "stop"));