mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

c0re: keep power intent in memory, drop agent_power/kv tables and the stop snapshot

Browse source
This commit is contained in:
damocles 2026-11-07 01:22:18 +01:00 • committed by mara
commit b77fa9edc3
2 changed files with 222 additions and 553 deletions
Download patch file Download diff file Expand all files Collapse all files

75
hive-c0re/src/workers/auto_update.rs
View file

@ -95,9 +95,9 @@ async fn classify(
.collect();
logical_names.sort();
// `get_or_seed` doubles as the one-time migration: an
// agent without an `agent_power` row is seeded from its observed
// state (running ⇒ Up), after which the DB is authoritative.
// `get_or_seed`: nothing remembers an intent across a restart, so an agent
// starts here with the one its observed state implies (running ⇒ Up) and
// the swarm's declaration (`workers::wanted`) moves it from there.
// stale ∧ wanted=Up → rebuild. The bool is the agent's observed
// running state, used to order running agents first before submit.
let mut rebuilds: Vec<(String, bool)> = Vec::new();
@ -106,13 +106,7 @@ async fn classify(
let mut skipped = 0usize;
for name in &logical_names {
let running = lifecycle::is_running(name).await;
let wanted = match coord.power.get_or_seed(name, running) {
Ok(w) => Some(w),
Err(e) => {
tracing::warn!(%name, error = ?e, "agent_power read failed — reconciling so it surfaces");
None
}
};
let wanted = coord.power.get_or_seed(name, running);
let pending = crate::attic::closure::pending_path(name).await;
let fresh = is_fresh(
pending.as_deref(),
@ -139,7 +133,6 @@ async fn classify(
drifted.push(name.clone());
}
}
BootAction::Unreadable => drifted.push(name.clone()),
}
}
@ -217,22 +210,9 @@ enum BootAction {
Defer { reconcile: bool },
/// Already running the declared closure, or nothing to deploy.
Skip { reconcile: bool },
/// The intent could not be read.
Unreadable,
}
/// `wanted` is `None` when the store could not answer for this agent.
///
/// That case has no neutral stand-in, which is why it is a variant rather
/// than a fallback value: `Wanted::from_running(running)` is precisely the
/// value for which `reconcile_action` returns `Noop`, so guessing it here
/// puts the agent beyond every check below — a corrupt row then leaves one
/// `warn!` per boot and no other trace. The queued reconcile's own
/// `get_or_seed` fails as a per-agent node instead.
fn boot_action(wanted: Option<crate::power::Wanted>, fresh: bool, running: bool) -> BootAction {
let Some(wanted) = wanted else {
return BootAction::Unreadable;
};
fn boot_action(wanted: crate::power::Wanted, fresh: bool, running: bool) -> BootAction {
let reconcile =
crate::power::reconcile_action(wanted, running) != crate::power::ReconcileAction::Noop;
if fresh {
@ -345,30 +325,14 @@ mod tests {
use super::{BootAction, boot_action, is_fresh};
use crate::power::Wanted;
// The regression the `Unreadable` variant exists for: the arm used to
// substitute `from_running(running)`, which is `Noop` against BOTH
// observations — so no combination of inputs could reach a reconcile.
#[test]
fn an_unreadable_intent_is_never_settled_whatever_the_agent_is_doing() {
for fresh in [true, false] {
for running in [true, false] {
assert_eq!(
boot_action(None, fresh, running),
BootAction::Unreadable,
"fresh={fresh} running={running}"
);
}
}
}
#[test]
fn a_fresh_agent_doing_what_it_should_is_skipped_without_a_reconcile() {
assert_eq!(
boot_action(Some(Wanted::Up), true, true),
boot_action(Wanted::Up, true, true),
BootAction::Skip { reconcile: false }
);
assert_eq!(
boot_action(Some(Wanted::Offline), true, false),
boot_action(Wanted::Offline, true, false),
BootAction::Skip { reconcile: false }
);
}
@ -376,35 +340,29 @@ mod tests {
#[test]
fn a_fresh_agent_that_drifted_from_its_intent_is_reconciled() {
assert_eq!(
boot_action(Some(Wanted::Up), true, false),
boot_action(Wanted::Up, true, false),
BootAction::Skip { reconcile: true }
);
assert_eq!(
boot_action(Some(Wanted::Offline), true, true),
boot_action(Wanted::Offline, true, true),
BootAction::Skip { reconcile: true }
);
}
#[test]
fn a_stale_agent_wanted_up_is_rebuilt_running_or_not() {
assert_eq!(
boot_action(Some(Wanted::Up), false, true),
BootAction::Rebuild
);
assert_eq!(
boot_action(Some(Wanted::Up), false, false),
BootAction::Rebuild
);
assert_eq!(boot_action(Wanted::Up, false, true), BootAction::Rebuild);
assert_eq!(boot_action(Wanted::Up, false, false), BootAction::Rebuild);
}
#[test]
fn a_stale_agent_wanted_offline_defers_and_still_reports_drift() {
assert_eq!(
boot_action(Some(Wanted::Offline), false, false),
boot_action(Wanted::Offline, false, false),
BootAction::Defer { reconcile: false }
);
assert_eq!(
boot_action(Some(Wanted::Offline), false, true),
boot_action(Wanted::Offline, false, true),
BootAction::Defer { reconcile: true }
);
}
@ -415,13 +373,10 @@ mod tests {
#[test]
fn a_changed_declaration_for_a_wanted_up_agent_is_rebuilt_unprompted() {
let fresh = is_fresh(Some("/nix/store/new"), None, false);
assert_eq!(
boot_action(Some(Wanted::Up), fresh, true),
BootAction::Rebuild
);
assert_eq!(boot_action(Wanted::Up, fresh, true), BootAction::Rebuild);
let fresh = is_fresh(None, None, false);
assert_eq!(
boot_action(Some(Wanted::Up), fresh, true),
boot_action(Wanted::Up, fresh, true),
BootAction::Skip { reconcile: false },
"control: nothing declared, nothing rebuilt"
);

101
hive-c0re/src/workers/wanted.rs
View file

@ -4,8 +4,7 @@
//! [`swarm_wire::wanted`]); this reads its own and acts on it. It is
//! the **repair** path, not the fast one: a deploy event
//! ([`crate::swarm_status`]) is core NATS, so a hive that was down never hears
//! it, and until this existed that left the hive on its old shape with nothing
//! looking wrong.
//! it and would stay on its old shape with nothing looking wrong.
//!
//! # Two things a declaration does not mean
//!
@ -21,13 +20,14 @@
//!
//! # What "diverged" is measured against
//!
//! The hive's **stored power intent**, not the container's observed running
//! state. An agent down while its intent says `Up` is already the boot
//! The power intent this process holds, not the container's observed running
//! state. An agent down while its intent says `Up` is already a queued
//! reconcile's work; a loop reading `is_running` would insert a start DAG
//! behind that reconcile's back on every boot.
//! behind that reconcile's back.
//!
//! One consequence, deliberate: a declaration outranks a local stop — an agent
//! declared `Up` returns at the next boot or watch attach, because the controller owns that.
//! A declaration outranks a local stop, since the controller owns lasting
//! power intent. Nothing is remembered across a restart, so the first pass
//! after one queues a (no-op) start or stop for each declared agent.
use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
@ -341,26 +341,17 @@ async fn converge(coord: &Arc<Coordinator>, declared: &HiveWanted) -> Result<()>
.map(|spec| spec.name)
.collect();
let mut intents = BTreeMap::new();
for agent in declared.agents.keys() {
match coord.power.get(agent) {
Ok(intent) => {
intents.insert(agent.clone(), intent);
}
// Left out of the map, which is what makes `plan` skip it.
Err(e) => {
tracing::warn!(%agent, error = ?e, "wanted state: cannot read power intent; skipping");
}
}
}
let intents: BTreeMap<String, Option<Wanted>> = declared
.agents
.keys()
.map(|agent| (agent.clone(), coord.power.get(agent)))
.collect();
// The pause marker is a stat, not a store read — it cannot fail the way
// `coord.power.get` above can, so there is no "unreadable, skip" branch
// to mirror. A name that does not parse as an `Ident` is skipped with a
// warning instead, the same defensive shape `crash_watch` uses for the
// same reason: a declaration is attacker-adjacent input (published by
// the swarm controller, not typed by an operator at this hive), so a
// malformed name should not panic the convergence loop.
// A name that does not parse as an `Ident` is skipped with a warning, the
// same defensive shape `crash_watch` uses for the same reason: a
// declaration is attacker-adjacent input (published by the swarm
// controller, not typed by an operator at this hive), so a malformed name
// should not panic the convergence loop.
let mut paused = BTreeMap::new();
for agent in declared.agents.keys() {
match hive_types::Ident::parse(agent) {
@ -463,10 +454,9 @@ struct Plan {
/// Turn one declaration into that plan — pure, so what the loop does with a
/// given declaration is testable without a queue, a container or a store.
///
/// `intents` is the hive's stored power intent per agent, and an agent
/// **missing from the map** is one whose intent could not be read: it is
/// skipped. That is not `Some(None)` — "no row yet" is a reading and gets
/// converged, "the read failed" is not and would be a guess.
/// `intents` is the power intent this process holds per agent: `None` is an
/// agent nothing has set an intent for since the process started, which gets
/// converged like any other that disagrees with the declaration.
///
/// An agent the declaration does not name reaches none of the three lists.
/// That is the whole of "absence is not a deletion order": the agents this
@ -479,10 +469,6 @@ fn plan(
) -> Plan {
let mut plan = Plan::default();
for (agent, decl) in &declared.agents {
// The pause axis does not gate on a readable power intent the way
// the match below does — it has no intent row to fail reading, only
// a stat that always answers — so it is evaluated unconditionally,
// even for an agent this pass otherwise skips.
let is_present = present.contains(agent);
let is_paused = paused.get(agent).copied().unwrap_or(false);
match decide_pause(decl.state, is_present, is_paused) {
@ -491,10 +477,8 @@ fn plan(
PauseConverge::Nothing => {}
}
let Some(intent) = intents.get(agent) else {
continue;
};
match decide(decl.state, is_present, *intent) {
let intent = intents.get(agent).copied().flatten();
match decide(decl.state, is_present, intent) {
Converge::Deploy => plan.deploy.push(agent.clone()),
Converge::Start => plan.start.push(agent.clone()),
Converge::Stop => plan.stop.push(agent.clone()),
@ -659,37 +643,6 @@ mod tests {
);
}
/// An intent that could not be read is skipped rather than treated as "no
/// row". `readable` is the control for the same call.
#[test]
fn an_unreadable_intent_skips_only_that_agent() {
let declared = declaration(&[("unreadable", AgentState::Up), ("readable", AgentState::Up)]);
let planned = plan(
&declared,
&set(&["unreadable", "readable"]),
&intents(&[("readable", None)]),
&paused(&[]),
);
assert_eq!(planned.start, vec!["readable".to_owned()]);
assert!(planned.deploy.is_empty() && planned.stop.is_empty());
}
/// The pause axis does not gate on a readable intent row at all — a
/// `paused` agent this hive has no power-intent row for still gets its
/// pause marker converged, unlike the power axis right above.
#[test]
fn pause_converges_even_when_the_power_intent_is_unreadable() {
let declared = declaration(&[("unreadable", AgentState::Paused)]);
let planned = plan(
&declared,
&set(&["unreadable"]),
&intents(&[]),
&paused(&[("unreadable", false)]),
);
assert_eq!(planned.pause, vec!["unreadable".to_owned()]);
assert!(planned.start.is_empty() && planned.deploy.is_empty());
}
/// A declared-paused agent that is present, running and already carries
/// the marker needs nothing on either axis.
#[test]
@ -738,7 +691,7 @@ mod tests {
#[test]
fn a_declared_agent_this_hive_does_not_have_is_deployed() {
assert_eq!(decide(AgentState::Up, false, None), Converge::Deploy);
// Even holding an intent: a row can outlive its container.
// Even holding an intent: it can outlive its container.
assert_eq!(
decide(AgentState::Up, false, Some(Wanted::Up)),
Converge::Deploy
@ -769,11 +722,11 @@ mod tests {
);
}
/// No row is not agreement. An agent that predates the power store is
/// converged to what it was declared as, not left on whatever it happens
/// to be doing.
/// No intent is not agreement. An agent nothing has set an intent for since
/// this process started is converged to what it was declared as, not left
/// on whatever it happens to be doing.
#[test]
fn an_agent_with_no_intent_row_is_converged() {
fn an_agent_with_no_intent_is_converged() {
assert_eq!(decide(AgentState::Up, true, None), Converge::Start);
assert_eq!(decide(AgentState::Offline, true, None), Converge::Stop);
}