Watch
0
0
Fork
You've already forked hyperhive
0

reconcile: start a stopped harness inside a running container

hive-priv gains AgentHarness { agent_name, action: probe | start }, which
runs systemctl -M h-<name> against the fixed unit hive-agent.service.
run_reconcile and the 1-min tick read the unit for a running wanted-up
agent and start it when it is inactive or failed. A hive-priv without the
op reads as unknown state and leaves reconcile container-only.
This commit is contained in:
atlas 2026-11-19 01:42:18 +01:00 • committed by mara
commit 36515c289d

View file

@ -1,11 +1,14 @@
//! Periodic declared ⇒ running convergence: an agent whose power intent is
//! `Up` and whose container is down gets a `Reconcile` queued, once a
//! [`TICK_INTERVAL`], without anyone having to ask.
//! `Up` and whose container is down, or running with its harness unit stopped,
//! gets a `Reconcile` queued, once a [`TICK_INTERVAL`], without anyone having
//! to ask.
//!
//! The tick queues the same `NodeKind::Reconcile` the boot sweep, a watch
//! attach and a declaration write end in, so what it starts, and how, is
//! `run_reconcile`'s decision. It only ever converges *toward a start*: an
//! agent wanted offline is left alone, and a running one is never touched.
//! agent wanted offline is left alone, and a running one is left alone unless
//! hive-priv reports its harness unit stopped or failed. A harness that cannot
//! be read (an older hive-priv, no answer) counts as running.
//! `crash_watch` keeps raising its alert; this worker does not replace it.
//!
//! # What the tick leaves alone
@ -29,7 +32,7 @@ use std::sync::Arc;
use std::time::Duration;
use crate::coordinator::Coordinator;
use crate::lifecycle::{self, AGENT_PREFIX};
use crate::lifecycle::{self, AGENT_PREFIX, HarnessState};
use crate::power::Wanted;
/// How often declared intent is compared with the observed containers. The
@ -51,6 +54,9 @@ struct Observation {
paused: bool,
/// An update has the agent down and not yet back up.
updating: bool,
/// The harness unit inside a running, wanted-up container; `Unknown`
/// where it was not read.
harness: HarnessState,
}
/// Why the tick did nothing for an agent.
@ -126,7 +132,8 @@ fn ticks_between(attempts: u32) -> u32 {
/// The decision for one agent. Pure but for the backoff counters it advances.
fn step(backoffs: &mut Backoffs, agent: &str, seen: Observation) -> Step {
if seen.running == Some(true) || seen.wanted != Some(Wanted::Up) {
let harness_down = seen.running == Some(true) && seen.harness == HarnessState::Stopped;
if (seen.running == Some(true) && !harness_down) || seen.wanted != Some(Wanted::Up) {
backoffs.settle(agent);
return Step::Leave(Leave::Converged);
}
@ -170,11 +177,20 @@ async fn tick(coord: &Arc<Coordinator>, backoffs: &mut Backoffs) {
let Some(name) = container.strip_prefix(AGENT_PREFIX) else {
continue;
};
let wanted = coord.power.get(name);
let running = lifecycle::running_state(name).await;
let updating = coord.is_updating(name);
let harness = if running == Some(true) && wanted == Some(Wanted::Up) && !updating {
lifecycle::harness_state(name).await
} else {
HarnessState::Unknown
};
let seen = Observation {
wanted: coord.power.get(name),
running: lifecycle::running_state(name).await,
wanted,
running,
paused: hive_types::Ident::parse(name).is_ok_and(|id| Coordinator::is_paused(&id)),
updating: coord.is_updating(name),
updating,
harness,
};
if step(backoffs, name, seen) != Step::Restart {
continue;
@ -184,7 +200,11 @@ async fn tick(coord: &Arc<Coordinator>, backoffs: &mut Backoffs) {
let attempt = backoffs.record_attempt(name);
tracing::info!(
agent = %name,
observed = "stopped",
observed = if seen.running == Some(true) {
"harness stopped"
} else {
"stopped"
},
wanted = "up",
attempt,
"reconcile tick: restarting agent"
@ -214,6 +234,7 @@ mod tests {
running: Some(false),
paused: false,
updating: false,
harness: HarnessState::Unknown,
};
#[test]
@ -284,6 +305,56 @@ mod tests {
}
}
/// A running, wanted-up agent whose harness unit is stopped is restarted;
/// every other reading of a running agent leaves it alone.
#[test]
fn a_running_agent_with_a_stopped_harness_is_restarted() {
let running = |harness| Observation {
running: Some(true),
harness,
..DOWN
};
assert_eq!(
step(
&mut Backoffs::default(),
"a",
running(HarnessState::Stopped)
),
Step::Restart
);
for harness in [HarnessState::Present, HarnessState::Unknown] {
assert_eq!(
step(&mut Backoffs::default(), "a", running(harness)),
Step::Leave(Leave::Converged),
"{harness:?}"
);
}
let wanted_offline = Observation {
wanted: Some(Wanted::Offline),
..running(HarnessState::Stopped)
};
assert_eq!(
step(&mut Backoffs::default(), "a", wanted_offline),
Step::Leave(Leave::Converged)
);
let mid_update = Observation {
updating: true,
..running(HarnessState::Stopped)
};
assert_eq!(
step(&mut Backoffs::default(), "a", mid_update),
Step::Leave(Leave::Held)
);
let mut backoffs = Backoffs::default();
backoffs.record_attempt("a");
backoffs.record_attempt("a");
assert_eq!(
step(&mut backoffs, "a", running(HarnessState::Stopped)),
Step::Leave(Leave::BackingOff),
"a harness that stays down backs off like a container"
);
}
/// Ticks between restarts of an agent that never comes up: 1, 2, 4, 8,
/// then the 15-tick cap, and it stays there.
#[test]