Watch
0
0
Fork
You've already forked hyperhive
0

reconcile: start a stopped harness inside a running container

hive-priv gains AgentHarness { agent_name, action: probe | start }, which
runs systemctl -M h-<name> against the fixed unit hive-agent.service.
run_reconcile and the 1-min tick read the unit for a running wanted-up
agent and start it when it is inactive or failed. A hive-priv without the
op reads as unknown state and leaves reconcile container-only.
This commit is contained in:
atlas 2026-11-19 01:42:18 +01:00 • committed by mara
commit 36515c289d
6 changed files with 517 additions and 32 deletions

View file

@ -15,7 +15,8 @@ use hive_jobq::NodeId;
use super::model::NodeKind;
use crate::coordinator::{AgentPaths, Coordinator, HiveEnv};
use crate::power::{ReconcileAction, reconcile_action};
use crate::lifecycle::HarnessState;
use crate::power::{ReconcileAction, Wanted, reconcile_action};
use crate::stats::sweep_health::{self, SweepHealth};
/// Max time `Drain` waits for the harness to run its stop-checkpoint
@ -40,6 +41,10 @@ const PAUSE_ACK_TIMEOUT: std::time::Duration = GRACEFUL_STOP_TIMEOUT;
pub(super) trait Host: Sync {
/// Whether the agent's container unit is active.
async fn is_running(&self, name: &str) -> bool;
/// The state of the agent's harness unit inside its running container.
async fn harness_state(&self, name: &str) -> HarnessState;
/// Queue a start of the agent's harness unit inside its running container.
async fn start_harness(&self, name: &str) -> Result<()>;
/// Stop the agent's container.
async fn kill(&self, name: &str) -> Result<()>;
/// The record the swarm's builder wrote for the agent; `None` when it wrote
@ -106,6 +111,14 @@ impl Host for RealHost {
crate::lifecycle::is_running(name).await
}
async fn harness_state(&self, name: &str) -> HarnessState {
crate::lifecycle::harness_state(name).await
}
async fn start_harness(&self, name: &str) -> Result<()> {
crate::lifecycle::start_harness(name).await
}
async fn kill(&self, name: &str) -> Result<()> {
crate::lifecycle::kill(name).await
}
@ -846,12 +859,35 @@ async fn run_reconcile(
agent: name.to_owned(),
}),
ReconcileAction::Noop => {
if running && wanted == Wanted::Up {
restart_stopped_harness(host, name).await;
}
tracing::debug!(%name, wanted = <&str>::from(wanted), running, "reconcile: noop");
None
}
})
}
/// Start the harness unit of a wanted-up agent whose container is running but
/// whose harness is stopped or failed. Nothing else would: the unit restarts
/// only on failure, and a graceful stop exits cleanly.
///
/// Runs inside the `Reconcile` node, which holds the agent's lease, so no
/// other node of the agent is mid-stop; the periodic tick queues a `Reconcile`
/// only for an agent no live node targets. A paused agent keeps its harness
/// active (the pause marker parks the turn loop), so it never reads as
/// stopped. A start that fails is logged, not failed: the node belongs to a
/// DAG that may report a successful rebuild, and the tick tries again.
async fn restart_stopped_harness(host: &impl Host, name: &str) {
if host.harness_state(name).await != HarnessState::Stopped {
return;
}
tracing::info!(%name, "harness stopped inside running container; starting it");
if let Err(e) = host.start_harness(name).await {
tracing::warn!(%name, error = ?e, "starting the stopped harness failed");
}
}
/// Mechanical container start — the sub-step a `Reconcile` planner fans
/// out when it observes `wanted = Up` and the container down.
async fn run_start(host: &impl Host, coord: &Arc<Coordinator>, name: &str) -> Result<()> {

View file

@ -18,6 +18,7 @@ use super::{
};
use crate::coordinator::{AgentPaths, Coordinator, HiveEnv};
use crate::job_queue::model::NodeKind;
use crate::lifecycle::HarnessState;
use crate::power::Wanted;
use crate::stats::sweep_health::SweepHealth;
use crate::warnings::snapshot;
@ -30,6 +31,8 @@ const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-
#[derive(Default)]
struct FakeHost {
running: bool,
/// The harness unit inside the container is stopped.
harness_stopped: bool,
/// What the swarm's builder recorded for the agent; `None` reads as no
/// record.
built: Mutex<Option<AgentClosure>>,
@ -66,6 +69,12 @@ impl FakeHost {
}
}
/// The harness unit is stopped inside the running container.
fn harness_down(mut self) -> Self {
self.harness_stopped = true;
self
}
/// The builder recorded the closure for the agent, pushed or not.
fn built(self, cached: bool) -> Self {
*self.built.lock().unwrap() = Some(closure(cached));
@ -126,6 +135,19 @@ impl Host for FakeHost {
self.running
}
async fn harness_state(&self, name: &str) -> HarnessState {
self.record(format!("harness_state {name}"));
if self.harness_stopped {
HarnessState::Stopped
} else {
HarnessState::Present
}
}
async fn start_harness(&self, name: &str) -> anyhow::Result<()> {
self.step("start_harness", format!("start_harness {name}"))
}
async fn kill(&self, name: &str) -> anyhow::Result<()> {
self.record(format!("kill {name}"));
if self.fails("kill") {
@ -683,6 +705,67 @@ async fn reconcile_is_a_noop_when_state_matches_intent() {
assert!(sub.is_none());
}
/// The harness unit is down inside a running container and the agent is
/// wanted up: reconcile starts it and plans no container work.
#[tokio::test]
async fn reconcile_starts_a_stopped_harness_of_a_wanted_up_running_agent() {
let (_dir, coord) = coordinator();
coord.power.set("alice", Wanted::Up);
let host = FakeHost::up().harness_down();
let sub = run_reconcile(&host, &coord, "alice").await.expect("ok");
assert!(sub.is_none(), "the container is left alone");
assert_eq!(
host.calls(),
[
"is_running alice",
"harness_state alice",
"start_harness alice"
]
);
}
/// Each case that must not start the harness, against the control above.
#[tokio::test]
async fn reconcile_leaves_the_harness_alone_unless_the_agent_is_wanted_up_and_running() {
let (_dir, coord) = coordinator();
// A running harness (a paused agent keeps its harness active too).
coord.power.set("alice", Wanted::Up);
let host = FakeHost::up();
run_reconcile(&host, &coord, "alice").await.expect("ok");
assert!(!host.calls().contains(&"start_harness alice".to_owned()));
// Wanted offline: the container is stopped instead, the harness untouched.
coord.power.set("alice", Wanted::Offline);
let host = FakeHost::up().harness_down();
let sub = run_reconcile(&host, &coord, "alice").await.expect("ok");
assert!(matches!(sub, Some(NodeKind::Stop { .. })));
assert_eq!(host.calls(), ["is_running alice"]);
// Wanted offline and already down: nothing to do, the harness is not read.
coord.power.set("alice", Wanted::Offline);
let host = FakeHost::down().harness_down();
let sub = run_reconcile(&host, &coord, "alice").await.expect("ok");
assert!(sub.is_none());
assert_eq!(host.calls(), ["is_running alice"]);
// Container down: it is started, and the harness is not read.
coord.power.set("alice", Wanted::Up);
let host = FakeHost::down().harness_down();
let sub = run_reconcile(&host, &coord, "alice").await.expect("ok");
assert!(matches!(sub, Some(NodeKind::Start { .. })));
assert_eq!(host.calls(), ["is_running alice"]);
}
/// A failed harness start is logged; the reconcile node still succeeds.
#[tokio::test]
async fn a_failed_harness_start_does_not_fail_the_reconcile() {
let (_dir, coord) = coordinator();
coord.power.set("alice", Wanted::Up);
let host = FakeHost::up().harness_down().failing("start_harness");
run_reconcile(&host, &coord, "alice").await.expect("ok");
assert!(host.calls().contains(&"start_harness alice".to_owned()));
}
#[tokio::test]
async fn reconcile_seeds_a_missing_intent_from_the_observed_state() {
let (_dir, coord) = coordinator();

View file

@ -604,6 +604,80 @@ pub async fn running_state(name: &str) -> Option<bool> {
.map(|out| out.status.success())
}
/// Whether an agent's harness unit (`hive-agent.service`) is up inside its
/// running container.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HarnessState {
/// Active, or on its way: activating, deactivating or reloading.
Present,
/// Loaded and `inactive` or `failed`: nothing is running it and nothing
/// is about to.
Stopped,
/// Not readable: hive-priv is older than the op, the container's systemd
/// did not answer, or the unit is not loaded.
Unknown,
}
/// Read the harness unit's state out of `AgentHarness` probe output
/// (`LoadState=` and `ActiveState=` lines).
fn parse_harness_state(out: &str) -> HarnessState {
let field = |key: &str| {
out.lines()
.find_map(|l| l.strip_prefix(key).and_then(|v| v.strip_prefix('=')))
.map(str::trim)
};
match (field("LoadState"), field("ActiveState")) {
(Some("loaded"), Some("inactive" | "failed")) => HarnessState::Stopped,
(Some("loaded"), Some("active" | "activating" | "deactivating" | "reloading")) => {
HarnessState::Present
}
_ => HarnessState::Unknown,
}
}
/// The state of `name`'s harness unit, asked of hive-priv. Bounded by
/// [`IS_ACTIVE_TIMEOUT`], because the tick calls it per agent inline.
///
/// Any failure reads as [`HarnessState::Unknown`], so reconcile keeps its
/// container-only behaviour: a hive-priv that predates the op answers an
/// error, and that is logged at `warn` the first time and `debug` after.
pub async fn harness_state(name: &str) -> HarnessState {
let probe = crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Probe);
if let Ok(answer) = tokio::time::timeout(IS_ACTIVE_TIMEOUT, probe).await {
harness_of_answer(name, answer)
} else {
tracing::debug!(%name, "reading the agent's harness state timed out");
HarnessState::Unknown
}
}
/// The state a probe answer means; an error is [`HarnessState::Unknown`].
fn harness_of_answer(name: &str, answer: Result<String>) -> HarnessState {
use std::sync::atomic::{AtomicBool, Ordering};
static WARNED: AtomicBool = AtomicBool::new(false);
match answer {
Ok(out) => parse_harness_state(&out),
Err(e) if !WARNED.swap(true, Ordering::Relaxed) => {
tracing::warn!(%name, error = ?e, "cannot read the agent's harness state; reconcile only watches the container (logged once)");
HarnessState::Unknown
}
Err(e) => {
tracing::debug!(%name, error = ?e, "cannot read the agent's harness state");
HarnessState::Unknown
}
}
}
/// Queue a start of `name`'s harness unit inside its running container.
///
/// # Errors
/// hive-priv is unreachable, refuses, or `systemctl` fails.
pub async fn start_harness(name: &str) -> Result<()> {
crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Start)
.await
.map(drop)
}
/// How long one `systemctl is-active` may run before the unit's state counts
/// as unreadable. Periodic workers make this call per agent inline.
const IS_ACTIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);

View file

@ -445,3 +445,45 @@ fn stop_kind_survives_a_clean_stop_between_failures() {
guards.remove("stopkind-c");
assert!(std::ptr::eq(first, failed(&mut guards)));
}
#[test]
fn harness_state_reads_load_and_active_state() {
let probe = |load: &str, active: &str| {
parse_harness_state(&format!("LoadState={load}\nActiveState={active}\n"))
};
for active in ["inactive", "failed"] {
assert_eq!(probe("loaded", active), HarnessState::Stopped, "{active}");
}
for active in ["active", "activating", "deactivating", "reloading"] {
assert_eq!(probe("loaded", active), HarnessState::Present, "{active}");
}
// A unit that is not loaded, or output that is not a probe, is not
// evidence the harness is down.
assert_eq!(probe("not-found", "inactive"), HarnessState::Unknown);
assert_eq!(parse_harness_state(""), HarnessState::Unknown);
assert_eq!(parse_harness_state("garbage"), HarnessState::Unknown);
}
/// A hive-priv that predates the op answers an error; that reads as unknown
/// state, which leaves reconcile with its container-only behaviour.
#[test]
fn an_unreadable_harness_is_unknown_never_stopped() {
let old_priv = anyhow::anyhow!("parse request: unknown variant `agent_harness`");
assert_eq!(
harness_of_answer("alice", Err(old_priv)),
HarnessState::Unknown
);
assert_eq!(
harness_of_answer("alice", Err(anyhow::anyhow!("again"))),
HarnessState::Unknown,
"later errors read the same"
);
assert_eq!(
harness_of_answer(
"alice",
Ok("LoadState=loaded\nActiveState=inactive\n".to_owned())
),
HarnessState::Stopped,
"control: a real answer is read"
);
}

View file

@ -8,8 +8,9 @@
use anyhow::{Context as _, Result, bail};
use hive_priv_sock::{
BindMount, CredentialMount, HiveFacts, InfraAction, InfraContainer, MEASURE_AGENT_DISK_TIMEOUT,
NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream,
BindMount, CredentialMount, HarnessAction, HiveFacts, InfraAction, InfraContainer,
MEASURE_AGENT_DISK_TIMEOUT, NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse,
PrivStream,
};
use hive_types::NixBuilds;
use std::path::Path;
@ -108,6 +109,20 @@ pub async fn start_failure_log(name: &str, since_unix: u64) -> Result<String> {
Ok(stdout)
}
/// Read (`Probe`, the unit's `LoadState` and `ActiveState` as `key=value`
/// lines) or queue a start of (`Start`) the harness unit inside `name`'s
/// container. An older hive-priv answers an error (it does not know the op).
pub async fn agent_harness(name: &str, action: HarnessAction) -> Result<String> {
let (stdout, _) = check(
call(&PrivRequest::AgentHarness {
agent_name: name.to_owned(),
action,
})
.await?,
)?;
Ok(stdout)
}
pub async fn stop_container(name: &str) -> Result<()> {
ok(call(&PrivRequest::StopContainer {
name: name.to_owned(),

View file

@ -1,11 +1,14 @@
//! Periodic declared ⇒ running convergence: an agent whose power intent is
//! `Up` and whose container is down gets a `Reconcile` queued, once a
//! [`TICK_INTERVAL`], without anyone having to ask.
//! `Up` and whose container is down, or running with its harness unit stopped,
//! gets a `Reconcile` queued, once a [`TICK_INTERVAL`], without anyone having
//! to ask.
//!
//! The tick queues the same `NodeKind::Reconcile` the boot sweep, a watch
//! attach and a declaration write end in, so what it starts, and how, is
//! `run_reconcile`'s decision. It only ever converges *toward a start*: an
//! agent wanted offline is left alone, and a running one is never touched.
//! agent wanted offline is left alone, and a running one is left alone unless
//! hive-priv reports its harness unit stopped or failed. A harness that cannot
//! be read (an older hive-priv, no answer) counts as running.
//! `crash_watch` keeps raising its alert; this worker does not replace it.
//!
//! # What the tick leaves alone
@ -29,7 +32,7 @@ use std::sync::Arc;
use std::time::Duration;
use crate::coordinator::Coordinator;
use crate::lifecycle::{self, AGENT_PREFIX};
use crate::lifecycle::{self, AGENT_PREFIX, HarnessState};
use crate::power::Wanted;
/// How often declared intent is compared with the observed containers. The
@ -51,6 +54,9 @@ struct Observation {
paused: bool,
/// An update has the agent down and not yet back up.
updating: bool,
/// The harness unit inside a running, wanted-up container; `Unknown`
/// where it was not read.
harness: HarnessState,
}
/// Why the tick did nothing for an agent.
@ -126,7 +132,8 @@ fn ticks_between(attempts: u32) -> u32 {
/// The decision for one agent. Pure but for the backoff counters it advances.
fn step(backoffs: &mut Backoffs, agent: &str, seen: Observation) -> Step {
if seen.running == Some(true) || seen.wanted != Some(Wanted::Up) {
let harness_down = seen.running == Some(true) && seen.harness == HarnessState::Stopped;
if (seen.running == Some(true) && !harness_down) || seen.wanted != Some(Wanted::Up) {
backoffs.settle(agent);
return Step::Leave(Leave::Converged);
}
@ -170,11 +177,20 @@ async fn tick(coord: &Arc<Coordinator>, backoffs: &mut Backoffs) {
let Some(name) = container.strip_prefix(AGENT_PREFIX) else {
continue;
};
let wanted = coord.power.get(name);
let running = lifecycle::running_state(name).await;
let updating = coord.is_updating(name);
let harness = if running == Some(true) && wanted == Some(Wanted::Up) && !updating {
lifecycle::harness_state(name).await
} else {
HarnessState::Unknown
};
let seen = Observation {
wanted: coord.power.get(name),
running: lifecycle::running_state(name).await,
wanted,
running,
paused: hive_types::Ident::parse(name).is_ok_and(|id| Coordinator::is_paused(&id)),
updating: coord.is_updating(name),
updating,
harness,
};
if step(backoffs, name, seen) != Step::Restart {
continue;
@ -184,7 +200,11 @@ async fn tick(coord: &Arc<Coordinator>, backoffs: &mut Backoffs) {
let attempt = backoffs.record_attempt(name);
tracing::info!(
agent = %name,
observed = "stopped",
observed = if seen.running == Some(true) {
"harness stopped"
} else {
"stopped"
},
wanted = "up",
attempt,
"reconcile tick: restarting agent"
@ -214,6 +234,7 @@ mod tests {
running: Some(false),
paused: false,
updating: false,
harness: HarnessState::Unknown,
};
#[test]
@ -284,6 +305,56 @@ mod tests {
}
}
/// A running, wanted-up agent whose harness unit is stopped is restarted;
/// every other reading of a running agent leaves it alone.
#[test]
fn a_running_agent_with_a_stopped_harness_is_restarted() {
let running = |harness| Observation {
running: Some(true),
harness,
..DOWN
};
assert_eq!(
step(
&mut Backoffs::default(),
"a",
running(HarnessState::Stopped)
),
Step::Restart
);
for harness in [HarnessState::Present, HarnessState::Unknown] {
assert_eq!(
step(&mut Backoffs::default(), "a", running(harness)),
Step::Leave(Leave::Converged),
"{harness:?}"
);
}
let wanted_offline = Observation {
wanted: Some(Wanted::Offline),
..running(HarnessState::Stopped)
};
assert_eq!(
step(&mut Backoffs::default(), "a", wanted_offline),
Step::Leave(Leave::Converged)
);
let mid_update = Observation {
updating: true,
..running(HarnessState::Stopped)
};
assert_eq!(
step(&mut Backoffs::default(), "a", mid_update),
Step::Leave(Leave::Held)
);
let mut backoffs = Backoffs::default();
backoffs.record_attempt("a");
backoffs.record_attempt("a");
assert_eq!(
step(&mut backoffs, "a", running(HarnessState::Stopped)),
Step::Leave(Leave::BackingOff),
"a harness that stays down backs off like a container"
);
}
/// Ticks between restarts of an agent that never comes up: 1, 2, 4, 8,
/// then the 15-tick cap, and it stays there.
#[test]