From 36515c289dc47c96fc137f38d48b1684d805447b Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 19 Nov 2026 01:42:18 +0100 Subject: [PATCH] reconcile: start a stopped harness inside a running container hive-priv gains AgentHarness { agent_name, action: probe | start }, which runs systemctl -M h- against the fixed unit hive-agent.service. run_reconcile and the 1-min tick read the unit for a running wanted-up agent and start it when it is inactive or failed. A hive-priv without the op reads as unknown state and leaves reconcile container-only. --- .../4-implementation/scheduler/coordinator.md | 16 ++- .../scheduler/observability.md | 18 +-- .../trust-boundary/hive-c0re-privilege.md | 1 + hive-c0re/src/job_queue/exec.rs | 38 +++++- hive-c0re/src/job_queue/exec/tests.rs | 83 +++++++++++++ hive-c0re/src/lifecycle/mod.rs | 74 ++++++++++++ hive-c0re/src/lifecycle/tests.rs | 42 +++++++ hive-c0re/src/priv_client.rs | 19 ++- hive-c0re/src/workers/reconcile_tick.rs | 89 ++++++++++++-- hive-priv-sock/src/lib.rs | 54 ++++++++- hive-priv/src/container.rs | 110 +++++++++++++++++- hive-priv/src/main.rs | 5 + 12 files changed, 517 insertions(+), 32 deletions(-) diff --git a/docs/4-implementation/scheduler/coordinator.md b/docs/4-implementation/scheduler/coordinator.md index a74cba57f..f91f14c5d 100644 --- a/docs/4-implementation/scheduler/coordinator.md +++ b/docs/4-implementation/scheduler/coordinator.md @@ -55,7 +55,7 @@ Cheap — no build slot: | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `MetaSync` | the rebuild's preamble — rebuild-dir prep, the archetype refresh and the agent-set refresh (`agent_set::refresh`), then reads the agent's closure record once and pins its store path for `Prebuild` and `Swap`; fails when the swarm has no closure to deploy; reads nothing from the forge. Declares no resources | | `Provision` | first-spawn pre-create provisioning — the state subvolume, agent-set refresh (`agent_set::refresh`); runs ahead of `Create`. Store-only, no container yet | -| `Reconcile` | idempotent power converge: read `wanted` (below) + observed state; start if `Up` & down (cold-start fallback included), stop if `Offline` & up, else noop | +| `Reconcile` | idempotent power converge: read `wanted` (below) + observed state; start if `Up` & down (cold-start fallback included), stop if `Offline` & up, else noop; an `Up` agent whose container runs but whose harness unit is stopped or failed gets the unit started | | `Start` | mechanical container start — runtime dir + drop-ins, `start_with_fallback`, MCP listener registration, the manager kick. Fanned out by a `Reconcile` that observed `wanted = Up` and the container down | | `Stop` | mechanical container stop — `nixos-container` kill, MCP listener unregister, the `Killed` manager notify. Fanned out by a `Reconcile` that observed `wanted = Offline` and up | | `StopForUpdate` | mechanical `nixos-container stop` for the profile swap; never touches `wanted`; noop if already stopped | @@ -324,15 +324,19 @@ boot with no operator action in this process never finds drift of its own. ### Periodic reconcile tick Once a minute (first tick one minute after boot), `workers::reconcile_tick` -looks for an agent that the hive wants up and whose container is down. It reads -the power intent, the container's unit state, the pause marker and -`Coordinator::is_updating`, then queues the same `Reconcile` node the boot power +looks for an agent that the hive wants up and whose container is down, or runs +with its harness unit (`hive-agent.service`) stopped or failed. It reads the +power intent, the container's unit state, the pause marker, +`Coordinator::is_updating` and, for a running wanted-up agent, the harness +unit's state through hive-priv (`AgentHarness`), then queues the same `Reconcile` node the boot power path uses, through `JobQueue::insert_reconcile_if_idle`. The tick never stops anything. It leaves an agent alone when: -- its intent is unknown or `Offline`, or its container runs; +- its intent is unknown or `Offline`, or its container runs with a harness that + is active, activating or unreadable (an older hive-priv doesn't know the op: hive-c0re logs + that once and the tick watches the container only); - the agent has its pause marker, an update in progress, or a container state the tick can't read in time; - any queued or running node targets it (update, create, destroy, power op or @@ -342,7 +346,7 @@ An agent still down on the tick after the tick queued a restart has failed to come up, so the tick spaces restarts of one agent 1, 2, 4, 8, then 15 ticks apart (a 15-minute cap). Seeing the agent running, or no longer wanted up, resets the spacing. Each restart logs `reconcile tick: restarting agent` with -the agent, `observed=stopped`, `wanted=up` and the attempt number, and bumps +the agent, `observed=stopped` (`harness stopped` for a running container), `wanted=up` and the attempt number, and bumps `hyperhive.hive.reconcile_tick.restarts`. `crash_watch` keeps raising its alert; the tick only converges. diff --git a/docs/4-implementation/scheduler/observability.md b/docs/4-implementation/scheduler/observability.md index 243f764d2..bf72fe64c 100644 --- a/docs/4-implementation/scheduler/observability.md +++ b/docs/4-implementation/scheduler/observability.md @@ -535,15 +535,15 @@ distinguishable from "this hive is gone." Select them with -| Metric | Unit | Kind | Meaning | -| ---------------------------------------- | ----------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `process.uptime` | `s` | gauge | seconds since this hive's `hive-c0re` started exporting; a restart reads as a drop to ~0 | -| `hyperhive.hive.degraded` | `1` | gauge | `1` while the hive reports itself unhealthy — the same verdict `/health/ready` gives and the swarm status view shows | -| `hyperhive.hive.warnings` | `1` | gauge | how many warnings are currently raised, split by a `level` attribute (`warn`, `crit`) | -| `hyperhive.hive.warning` | `1` | gauge | `1` per raised warning, with `kind` and `level` attributes; a cleared warning stops reporting | -| `hyperhive.hive.nix_store.utilization` | `1` | gauge | used fraction (0–1) of the filesystem that holds the nix store; no sample when the probe fails | -| `hyperhive.hive.pressure.stall` | `us` | counter | host-wide `/proc/pressure/{cpu,memory,io}` `total` in microseconds, split by `resource` (`cpu`, `memory`, `io`) and `kind` (`some`, `full`); a rate is the stalled share of time; no sample without PSI | -| `hyperhive.hive.reconcile_tick.restarts` | `{restart}` | counter | restarts the periodic reconcile tick has queued since `hive-c0re` started, for agents declared up and found down; the agent is in the `reconcile tick: restarting agent` log line | +| Metric | Unit | Kind | Meaning | +| ---------------------------------------- | ----------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `process.uptime` | `s` | gauge | seconds since this hive's `hive-c0re` started exporting; a restart reads as a drop to ~0 | +| `hyperhive.hive.degraded` | `1` | gauge | `1` while the hive reports itself unhealthy — the same verdict `/health/ready` gives and the swarm status view shows | +| `hyperhive.hive.warnings` | `1` | gauge | how many warnings are currently raised, split by a `level` attribute (`warn`, `crit`) | +| `hyperhive.hive.warning` | `1` | gauge | `1` per raised warning, with `kind` and `level` attributes; a cleared warning stops reporting | +| `hyperhive.hive.nix_store.utilization` | `1` | gauge | used fraction (0–1) of the filesystem that holds the nix store; no sample when the probe fails | +| `hyperhive.hive.pressure.stall` | `us` | counter | host-wide `/proc/pressure/{cpu,memory,io}` `total` in microseconds, split by `resource` (`cpu`, `memory`, `io`) and `kind` (`some`, `full`); a rate is the stalled share of time; no sample without PSI | +| `hyperhive.hive.reconcile_tick.restarts` | `{restart}` | counter | restarts the periodic reconcile tick has queued since `hive-c0re` started, for agents declared up and found down, or running with a stopped harness; the agent is in the `reconcile tick: restarting agent` log line | hive-c0re reports both levels of `hyperhive.hive.warnings` every cycle, `0` included, so a healthy hive is visible as zeros rather than as missing diff --git a/docs/4-implementation/trust-boundary/hive-c0re-privilege.md b/docs/4-implementation/trust-boundary/hive-c0re-privilege.md index c449cefb8..23718fdcc 100644 --- a/docs/4-implementation/trust-boundary/hive-c0re-privilege.md +++ b/docs/4-implementation/trust-boundary/hive-c0re-privilege.md @@ -46,6 +46,7 @@ known operations; there is no arbitrary command pass-through: | `DaemonReload` | `systemctl daemon-reload` | | `ControlInfraContainer` | `systemctl nginx.service` for the gateway — the `InfraContainer` enum is the allowlist, and serde rejects unknown names at the wire boundary (`hive-c0re`, the forge and the matrix have no variant, so no request can name them) | | `SyncAgentTmpfiles` | unlink `/etc/tmpfiles.d/hyperhive-agents.conf` and return `Ok` | +| `AgentHarness` | `systemctl -M h- show --property=LoadState --property=ActiveState hive-agent.service` (`Probe`) or `systemctl -M h- --no-block start hive-agent.service` (`Start`). The unit name is a constant in hive-priv, the agent name goes through the same validator as the other per-agent ops, and each call stops after 5 s. It reads and starts one unit inside one agent container: the container's own systemd answers, so hive-priv trusts that reply for the probe only. A hive-priv that predates the op answers an unknown-op error, which hive-c0re reads as "harness state unknown" | | `SetAgentPaused` | create / remove the `//harness/paused` marker that parks an agent's turn loop | | `EnsureAgentSubvolume` | `btrfs subvolume create /` for a new agent — no-op when the path exists or the filesystem isn't btrfs | | `UpgradeAgentSubvolume` | migrate an existing plain state dir into a subvolume: create, `cp -a --reflink=auto`, atomic swap. Operator opt-in, and the caller stops the agent first | diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 28999f5e2..cc1348666 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -15,7 +15,8 @@ use hive_jobq::NodeId; use super::model::NodeKind; use crate::coordinator::{AgentPaths, Coordinator, HiveEnv}; -use crate::power::{ReconcileAction, reconcile_action}; +use crate::lifecycle::HarnessState; +use crate::power::{ReconcileAction, Wanted, reconcile_action}; use crate::stats::sweep_health::{self, SweepHealth}; /// Max time `Drain` waits for the harness to run its stop-checkpoint @@ -40,6 +41,10 @@ const PAUSE_ACK_TIMEOUT: std::time::Duration = GRACEFUL_STOP_TIMEOUT; pub(super) trait Host: Sync { /// Whether the agent's container unit is active. async fn is_running(&self, name: &str) -> bool; + /// The state of the agent's harness unit inside its running container. + async fn harness_state(&self, name: &str) -> HarnessState; + /// Queue a start of the agent's harness unit inside its running container. + async fn start_harness(&self, name: &str) -> Result<()>; /// Stop the agent's container. async fn kill(&self, name: &str) -> Result<()>; /// The record the swarm's builder wrote for the agent; `None` when it wrote @@ -106,6 +111,14 @@ impl Host for RealHost { crate::lifecycle::is_running(name).await } + async fn harness_state(&self, name: &str) -> HarnessState { + crate::lifecycle::harness_state(name).await + } + + async fn start_harness(&self, name: &str) -> Result<()> { + crate::lifecycle::start_harness(name).await + } + async fn kill(&self, name: &str) -> Result<()> { crate::lifecycle::kill(name).await } @@ -846,12 +859,35 @@ async fn run_reconcile( agent: name.to_owned(), }), ReconcileAction::Noop => { + if running && wanted == Wanted::Up { + restart_stopped_harness(host, name).await; + } tracing::debug!(%name, wanted = <&str>::from(wanted), running, "reconcile: noop"); None } }) } +/// Start the harness unit of a wanted-up agent whose container is running but +/// whose harness is stopped or failed. Nothing else would: the unit restarts +/// only on failure, and a graceful stop exits cleanly. +/// +/// Runs inside the `Reconcile` node, which holds the agent's lease, so no +/// other node of the agent is mid-stop; the periodic tick queues a `Reconcile` +/// only for an agent no live node targets. A paused agent keeps its harness +/// active (the pause marker parks the turn loop), so it never reads as +/// stopped. A start that fails is logged, not failed: the node belongs to a +/// DAG that may report a successful rebuild, and the tick tries again. +async fn restart_stopped_harness(host: &impl Host, name: &str) { + if host.harness_state(name).await != HarnessState::Stopped { + return; + } + tracing::info!(%name, "harness stopped inside running container; starting it"); + if let Err(e) = host.start_harness(name).await { + tracing::warn!(%name, error = ?e, "starting the stopped harness failed"); + } +} + /// Mechanical container start — the sub-step a `Reconcile` planner fans /// out when it observes `wanted = Up` and the container down. async fn run_start(host: &impl Host, coord: &Arc, name: &str) -> Result<()> { diff --git a/hive-c0re/src/job_queue/exec/tests.rs b/hive-c0re/src/job_queue/exec/tests.rs index 389e8a1e8..e5da3e65d 100644 --- a/hive-c0re/src/job_queue/exec/tests.rs +++ b/hive-c0re/src/job_queue/exec/tests.rs @@ -18,6 +18,7 @@ use super::{ }; use crate::coordinator::{AgentPaths, Coordinator, HiveEnv}; use crate::job_queue::model::NodeKind; +use crate::lifecycle::HarnessState; use crate::power::Wanted; use crate::stats::sweep_health::SweepHealth; use crate::warnings::snapshot; @@ -30,6 +31,8 @@ const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system- #[derive(Default)] struct FakeHost { running: bool, + /// The harness unit inside the container is stopped. + harness_stopped: bool, /// What the swarm's builder recorded for the agent; `None` reads as no /// record. built: Mutex>, @@ -66,6 +69,12 @@ impl FakeHost { } } + /// The harness unit is stopped inside the running container. + fn harness_down(mut self) -> Self { + self.harness_stopped = true; + self + } + /// The builder recorded the closure for the agent, pushed or not. fn built(self, cached: bool) -> Self { *self.built.lock().unwrap() = Some(closure(cached)); @@ -126,6 +135,19 @@ impl Host for FakeHost { self.running } + async fn harness_state(&self, name: &str) -> HarnessState { + self.record(format!("harness_state {name}")); + if self.harness_stopped { + HarnessState::Stopped + } else { + HarnessState::Present + } + } + + async fn start_harness(&self, name: &str) -> anyhow::Result<()> { + self.step("start_harness", format!("start_harness {name}")) + } + async fn kill(&self, name: &str) -> anyhow::Result<()> { self.record(format!("kill {name}")); if self.fails("kill") { @@ -683,6 +705,67 @@ async fn reconcile_is_a_noop_when_state_matches_intent() { assert!(sub.is_none()); } +/// The harness unit is down inside a running container and the agent is +/// wanted up: reconcile starts it and plans no container work. +#[tokio::test] +async fn reconcile_starts_a_stopped_harness_of_a_wanted_up_running_agent() { + let (_dir, coord) = coordinator(); + coord.power.set("alice", Wanted::Up); + let host = FakeHost::up().harness_down(); + let sub = run_reconcile(&host, &coord, "alice").await.expect("ok"); + assert!(sub.is_none(), "the container is left alone"); + assert_eq!( + host.calls(), + [ + "is_running alice", + "harness_state alice", + "start_harness alice" + ] + ); +} + +/// Each case that must not start the harness, against the control above. +#[tokio::test] +async fn reconcile_leaves_the_harness_alone_unless_the_agent_is_wanted_up_and_running() { + let (_dir, coord) = coordinator(); + // A running harness (a paused agent keeps its harness active too). + coord.power.set("alice", Wanted::Up); + let host = FakeHost::up(); + run_reconcile(&host, &coord, "alice").await.expect("ok"); + assert!(!host.calls().contains(&"start_harness alice".to_owned())); + + // Wanted offline: the container is stopped instead, the harness untouched. + coord.power.set("alice", Wanted::Offline); + let host = FakeHost::up().harness_down(); + let sub = run_reconcile(&host, &coord, "alice").await.expect("ok"); + assert!(matches!(sub, Some(NodeKind::Stop { .. }))); + assert_eq!(host.calls(), ["is_running alice"]); + + // Wanted offline and already down: nothing to do, the harness is not read. + coord.power.set("alice", Wanted::Offline); + let host = FakeHost::down().harness_down(); + let sub = run_reconcile(&host, &coord, "alice").await.expect("ok"); + assert!(sub.is_none()); + assert_eq!(host.calls(), ["is_running alice"]); + + // Container down: it is started, and the harness is not read. + coord.power.set("alice", Wanted::Up); + let host = FakeHost::down().harness_down(); + let sub = run_reconcile(&host, &coord, "alice").await.expect("ok"); + assert!(matches!(sub, Some(NodeKind::Start { .. }))); + assert_eq!(host.calls(), ["is_running alice"]); +} + +/// A failed harness start is logged; the reconcile node still succeeds. +#[tokio::test] +async fn a_failed_harness_start_does_not_fail_the_reconcile() { + let (_dir, coord) = coordinator(); + coord.power.set("alice", Wanted::Up); + let host = FakeHost::up().harness_down().failing("start_harness"); + run_reconcile(&host, &coord, "alice").await.expect("ok"); + assert!(host.calls().contains(&"start_harness alice".to_owned())); +} + #[tokio::test] async fn reconcile_seeds_a_missing_intent_from_the_observed_state() { let (_dir, coord) = coordinator(); diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index 53dd27614..83544a48c 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -604,6 +604,80 @@ pub async fn running_state(name: &str) -> Option { .map(|out| out.status.success()) } +/// Whether an agent's harness unit (`hive-agent.service`) is up inside its +/// running container. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HarnessState { + /// Active, or on its way: activating, deactivating or reloading. + Present, + /// Loaded and `inactive` or `failed`: nothing is running it and nothing + /// is about to. + Stopped, + /// Not readable: hive-priv is older than the op, the container's systemd + /// did not answer, or the unit is not loaded. + Unknown, +} + +/// Read the harness unit's state out of `AgentHarness` probe output +/// (`LoadState=` and `ActiveState=` lines). +fn parse_harness_state(out: &str) -> HarnessState { + let field = |key: &str| { + out.lines() + .find_map(|l| l.strip_prefix(key).and_then(|v| v.strip_prefix('='))) + .map(str::trim) + }; + match (field("LoadState"), field("ActiveState")) { + (Some("loaded"), Some("inactive" | "failed")) => HarnessState::Stopped, + (Some("loaded"), Some("active" | "activating" | "deactivating" | "reloading")) => { + HarnessState::Present + } + _ => HarnessState::Unknown, + } +} + +/// The state of `name`'s harness unit, asked of hive-priv. Bounded by +/// [`IS_ACTIVE_TIMEOUT`], because the tick calls it per agent inline. +/// +/// Any failure reads as [`HarnessState::Unknown`], so reconcile keeps its +/// container-only behaviour: a hive-priv that predates the op answers an +/// error, and that is logged at `warn` the first time and `debug` after. +pub async fn harness_state(name: &str) -> HarnessState { + let probe = crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Probe); + if let Ok(answer) = tokio::time::timeout(IS_ACTIVE_TIMEOUT, probe).await { + harness_of_answer(name, answer) + } else { + tracing::debug!(%name, "reading the agent's harness state timed out"); + HarnessState::Unknown + } +} + +/// The state a probe answer means; an error is [`HarnessState::Unknown`]. +fn harness_of_answer(name: &str, answer: Result) -> HarnessState { + use std::sync::atomic::{AtomicBool, Ordering}; + static WARNED: AtomicBool = AtomicBool::new(false); + match answer { + Ok(out) => parse_harness_state(&out), + Err(e) if !WARNED.swap(true, Ordering::Relaxed) => { + tracing::warn!(%name, error = ?e, "cannot read the agent's harness state; reconcile only watches the container (logged once)"); + HarnessState::Unknown + } + Err(e) => { + tracing::debug!(%name, error = ?e, "cannot read the agent's harness state"); + HarnessState::Unknown + } + } +} + +/// Queue a start of `name`'s harness unit inside its running container. +/// +/// # Errors +/// hive-priv is unreachable, refuses, or `systemctl` fails. +pub async fn start_harness(name: &str) -> Result<()> { + crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Start) + .await + .map(drop) +} + /// How long one `systemctl is-active` may run before the unit's state counts /// as unreadable. Periodic workers make this call per agent inline. const IS_ACTIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); diff --git a/hive-c0re/src/lifecycle/tests.rs b/hive-c0re/src/lifecycle/tests.rs index 9e5da826b..ef04cf407 100644 --- a/hive-c0re/src/lifecycle/tests.rs +++ b/hive-c0re/src/lifecycle/tests.rs @@ -445,3 +445,45 @@ fn stop_kind_survives_a_clean_stop_between_failures() { guards.remove("stopkind-c"); assert!(std::ptr::eq(first, failed(&mut guards))); } + +#[test] +fn harness_state_reads_load_and_active_state() { + let probe = |load: &str, active: &str| { + parse_harness_state(&format!("LoadState={load}\nActiveState={active}\n")) + }; + for active in ["inactive", "failed"] { + assert_eq!(probe("loaded", active), HarnessState::Stopped, "{active}"); + } + for active in ["active", "activating", "deactivating", "reloading"] { + assert_eq!(probe("loaded", active), HarnessState::Present, "{active}"); + } + // A unit that is not loaded, or output that is not a probe, is not + // evidence the harness is down. + assert_eq!(probe("not-found", "inactive"), HarnessState::Unknown); + assert_eq!(parse_harness_state(""), HarnessState::Unknown); + assert_eq!(parse_harness_state("garbage"), HarnessState::Unknown); +} + +/// A hive-priv that predates the op answers an error; that reads as unknown +/// state, which leaves reconcile with its container-only behaviour. +#[test] +fn an_unreadable_harness_is_unknown_never_stopped() { + let old_priv = anyhow::anyhow!("parse request: unknown variant `agent_harness`"); + assert_eq!( + harness_of_answer("alice", Err(old_priv)), + HarnessState::Unknown + ); + assert_eq!( + harness_of_answer("alice", Err(anyhow::anyhow!("again"))), + HarnessState::Unknown, + "later errors read the same" + ); + assert_eq!( + harness_of_answer( + "alice", + Ok("LoadState=loaded\nActiveState=inactive\n".to_owned()) + ), + HarnessState::Stopped, + "control: a real answer is read" + ); +} diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index bbcd5add7..a60e86bb9 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -8,8 +8,9 @@ use anyhow::{Context as _, Result, bail}; use hive_priv_sock::{ - BindMount, CredentialMount, HiveFacts, InfraAction, InfraContainer, MEASURE_AGENT_DISK_TIMEOUT, - NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream, + BindMount, CredentialMount, HarnessAction, HiveFacts, InfraAction, InfraContainer, + MEASURE_AGENT_DISK_TIMEOUT, NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, + PrivStream, }; use hive_types::NixBuilds; use std::path::Path; @@ -108,6 +109,20 @@ pub async fn start_failure_log(name: &str, since_unix: u64) -> Result { Ok(stdout) } +/// Read (`Probe`, the unit's `LoadState` and `ActiveState` as `key=value` +/// lines) or queue a start of (`Start`) the harness unit inside `name`'s +/// container. An older hive-priv answers an error (it does not know the op). +pub async fn agent_harness(name: &str, action: HarnessAction) -> Result { + let (stdout, _) = check( + call(&PrivRequest::AgentHarness { + agent_name: name.to_owned(), + action, + }) + .await?, + )?; + Ok(stdout) +} + pub async fn stop_container(name: &str) -> Result<()> { ok(call(&PrivRequest::StopContainer { name: name.to_owned(), diff --git a/hive-c0re/src/workers/reconcile_tick.rs b/hive-c0re/src/workers/reconcile_tick.rs index 819e8d4f9..6013534a4 100644 --- a/hive-c0re/src/workers/reconcile_tick.rs +++ b/hive-c0re/src/workers/reconcile_tick.rs @@ -1,11 +1,14 @@ //! Periodic declared ⇒ running convergence: an agent whose power intent is -//! `Up` and whose container is down gets a `Reconcile` queued, once a -//! [`TICK_INTERVAL`], without anyone having to ask. +//! `Up` and whose container is down, or running with its harness unit stopped, +//! gets a `Reconcile` queued, once a [`TICK_INTERVAL`], without anyone having +//! to ask. //! //! The tick queues the same `NodeKind::Reconcile` the boot sweep, a watch //! attach and a declaration write end in, so what it starts, and how, is //! `run_reconcile`'s decision. It only ever converges *toward a start*: an -//! agent wanted offline is left alone, and a running one is never touched. +//! agent wanted offline is left alone, and a running one is left alone unless +//! hive-priv reports its harness unit stopped or failed. A harness that cannot +//! be read (an older hive-priv, no answer) counts as running. //! `crash_watch` keeps raising its alert; this worker does not replace it. //! //! # What the tick leaves alone @@ -29,7 +32,7 @@ use std::sync::Arc; use std::time::Duration; use crate::coordinator::Coordinator; -use crate::lifecycle::{self, AGENT_PREFIX}; +use crate::lifecycle::{self, AGENT_PREFIX, HarnessState}; use crate::power::Wanted; /// How often declared intent is compared with the observed containers. The @@ -51,6 +54,9 @@ struct Observation { paused: bool, /// An update has the agent down and not yet back up. updating: bool, + /// The harness unit inside a running, wanted-up container; `Unknown` + /// where it was not read. + harness: HarnessState, } /// Why the tick did nothing for an agent. @@ -126,7 +132,8 @@ fn ticks_between(attempts: u32) -> u32 { /// The decision for one agent. Pure but for the backoff counters it advances. fn step(backoffs: &mut Backoffs, agent: &str, seen: Observation) -> Step { - if seen.running == Some(true) || seen.wanted != Some(Wanted::Up) { + let harness_down = seen.running == Some(true) && seen.harness == HarnessState::Stopped; + if (seen.running == Some(true) && !harness_down) || seen.wanted != Some(Wanted::Up) { backoffs.settle(agent); return Step::Leave(Leave::Converged); } @@ -170,11 +177,20 @@ async fn tick(coord: &Arc, backoffs: &mut Backoffs) { let Some(name) = container.strip_prefix(AGENT_PREFIX) else { continue; }; + let wanted = coord.power.get(name); + let running = lifecycle::running_state(name).await; + let updating = coord.is_updating(name); + let harness = if running == Some(true) && wanted == Some(Wanted::Up) && !updating { + lifecycle::harness_state(name).await + } else { + HarnessState::Unknown + }; let seen = Observation { - wanted: coord.power.get(name), - running: lifecycle::running_state(name).await, + wanted, + running, paused: hive_types::Ident::parse(name).is_ok_and(|id| Coordinator::is_paused(&id)), - updating: coord.is_updating(name), + updating, + harness, }; if step(backoffs, name, seen) != Step::Restart { continue; @@ -184,7 +200,11 @@ async fn tick(coord: &Arc, backoffs: &mut Backoffs) { let attempt = backoffs.record_attempt(name); tracing::info!( agent = %name, - observed = "stopped", + observed = if seen.running == Some(true) { + "harness stopped" + } else { + "stopped" + }, wanted = "up", attempt, "reconcile tick: restarting agent" @@ -214,6 +234,7 @@ mod tests { running: Some(false), paused: false, updating: false, + harness: HarnessState::Unknown, }; #[test] @@ -284,6 +305,56 @@ mod tests { } } + /// A running, wanted-up agent whose harness unit is stopped is restarted; + /// every other reading of a running agent leaves it alone. + #[test] + fn a_running_agent_with_a_stopped_harness_is_restarted() { + let running = |harness| Observation { + running: Some(true), + harness, + ..DOWN + }; + assert_eq!( + step( + &mut Backoffs::default(), + "a", + running(HarnessState::Stopped) + ), + Step::Restart + ); + for harness in [HarnessState::Present, HarnessState::Unknown] { + assert_eq!( + step(&mut Backoffs::default(), "a", running(harness)), + Step::Leave(Leave::Converged), + "{harness:?}" + ); + } + let wanted_offline = Observation { + wanted: Some(Wanted::Offline), + ..running(HarnessState::Stopped) + }; + assert_eq!( + step(&mut Backoffs::default(), "a", wanted_offline), + Step::Leave(Leave::Converged) + ); + let mid_update = Observation { + updating: true, + ..running(HarnessState::Stopped) + }; + assert_eq!( + step(&mut Backoffs::default(), "a", mid_update), + Step::Leave(Leave::Held) + ); + let mut backoffs = Backoffs::default(); + backoffs.record_attempt("a"); + backoffs.record_attempt("a"); + assert_eq!( + step(&mut backoffs, "a", running(HarnessState::Stopped)), + Step::Leave(Leave::BackingOff), + "a harness that stays down backs off like a container" + ); + } + /// Ticks between restarts of an agent that never comes up: 1, 2, 4, 8, /// then the 15-tick cap, and it stays there. #[test] diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index e35e5b90b..684467b3d 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -29,6 +29,18 @@ pub const PAUSED_MARKER_FILE: &str = "paused"; /// Sub-agent container prefix. System container name = `h-`. pub const AGENT_PREFIX: &str = "h-"; +/// What [`PrivRequest::AgentHarness`] does to the agent's harness unit +/// (`hive-agent.service` inside the agent's container). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HarnessAction { + /// Read the unit's `LoadState` and `ActiveState`, returned in `stdout` + /// as `key=value` lines. + Probe, + /// Queue a start of the unit without waiting for it to come up. + Start, +} + /// Lifecycle verb for [`PrivRequest::ControlInfraContainer`]. Maps directly /// to `systemctl `, where the unit comes from /// [`InfraContainer::service_unit`]. @@ -459,6 +471,16 @@ pub enum PrivRequest { paused: bool, }, + /// Probe or start the harness unit `hive-agent.service` inside `agent_name`'s + /// container, through `systemctl -M h-`. The unit name is + /// fixed in hive-priv; the request carries only the agent and the verb. + /// An older hive-priv answers an unknown-op error. + AgentHarness { + /// Logical agent name (validated by `validate_agent_name`). + agent_name: String, + action: HarnessAction, + }, + /// Start / stop a hive infrastructure service on the host via /// `systemctl `, the unit derived from the variant. The /// [`InfraContainer`] enum is the allowlist — serde rejects unknown / @@ -716,7 +738,37 @@ pub enum PrivEvent { mod tests { use hive_types::NixBuilds; - use super::{InfraContainer, PrivRequest, valid_closure_path, valid_meta_rev}; + use super::{HarnessAction, InfraContainer, PrivRequest, valid_closure_path, valid_meta_rev}; + + /// The harness op's wire form: the verb is lowercase and the request + /// carries no unit name; an unknown verb is refused at decode. + #[test] + fn the_harness_op_carries_an_agent_and_a_verb_only() { + for (verb, action) in [ + ("probe", HarnessAction::Probe), + ("start", HarnessAction::Start), + ] { + let wire = + serde_json::json!({"op": "agent_harness", "agent_name": "iris", "action": verb}); + let PrivRequest::AgentHarness { + agent_name, + action: got, + } = serde_json::from_value(wire.clone()).expect("decodes") + else { + panic!("variant"); + }; + assert_eq!((agent_name.as_str(), got), ("iris", action)); + let again = serde_json::to_value(PrivRequest::AgentHarness { + agent_name, + action: got, + }) + .expect("encodes"); + assert_eq!(again, wire); + } + let stop = + serde_json::json!({"op": "agent_harness", "agent_name": "iris", "action": "stop"}); + assert!(serde_json::from_value::(stop).is_err()); + } /// A request from a hive-c0re that predates the field decodes with the /// agent's namespace left to hive-priv, and the field round-trips. diff --git a/hive-priv/src/container.rs b/hive-priv/src/container.rs index 942704f3e..ce8cbb3fb 100644 --- a/hive-priv/src/container.rs +++ b/hive-priv/src/container.rs @@ -1,13 +1,14 @@ //! Container lifecycle ops: `StartContainer`, `StopContainer`, -//! `KillContainer`, `CreateContainer`, `UpdateContainer`, `DestroyContainer` -//! and `ListContainers`. Every op takes a logical agent name and reaches only -//! the `h-`-prefixed container [`container_system_name`] derives from it. +//! `KillContainer`, `CreateContainer`, `UpdateContainer`, `DestroyContainer`, +//! `ListContainers` and `AgentHarness`. Every op takes a logical agent name and +//! reaches only the `h-`-prefixed container [`container_system_name`] derives +//! from it. //! //! `CreateContainer` / `UpdateContainer` apply a closure already in the store //! and stream progress to the caller as [`PrivEvent::Line`]s when asked to. use anyhow::{Context as _, Result, bail}; -use hive_priv_sock::{PrivEvent, PrivStream, PrivStreamLine}; +use hive_priv_sock::{HarnessAction, PrivEvent, PrivStream, PrivStreamLine}; use tokio::io::{AsyncWriteExt, BufReader}; use tokio::net::unix::OwnedWriteHalf; use tokio::process::Command; @@ -182,6 +183,54 @@ async fn stop_container_with(host: &Host, name: &str) -> Result<(Str stop_and_release(host, &container_system_name(name)).await } +/// The unit `AgentHarness` acts on. Fixed here so no request names a unit. +const HARNESS_UNIT: &str = "hive-agent.service"; + +/// `AgentHarness`: read or start [`HARNESS_UNIT`] inside the agent's container. +pub(crate) async fn agent_harness(name: &str, action: HarnessAction) -> Result<(String, String)> { + agent_harness_with(&Host::REAL, name, action).await +} + +/// `systemctl -M h-` talks to the container's own systemd through +/// machined, so it reaches exactly the machine the `h-` prefix names. A start +/// is `--no-block`: it returns once the job is queued, and the next probe +/// shows whether the unit came up. Both verbs are bounded by +/// `Host::probe_timeout`. +async fn agent_harness_with( + host: &Host, + name: &str, + action: HarnessAction, +) -> Result<(String, String)> { + validate_agent_name(name)?; + let machine = container_system_name(name); + let verb: &[&str] = match action { + HarnessAction::Probe => &["show", "--property=LoadState", "--property=ActiveState"], + HarnessAction::Start => &["--no-block", "start"], + }; + let call = host + .command("systemctl") + .args(["-M", &machine]) + .args(verb) + .arg(HARNESS_UNIT) + .kill_on_drop(true) + .output(); + let out = tokio::time::timeout(host.probe_timeout, call) + .await + .context("systemctl timed out")? + .context("invoke systemctl")?; + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + if !out.status.success() { + bail!( + "systemctl -M {machine} {} {HARNESS_UNIT} failed ({}): {}", + verb.join(" "), + out.status, + stderr.trim() + ); + } + Ok((stdout, stderr)) +} + /// `KillContainer`: `nixos-container` has no kill verb, so this uses /// `machinectl` to send `SIGKILL` to every process in the container — the /// right semantics for a forced shutdown after a graceful stop has already @@ -1054,6 +1103,59 @@ mod host_tests { ); } + // ---- agent harness ---- + + #[tokio::test] + async fn a_probe_reads_the_fixed_unit_inside_the_prefixed_machine() { + let stub = Stub::new("echo LoadState=loaded; echo ActiveState=inactive"); + let (stdout, _) = agent_harness_with(&stub.host(), "bob", HarnessAction::Probe) + .await + .unwrap(); + assert_eq!(stdout, "LoadState=loaded\nActiveState=inactive\n"); + assert_eq!( + stub.calls(), + [ + "systemctl -M h-bob show --property=LoadState --property=ActiveState hive-agent.service" + ] + ); + } + + #[tokio::test] + async fn a_start_queues_the_fixed_unit_without_waiting() { + let stub = Stub::new("exit 0"); + agent_harness_with(&stub.host(), "bob", HarnessAction::Start) + .await + .unwrap(); + assert_eq!( + stub.calls(), + ["systemctl -M h-bob --no-block start hive-agent.service"] + ); + } + + #[tokio::test] + async fn a_harness_op_refuses_a_bad_name_before_running_anything() { + for bad in ["", "Bob", "bob.x", "bob/../x", "bob -M h-other"] { + let stub = Stub::new("exit 0"); + let msg = err_text(agent_harness_with(&stub.host(), bad, HarnessAction::Start).await); + assert!(msg.contains("invalid name"), "{bad:?}: {msg}"); + assert!(stub.calls().is_empty(), "{bad:?} ran {:?}", stub.calls()); + } + } + + #[tokio::test] + async fn a_failing_or_hung_systemctl_is_an_error() { + let stub = Stub::new("echo no bus >&2; exit 1"); + let msg = err_text(agent_harness_with(&stub.host(), "bob", HarnessAction::Probe).await); + assert!(msg.contains("systemctl -M h-bob show"), "{msg}"); + assert!(msg.contains("no bus"), "{msg}"); + + let stub = Stub::new("sleep 5"); + let mut host = stub.host(); + host.probe_timeout = Duration::from_millis(100); + let msg = err_text(agent_harness_with(&host, "bob", HarnessAction::Probe).await); + assert!(msg.contains("timed out"), "{msg}"); + } + // ---- kill / destroy ---- #[tokio::test] diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index fa878dc68..4eca3d278 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -216,6 +216,11 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String, PrivRequest::KillContainer { ref name } => container::kill_container(name).await, + PrivRequest::AgentHarness { + ref agent_name, + action, + } => container::agent_harness(agent_name, action).await, + PrivRequest::UpdateContainer { ref name, stream,