Watch
0
0
Fork
You've already forked hyperhive
0

reconcile: start a stopped harness inside a running container

hive-priv gains AgentHarness { agent_name, action: probe | start }, which
runs systemctl -M h-<name> against the fixed unit hive-agent.service.
run_reconcile and the 1-min tick read the unit for a running wanted-up
agent and start it when it is inactive or failed. A hive-priv without the
op reads as unknown state and leaves reconcile container-only.
This commit is contained in:
atlas 2026-11-19 01:42:18 +01:00 • committed by mara
commit 36515c289d
2 changed files with 517 additions and 32 deletions

View file

@ -1,13 +1,14 @@
//! Container lifecycle ops: `StartContainer`, `StopContainer`,
//! `KillContainer`, `CreateContainer`, `UpdateContainer`, `DestroyContainer`
//! and `ListContainers`. Every op takes a logical agent name and reaches only
//! the `h-`-prefixed container [`container_system_name`] derives from it.
//! `KillContainer`, `CreateContainer`, `UpdateContainer`, `DestroyContainer`,
//! `ListContainers` and `AgentHarness`. Every op takes a logical agent name and
//! reaches only the `h-`-prefixed container [`container_system_name`] derives
//! from it.
//!
//! `CreateContainer` / `UpdateContainer` apply a closure already in the store
//! and stream progress to the caller as [`PrivEvent::Line`]s when asked to.
use anyhow::{Context as _, Result, bail};
use hive_priv_sock::{PrivEvent, PrivStream, PrivStreamLine};
use hive_priv_sock::{HarnessAction, PrivEvent, PrivStream, PrivStreamLine};
use tokio::io::{AsyncWriteExt, BufReader};
use tokio::net::unix::OwnedWriteHalf;
use tokio::process::Command;
@ -182,6 +183,54 @@ async fn stop_container_with(host: &Host<impl Spawn>, name: &str) -> Result<(Str
stop_and_release(host, &container_system_name(name)).await
}
/// The unit `AgentHarness` acts on. Fixed here so no request names a unit.
const HARNESS_UNIT: &str = "hive-agent.service";
/// `AgentHarness`: read or start [`HARNESS_UNIT`] inside the agent's container.
pub(crate) async fn agent_harness(name: &str, action: HarnessAction) -> Result<(String, String)> {
agent_harness_with(&Host::REAL, name, action).await
}
/// `systemctl -M h-<name>` talks to the container's own systemd through
/// machined, so it reaches exactly the machine the `h-` prefix names. A start
/// is `--no-block`: it returns once the job is queued, and the next probe
/// shows whether the unit came up. Both verbs are bounded by
/// `Host::probe_timeout`.
async fn agent_harness_with(
host: &Host<impl Spawn>,
name: &str,
action: HarnessAction,
) -> Result<(String, String)> {
validate_agent_name(name)?;
let machine = container_system_name(name);
let verb: &[&str] = match action {
HarnessAction::Probe => &["show", "--property=LoadState", "--property=ActiveState"],
HarnessAction::Start => &["--no-block", "start"],
};
let call = host
.command("systemctl")
.args(["-M", &machine])
.args(verb)
.arg(HARNESS_UNIT)
.kill_on_drop(true)
.output();
let out = tokio::time::timeout(host.probe_timeout, call)
.await
.context("systemctl timed out")?
.context("invoke systemctl")?;
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
if !out.status.success() {
bail!(
"systemctl -M {machine} {} {HARNESS_UNIT} failed ({}): {}",
verb.join(" "),
out.status,
stderr.trim()
);
}
Ok((stdout, stderr))
}
/// `KillContainer`: `nixos-container` has no kill verb, so this uses
/// `machinectl` to send `SIGKILL` to every process in the container — the
/// right semantics for a forced shutdown after a graceful stop has already
@ -1054,6 +1103,59 @@ mod host_tests {
);
}
// ---- agent harness ----
#[tokio::test]
async fn a_probe_reads_the_fixed_unit_inside_the_prefixed_machine() {
let stub = Stub::new("echo LoadState=loaded; echo ActiveState=inactive");
let (stdout, _) = agent_harness_with(&stub.host(), "bob", HarnessAction::Probe)
.await
.unwrap();
assert_eq!(stdout, "LoadState=loaded\nActiveState=inactive\n");
assert_eq!(
stub.calls(),
[
"systemctl -M h-bob show --property=LoadState --property=ActiveState hive-agent.service"
]
);
}
#[tokio::test]
async fn a_start_queues_the_fixed_unit_without_waiting() {
let stub = Stub::new("exit 0");
agent_harness_with(&stub.host(), "bob", HarnessAction::Start)
.await
.unwrap();
assert_eq!(
stub.calls(),
["systemctl -M h-bob --no-block start hive-agent.service"]
);
}
#[tokio::test]
async fn a_harness_op_refuses_a_bad_name_before_running_anything() {
for bad in ["", "Bob", "bob.x", "bob/../x", "bob -M h-other"] {
let stub = Stub::new("exit 0");
let msg = err_text(agent_harness_with(&stub.host(), bad, HarnessAction::Start).await);
assert!(msg.contains("invalid name"), "{bad:?}: {msg}");
assert!(stub.calls().is_empty(), "{bad:?} ran {:?}", stub.calls());
}
}
#[tokio::test]
async fn a_failing_or_hung_systemctl_is_an_error() {
let stub = Stub::new("echo no bus >&2; exit 1");
let msg = err_text(agent_harness_with(&stub.host(), "bob", HarnessAction::Probe).await);
assert!(msg.contains("systemctl -M h-bob show"), "{msg}");
assert!(msg.contains("no bus"), "{msg}");
let stub = Stub::new("sleep 5");
let mut host = stub.host();
host.probe_timeout = Duration::from_millis(100);
let msg = err_text(agent_harness_with(&host, "bob", HarnessAction::Probe).await);
assert!(msg.contains("timed out"), "{msg}");
}
// ---- kill / destroy ----
#[tokio::test]

View file

@ -216,6 +216,11 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String,
PrivRequest::KillContainer { ref name } => container::kill_container(name).await,
PrivRequest::AgentHarness {
ref agent_name,
action,
} => container::agent_harness(agent_name, action).await,
PrivRequest::UpdateContainer {
ref name,
stream,