reconcile: start a stopped harness inside a running container
hive-priv gains AgentHarness { agent_name, action: probe | start }, which
runs systemctl -M h-<name> against the fixed unit hive-agent.service.
run_reconcile and the 1-min tick read the unit for a running wanted-up
agent and start it when it is inactive or failed. A hive-priv without the
op reads as unknown state and leaves reconcile container-only.
This commit is contained in:
parent
ca734f8ea0
commit
36515c289d
2 changed files with 517 additions and 32 deletions
|
|
@ -1,13 +1,14 @@
|
|||
//! Container lifecycle ops: `StartContainer`, `StopContainer`,
|
||||
//! `KillContainer`, `CreateContainer`, `UpdateContainer`, `DestroyContainer`
|
||||
//! and `ListContainers`. Every op takes a logical agent name and reaches only
|
||||
//! the `h-`-prefixed container [`container_system_name`] derives from it.
|
||||
//! `KillContainer`, `CreateContainer`, `UpdateContainer`, `DestroyContainer`,
|
||||
//! `ListContainers` and `AgentHarness`. Every op takes a logical agent name and
|
||||
//! reaches only the `h-`-prefixed container [`container_system_name`] derives
|
||||
//! from it.
|
||||
//!
|
||||
//! `CreateContainer` / `UpdateContainer` apply a closure already in the store
|
||||
//! and stream progress to the caller as [`PrivEvent::Line`]s when asked to.
|
||||
|
||||
use anyhow::{Context as _, Result, bail};
|
||||
use hive_priv_sock::{PrivEvent, PrivStream, PrivStreamLine};
|
||||
use hive_priv_sock::{HarnessAction, PrivEvent, PrivStream, PrivStreamLine};
|
||||
use tokio::io::{AsyncWriteExt, BufReader};
|
||||
use tokio::net::unix::OwnedWriteHalf;
|
||||
use tokio::process::Command;
|
||||
|
|
@ -182,6 +183,54 @@ async fn stop_container_with(host: &Host<impl Spawn>, name: &str) -> Result<(Str
|
|||
stop_and_release(host, &container_system_name(name)).await
|
||||
}
|
||||
|
||||
/// The unit `AgentHarness` acts on. Fixed here so no request names a unit.
|
||||
const HARNESS_UNIT: &str = "hive-agent.service";
|
||||
|
||||
/// `AgentHarness`: read or start [`HARNESS_UNIT`] inside the agent's container.
|
||||
pub(crate) async fn agent_harness(name: &str, action: HarnessAction) -> Result<(String, String)> {
|
||||
agent_harness_with(&Host::REAL, name, action).await
|
||||
}
|
||||
|
||||
/// `systemctl -M h-<name>` talks to the container's own systemd through
|
||||
/// machined, so it reaches exactly the machine the `h-` prefix names. A start
|
||||
/// is `--no-block`: it returns once the job is queued, and the next probe
|
||||
/// shows whether the unit came up. Both verbs are bounded by
|
||||
/// `Host::probe_timeout`.
|
||||
async fn agent_harness_with(
|
||||
host: &Host<impl Spawn>,
|
||||
name: &str,
|
||||
action: HarnessAction,
|
||||
) -> Result<(String, String)> {
|
||||
validate_agent_name(name)?;
|
||||
let machine = container_system_name(name);
|
||||
let verb: &[&str] = match action {
|
||||
HarnessAction::Probe => &["show", "--property=LoadState", "--property=ActiveState"],
|
||||
HarnessAction::Start => &["--no-block", "start"],
|
||||
};
|
||||
let call = host
|
||||
.command("systemctl")
|
||||
.args(["-M", &machine])
|
||||
.args(verb)
|
||||
.arg(HARNESS_UNIT)
|
||||
.kill_on_drop(true)
|
||||
.output();
|
||||
let out = tokio::time::timeout(host.probe_timeout, call)
|
||||
.await
|
||||
.context("systemctl timed out")?
|
||||
.context("invoke systemctl")?;
|
||||
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
|
||||
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
||||
if !out.status.success() {
|
||||
bail!(
|
||||
"systemctl -M {machine} {} {HARNESS_UNIT} failed ({}): {}",
|
||||
verb.join(" "),
|
||||
out.status,
|
||||
stderr.trim()
|
||||
);
|
||||
}
|
||||
Ok((stdout, stderr))
|
||||
}
|
||||
|
||||
/// `KillContainer`: `nixos-container` has no kill verb, so this uses
|
||||
/// `machinectl` to send `SIGKILL` to every process in the container — the
|
||||
/// right semantics for a forced shutdown after a graceful stop has already
|
||||
|
|
@ -1054,6 +1103,59 @@ mod host_tests {
|
|||
);
|
||||
}
|
||||
|
||||
// ---- agent harness ----
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_probe_reads_the_fixed_unit_inside_the_prefixed_machine() {
|
||||
let stub = Stub::new("echo LoadState=loaded; echo ActiveState=inactive");
|
||||
let (stdout, _) = agent_harness_with(&stub.host(), "bob", HarnessAction::Probe)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stdout, "LoadState=loaded\nActiveState=inactive\n");
|
||||
assert_eq!(
|
||||
stub.calls(),
|
||||
[
|
||||
"systemctl -M h-bob show --property=LoadState --property=ActiveState hive-agent.service"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_start_queues_the_fixed_unit_without_waiting() {
|
||||
let stub = Stub::new("exit 0");
|
||||
agent_harness_with(&stub.host(), "bob", HarnessAction::Start)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
stub.calls(),
|
||||
["systemctl -M h-bob --no-block start hive-agent.service"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_harness_op_refuses_a_bad_name_before_running_anything() {
|
||||
for bad in ["", "Bob", "bob.x", "bob/../x", "bob -M h-other"] {
|
||||
let stub = Stub::new("exit 0");
|
||||
let msg = err_text(agent_harness_with(&stub.host(), bad, HarnessAction::Start).await);
|
||||
assert!(msg.contains("invalid name"), "{bad:?}: {msg}");
|
||||
assert!(stub.calls().is_empty(), "{bad:?} ran {:?}", stub.calls());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failing_or_hung_systemctl_is_an_error() {
|
||||
let stub = Stub::new("echo no bus >&2; exit 1");
|
||||
let msg = err_text(agent_harness_with(&stub.host(), "bob", HarnessAction::Probe).await);
|
||||
assert!(msg.contains("systemctl -M h-bob show"), "{msg}");
|
||||
assert!(msg.contains("no bus"), "{msg}");
|
||||
|
||||
let stub = Stub::new("sleep 5");
|
||||
let mut host = stub.host();
|
||||
host.probe_timeout = Duration::from_millis(100);
|
||||
let msg = err_text(agent_harness_with(&host, "bob", HarnessAction::Probe).await);
|
||||
assert!(msg.contains("timed out"), "{msg}");
|
||||
}
|
||||
|
||||
// ---- kill / destroy ----
|
||||
|
||||
#[tokio::test]
|
||||
|
|
|
|||
|
|
@ -216,6 +216,11 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String,
|
|||
|
||||
PrivRequest::KillContainer { ref name } => container::kill_container(name).await,
|
||||
|
||||
PrivRequest::AgentHarness {
|
||||
ref agent_name,
|
||||
action,
|
||||
} => container::agent_harness(agent_name, action).await,
|
||||
|
||||
PrivRequest::UpdateContainer {
|
||||
ref name,
|
||||
stream,
|
||||
|
|
|
|||
Loading…
Reference in a new issue