6b800600e2
hive-priv applies only the path its last RealiseClosure returned for that agent. On drift the swap and create record the pins of the commit the closure was built from.
637 lines
26 KiB
Rust
637 lines
26 KiB
Rust
//! Container lifecycle ops: `StartContainer`, `StopContainer`,
|
|
//! `KillContainer`, `CreateContainer`, `UpdateContainer`, `DestroyContainer`
|
|
//! and `ListContainers`. Every op takes a logical agent name and reaches only
|
|
//! the `h-`-prefixed container [`container_system_name`] derives from it.
|
|
//!
|
|
//! `CreateContainer` / `UpdateContainer` apply a closure already in the store
|
|
//! and stream progress to the caller as [`PrivEvent::Line`]s when asked to.
|
|
|
|
use anyhow::{Context as _, Result, bail};
|
|
use hive_priv_sock::{PrivEvent, PrivStream, PrivStreamLine};
|
|
use tokio::io::{AsyncWriteExt, BufReader};
|
|
use tokio::net::unix::OwnedWriteHalf;
|
|
use tokio::process::Command;
|
|
|
|
use crate::shared::{container_system_name, validate_agent_name};
|
|
|
|
/// Write one `PrivEvent::Line` to the client. Best-effort: a write
|
|
/// failure is logged but doesn't abort the running subprocess.
|
|
async fn write_line_event(writer: &mut OwnedWriteHalf, stream: PrivStream, data: &str) {
|
|
let event = PrivEvent::Line(PrivStreamLine {
|
|
stream,
|
|
data: data.to_owned(),
|
|
});
|
|
if let Ok(mut json) = serde_json::to_string(&event) {
|
|
json.push('\n');
|
|
if let Err(e) = writer.write_all(json.as_bytes()).await {
|
|
tracing::warn!(error = %e, "write_line_event: write failed");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// `StartContainer`: clear any start-limit lockout left by earlier failures
|
|
/// so a now-correct start isn't blocked (`nixos-container start` does not do
|
|
/// this itself). Best-effort — if the unit doesn't exist yet (first-time
|
|
/// create), `reset-failed` is a no-op and the start proceeds regardless.
|
|
pub(crate) async fn start_container(name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(name)?;
|
|
let machine = container_system_name(name);
|
|
// Silenced: a first-time create means the unit doesn't exist yet, and
|
|
// `systemctl`'s own "Unit ... not loaded" complaint for that case would
|
|
// otherwise land in the journal at error priority on every such start —
|
|
// routine, not a signal, since the result is already discarded below.
|
|
let _ = Command::new("systemctl")
|
|
.args(["reset-failed", &format!("container@{machine}.service")])
|
|
.stdout(std::process::Stdio::null())
|
|
.stderr(std::process::Stdio::null())
|
|
.status()
|
|
.await;
|
|
container_run(&["start", &machine]).await
|
|
}
|
|
|
|
/// `StopContainer`.
|
|
pub(crate) async fn stop_container(name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(name)?;
|
|
stop_and_release(&container_system_name(name)).await
|
|
}
|
|
|
|
/// `KillContainer`: `nixos-container` has no kill verb, so this uses
|
|
/// `machinectl` to send `SIGKILL` to every process in the container — the
|
|
/// right semantics for a forced shutdown after a graceful stop has already
|
|
/// been attempted.
|
|
pub(crate) async fn kill_container(name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(name)?;
|
|
let machine = container_system_name(name);
|
|
machinectl_run(&["kill", &machine, "--signal=SIGKILL"]).await
|
|
}
|
|
|
|
/// `DestroyContainer`.
|
|
pub(crate) async fn destroy_container(name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(name)?;
|
|
container_run(&["destroy", &container_system_name(name)]).await
|
|
}
|
|
|
|
/// Shared body for `CreateContainer` / `UpdateContainer`: validate the name
|
|
/// and that `toplevel` is the closure realised for the agent, and apply it (streaming line events to `writer` when
|
|
/// `stream` is set). Nothing is built here: `toplevel` is a closure already in
|
|
/// the store, which `RealiseClosure` put there.
|
|
///
|
|
/// `create` passes `--system-path` so `nixos-container`'s own `buildFlake()`
|
|
/// never runs; it builds to a relative `.tmp` out-link that concurrent calls
|
|
/// share, which handed an agent another agent's closure.
|
|
///
|
|
/// `update` does not call `nixos-container update` at all: that action's
|
|
/// version-compat probe runs before `--system-path` is honored and dies on every
|
|
/// agent. [`swap_container_profile`] does what the action does past that probe:
|
|
/// `nix-env --set` the per-container profile, then `systemctl reload` if the
|
|
/// container is running.
|
|
pub(crate) async fn container_toplevel_action(
|
|
verb: &str,
|
|
name: &str,
|
|
toplevel: &str,
|
|
stream: bool,
|
|
writer: &mut OwnedWriteHalf,
|
|
) -> Result<(String, String)> {
|
|
validate_agent_name(name)?;
|
|
let toplevel = crate::closure::applicable_for(name, toplevel)?;
|
|
let system_name = container_system_name(name);
|
|
if verb == "update" {
|
|
return swap_container_profile(&system_name, toplevel, stream, writer).await;
|
|
}
|
|
let args = [verb, &system_name, "--system-path", toplevel];
|
|
if stream {
|
|
container_run_streaming(&args, writer).await
|
|
} else {
|
|
container_run(&args).await
|
|
}
|
|
}
|
|
|
|
/// Apply a prebuilt toplevel to an existing container directly, without
|
|
/// going through `nixos-container update` (see `container_toplevel_action`'s
|
|
/// doc comment for why). Mirrors `nixos-container.pl`'s own `update`
|
|
/// action verbatim, past its version probe: point the per-container `nix-
|
|
/// env` profile at the new toplevel, then reload the container unit *if*
|
|
/// it's currently running — the container's own next start already reads
|
|
/// from the profile, so a stopped container needs nothing further.
|
|
///
|
|
/// Both steps are near-instant in the happy case, but still stream-
|
|
/// forwarded like the rest of this operation: a failure here (e.g. a
|
|
/// wedged `systemctl reload`) is exactly when the caller most wants the
|
|
/// live line in the build log, not just a summary error afterward.
|
|
async fn swap_container_profile(
|
|
system_name: &str,
|
|
toplevel: &str,
|
|
stream: bool,
|
|
writer: &mut OwnedWriteHalf,
|
|
) -> Result<(String, String)> {
|
|
let profile = format!("/nix/var/nix/profiles/per-container/{system_name}/system");
|
|
let set_out = Command::new("nix-env")
|
|
.args(["-p", &profile, "--set", toplevel])
|
|
.output()
|
|
.await
|
|
.context("invoke nix-env --set")?;
|
|
let mut stdout = String::from_utf8_lossy(&set_out.stdout).into_owned();
|
|
let mut stderr = String::from_utf8_lossy(&set_out.stderr).into_owned();
|
|
log_and_forward(&stdout, &stderr, stream, writer).await;
|
|
if !set_out.status.success() {
|
|
bail!(
|
|
"nix-env -p {profile} --set failed ({}): {}",
|
|
set_out.status,
|
|
stderr.trim()
|
|
);
|
|
}
|
|
|
|
let unit = format!("container@{system_name}");
|
|
let state_out = Command::new("systemctl")
|
|
.args(["show", "--property=ActiveState", "--value", &unit])
|
|
.output()
|
|
.await
|
|
.context("query container ActiveState")?;
|
|
let active = String::from_utf8_lossy(&state_out.stdout).trim() == "active";
|
|
if active {
|
|
let reload_out = Command::new("systemctl")
|
|
.args(["reload", &unit])
|
|
.output()
|
|
.await
|
|
.context("invoke systemctl reload")?;
|
|
let r_stdout = String::from_utf8_lossy(&reload_out.stdout).into_owned();
|
|
let r_stderr = String::from_utf8_lossy(&reload_out.stderr).into_owned();
|
|
log_and_forward(&r_stdout, &r_stderr, stream, writer).await;
|
|
if !reload_out.status.success() {
|
|
bail!(
|
|
"systemctl reload {unit} failed ({}): {}",
|
|
reload_out.status,
|
|
r_stderr.trim()
|
|
);
|
|
}
|
|
stdout.push_str(&r_stdout);
|
|
stderr.push_str(&r_stderr);
|
|
}
|
|
Ok((stdout, stderr))
|
|
}
|
|
|
|
/// Log a command's captured stdout/stderr the same way every other
|
|
/// `nixos-container`-adjacent shellout in this file does, and — when
|
|
/// `stream` is set — forward each line to the client as a live
|
|
/// [`PrivEvent::Line`] too, so a caller watching the build log sees these
|
|
/// lines exactly like any other step's, not just a summary on failure.
|
|
async fn log_and_forward(stdout: &str, stderr: &str, stream: bool, writer: &mut OwnedWriteHalf) {
|
|
for line in stdout.lines() {
|
|
tracing::info!(target: "nixos-container", "{line}");
|
|
if stream {
|
|
write_line_event(writer, PrivStream::Stdout, line).await;
|
|
}
|
|
}
|
|
for line in stderr.lines() {
|
|
tracing::warn!(target: "nixos-container", "{line}");
|
|
if stream {
|
|
write_line_event(writer, PrivStream::Stderr, line).await;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// How a [`run_bounded`] child ended.
|
|
pub(crate) enum BoundedRun {
|
|
Exited {
|
|
status: std::process::ExitStatus,
|
|
stderr: String,
|
|
},
|
|
/// Still running at the limit; its whole process group was killed.
|
|
TimedOut,
|
|
}
|
|
|
|
/// Run `cmd` to completion or until `limit`, draining both pipes: stdout is
|
|
/// discarded, stderr is kept and also logged as it arrives.
|
|
///
|
|
/// The child leads its own process group and a timeout `SIGKILL`s the whole
|
|
/// group, so a grandchild still holding the pipes (or the daemon connection)
|
|
/// dies with it instead of outliving the error.
|
|
pub(crate) async fn run_bounded(
|
|
mut cmd: Command,
|
|
limit: std::time::Duration,
|
|
) -> std::io::Result<BoundedRun> {
|
|
use tokio::io::AsyncBufReadExt as _;
|
|
|
|
let mut child = cmd
|
|
.stdout(std::process::Stdio::piped())
|
|
.stderr(std::process::Stdio::piped())
|
|
.process_group(0)
|
|
.spawn()?;
|
|
// Group leader, so its pid is the pgid.
|
|
let pgid = child.id().and_then(|pid| i32::try_from(pid).ok());
|
|
|
|
let stdout = child.stdout.take().expect("stdout piped");
|
|
let stderr = child.stderr.take().expect("stderr piped");
|
|
let run = async {
|
|
let mut stdout_lines = BufReader::new(stdout).lines();
|
|
let mut stderr_lines = BufReader::new(stderr).lines();
|
|
let mut stderr_buf = String::new();
|
|
|
|
// ⚠️ Both pipes are drained concurrently: reading one alone would let
|
|
// the other fill its pipe buffer and deadlock the child.
|
|
//
|
|
// ⚠️ Each stream's EOF is tracked separately rather than breaking on
|
|
// the first `None`: `next_line()` on a closed stream returns
|
|
// `Ok(None)` immediately and forever, so a loop that keeps polling a
|
|
// finished stream spins hot until the other one ends too.
|
|
let mut stdout_done = false;
|
|
let mut stderr_done = false;
|
|
while !(stdout_done && stderr_done) {
|
|
tokio::select! {
|
|
line = stdout_lines.next_line(), if !stdout_done => {
|
|
match line {
|
|
Ok(Some(_)) => {}
|
|
Ok(None) => stdout_done = true,
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "child stdout read error");
|
|
stdout_done = true;
|
|
}
|
|
}
|
|
}
|
|
line = stderr_lines.next_line(), if !stderr_done => {
|
|
match line {
|
|
Ok(Some(l)) => {
|
|
tracing::info!(target: "nix-store", "{l}");
|
|
if !stderr_buf.is_empty() {
|
|
stderr_buf.push('\n');
|
|
}
|
|
stderr_buf.push_str(&l);
|
|
}
|
|
Ok(None) => stderr_done = true,
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "child stderr read error");
|
|
stderr_done = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
let status = child.wait().await?;
|
|
Ok::<_, std::io::Error>(BoundedRun::Exited {
|
|
status,
|
|
stderr: stderr_buf,
|
|
})
|
|
};
|
|
if let Ok(exited) = tokio::time::timeout(limit, run).await {
|
|
return exited;
|
|
}
|
|
if let Some(pgid) = pgid {
|
|
// SAFETY: plain `kill(2)`; `-pgid` targets the child's own process
|
|
// group, which `process_group(0)` made it lead.
|
|
unsafe {
|
|
libc::kill(-pgid, libc::SIGKILL);
|
|
}
|
|
}
|
|
child.wait().await?;
|
|
Ok(BoundedRun::TimedOut)
|
|
}
|
|
|
|
/// Invoke `nixos-container` with the given args, log output to journald.
|
|
pub(crate) async fn container_run(args: &[&str]) -> Result<(String, String)> {
|
|
let out = Command::new("nixos-container")
|
|
.args(args)
|
|
.output()
|
|
.await
|
|
.context("invoke nixos-container")?;
|
|
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
|
|
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
|
// `list` is a read-only enumeration called on the hot path (dashboard
|
|
// rescan, forge + boot sweeps) — its stdout is the return value, not
|
|
// progress, so logging every container name on every call floods the
|
|
// journal. Log stdout only for the mutating ops, where each line is
|
|
// genuine progress. stderr is always logged (errors matter regardless).
|
|
if args.first() != Some(&"list") {
|
|
for line in stdout.lines() {
|
|
tracing::info!(target: "nixos-container", "{line}");
|
|
}
|
|
}
|
|
for line in stderr.lines() {
|
|
tracing::warn!(target: "nixos-container", "{line}");
|
|
}
|
|
if !out.status.success() {
|
|
bail!(
|
|
"nixos-container {} failed ({}): {}",
|
|
args.join(" "),
|
|
out.status,
|
|
stderr.trim()
|
|
);
|
|
}
|
|
Ok((stdout, stderr))
|
|
}
|
|
|
|
/// Invoke `machinectl` with the given args, log output to journald.
|
|
/// Used for operations that nixos-container doesn't expose (e.g. sending
|
|
/// signals to running containers).
|
|
async fn machinectl_run(args: &[&str]) -> Result<(String, String)> {
|
|
let out = Command::new("machinectl")
|
|
.args(args)
|
|
.output()
|
|
.await
|
|
.context("invoke machinectl")?;
|
|
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
|
|
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
|
for line in stdout.lines() {
|
|
tracing::info!(target: "machinectl", "{line}");
|
|
}
|
|
for line in stderr.lines() {
|
|
tracing::warn!(target: "machinectl", "{line}");
|
|
}
|
|
if !out.status.success() {
|
|
bail!(
|
|
"machinectl {} failed ({}): {}",
|
|
args.join(" "),
|
|
out.status,
|
|
stderr.trim()
|
|
);
|
|
}
|
|
Ok((stdout, stderr))
|
|
}
|
|
|
|
/// How long to wait for machined to drop a machine's registration after a
|
|
/// shutdown has been asked for. Generous: a container with slow-stopping
|
|
/// units legitimately takes a while, and escalating early would SIGKILL a
|
|
/// shutdown that was going to finish on its own.
|
|
const NAME_RELEASE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
|
|
|
|
/// Poll cadence while waiting for the registration to go away.
|
|
const NAME_RELEASE_POLL: std::time::Duration = std::time::Duration::from_millis(500);
|
|
|
|
/// Cap on a single registration probe. The probe is a D-Bus round trip to
|
|
/// machined; if machined itself is wedged the call would otherwise sit on
|
|
/// the D-Bus method timeout, which is far longer than the whole stop
|
|
/// sequence should take.
|
|
const MACHINE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
|
|
|
|
/// True when machined still holds a registration for `machine`.
|
|
///
|
|
/// This asks machined the same question the registration itself answers —
|
|
/// `machinectl show` resolves the name through `GetMachine`, the very lookup
|
|
/// that makes a later boot fail with `Failed to register machine: already
|
|
/// exists`. So a positive answer here is exactly the condition that breaks
|
|
/// the next start, not a proxy for it.
|
|
///
|
|
/// Deliberately NOT the container's systemd unit state: the unit can be
|
|
/// `inactive` while the registration is still held, and that gap is the
|
|
/// whole bug this probe exists to catch.
|
|
///
|
|
/// Errors and timeouts answer "still registered". Being wrong that way costs
|
|
/// a redundant SIGKILL to something already gone; being wrong the other way
|
|
/// hands back a stop that silently leaked the name.
|
|
async fn machine_registered(machine: &str) -> bool {
|
|
let probe = Command::new("machinectl")
|
|
.args(["show", machine, "--property=Name"])
|
|
// Don't leave a probe behind when the timeout below fires.
|
|
.kill_on_drop(true)
|
|
.output();
|
|
match tokio::time::timeout(MACHINE_PROBE_TIMEOUT, probe).await {
|
|
Ok(Ok(out)) => out.status.success(),
|
|
Ok(Err(e)) => {
|
|
tracing::warn!(%machine, error = %e, "machinectl show failed to run; assuming still registered");
|
|
true
|
|
}
|
|
Err(_) => {
|
|
tracing::warn!(%machine, "machinectl show timed out; assuming still registered");
|
|
true
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Poll [`machine_registered`] until the name is free or the timeout expires.
|
|
/// Returns true once the registration is gone.
|
|
async fn wait_name_released(machine: &str) -> bool {
|
|
let deadline = tokio::time::Instant::now() + NAME_RELEASE_TIMEOUT;
|
|
loop {
|
|
if !machine_registered(machine).await {
|
|
return true;
|
|
}
|
|
if tokio::time::Instant::now() >= deadline {
|
|
return false;
|
|
}
|
|
tokio::time::sleep(NAME_RELEASE_POLL).await;
|
|
}
|
|
}
|
|
|
|
/// Stop `machine` and only report success once machined has actually released
|
|
/// the name.
|
|
///
|
|
/// `nixos-container stop` exiting 0 does not mean the machine is gone. A
|
|
/// process that sits in the machine's cgroup without being a child of the
|
|
/// container's init — a shell exec'd in from outside, say — never receives
|
|
/// the shutdown's SIGTERM if it has been stopped with SIGSTOP, so the
|
|
/// registration outlives the "successful" stop. Every later start of that
|
|
/// container then fails with `Failed to register machine: already exists`,
|
|
/// and machined re-persists the stale record across its own restart, so
|
|
/// there is no cleaning it up after the fact. The only place to catch it is
|
|
/// here, in the stop.
|
|
///
|
|
/// So: ask for the stop, wait for the name, and fail loudly if it is still
|
|
/// held — a caller that is told the stop worked will go on to start the
|
|
/// container and hit the confusing registration error instead of this one.
|
|
///
|
|
/// This used to escalate to `machinectl kill --signal=SIGKILL` here. Dropped
|
|
/// per real incident data: the case this guards is a container genuinely
|
|
/// wedged (e.g. a root-login process survived the stop), and SIGKILL
|
|
/// doesn't recover that in practice — only a host-level reboot has.
|
|
/// Pretending a kill attempt handled it hides a condition that needs a human
|
|
/// to look at the host, so this now just reports the failure instead of
|
|
/// quietly (and ineffectually) trying to force it.
|
|
///
|
|
/// The verify-then-fail lives in the helper rather than at a call site so
|
|
/// that every stop gets it: dashboard, reconcile, destroy, cold-start
|
|
/// fallback. The start path already distrusts its own exit code the same way;
|
|
/// this is the missing half of that pair.
|
|
async fn stop_and_release(machine: &str) -> Result<(String, String)> {
|
|
let stop = container_run(&["stop", machine]).await;
|
|
if wait_name_released(machine).await {
|
|
// Happy path, and also the path where a stop that reported failure
|
|
// nonetheless brought the machine down. Either way the caller gets
|
|
// the original result untouched.
|
|
return stop;
|
|
}
|
|
|
|
tracing::error!(
|
|
%machine,
|
|
"stop finished but machined still holds the registration — the \
|
|
container is likely wedged (a process outside its init tree \
|
|
survived the stop); this needs a host-level look, not another \
|
|
stop attempt"
|
|
);
|
|
bail!(
|
|
"stop {machine}: machined still holds the machine name after {}s. \
|
|
This container is likely wedged and starting it again will fail to \
|
|
register — needs host-level intervention (a reboot has been the \
|
|
only reliable fix in practice).",
|
|
NAME_RELEASE_TIMEOUT.as_secs()
|
|
)
|
|
}
|
|
|
|
/// Invoke `nixos-container` with the given args and forward output lines
|
|
/// to the caller as `PrivEvent::Line` messages in real time, logging each
|
|
/// line to journald as it arrives. Returns `(String::new(), String::new())`
|
|
/// on success (all output was streamed); the error string includes stderr
|
|
/// tail on failure.
|
|
async fn container_run_streaming(
|
|
args: &[&str],
|
|
writer: &mut OwnedWriteHalf,
|
|
) -> Result<(String, String)> {
|
|
use tokio::io::AsyncBufReadExt as _;
|
|
use tokio::process::Command;
|
|
|
|
let mut child = Command::new("nixos-container")
|
|
.args(args)
|
|
.stdout(std::process::Stdio::piped())
|
|
.stderr(std::process::Stdio::piped())
|
|
.spawn()
|
|
.context("invoke nixos-container (streaming)")?;
|
|
|
|
let stdout = child.stdout.take().expect("stdout piped");
|
|
let stderr = child.stderr.take().expect("stderr piped");
|
|
|
|
let mut stdout_lines = BufReader::new(stdout).lines();
|
|
let mut stderr_lines = BufReader::new(stderr).lines();
|
|
|
|
// Collect stderr for the error message; stream both to the client.
|
|
let mut stderr_buf = String::new();
|
|
|
|
// Drive stdout and stderr concurrently. `tokio::select!` interleaves
|
|
// them without bias — both streams drain at roughly the same rate
|
|
// as the subprocess produces output.
|
|
loop {
|
|
tokio::select! {
|
|
line = stdout_lines.next_line() => {
|
|
match line {
|
|
Ok(Some(l)) => {
|
|
tracing::info!(target: "nixos-container", "{l}");
|
|
write_line_event(writer, PrivStream::Stdout, &l).await;
|
|
}
|
|
Ok(None) => break,
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "nixos-container stdout read error");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
line = stderr_lines.next_line() => {
|
|
match line {
|
|
Ok(Some(l)) => {
|
|
tracing::warn!(target: "nixos-container", "{l}");
|
|
write_line_event(writer, PrivStream::Stderr, &l).await;
|
|
if !stderr_buf.is_empty() {
|
|
stderr_buf.push('\n');
|
|
}
|
|
stderr_buf.push_str(&l);
|
|
}
|
|
Ok(None) => {}
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "nixos-container stderr read error");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Drain any remaining stderr after stdout closed.
|
|
while let Ok(Some(l)) = stderr_lines.next_line().await {
|
|
tracing::warn!(target: "nixos-container", "{l}");
|
|
write_line_event(writer, PrivStream::Stderr, &l).await;
|
|
if !stderr_buf.is_empty() {
|
|
stderr_buf.push('\n');
|
|
}
|
|
stderr_buf.push_str(&l);
|
|
}
|
|
|
|
let status = child.wait().await.context("wait nixos-container")?;
|
|
if !status.success() {
|
|
// Only the last stderr line is embedded — the full stderr was
|
|
// already forwarded line-by-line as PrivEvent::Line messages and
|
|
// is captured in build_logs.sqlite by the caller. Keeping the
|
|
// error message short avoids bloating the anyhow chain.
|
|
bail!(
|
|
"nixos-container {} failed ({}): {}",
|
|
args.join(" "),
|
|
status,
|
|
stderr_buf.lines().last().unwrap_or("").trim()
|
|
);
|
|
}
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::{BoundedRun, run_bounded};
|
|
use crate::shared::test_support::scratch;
|
|
|
|
/// Whether `pid` is gone, a zombie (`Z`), or being reaped (`X`): none of
|
|
/// these runs again, and a process passes through them in that order,
|
|
/// so once this is true it stays true.
|
|
fn is_dead(pid: &str) -> bool {
|
|
match std::fs::read_to_string(format!("/proc/{pid}/stat")) {
|
|
Err(_) => true,
|
|
Ok(stat) => stat_is_dead(&stat),
|
|
}
|
|
}
|
|
|
|
/// The state letter of a `/proc/<pid>/stat` line, which follows the
|
|
/// parenthesised command name, is `Z` or `X`.
|
|
fn stat_is_dead(stat: &str) -> bool {
|
|
stat.rsplit_once(") ")
|
|
.is_some_and(|(_, rest)| rest.starts_with(['Z', 'X']))
|
|
}
|
|
|
|
#[test]
|
|
fn zombie_and_reaped_states_count_as_dead() {
|
|
assert!(stat_is_dead("42 (sleep) Z 1 42 42"));
|
|
assert!(stat_is_dead("42 (sleep) X 1 42 42"));
|
|
assert!(!stat_is_dead("42 (sleep) S 1 42 42"));
|
|
assert!(!stat_is_dead("42 (sleep) R 1 42 42"));
|
|
}
|
|
|
|
/// A child still running at the limit comes back `TimedOut`, and the kill
|
|
/// reaches its whole process group: the backgrounded grandchild dies too,
|
|
/// not just the shell that started it.
|
|
#[tokio::test]
|
|
async fn a_child_past_its_limit_is_killed_with_its_group() {
|
|
use std::time::{Duration, Instant};
|
|
let dir = scratch();
|
|
let pidfile = dir.join("grandchild");
|
|
let mut cmd = tokio::process::Command::new("sh");
|
|
cmd.arg("-c")
|
|
.arg(format!("sleep 600 & echo $! > {}; wait", pidfile.display()));
|
|
|
|
let started = Instant::now();
|
|
let run = run_bounded(cmd, Duration::from_secs(2)).await.unwrap();
|
|
assert!(matches!(run, BoundedRun::TimedOut), "must time out");
|
|
assert!(
|
|
started.elapsed() < Duration::from_mins(1),
|
|
"the limit did not bound the run"
|
|
);
|
|
|
|
let pid = std::fs::read_to_string(&pidfile).unwrap();
|
|
let pid = pid.trim();
|
|
let deadline = Instant::now() + Duration::from_secs(5);
|
|
while !is_dead(pid) && Instant::now() < deadline {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
}
|
|
assert!(is_dead(pid), "grandchild {pid} outlived the timeout");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The control: a child that exits inside its limit is `Exited`, with its
|
|
/// status and its stderr.
|
|
#[tokio::test]
|
|
async fn a_child_inside_its_limit_returns_its_status_and_stderr() {
|
|
let mut cmd = tokio::process::Command::new("sh");
|
|
cmd.arg("-c")
|
|
.arg("echo progress >&2; echo /nix/store/abc-toplevel");
|
|
match run_bounded(cmd, std::time::Duration::from_mins(1))
|
|
.await
|
|
.unwrap()
|
|
{
|
|
BoundedRun::Exited { status, stderr } => {
|
|
assert!(status.success());
|
|
assert_eq!(stderr, "progress");
|
|
}
|
|
BoundedRun::TimedOut => panic!("a fast child must not time out"),
|
|
}
|
|
}
|
|
}
|