diff --git a/hive-priv/README.md b/hive-priv/README.md index 7718aa31..033c1548 100644 --- a/hive-priv/README.md +++ b/hive-priv/README.md @@ -30,11 +30,21 @@ The wire contract (`PrivRequest` / response types) lives in the separate `hive-priv-sock` crate so this root binary depends on just the protocol shapes, not the whole daemon-shared crate. +## Layout + +`src/main.rs` holds the socket loop and the dispatch table, one arm per +`PrivRequest` variant. Each op family, with its validation, has its own module: +`container` (lifecycle), `container_config` (nspawn flags, systemd drop-ins), +`storage` (btrfs subvolumes, quotas, snapshots, disk measurement), `files` +(pause marker, socket dir, legacy tmpfiles) and `infra` (gateway nginx, forge +admin, infra containers). Helpers more than one family calls live in `shared`. +Each module's `//!` header lists its ops. + ## Implementation notes ### Container toplevel builds (create/update) -`container_flake_action` (in `src/main.rs`) builds +`container_flake_action` (in `src/container.rs`) builds `nixosConfigurations..config.system.build.toplevel` itself (`nix_build_toplevel`) and passes the resolved store path to `nixos-container create`/`update` via `--system-path`, for both verbs. diff --git a/hive-priv/src/container.rs b/hive-priv/src/container.rs new file mode 100644 index 00000000..7c0bb74e --- /dev/null +++ b/hive-priv/src/container.rs @@ -0,0 +1,798 @@ +//! Container lifecycle ops: `StartContainer`, `StopContainer`, +//! `KillContainer`, `CreateContainer`, `UpdateContainer`, `DestroyContainer` +//! and `ListContainers`. Every op takes a logical agent name and reaches only +//! the `h-`-prefixed container [`container_system_name`] derives from it. +//! +//! `CreateContainer` / `UpdateContainer` build the toplevel here +//! ([`nix_build_toplevel`]) and stream progress to the caller as +//! [`PrivEvent::Line`]s when asked to. + +use anyhow::{Context as _, Result, anyhow, bail}; +use hive_priv_sock::{META_DIR, PrivEvent, PrivStream, PrivStreamLine}; +use tokio::io::{AsyncWriteExt, BufReader}; +use tokio::net::unix::OwnedWriteHalf; +use tokio::process::Command; + +use crate::shared::{container_system_name, validate_agent_name}; + +/// Write one `PrivEvent::Line` to the client. Best-effort: a write +/// failure is logged but doesn't abort the running subprocess. +async fn write_line_event(writer: &mut OwnedWriteHalf, stream: PrivStream, data: &str) { + let event = PrivEvent::Line(PrivStreamLine { + stream, + data: data.to_owned(), + }); + if let Ok(mut json) = serde_json::to_string(&event) { + json.push('\n'); + if let Err(e) = writer.write_all(json.as_bytes()).await { + tracing::warn!(error = %e, "write_line_event: write failed"); + } + } +} + +/// `StartContainer`: clear any start-limit lockout left by earlier failures +/// so a now-correct start isn't blocked (`nixos-container start` does not do +/// this itself). Best-effort — if the unit doesn't exist yet (first-time +/// create), `reset-failed` is a no-op and the start proceeds regardless. +pub(crate) async fn start_container(name: &str) -> Result<(String, String)> { + validate_agent_name(name)?; + let machine = container_system_name(name); + // Silenced: a first-time create means the unit doesn't exist yet, and + // `systemctl`'s own "Unit ... not loaded" complaint for that case would + // otherwise land in the journal at error priority on every such start — + // routine, not a signal, since the result is already discarded below. + let _ = Command::new("systemctl") + .args(["reset-failed", &format!("container@{machine}.service")]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .await; + container_run(&["start", &machine]).await +} + +/// `StopContainer`. +pub(crate) async fn stop_container(name: &str) -> Result<(String, String)> { + validate_agent_name(name)?; + stop_and_release(&container_system_name(name)).await +} + +/// `KillContainer`: `nixos-container` has no kill verb, so this uses +/// `machinectl` to send `SIGKILL` to every process in the container — the +/// right semantics for a forced shutdown after a graceful stop has already +/// been attempted. +pub(crate) async fn kill_container(name: &str) -> Result<(String, String)> { + validate_agent_name(name)?; + let machine = container_system_name(name); + machinectl_run(&["kill", &machine, "--signal=SIGKILL"]).await +} + +/// `DestroyContainer`. +pub(crate) async fn destroy_container(name: &str) -> Result<(String, String)> { + validate_agent_name(name)?; + container_run(&["destroy", &container_system_name(name)]).await +} + +/// Shared body for `CreateContainer` / `UpdateContainer`: validate the +/// name, build the toplevel ourselves, and apply it (streaming line +/// events to `writer` when `stream` is set). +/// +/// **Both verbs build explicitly now — not just `update`.** The first cut +/// of this fix only rewrote `update`, reasoning that `create` was already +/// safe: it wraps its whole action in an exclusive `flock` before calling +/// `nixos-container`'s own `buildFlake()`, and once `update` stopped +/// writing to `buildFlake()`'s shared `.tmp` out-link, concurrent +/// `create`s were the only remaining writers — mutually excluded by that +/// lock. True, but it leaves `create`'s safety resting on an internal +/// implementation detail of a script we don't own (its current locking +/// behavior, which could change upstream without notice) instead of on +/// something we control. Building here for both verbs removes `buildFlake()` +/// from the picture entirely — there's no shared `.tmp` left to race on, +/// so there's nothing left to reason about staying in sync with. +/// +/// **`update` no longer calls `nixos-container update` at all.** That +/// action's own version-compat probe runs unconditionally before +/// `--system-path` is ever honored, dying on every agent's update. +/// [`swap_container_profile`] replicates exactly what `update`'s own +/// action does *past* that probe (confirmed against its source): `nix-env +/// --set` the per-container profile, then `systemctl reload` if the +/// container is running. `create` is untouched — it isn't the failing +/// verb. +pub(crate) async fn container_flake_action( + verb: &str, + name: &str, + stream: bool, + writer: &mut OwnedWriteHalf, +) -> Result<(String, String)> { + validate_agent_name(name)?; + // The build is the multi-minute phase of this operation — give it the + // same live-line treatment `container_run_streaming` gives + // `nixos-container` itself when the caller asked for it. Without + // this, moving the build out of the streamed `nixos-container` call + // (which is the whole point of this fix) would silently regress every + // UI that shows build progress: nothing until the longest phase + // finishes, then everything at once. + let toplevel = nix_build_toplevel(name, stream.then_some(&mut *writer)).await?; + let system_name = container_system_name(name); + if verb == "update" { + return swap_container_profile(&system_name, &toplevel, stream, writer).await; + } + let args = [verb, &system_name, "--system-path", &toplevel]; + if stream { + container_run_streaming(&args, writer).await + } else { + container_run(&args).await + } +} + +/// Apply a prebuilt toplevel to an existing container directly, without +/// going through `nixos-container update` (see `container_flake_action`'s +/// doc comment for why). Mirrors `nixos-container.pl`'s own `update` +/// action verbatim, past its version probe: point the per-container `nix- +/// env` profile at the new toplevel, then reload the container unit *if* +/// it's currently running — the container's own next start already reads +/// from the profile, so a stopped container needs nothing further. +/// +/// Both steps are near-instant in the happy case, but still stream- +/// forwarded like the rest of this operation: a failure here (e.g. a +/// wedged `systemctl reload`) is exactly when the caller most wants the +/// live line in the build log, not just a summary error afterward. +async fn swap_container_profile( + system_name: &str, + toplevel: &str, + stream: bool, + writer: &mut OwnedWriteHalf, +) -> Result<(String, String)> { + let profile = format!("/nix/var/nix/profiles/per-container/{system_name}/system"); + let set_out = Command::new("nix-env") + .args(["-p", &profile, "--set", toplevel]) + .output() + .await + .context("invoke nix-env --set")?; + let mut stdout = String::from_utf8_lossy(&set_out.stdout).into_owned(); + let mut stderr = String::from_utf8_lossy(&set_out.stderr).into_owned(); + log_and_forward(&stdout, &stderr, stream, writer).await; + if !set_out.status.success() { + bail!( + "nix-env -p {profile} --set failed ({}): {}", + set_out.status, + stderr.trim() + ); + } + + let unit = format!("container@{system_name}"); + let state_out = Command::new("systemctl") + .args(["show", "--property=ActiveState", "--value", &unit]) + .output() + .await + .context("query container ActiveState")?; + let active = String::from_utf8_lossy(&state_out.stdout).trim() == "active"; + if active { + let reload_out = Command::new("systemctl") + .args(["reload", &unit]) + .output() + .await + .context("invoke systemctl reload")?; + let r_stdout = String::from_utf8_lossy(&reload_out.stdout).into_owned(); + let r_stderr = String::from_utf8_lossy(&reload_out.stderr).into_owned(); + log_and_forward(&r_stdout, &r_stderr, stream, writer).await; + if !reload_out.status.success() { + bail!( + "systemctl reload {unit} failed ({}): {}", + reload_out.status, + r_stderr.trim() + ); + } + stdout.push_str(&r_stdout); + stderr.push_str(&r_stderr); + } + Ok((stdout, stderr)) +} + +/// Log a command's captured stdout/stderr the same way every other +/// `nixos-container`-adjacent shellout in this file does, and — when +/// `stream` is set — forward each line to the client as a live +/// [`PrivEvent::Line`] too, so a caller watching the build log sees these +/// lines exactly like any other step's, not just a summary on failure. +async fn log_and_forward(stdout: &str, stderr: &str, stream: bool, writer: &mut OwnedWriteHalf) { + for line in stdout.lines() { + tracing::info!(target: "nixos-container", "{line}"); + if stream { + write_line_event(writer, PrivStream::Stdout, line).await; + } + } + for line in stderr.lines() { + tracing::warn!(target: "nixos-container", "{line}"); + if stream { + write_line_event(writer, PrivStream::Stderr, line).await; + } + } +} + +/// The explicit `nixosConfigurations..config.system.build.toplevel` +/// flake attr path — same construction `hive-c0re`'s own +/// `lifecycle::prebuild_toplevel` uses, kept here as a pure function so +/// the exact string shape is unit-tested without needing to run `nix`. +fn toplevel_attr(name: &str) -> String { + format!("{META_DIR}#nixosConfigurations.{name}.config.system.build.toplevel") +} + +/// Wall-clock bound on [`nix_build_toplevel`]. Four times CI's observed +/// cold-cache `nix flake check` (~15 min, `.forgejo/workflows/ci.yml`), which +/// compiles this workspace's crates, the part of an agent toplevel no binary +/// cache holds; the rebuild path has usually warmed this exact attr already +/// (`hive-c0re`'s `prebuild_toplevel`). A stalled download is bounded by nix's +/// own `stalled-download-timeout`; this covers the rest, such as a wedged +/// nix-daemon holding the caller's build slot forever. +const NIX_BUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_hours(1); + +/// How a [`run_bounded`] child ended. +enum BoundedRun { + Exited { + status: std::process::ExitStatus, + stdout: String, + stderr: String, + }, + /// Still running at the limit; its whole process group was killed. + TimedOut, +} + +/// Run `cmd` to completion or until `limit`, draining both pipes: stdout is +/// captured silently, stderr is logged and, when `writer` is set, forwarded +/// live as [`PrivEvent::Line`]s. +/// +/// The child leads its own process group and a timeout `SIGKILL`s the whole +/// group, so a grandchild still holding the pipes (or the daemon connection) +/// dies with it instead of outliving the error. +async fn run_bounded( + mut cmd: Command, + limit: std::time::Duration, + mut writer: Option<&mut OwnedWriteHalf>, +) -> std::io::Result { + use tokio::io::AsyncBufReadExt as _; + + let mut child = cmd + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .process_group(0) + .spawn()?; + // Group leader, so its pid is the pgid. + let pgid = child.id().and_then(|pid| i32::try_from(pid).ok()); + + let stdout = child.stdout.take().expect("stdout piped"); + let stderr = child.stderr.take().expect("stderr piped"); + let run = async { + let mut stdout_lines = BufReader::new(stdout).lines(); + let mut stderr_lines = BufReader::new(stderr).lines(); + let mut stdout_buf = String::new(); + let mut stderr_buf = String::new(); + + // ⚠️ Both pipes are drained concurrently even though only one is + // streamed: reading stderr alone would let stdout fill its pipe + // buffer and deadlock the child on a build with enough stdout output + // to fill it. + // + // ⚠️ Each stream's EOF is tracked separately rather than breaking on + // the first `None`: `next_line()` on a closed stream returns + // `Ok(None)` immediately and forever, so a loop that keeps polling a + // finished stream spins hot until the other one ends too. + let mut stdout_done = false; + let mut stderr_done = false; + while !(stdout_done && stderr_done) { + tokio::select! { + line = stdout_lines.next_line(), if !stdout_done => { + match line { + // Captured, never streamed — this is the store path. + Ok(Some(l)) => { + stdout_buf.push_str(&l); + stdout_buf.push('\n'); + } + Ok(None) => stdout_done = true, + Err(e) => { + tracing::warn!(error = %e, "nix build stdout read error"); + stdout_done = true; + } + } + } + line = stderr_lines.next_line(), if !stderr_done => { + match line { + // Streamed as it arrives, so the dashboard and + // `journalctl -f` show build progress live. + Ok(Some(l)) => { + tracing::info!(target: "nix-build-toplevel", "{l}"); + if let Some(w) = writer.as_deref_mut() { + write_line_event(w, PrivStream::Stderr, &l).await; + } + if !stderr_buf.is_empty() { + stderr_buf.push('\n'); + } + stderr_buf.push_str(&l); + } + Ok(None) => stderr_done = true, + Err(e) => { + tracing::warn!(error = %e, "nix build stderr read error"); + stderr_done = true; + } + } + } + } + } + let status = child.wait().await?; + Ok::<_, std::io::Error>(BoundedRun::Exited { + status, + stdout: stdout_buf, + stderr: stderr_buf, + }) + }; + if let Ok(exited) = tokio::time::timeout(limit, run).await { + return exited; + } + if let Some(pgid) = pgid { + // SAFETY: plain `kill(2)`; `-pgid` targets the child's own process + // group, which `process_group(0)` made it lead. + unsafe { + libc::kill(-pgid, libc::SIGKILL); + } + } + child.wait().await?; + Ok(BoundedRun::TimedOut) +} + +/// Build `nixosConfigurations..config.system.build.toplevel` and +/// return the resulting store path, so `create`/`update` can hand +/// `nixos-container` an explicit `--system-path` instead of letting its +/// own `buildFlake()` build to a racy shared out-link. See "Container +/// toplevel builds" in this crate's README for the full story — the +/// concurrency bug this closes (the "agent container gets closure of +/// other agent" mystery bug) and why stdout/stderr are drained +/// concurrently but handled asymmetrically (stderr streamed live, +/// stdout captured and required to be exactly one line). +/// +/// `writer` is `None` for the non-streaming call shape (`stream: false`); +/// stderr still logs to journald either way, just without the +/// `PrivEvent::Line` forwarding. Bounded by [`NIX_BUILD_TIMEOUT`]. +async fn nix_build_toplevel(name: &str, writer: Option<&mut OwnedWriteHalf>) -> Result { + let attr = toplevel_attr(name); + let mut cmd = Command::new("nix"); + cmd.args([ + "--extra-experimental-features", + "nix-command flakes", + "build", + "--no-link", + "--print-out-paths", + &attr, + ]); + let (status, stdout_buf, stderr_buf) = match run_bounded(cmd, NIX_BUILD_TIMEOUT, writer) + .await + .with_context(|| format!("run nix build {attr}"))? + { + BoundedRun::Exited { + status, + stdout, + stderr, + } => (status, stdout, stderr), + BoundedRun::TimedOut => bail!( + "nix build {attr} timed out after {}s; killed it", + NIX_BUILD_TIMEOUT.as_secs() + ), + }; + + // ⚠️ Success is decided by the exit status, not by "we parsed a + // path" — a build can print to stdout and still fail. + if !status.success() { + bail!( + "nix build {attr} failed ({status}): {}", + stderr_buf.lines().last().unwrap_or("").trim() + ); + } + single_output_path(&stdout_buf) + .map(str::to_owned) + .map_err(|count| { + anyhow!("nix build {attr} produced {count} output path(s), expected exactly 1: {stdout_buf:?}") + }) +} + +/// Parse `nix build --print-out-paths`' stdout down to the single output +/// path this function's caller expects. `--print-out-paths` prints one +/// line *per output*, not one line total (`nix build --no-link +/// --print-out-paths nixpkgs#openssl` prints two: `…-bin`, `…-man`); +/// `config.system.build.toplevel` is single-output today, so this is one +/// line in practice — but a bare whole-buffer `.trim()` would silently +/// hand a multi-line string on to `--system-path` the day that ever +/// changes, the same corrupted-argument failure this function exists to +/// avoid. Trims and drops empty lines *before* counting, so a lone +/// `"\n"` (or trailing whitespace on the real line) can't be mistaken +/// for a present-but-blank path — see the unit tests below for the exact +/// table this closes. `Err` carries the surviving line count, for the +/// caller's error message. +fn single_output_path(stdout: &str) -> Result<&str, usize> { + let lines: Vec<&str> = stdout + .lines() + .map(str::trim) + .filter(|l| !l.is_empty()) + .collect(); + match lines[..] { + [path] => Ok(path), + _ => Err(lines.len()), + } +} + +/// Invoke `nixos-container` with the given args, log output to journald. +pub(crate) async fn container_run(args: &[&str]) -> Result<(String, String)> { + let out = Command::new("nixos-container") + .args(args) + .output() + .await + .context("invoke nixos-container")?; + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + // `list` is a read-only enumeration called on the hot path (dashboard + // rescan, forge + boot sweeps) — its stdout is the return value, not + // progress, so logging every container name on every call floods the + // journal. Log stdout only for the mutating ops, where each line is + // genuine progress. stderr is always logged (errors matter regardless). + if args.first() != Some(&"list") { + for line in stdout.lines() { + tracing::info!(target: "nixos-container", "{line}"); + } + } + for line in stderr.lines() { + tracing::warn!(target: "nixos-container", "{line}"); + } + if !out.status.success() { + bail!( + "nixos-container {} failed ({}): {}", + args.join(" "), + out.status, + stderr.trim() + ); + } + Ok((stdout, stderr)) +} + +/// Invoke `machinectl` with the given args, log output to journald. +/// Used for operations that nixos-container doesn't expose (e.g. sending +/// signals to running containers). +async fn machinectl_run(args: &[&str]) -> Result<(String, String)> { + let out = Command::new("machinectl") + .args(args) + .output() + .await + .context("invoke machinectl")?; + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + for line in stdout.lines() { + tracing::info!(target: "machinectl", "{line}"); + } + for line in stderr.lines() { + tracing::warn!(target: "machinectl", "{line}"); + } + if !out.status.success() { + bail!( + "machinectl {} failed ({}): {}", + args.join(" "), + out.status, + stderr.trim() + ); + } + Ok((stdout, stderr)) +} + +/// How long to wait for machined to drop a machine's registration after a +/// shutdown has been asked for. Generous: a container with slow-stopping +/// units legitimately takes a while, and escalating early would SIGKILL a +/// shutdown that was going to finish on its own. +const NAME_RELEASE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); + +/// Poll cadence while waiting for the registration to go away. +const NAME_RELEASE_POLL: std::time::Duration = std::time::Duration::from_millis(500); + +/// Cap on a single registration probe. The probe is a D-Bus round trip to +/// machined; if machined itself is wedged the call would otherwise sit on +/// the D-Bus method timeout, which is far longer than the whole stop +/// sequence should take. +const MACHINE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// True when machined still holds a registration for `machine`. +/// +/// This asks machined the same question the registration itself answers — +/// `machinectl show` resolves the name through `GetMachine`, the very lookup +/// that makes a later boot fail with `Failed to register machine: already +/// exists`. So a positive answer here is exactly the condition that breaks +/// the next start, not a proxy for it. +/// +/// Deliberately NOT the container's systemd unit state: the unit can be +/// `inactive` while the registration is still held, and that gap is the +/// whole bug this probe exists to catch. +/// +/// Errors and timeouts answer "still registered". Being wrong that way costs +/// a redundant SIGKILL to something already gone; being wrong the other way +/// hands back a stop that silently leaked the name. +async fn machine_registered(machine: &str) -> bool { + let probe = Command::new("machinectl") + .args(["show", machine, "--property=Name"]) + // Don't leave a probe behind when the timeout below fires. + .kill_on_drop(true) + .output(); + match tokio::time::timeout(MACHINE_PROBE_TIMEOUT, probe).await { + Ok(Ok(out)) => out.status.success(), + Ok(Err(e)) => { + tracing::warn!(%machine, error = %e, "machinectl show failed to run; assuming still registered"); + true + } + Err(_) => { + tracing::warn!(%machine, "machinectl show timed out; assuming still registered"); + true + } + } +} + +/// Poll [`machine_registered`] until the name is free or the timeout expires. +/// Returns true once the registration is gone. +async fn wait_name_released(machine: &str) -> bool { + let deadline = tokio::time::Instant::now() + NAME_RELEASE_TIMEOUT; + loop { + if !machine_registered(machine).await { + return true; + } + if tokio::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(NAME_RELEASE_POLL).await; + } +} + +/// Stop `machine` and only report success once machined has actually released +/// the name. +/// +/// `nixos-container stop` exiting 0 does not mean the machine is gone. A +/// process that sits in the machine's cgroup without being a child of the +/// container's init — a shell exec'd in from outside, say — never receives +/// the shutdown's SIGTERM if it has been stopped with SIGSTOP, so the +/// registration outlives the "successful" stop. Every later start of that +/// container then fails with `Failed to register machine: already exists`, +/// and machined re-persists the stale record across its own restart, so +/// there is no cleaning it up after the fact. The only place to catch it is +/// here, in the stop. +/// +/// So: ask for the stop, wait for the name, and fail loudly if it is still +/// held — a caller that is told the stop worked will go on to start the +/// container and hit the confusing registration error instead of this one. +/// +/// This used to escalate to `machinectl kill --signal=SIGKILL` here. Dropped +/// per real incident data: the case this guards is a container genuinely +/// wedged (e.g. a root-login process survived the stop), and SIGKILL +/// doesn't recover that in practice — only a host-level reboot has. +/// Pretending a kill attempt handled it hides a condition that needs a human +/// to look at the host, so this now just reports the failure instead of +/// quietly (and ineffectually) trying to force it. +/// +/// The verify-then-fail lives in the helper rather than at a call site so +/// that every stop gets it: dashboard, reconcile, destroy, cold-start +/// fallback. The start path already distrusts its own exit code the same way; +/// this is the missing half of that pair. +async fn stop_and_release(machine: &str) -> Result<(String, String)> { + let stop = container_run(&["stop", machine]).await; + if wait_name_released(machine).await { + // Happy path, and also the path where a stop that reported failure + // nonetheless brought the machine down. Either way the caller gets + // the original result untouched. + return stop; + } + + tracing::error!( + %machine, + "stop finished but machined still holds the registration — the \ + container is likely wedged (a process outside its init tree \ + survived the stop); this needs a host-level look, not another \ + stop attempt" + ); + bail!( + "stop {machine}: machined still holds the machine name after {}s. \ + This container is likely wedged and starting it again will fail to \ + register — needs host-level intervention (a reboot has been the \ + only reliable fix in practice).", + NAME_RELEASE_TIMEOUT.as_secs() + ) +} + +/// Invoke `nixos-container` with the given args and forward output lines +/// to the caller as `PrivEvent::Line` messages in real time, logging each +/// line to journald as it arrives. Returns `(String::new(), String::new())` +/// on success (all output was streamed); the error string includes stderr +/// tail on failure. +async fn container_run_streaming( + args: &[&str], + writer: &mut OwnedWriteHalf, +) -> Result<(String, String)> { + use tokio::io::AsyncBufReadExt as _; + use tokio::process::Command; + + let mut child = Command::new("nixos-container") + .args(args) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .context("invoke nixos-container (streaming)")?; + + let stdout = child.stdout.take().expect("stdout piped"); + let stderr = child.stderr.take().expect("stderr piped"); + + let mut stdout_lines = BufReader::new(stdout).lines(); + let mut stderr_lines = BufReader::new(stderr).lines(); + + // Collect stderr for the error message; stream both to the client. + let mut stderr_buf = String::new(); + + // Drive stdout and stderr concurrently. `tokio::select!` interleaves + // them without bias — both streams drain at roughly the same rate + // as the subprocess produces output. + loop { + tokio::select! { + line = stdout_lines.next_line() => { + match line { + Ok(Some(l)) => { + tracing::info!(target: "nixos-container", "{l}"); + write_line_event(writer, PrivStream::Stdout, &l).await; + } + Ok(None) => break, + Err(e) => { + tracing::warn!(error = %e, "nixos-container stdout read error"); + break; + } + } + } + line = stderr_lines.next_line() => { + match line { + Ok(Some(l)) => { + tracing::warn!(target: "nixos-container", "{l}"); + write_line_event(writer, PrivStream::Stderr, &l).await; + if !stderr_buf.is_empty() { + stderr_buf.push('\n'); + } + stderr_buf.push_str(&l); + } + Ok(None) => {} + Err(e) => { + tracing::warn!(error = %e, "nixos-container stderr read error"); + } + } + } + } + } + + // Drain any remaining stderr after stdout closed. + while let Ok(Some(l)) = stderr_lines.next_line().await { + tracing::warn!(target: "nixos-container", "{l}"); + write_line_event(writer, PrivStream::Stderr, &l).await; + if !stderr_buf.is_empty() { + stderr_buf.push('\n'); + } + stderr_buf.push_str(&l); + } + + let status = child.wait().await.context("wait nixos-container")?; + if !status.success() { + // Only the last stderr line is embedded — the full stderr was + // already forwarded line-by-line as PrivEvent::Line messages and + // is captured in build_logs.sqlite by the caller. Keeping the + // error message short avoids bloating the anyhow chain. + bail!( + "nixos-container {} failed ({}): {}", + args.join(" "), + status, + stderr_buf.lines().last().unwrap_or("").trim() + ); + } + Ok((String::new(), String::new())) +} +#[cfg(test)] +mod tests { + use super::{BoundedRun, run_bounded, single_output_path, toplevel_attr}; + use crate::shared::test_support::scratch; + + /// Pins the exact attr path we hand to `nix build` — it has to match + /// `hive-c0re`'s own `lifecycle::prebuild_toplevel` construction, since + /// that step's whole point is warming the store for this later build. + /// A drifted attr path defeats the cache-warming silently — no error, + /// just a slower `update`. + #[test] + fn toplevel_attr_matches_prebuild_toplevels_construction() { + assert_eq!( + toplevel_attr("atlas"), + "/var/lib/hyperhive/meta#nixosConfigurations.atlas.config.system.build.toplevel" + ); + } + + /// `--print-out-paths`' exact per-shape table (measured against real + /// `rustc` semantics, not just reasoned about) — a lone `"\n"` and a + /// trailing-space path are the two shapes a bare `.lines().collect()` + /// gets wrong, both catchable only by trimming + dropping empties + /// before counting rather than after. + #[test] + fn single_output_path_rejects_blank_and_trims_whitespace() { + assert_eq!(single_output_path(""), Err(0)); + assert_eq!(single_output_path("\n"), Err(0)); + assert_eq!(single_output_path("\n\n"), Err(0)); + assert_eq!(single_output_path("/nix/store/abc\n"), Ok("/nix/store/abc")); + assert_eq!( + single_output_path("/nix/store/abc \n"), + Ok("/nix/store/abc") + ); + assert_eq!( + single_output_path("/nix/store/abc\r\n"), + Ok("/nix/store/abc") + ); + assert_eq!( + single_output_path("/nix/store/abc\n/nix/store/def\n"), + Err(2) + ); + } + + /// Whether `pid` is gone, or only a zombie awaiting its reaper: either + /// way it is no longer running. + fn is_dead(pid: &str) -> bool { + match std::fs::read_to_string(format!("/proc/{pid}/stat")) { + Err(_) => true, + // The state letter follows the parenthesised command name. + Ok(stat) => stat + .rsplit_once(") ") + .is_some_and(|(_, rest)| rest.starts_with('Z')), + } + } + + /// A child still running at the limit comes back `TimedOut`, and the kill + /// reaches its whole process group: the backgrounded grandchild dies too, + /// not just the shell that started it. + #[tokio::test] + async fn a_build_past_its_limit_is_killed_with_its_group() { + use std::time::{Duration, Instant}; + let dir = scratch(); + let pidfile = dir.join("grandchild"); + let mut cmd = tokio::process::Command::new("sh"); + cmd.arg("-c") + .arg(format!("sleep 600 & echo $! > {}; wait", pidfile.display())); + + let started = Instant::now(); + let run = run_bounded(cmd, Duration::from_secs(2), None) + .await + .unwrap(); + assert!(matches!(run, BoundedRun::TimedOut), "must time out"); + assert!( + started.elapsed() < Duration::from_mins(1), + "the limit did not bound the run" + ); + + let pid = std::fs::read_to_string(&pidfile).unwrap(); + let pid = pid.trim(); + let deadline = Instant::now() + Duration::from_secs(5); + while !is_dead(pid) && Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!(is_dead(pid), "grandchild {pid} outlived the timeout"); + std::fs::remove_dir_all(&dir).ok(); + } + + /// The control: a child that exits inside its limit is `Exited`, with its + /// status and both streams kept apart. + #[tokio::test] + async fn a_build_inside_its_limit_returns_its_output() { + let mut cmd = tokio::process::Command::new("sh"); + cmd.arg("-c") + .arg("echo progress >&2; echo /nix/store/abc-toplevel"); + match run_bounded(cmd, std::time::Duration::from_mins(1), None) + .await + .unwrap() + { + BoundedRun::Exited { + status, + stdout, + stderr, + } => { + assert!(status.success()); + assert_eq!(stdout, "/nix/store/abc-toplevel\n"); + assert_eq!(stderr, "progress"); + } + BoundedRun::TimedOut => panic!("a fast child must not time out"), + } + } +} diff --git a/hive-priv/src/container_config.rs b/hive-priv/src/container_config.rs new file mode 100644 index 00000000..64352042 --- /dev/null +++ b/hive-priv/src/container_config.rs @@ -0,0 +1,774 @@ +//! Container config writes: `WriteNspawnFlags` (the container's +//! `/etc/nixos-containers/.conf`, plus the bridge-DNS marker and +//! hive-facts file in its rootfs), `WriteResourceLimits` / +//! `RemoveServiceDropin` (the `container@.service` drop-in) and +//! `DaemonReload`. +//! +//! Every op that names a container takes its system name and refuses anything +//! [`validate_container_system_name`] does not accept. + +use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, Result, bail}; +use hive_priv_sock::{ + AGENT_PREFIX, AGENT_RUNTIME_ROOT, BindMount, CredentialMount, HiveFacts, NetworkIsolation, +}; +use tokio::process::Command; + +use crate::shared::{StagedFile, open_dir, validate_credential_name, validate_name_chars}; + +/// `WriteNspawnFlags` — validate the container + every bind path + every +/// credential entry, then write the container's nspawn flag overrides. +pub(crate) fn handle_write_nspawn_flags( + container: &str, + binds: &[BindMount], + isolation: &NetworkIsolation, + load_credentials: &[CredentialMount], + hive_facts: &HiveFacts, +) -> Result<(String, String)> { + validate_container_system_name(container)?; + for bind in binds { + validate_bind_path(&bind.host_path)?; + validate_bind_path(&bind.container_path)?; + } + for cred in load_credentials { + validate_credential_name(&cred.name)?; + // Same path rules as binds (absolute, no colon/newline/quote/null): + // the colon ban is essential since `--load-credential=name:path` + // uses `:` as the name/path separator. + validate_bind_path(&cred.host_path)?; + } + let hive_facts = render_hive_facts(hive_facts)?; + write_nspawn_flags(container, binds, isolation, load_credentials, &hive_facts)?; + Ok((String::new(), String::new())) +} + +/// Validate a system-level container name (`h-` prefix for all agents, +/// the manager included) and return the logical agent name after it. +/// Anything else is refused, the swarm's forge and matrix containers +/// included: they belong to the host that runs them, not to a hive. +fn validate_container_system_name(name: &str) -> Result<&str> { + let Some(suffix) = name.strip_prefix(AGENT_PREFIX) else { + bail!("container name {name:?} is not managed by hive"); + }; + validate_name_chars(suffix)?; + Ok(suffix) +} + +/// `RemoveServiceDropin` — remove the container service's drop-in dir +/// if present (idempotent). +pub(crate) fn remove_service_dropin(container: &str) -> Result<(String, String)> { + validate_container_system_name(container)?; + let dir = format!("/run/systemd/system/container@{container}.service.d"); + if Path::new(&dir).exists() { + std::fs::remove_dir_all(&dir).with_context(|| format!("remove {dir}"))?; + } + Ok((String::new(), String::new())) +} + +/// `WriteResourceLimits` — drop the systemd resource settings into the +/// container service's drop-in dir, together with a +/// `ConditionPathIsDirectory=` guard on the agent's MCP runtime dir. +/// +/// Two different kinds of setting land in the same file. `MemoryMax=` / +/// `CPUQuota=` are hard caps that throttle even on an idle host; +/// `CPUWeight=` / `IOWeight=` are cgroup v2 relative shares that only +/// decide who yields *under contention*. A weight of `None` means "not +/// configured" and omits the line, so a hive-c0re built before the weights +/// existed keeps producing the old two-line drop-in. +/// +/// The condition causes systemd to *skip* (not *fail*) the unit when the +/// bind-mount source dir is absent — result is `condition`, which does not +/// increment the start-limit counter. This is belt-and-braces on top of +/// hive-c0re creating the dir in its start preamble: if it is missing at +/// start time, the unit idles rather than restart-looping into +/// `start-limit-hit`. +pub(crate) fn write_resource_limits( + container: &str, + memory_max: &str, + cpu_quota: &str, + cpu_weight: Option, + io_weight: Option, +) -> Result<(String, String)> { + let logical = validate_container_system_name(container)?; + let runtime_dir = format!("{AGENT_RUNTIME_ROOT}/{logical}"); + let dir = format!("/run/systemd/system/container@{container}.service.d"); + std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?; + let path = format!("{dir}/hyperhive-limits.conf"); + let content = limits_dropin_body(&runtime_dir, memory_max, cpu_quota, cpu_weight, io_weight); + // Published whole: a `daemon-reload` between truncate and write would + // otherwise load the unit with its limits and start guard missing. + publish_file(Path::new(&path), content.as_bytes(), 0o644, None)?; + Ok((String::new(), String::new())) +} + +/// How long a window the start-limit counts over, and how many starts it +/// allows inside it. +/// +/// `container@.service` sets `Restart=on-failure` and **no** start limit, so +/// systemd's defaults apply: 5 starts per 10s, `RestartSec` 100ms. That +/// makes the bound depend on *how fast* a container dies — one that fails +/// instantly trips the limit in under a second, one that takes longer than +/// ~2s never trips it and restarts forever. Whether an agent gets bounded +/// is not meant to be a function of its failure speed. +/// +/// The window has to exceed the worst-case time to burn the burst, or the +/// counter ages out between attempts and the limit is again unreachable: +/// `TimeoutStartSec` is 1min, so `BURST` slow failures plus their backoff +/// can span several minutes. 10min covers that with room. +/// +/// Giving up is cheap here **because it is not terminal** — hive-c0re's +/// reconcile sweep retries later, and `reset-failed` (see `StartContainer`) +/// clears the latch first. That is what makes a tight burst safe. +const START_LIMIT_INTERVAL_SEC: u32 = 600; +/// One start plus two retries — the operator's ruling was "retry once or +/// twice", with the reconcile sweep as the slow path after that. +const START_LIMIT_BURST: u32 = 3; +/// Backoff between those retries. The 100ms default is for processes that +/// respawn instantly; a container that just failed to boot gains nothing +/// from being retried a tenth of a second later. +const RESTART_SEC: u32 = 5; + +/// Render the body of `hyperhive-limits.conf`. +/// +/// `[Unit]`: the condition is checked at start time — it skips (not fails) +/// the unit when the MCP socket dir is absent, avoiding restart loops — +/// plus the bounded start limit (see the constants above; `StartLimit*` are +/// `[Unit]` settings since systemd 229 and are silently ignored under +/// `[Service]`). +/// `[Service]`: the restart backoff, the hard caps, then the relative +/// weights. A weight of `None` means "not configured" and omits its line +/// entirely, so a request from a hive-c0re built before the weights +/// existed — or one whose nix option is `null` — renders no weight lines. +fn limits_dropin_body( + runtime_dir: &str, + memory_max: &str, + cpu_quota: &str, + cpu_weight: Option, + io_weight: Option, +) -> String { + // Built as two possibly-empty lines rather than pushed onto the + // string: `format!` appended to a `String` trips clippy::pedantic's + // `format_push_string`, and a `write!` would need an unwrap. + let cpu_weight_line = cpu_weight.map_or_else(String::new, |w| format!("CPUWeight={w}\n")); + let io_weight_line = io_weight.map_or_else(String::new, |w| format!("IOWeight={w}\n")); + format!( + "[Unit]\n\ + ConditionPathIsDirectory={runtime_dir}\n\ + StartLimitIntervalSec={START_LIMIT_INTERVAL_SEC}\n\ + StartLimitBurst={START_LIMIT_BURST}\n\ + \n\ + [Service]\n\ + RestartSec={RESTART_SEC}\n\ + MemoryMax={memory_max}\n\ + CPUQuota={cpu_quota}\n\ + {cpu_weight_line}{io_weight_line}" + ) +} + +/// `DaemonReload` — `systemctl daemon-reload` on the host. +pub(crate) async fn daemon_reload() -> Result<(String, String)> { + let out = Command::new("systemctl") + .arg("daemon-reload") + .output() + .await + .context("invoke systemctl daemon-reload")?; + if !out.status.success() { + bail!( + "systemctl daemon-reload failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + Ok((String::new(), String::new())) +} + +/// Publish `content` at `path` through a [`StagedFile`], at `mode` and, when +/// given, owned by `owner` (`(uid, gid)`). +fn publish_file(path: &Path, content: &[u8], mode: u32, owner: Option<(u32, u32)>) -> Result<()> { + let (Some(dir_path), Some(filename)) = + (path.parent(), path.file_name().and_then(|n| n.to_str())) + else { + bail!( + "publish_file: {} has no directory or file name", + path.display() + ); + }; + let dir = open_dir(dir_path)?; + let staged = StagedFile::write(&dir, dir_path, filename, content, mode)?; + if let Some((uid, gid)) = owner { + std::os::unix::fs::fchown(staged.file(), Some(uid), Some(gid)) + .with_context(|| format!("chown {}", path.display()))?; + } + staged.publish() +} + +/// Validate a bind-mount path: must be absolute, non-empty, and contain +/// no newlines, null bytes, double-quotes, or colons. +/// +/// The first three would break the `EXTRA_NSPAWN_FLAGS="..."` conf line +/// format. The colon is the one that matters most and had been left out +/// of this list: `--bind=SRC:DST` is colon-separated, so a path carrying +/// one does not corrupt the line — it silently becomes a *different +/// mount* than the caller asked for. +fn validate_bind_path(path: &str) -> Result<()> { + if path.is_empty() + || !path.starts_with('/') + || path + .bytes() + .any(|b| b == 0 || b == b'\n' || b == b'"' || b == b':') + { + bail!( + "invalid bind path {path:?}: must be an absolute path with no colons, newlines, null bytes, or double-quotes" + ); + } + Ok(()) +} + +/// `--tmpfs=/.git` for every bound git repo, hiding its metadata +/// from inside the container. +/// +/// Two kinds of mount qualify, for one reason: **the agent is given a +/// working tree, never a repository.** `/knowledge` is the hive's shared +/// docs — whose `.git/config` has held a credential the host-side worker +/// embedded — and `/agents//config` is a config repo, an agent's own +/// or a parent's read-only view of a child's. In both cases `.git` carries +/// every branch and the full history of a document whose *currently +/// deployed* value is the only thing a reader may act on, and an abandoned +/// branch is indistinguishable from a live one. +/// +/// An overlay rather than an exported copy: there is no second tree to +/// keep in sync, so nothing can go stale, and no code path has to remember +/// to refresh it. +/// +/// ⚠️ Ordering matters — these must be appended **after** the `--bind` +/// flags so nspawn mounts them on top of the already-mounted trees. +/// Config mounts are matched by shape, not by a name list: the set is +/// dynamic, growing with each child bound into a parent. +fn git_overlay_flags(binds: &[BindMount]) -> Vec { + binds + .iter() + .map(|b| b.container_path.as_str()) + .filter(|p| *p == "/knowledge" || (p.starts_with("/agents/") && p.ends_with("/config"))) + .map(|p| format!("--tmpfs={p}/.git")) + .collect() +} + +/// Update `/etc/nixos-containers/.conf`: strip old network vars +/// (`PRIVATE_NETWORK`, `HOST_ADDRESS*`, `LOCAL_ADDRESS*`, `HOST_BRIDGE`), +/// write the current network-isolation settings, then append +/// `EXTRA_NSPAWN_FLAGS`. Always writes `PRIVATE_NETWORK=1` + veth +/// wiring — isolation is the only mode, so there is no branch that +/// leaves a container on the host's network namespace. +fn write_nspawn_flags( + container: &str, + binds: &[BindMount], + isolation: &NetworkIsolation, + load_credentials: &[CredentialMount], + hive_facts: &str, +) -> Result<()> { + use std::fmt::Write as _; + use std::os::unix::fs::MetadataExt as _; + let path = format!("/etc/nixos-containers/{container}.conf"); + let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?; + let lines: Vec<&str> = original + .lines() + .filter(|line| { + let t = line.trim_start(); + !t.starts_with("EXTRA_NSPAWN_FLAGS=") + && !t.starts_with("PRIVATE_NETWORK=") + && !t.starts_with("HOST_ADDRESS=") + && !t.starts_with("LOCAL_ADDRESS=") + && !t.starts_with("HOST_ADDRESS6=") + && !t.starts_with("LOCAL_ADDRESS6=") + && !t.starts_with("HOST_BRIDGE=") + }) + .collect(); + let mut out = lines.join("\n"); + if !out.is_empty() { + out.push('\n'); + } + { + let iso = isolation; + out.push_str("PRIVATE_NETWORK=1\n"); + // HOST_ADDRESS = the bridge gateway IP. nixos-container's + // container-side setup only installs a default route + // (`ip route add default via $HOST_ADDRESS`) when HOST_ADDRESS is + // non-empty; leaving it blank gave the container an address but no + // route off the bridge subnet (no internet, no api.anthropic.com). + // In bridge mode (HOST_BRIDGE set) the host-side address/route + // setup is skipped, so this only affects the container's route — + // exactly what we want. + let _ = writeln!(out, "HOST_ADDRESS={}", iso.gateway_ip); + // LOCAL_ADDRESS is intentionally empty: containers take their IP by + // DHCP from the bridge dnsmasq pool. (Why HOST_ADDRESS is still + // written: the comment directly above.) + out.push_str("LOCAL_ADDRESS=\n"); + out.push_str("HOST_ADDRESS6=\n"); + out.push_str("LOCAL_ADDRESS6=\n"); + let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge); + } + let mut flags: Vec = binds + .iter() + .map(|b| { + let flag = if b.read_only { "--bind-ro" } else { "--bind" }; + format!("{flag}={}:{}", b.host_path, b.container_path) + }) + .collect(); + flags.extend(git_overlay_flags(binds)); + // Credential forwarding: nspawn loads each host secret into the + // container's credential store under ``; inner units inherit it + // via `LoadCredential=`. Validated (name charset + bind-path + // rules) in handle_write_nspawn_flags above. + for cred in load_credentials { + flags.push(format!( + "--load-credential={}:{}", + cred.name, cred.host_path + )); + } + let flags_joined = flags.join(" "); + let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\""); + // Keeps the owner and mode `nixos-container` created the file with; this + // helper only rewrites its lines. + let meta = std::fs::metadata(&path).with_context(|| format!("stat {path}"))?; + publish_file( + Path::new(&path), + out.as_bytes(), + meta.mode() & 0o7777, + Some((meta.uid(), meta.gid())), + )?; + + // DNS marker for the in-container resolver oneshot: the oneshot only + // rewrites the container's resolv.conf when this marker exists, and the + // marker carries the gateway IP so the container need not re-derive it. + // Always written — every container is isolated, so there is no mode in + // which the marker should be absent. Why the rewrite is needed at all, + // and which unit does it: `docs/networking/network.md` § *How the isolated + // container gets its resolver*. + write_bridge_dns_marker(container, isolation)?; + // Read by the agent's environment generator at boot + // (`nix/agent-modules/hive-facts.nix`). + write_container_etc_file_in(&container_rootfs(container), HIVE_FACTS_FILE, hive_facts)?; + Ok(()) +} + +/// Filename of the in-container DNS marker, in the container's own `/etc`. +const BRIDGE_DNS_MARKER: &str = "hyperhive-bridge-dns"; + +/// Filename of the hive-facts environment file, in the container's own `/etc`. +const HIVE_FACTS_FILE: &str = "hyperhive-hive-facts"; + +/// The container's `/` on the host. +fn container_rootfs(container: &str) -> PathBuf { + PathBuf::from(format!("/var/lib/nixos-containers/{container}")) +} + +/// Render [`HiveFacts`] as the systemd environment file the agent loads, +/// one `KEY=value` line per present fact. +/// +/// A value is refused unless it is DNS-name characters only, so it can carry +/// neither a newline (a second assignment) nor anything an environment file +/// or a shell would interpret. +fn render_hive_facts(facts: &HiveFacts) -> Result { + use std::fmt::Write as _; + let mut out = String::new(); + for (key, value) in [ + ("HYPERHIVE_HIVE_NAME", &facts.name), + ("HYPERHIVE_HIVE_DOMAIN", &facts.domain), + ] { + let Some(value) = value else { continue }; + if value.is_empty() + || value.len() > 253 + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'_')) + { + bail!("invalid {key} value {value:?}: must be 1-253 chars of [A-Za-z0-9._-]"); + } + let _ = writeln!(out, "{key}={value}"); + } + Ok(out) +} + +/// Write the bridge-DNS marker the `hyperhive-isolated-dns` oneshot keys +/// off. The marker file contains just the gateway IP. Always written: +/// every container is isolated, so there is no host-netns case that +/// wants the marker absent. +fn write_bridge_dns_marker(container: &str, isolation: &NetworkIsolation) -> Result<()> { + write_bridge_dns_marker_in(&container_rootfs(container), &isolation.gateway_ip) +} + +/// Write `/etc/hyperhive-bridge-dns`; see [`write_container_etc_file_in`]. +fn write_bridge_dns_marker_in(rootfs: &Path, gateway_ip: &str) -> Result<()> { + write_container_etc_file_in(rootfs, BRIDGE_DNS_MARKER, &format!("{gateway_ip}\n")) +} + +/// Write `/etc/` without following a symlink. +/// +/// Root inside the container owns its `/etc`, and this write runs as host +/// root, where an absolute symlink planted in the container resolves against +/// the host's `/`. So this opens `etc` with `O_DIRECTORY|O_NOFOLLOW` (a +/// symlinked `etc` fails with `ELOOP`) and publishes the file through a +/// [`StagedFile`] relative to that fd: whatever the container planted at the +/// leaf, symlink or FIFO or device node, is replaced by the rename, never +/// written through or opened. +/// +/// On a fresh install the container's `/etc` may not exist yet (nixos-container +/// materialises the rootfs on the first start), so this creates it first; the +/// dir and the file persist through that start. Mode 0644: nothing written +/// here is a secret. +fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &str) -> Result<()> { + use std::os::unix::fs::OpenOptionsExt as _; + + let etc_path = rootfs.join("etc"); + // The rootfs dir is the container's `/`, which the container cannot + // replace, so creating it by path is safe. + std::fs::create_dir_all(rootfs) + .with_context(|| format!("create container rootfs {}", rootfs.display()))?; + let root = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) + .open(rootfs) + .with_context(|| format!("open container rootfs {}", rootfs.display()))?; + // SAFETY: `root` is an open directory fd and the name a NUL-terminated + // literal. + if unsafe { libc::mkdirat(root.as_raw_fd(), c"etc".as_ptr(), 0o755) } != 0 { + let e = std::io::Error::last_os_error(); + if e.kind() != std::io::ErrorKind::AlreadyExists { + return Err(e).with_context(|| format!("create {}", etc_path.display())); + } + } + // SAFETY: as above; this checks the result before wrapping it. + let etc_fd = unsafe { + libc::openat( + root.as_raw_fd(), + c"etc".as_ptr(), + libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC, + ) + }; + if etc_fd < 0 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("open (no-follow) {}", etc_path.display())); + } + // SAFETY: `etc_fd` is a fresh, valid fd that nothing else owns. + let etc = std::fs::File::from(unsafe { OwnedFd::from_raw_fd(etc_fd) }); + StagedFile::write(&etc, &etc_path, name, contents.as_bytes(), 0o644)?.publish() +} +#[cfg(test)] +mod tests { + use super::{ + BindMount, HiveFacts, NetworkIsolation, git_overlay_flags, handle_write_nspawn_flags, + limits_dropin_body, publish_file, remove_service_dropin, render_hive_facts, + validate_container_system_name, write_bridge_dns_marker_in, write_resource_limits, + }; + use crate::shared::test_support::{entries, scratch}; + + fn bind(container_path: &str) -> BindMount { + BindMount { + host_path: "/var/lib/hyperhive/whatever".to_owned(), + container_path: container_path.to_owned(), + read_only: true, + } + } + + /// Every bound git repo gets its `.git` overlaid — the knowledge tree + /// and *each* config mount, an agent's own plus every child's. + /// + /// The child case is the one worth pinning: that set grows at runtime + /// as agents gain children, so a rule written as a list of names would + /// silently stop covering new ones. + #[test] + fn every_bound_git_repo_gets_its_dot_git_hidden() { + let flags = git_overlay_flags(&[ + bind("/knowledge"), + bind("/agents/atlas/config"), + bind("/agents/kiddo/config"), + ]); + assert_eq!( + flags, + [ + "--tmpfs=/knowledge/.git", + "--tmpfs=/agents/atlas/config/.git", + "--tmpfs=/agents/kiddo/config/.git", + ] + ); + } + + /// ...and nothing else does. A blanket "overlay .git on every bind" + /// would mask a real `.git` under `state/`, where an agent legitimately + /// keeps working clones of its own. + #[test] + fn non_repo_mounts_are_left_alone() { + let flags = git_overlay_flags(&[ + bind("/agents/atlas/state"), + bind("/shared"), + bind("/applied"), + bind("/agents/atlas/config-notes"), + ]); + assert!(flags.is_empty(), "overlaid a non-repo mount: {flags:?}"); + } + + /// The whole rendered body, pinned literally — including the values, + /// so changing the restart policy is a visible test edit rather than a + /// silent one. + /// + /// Placement is the part most worth pinning: `StartLimit*` are `[Unit]` + /// settings and systemd **silently ignores** them under `[Service]`, so + /// a bound that moved sections would look configured and do nothing. + /// An unset weight still emits no line at all, which is what keeps a + /// hive-c0re older than that field from changing what lands on disk. + #[test] + fn dropin_body_is_pinned_exactly() { + assert_eq!( + limits_dropin_body("/run/hyperhive/agents/iris", "4G", "200%", None, None), + "[Unit]\n\ + ConditionPathIsDirectory=/run/hyperhive/agents/iris\n\ + StartLimitIntervalSec=600\n\ + StartLimitBurst=3\n\ + \n\ + [Service]\n\ + RestartSec=5\n\ + MemoryMax=4G\n\ + CPUQuota=200%\n" + ); + } + + /// The bound is hive-wide **policy**, not a per-agent parameter: it is + /// rendered from constants and no caller-supplied value can omit or + /// alter it. This is the property that justifies keeping it out of the + /// wire protocol — if it ever varies by request, that argument is gone. + #[test] + fn the_start_limit_is_present_whatever_the_caller_passes() { + for (mem, cpu, cw, iw) in [ + ("4G", "200%", None, None), + ("512M", "50%", Some(10), Some(10)), + ("infinity", "infinity", Some(10_000), None), + ] { + let body = limits_dropin_body("/rt/x", mem, cpu, cw, iw); + let unit = body + .split("[Service]") + .next() + .expect("the body always has a [Unit] section before [Service]"); + assert!( + unit.contains("StartLimitIntervalSec=600") && unit.contains("StartLimitBurst=3"), + "start limit missing from [Unit] for ({mem}, {cpu}): {body}" + ); + assert!( + body.contains("\nRestartSec=5\n"), + "restart backoff missing for ({mem}, {cpu}): {body}" + ); + } + } + + /// Weights are appended to the `[Service]` section, each omitted + /// independently when `None`. + #[test] + fn weights_are_emitted_only_when_set() { + let both = limits_dropin_body("/rt/x", "4G", "200%", Some(80), Some(80)); + assert!( + both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"), + "{both}" + ); + + let cpu_only = limits_dropin_body("/rt/x", "4G", "200%", Some(80), None); + assert!( + cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"), + "{cpu_only}" + ); + assert!(!cpu_only.contains("IOWeight"), "{cpu_only}"); + + let io_only = limits_dropin_body("/rt/x", "4G", "200%", None, Some(80)); + assert!( + io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"), + "{io_only}" + ); + assert!(!io_only.contains("CPUWeight"), "{io_only}"); + } + + /// The mode lands exactly as asked, whatever the umask would have made + /// of it (0666 under the usual 022 would otherwise come out 0644), and + /// the requested owner is on the file. + #[test] + fn publish_file_applies_mode_and_owner() { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + let dir = scratch(); + let path = dir.join("c.conf"); + let owner = std::fs::metadata(&dir).unwrap(); + + publish_file(&path, b"x\n", 0o666, Some((owner.uid(), owner.gid()))).unwrap(); + let meta = std::fs::metadata(&path).unwrap(); + assert_eq!(meta.permissions().mode() & 0o7777, 0o666); + assert_eq!((meta.uid(), meta.gid()), (owner.uid(), owner.gid())); + + publish_file(&path, b"y\n", 0o600, None).unwrap(); + let meta = std::fs::metadata(&path).unwrap(); + assert_eq!(meta.permissions().mode() & 0o7777, 0o600); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "y\n"); + assert_eq!(entries(&dir), ["c.conf"], "temp left behind"); + std::fs::remove_dir_all(&dir).ok(); + } + + /// The container plants a symlink where the bridge-DNS marker goes; the + /// rename replaces it without writing through it. + #[test] + fn bridge_dns_marker_replaces_symlink_leaf() { + let dir = scratch(); + let rootfs = dir.join("rootfs"); + std::fs::create_dir_all(rootfs.join("etc")).unwrap(); + let target = dir.join("target"); + std::fs::write(&target, "original").unwrap(); + let leaf = rootfs.join("etc/hyperhive-bridge-dns"); + std::os::unix::fs::symlink(&target, &leaf).unwrap(); + + write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap(); + assert_eq!( + std::fs::read_to_string(&target).unwrap(), + "original", + "symlink target must be untouched" + ); + assert!( + std::fs::symlink_metadata(&leaf) + .unwrap() + .file_type() + .is_file() + ); + assert_eq!(std::fs::read_to_string(&leaf).unwrap(), "10.0.0.1\n"); + std::fs::remove_dir_all(&dir).ok(); + } + + /// The container replaces its whole `/etc` with a symlink, so the leaf + /// name resolves inside a directory of its choosing. + #[test] + fn bridge_dns_marker_refuses_symlink_etc() { + let dir = scratch(); + let rootfs = dir.join("rootfs"); + std::fs::create_dir_all(&rootfs).unwrap(); + let elsewhere = dir.join("elsewhere"); + std::fs::create_dir_all(&elsewhere).unwrap(); + let target = elsewhere.join("hyperhive-bridge-dns"); + std::fs::write(&target, "original").unwrap(); + std::os::unix::fs::symlink(&elsewhere, rootfs.join("etc")).unwrap(); + + let res = write_bridge_dns_marker_in(&rootfs, "10.0.0.1"); + assert!(res.is_err(), "O_NOFOLLOW must refuse a symlinked etc"); + assert_eq!( + std::fs::read_to_string(&target).unwrap(), + "original", + "file behind the symlinked etc must be untouched" + ); + std::fs::remove_dir_all(&dir).ok(); + } + + /// A FIFO at the leaf is replaced, never opened, so it cannot block the + /// helper. + #[test] + fn bridge_dns_marker_replaces_fifo_leaf() { + use std::os::unix::ffi::OsStrExt as _; + let dir = scratch(); + let rootfs = dir.join("rootfs"); + std::fs::create_dir_all(rootfs.join("etc")).unwrap(); + let fifo = rootfs.join("etc/hyperhive-bridge-dns"); + let c_fifo = std::ffi::CString::new(fifo.as_os_str().as_bytes()).unwrap(); + // SAFETY: `c_fifo` is a valid NUL-terminated path. + assert_eq!(unsafe { libc::mkfifo(c_fifo.as_ptr(), 0o644) }, 0); + + write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap(); + assert!( + std::fs::symlink_metadata(&fifo) + .unwrap() + .file_type() + .is_file() + ); + assert_eq!(std::fs::read_to_string(&fifo).unwrap(), "10.0.0.1\n"); + std::fs::remove_dir_all(&dir).ok(); + } + + /// The control: a plain rootfs with no `etc` yet (a fresh install) gets + /// the dir and a 0644 marker, and a second write replaces it. + #[test] + fn bridge_dns_marker_written_into_plain_etc() { + use std::os::unix::fs::PermissionsExt as _; + let dir = scratch(); + let rootfs = dir.join("rootfs"); + std::fs::create_dir_all(&rootfs).unwrap(); + + write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap(); + let path = rootfs.join("etc/hyperhive-bridge-dns"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "10.0.0.1\n"); + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o644, "marker must stay readable in the container"); + + write_bridge_dns_marker_in(&rootfs, "10.9.8.7").unwrap(); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "10.9.8.7\n"); + std::fs::remove_dir_all(&dir).ok(); + } + + /// Each present fact is one environment-file line; an absent one writes + /// nothing, so the agent sees the variable unset rather than empty. + #[test] + fn hive_facts_render_one_line_per_present_fact() { + let both = HiveFacts { + name: Some("pr1ma".to_owned()), + domain: Some("pr1ma.darkest.space".to_owned()), + }; + assert_eq!( + render_hive_facts(&both).unwrap(), + "HYPERHIVE_HIVE_NAME=pr1ma\nHYPERHIVE_HIVE_DOMAIN=pr1ma.darkest.space\n" + ); + let domain_only = HiveFacts { + name: None, + domain: Some("h1.example.test".to_owned()), + }; + assert_eq!( + render_hive_facts(&domain_only).unwrap(), + "HYPERHIVE_HIVE_DOMAIN=h1.example.test\n" + ); + assert_eq!(render_hive_facts(&HiveFacts::default()).unwrap(), ""); + } + + /// A value that could end its line or be read as syntax is refused, not + /// escaped: a newline would add an assignment of the caller's choosing. + #[test] + fn hive_facts_refuse_values_outside_dns_characters() { + for bad in ["", "pr1ma\nLD_PRELOAD=/x", "a b", "a\"b", "$(x)"] { + let facts = HiveFacts { + name: Some(bad.to_owned()), + domain: None, + }; + assert!( + render_hive_facts(&facts).is_err(), + "{bad:?} must be refused" + ); + } + } + + /// The swarm's forge and matrix containers are not the hive's: every + /// request that names a container as a string refuses them before + /// touching the filesystem. Control: an agent container name passes. + #[test] + fn container_name_requests_refuse_swarm_service_containers() { + let isolation = NetworkIsolation { + bridge: "hive0".to_owned(), + gateway_ip: "10.42.0.1".to_owned(), + }; + let facts = HiveFacts { + name: None, + domain: None, + }; + for name in ["hive-forge", "hive-matrix"] { + let errs = [ + handle_write_nspawn_flags(name, &[], &isolation, &[], &facts).unwrap_err(), + write_resource_limits(name, "4G", "200%", None, None).unwrap_err(), + remove_service_dropin(name).unwrap_err(), + ]; + for e in errs { + assert!( + e.to_string().contains("is not managed by hive"), + "{name}: {e:#}" + ); + } + } + assert_eq!(validate_container_system_name("h-atlas").unwrap(), "atlas"); + } +} diff --git a/hive-priv/src/files.rs b/hive-priv/src/files.rs new file mode 100644 index 00000000..fb9a3b01 --- /dev/null +++ b/hive-priv/src/files.rs @@ -0,0 +1,491 @@ +//! Root-side file ops in agent-owned or agent-reachable directories: +//! `SetAgentPaused` (the pause marker in the agent's `harness/`), +//! `EnsureAgentSocketDir` (`/run/hive-agent/`) and the legacy +//! `SyncAgentTmpfiles`, which only removes the tmpfiles file older builds +//! wrote. + +use std::os::fd::AsRawFd as _; +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, Result, bail}; +use hive_priv_sock::{AGENT_STATE_ROOT, PAUSED_MARKER_FILE}; + +use crate::shared::{StagedFile, ensure_plain_filename, open_dir, validate_agent_name}; + +/// Root of the per-agent unix-socket dirs on the host. +const SOCKET_DIR_ROOT: &str = "/run/hive-agent"; + +/// `SetAgentPaused`. +pub(crate) fn exec_set_agent_paused(agent_name: &str, paused: bool) -> Result<(String, String)> { + validate_agent_name(agent_name)?; + set_agent_paused(agent_name, paused) +} + +/// Create or remove an agent's pause marker under its harness dir. The +/// marker is written empty and chowned to the harness dir's owner (the +/// agent), matching how the harness itself would have created it. +/// +/// Both directions are idempotent: re-pausing truncates the existing empty +/// marker rather than failing, and a `NotFound` on removal is the +/// already-resumed case, not an error. +fn set_agent_paused(agent_name: &str, paused: bool) -> Result<(String, String)> { + let harness_dir = PathBuf::from(AGENT_STATE_ROOT) + .join(agent_name) + .join("harness"); + if paused { + return write_agent_dir_file(agent_name, &harness_dir, PAUSED_MARKER_FILE, ""); + } + remove_marker_in(&harness_dir, PAUSED_MARKER_FILE)?; + tracing::info!(agent = %agent_name, "cleared pause marker"); + Ok((String::new(), String::new())) +} + +/// Unlink `dir/filename`, treating "already gone" as success. +/// +/// `remove_file` unlinks the leaf itself and never follows a symlink, so an +/// agent-planted link at the marker path cannot redirect this root unlink +/// at another file — the same threat [`StagedFile`] closes on the create side. +fn remove_marker_in(dir: &Path, filename: &str) -> Result<()> { + let path = dir.join(filename); + match std::fs::remove_file(&path) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).with_context(|| format!("remove {}", path.display())), + } +} + +/// Write `content` to `dir/filename` as root, chowning the result to `dir`'s +/// owner so the agent process can read it back. Its caller is the pause +/// marker, in `harness/`, a directory owned by the agent, which is precisely +/// why it needs hive-priv at all. +/// +/// The file is published through a [`StagedFile`], so a reader woken by its +/// appearance cannot catch it empty or half-written: several of these paths +/// have a `systemd.path` unit watching them, and one of those triggers on the +/// file existing rather than changing. +fn write_agent_dir_file( + agent_name: &str, + dir: &Path, + filename: &str, + content: &str, +) -> Result<(String, String)> { + use std::os::fd::AsRawFd as _; + use std::os::unix::fs::MetadataExt as _; + + // Checked before `create_dir_all`, so a bad leaf does no filesystem work. + ensure_plain_filename("write_agent_dir_file", filename)?; + + let state_dir = dir.to_path_buf(); + // NOTE: `create_dir_all` is normally a no-op — lifecycle creates and chowns + // the state dir during spawn. On the rare edge where the dir doesn't exist + // yet (container being provisioned for the first time), the newly created dir + // is root:root. The `stat state_dir` chown below will then see uid=0 and + // leave the file root-owned (0600). The agent won't be able to read it until + // its lifecycle completes. + std::fs::create_dir_all(&state_dir) + .with_context(|| format!("create state dir {}", state_dir.display()))?; + + // Security-critical: the (state/-owning) agent may plant a symlink or + // hardlink in this dir, and `StagedFile` neither follows nor reuses one, + // so this root-privileged create/write/chmod/chown can't be redirected at + // an arbitrary file. + let path = state_dir.join(filename); + let dir_fd = open_dir(&state_dir)?; + let staged = StagedFile::write(&dir_fd, &state_dir, filename, content.as_bytes(), 0o600)?; + + // Chown to the state dir's owner so the agent process can read the file. + // fchown on the temp's fd — TOCTOU-immune (the inode the write hit, never a + // swapped path). If stat fails (e.g. dir just created, owner is root), the + // file stays root-owned and 0600 — still unreadable by others, just not + // agent-readable. Log a warning so operators can diagnose. + match dir_fd.metadata() { + Ok(meta) => { + // SAFETY: the temp's fd is open and owned by `staged` for the whole + // call; `fchown` only mutates that inode's uid/gid. + let rc = unsafe { libc::fchown(staged.file().as_raw_fd(), meta.uid(), meta.gid()) }; + if rc != 0 { + let e = std::io::Error::last_os_error(); + tracing::warn!( + agent = %agent_name, + path = %path.display(), + error = %e, + "write_agent_dir_file: fchown failed" + ); + } + } + Err(e) => { + tracing::warn!( + agent = %agent_name, + error = %e, + "write_agent_dir_file: stat state_dir failed, leaving file root-owned" + ); + } + } + // After the chown, never before: the file must never be visible under its + // final name while still root-owned. A failed publish drops `staged`, + // which removes the temp and the credential in it. + staged.publish()?; + + tracing::info!( + agent = %agent_name, + dir = %state_dir.display(), + file = %filename, + "wrote agent file" + ); + Ok((String::new(), String::new())) +} + +/// Written by older hive-priv builds. Nothing writes it now, but left on disk +/// systemd-tmpfiles would still apply it at every boot. +const LEGACY_TMPFILES_PATH: &str = "/etc/tmpfiles.d/hyperhive-agents.conf"; + +/// Unlink [`LEGACY_TMPFILES_PATH`]; already absent is success. Run at every +/// hive-priv start and by the legacy `SyncAgentTmpfiles` request. +pub(crate) fn remove_legacy_tmpfiles() -> Result<()> { + match std::fs::remove_file(LEGACY_TMPFILES_PATH) { + Ok(()) => { + tracing::info!("removed legacy {LEGACY_TMPFILES_PATH}"); + Ok(()) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).with_context(|| format!("remove {LEGACY_TMPFILES_PATH}")), + } +} + +/// `EnsureAgentSocketDir` — create `/run/hive-agent/` if missing. +pub(crate) fn ensure_agent_socket_dir(name: &str) -> Result<(String, String)> { + ensure_socket_dir_in(Path::new(SOCKET_DIR_ROOT), name)?; + Ok((String::new(), String::new())) +} + +/// Create `/` as a `0751` directory owned by the caller (root in +/// production), or accept an existing directory untouched. +/// `name` must be an agent ident; it is validated here, before any path is +/// built from it. +/// +/// The result is an nspawn bind source, so a symlink here would bind a host +/// path of the planter's choosing into the container. Every step is relative +/// to an `O_DIRECTORY|O_NOFOLLOW` fd for `root`, and an existing entry is +/// checked with `AT_SYMLINK_NOFOLLOW` and refused unless it is a directory. +/// +/// An existing directory keeps its owner and mode: the container's activation +/// hands it to the agent user, and re-asserting root here would undo that on +/// every start. +fn ensure_socket_dir_in(root: &Path, name: &str) -> Result<()> { + use std::os::unix::fs::OpenOptionsExt as _; + + validate_agent_name(name)?; + let path = root.join(name); + let c_name = std::ffi::CString::new(name) + .with_context(|| format!("agent name {name:?} contains a NUL byte"))?; + let root_dir = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(root) + .with_context(|| format!("open (no-follow) {}", root.display()))?; + // SAFETY: `root_dir` is an open directory fd and `c_name` NUL-terminated. + if unsafe { libc::mkdirat(root_dir.as_raw_fd(), c_name.as_ptr(), 0o751) } == 0 { + // mkdirat's mode is masked by the umask; the mode is part of the + // contract, so it is set explicitly. + // SAFETY: as above. + let rc = unsafe { + libc::fchmodat( + root_dir.as_raw_fd(), + c_name.as_ptr(), + 0o751, + libc::AT_SYMLINK_NOFOLLOW, + ) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("chmod 0751 {}", path.display())); + } + tracing::info!(path = %path.display(), "created agent socket dir"); + return Ok(()); + } + let err = std::io::Error::last_os_error(); + if err.kind() != std::io::ErrorKind::AlreadyExists { + return Err(err).with_context(|| format!("create {}", path.display())); + } + // SAFETY: `stat` is plain old data; the zeroed value is only read after + // `fstatat` has filled it in. + let mut st: libc::stat = unsafe { std::mem::zeroed() }; + // SAFETY: as for `mkdirat`; `st` is a valid, writable `stat`. + let rc = unsafe { + libc::fstatat( + root_dir.as_raw_fd(), + c_name.as_ptr(), + &raw mut st, + libc::AT_SYMLINK_NOFOLLOW, + ) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("stat (no-follow) {}", path.display())); + } + if st.st_mode & libc::S_IFMT != libc::S_IFDIR { + bail!( + "{} exists but is not a directory; refusing it as a bind source", + path.display() + ); + } + Ok(()) +} +#[cfg(test)] +mod tests { + use super::{PAUSED_MARKER_FILE, ensure_socket_dir_in, remove_marker_in, write_agent_dir_file}; + use crate::shared::test_support::{entries, scratch}; + use crate::shared::{StagedFile, open_dir}; + use std::path::PathBuf; + + #[test] + fn rejects_non_plain_filenames() { + let dir = scratch(); + let dir_fd = open_dir(&dir).unwrap(); + for bad in ["", ".", "..", "a/b", "/etc/passwd", "../escape", "sub/tok"] { + assert!( + StagedFile::write(&dir_fd, &dir, bad, b"x", 0o600).is_err(), + "must reject filename {bad:?}" + ); + // The wrapper's own check runs before its `create_dir_all`. + // Pointing it at a directory that does not exist yet is what makes + // this arm bite: the property worth pinning is that a bad leaf + // fails BEFORE any root-privileged filesystem work. + let absent = dir.join("never-created"); + assert!( + write_agent_dir_file("a", &absent, bad, "x").is_err(), + "wrapper must reject filename {bad:?}" + ); + assert!( + !absent.exists(), + "rejected write created a directory for {bad:?}" + ); + } + std::fs::remove_dir_all(&dir).ok(); + } + + /// A published file arrives complete and alone. The residue arm is the + /// load-bearing one: a stranded temp holds the same secret, is owned by + /// root, and nothing else in the system would ever remove it. + #[test] + fn publishing_leaves_only_the_finished_file() { + use std::os::unix::fs::PermissionsExt as _; + let dir = scratch(); + let names = |d: &PathBuf| { + let mut v: Vec = std::fs::read_dir(d) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + v.sort(); + v + }; + assert!(names(&dir).is_empty(), "control: scratch starts empty"); + + let path = dir.join("matrix-token-alice"); + for content in ["SECRET", "ROTATED"] { + write_agent_dir_file("a", &dir, "matrix-token-alice", content).unwrap(); + assert_eq!(names(&dir), ["matrix-token-alice"], "temp left behind"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), content); + } + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "credential must be 0600 once published"); + + std::fs::remove_dir_all(&dir).ok(); + } + + /// A failed publish must not strand the temp. + #[test] + fn a_failed_publish_removes_the_temp() { + let dir = scratch(); + // A directory at the destination fails the `rename` (EISDIR) without + // needing to drop privileges; the write and chown ahead of it still run. + std::fs::create_dir(dir.join("forge-token")).unwrap(); + + let err = write_agent_dir_file("a", &dir, "forge-token", "SECRET") + .expect_err("publishing onto a directory must fail"); + assert!( + format!("{err:#}").contains("publishing"), + "must fail AT the publish -- failing earlier leaves no temp to \ + clean up and makes the next assertion vacuous. got: {err:#}" + ); + assert_eq!( + entries(&dir), + ["forge-token"], + "temp still holds the secret after a failed publish" + ); + + std::fs::remove_dir_all(&dir).ok(); + } + + /// The agent plants a symlink where the token goes. The rename replaces + /// the link; the root-privileged write never reaches its target. + #[test] + fn a_symlink_at_the_name_is_replaced_not_followed() { + let dir = scratch(); + let target = dir.join("target"); + std::fs::write(&target, "original").unwrap(); + std::os::unix::fs::symlink(&target, dir.join("forge-token")).unwrap(); + + write_agent_dir_file("a", &dir, "forge-token", "SECRET").unwrap(); + assert_eq!( + std::fs::read_to_string(&target).unwrap(), + "original", + "symlink target must be untouched" + ); + let meta = std::fs::symlink_metadata(dir.join("forge-token")).unwrap(); + assert!(meta.file_type().is_file(), "the link must be replaced"); + assert_eq!( + std::fs::read_to_string(dir.join("forge-token")).unwrap(), + "SECRET" + ); + std::fs::remove_dir_all(&dir).ok(); + } + + /// A missing socket dir is created `0751`, owned by the caller (root in + /// production), which is the mode the container's activation expects to + /// hand over and the one dialers traverse. + #[test] + fn socket_dir_created_0751_when_absent() { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + let dir = scratch(); + ensure_socket_dir_in(&dir, "probe").unwrap(); + let meta = std::fs::symlink_metadata(dir.join("probe")).unwrap(); + assert!(meta.is_dir()); + assert_eq!(meta.permissions().mode() & 0o7777, 0o751); + // SAFETY: `geteuid` has no preconditions. + assert_eq!(meta.uid(), unsafe { libc::geteuid() }); + std::fs::remove_dir_all(&dir).ok(); + } + + /// An existing directory keeps its mode and contents: by the second start + /// it belongs to the agent user, and a live socket may sit inside it. + #[test] + fn socket_dir_existing_directory_left_alone() { + use std::os::unix::fs::PermissionsExt as _; + let dir = scratch(); + let sub = dir.join("probe"); + std::fs::create_dir(&sub).unwrap(); + std::fs::set_permissions(&sub, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::write(sub.join("web.sock"), "").unwrap(); + + ensure_socket_dir_in(&dir, "probe").unwrap(); + let mode = std::fs::metadata(&sub).unwrap().permissions().mode() & 0o7777; + assert_eq!(mode, 0o700, "an existing dir's mode must not be reset"); + assert!(sub.join("web.sock").exists()); + std::fs::remove_dir_all(&dir).ok(); + } + + /// A symlink at the leaf would make nspawn bind wherever it points; it is + /// refused, whether it points at a directory or nowhere. + #[test] + fn socket_dir_refuses_symlink_leaf() { + use std::os::unix::fs::PermissionsExt as _; + let dir = scratch(); + let elsewhere = dir.join("elsewhere"); + std::fs::create_dir(&elsewhere).unwrap(); + std::fs::set_permissions(&elsewhere, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::os::unix::fs::symlink(&elsewhere, dir.join("probe")).unwrap(); + std::os::unix::fs::symlink(dir.join("missing"), dir.join("dangling")).unwrap(); + + assert!(ensure_socket_dir_in(&dir, "probe").is_err()); + assert!(ensure_socket_dir_in(&dir, "dangling").is_err()); + let mode = std::fs::metadata(&elsewhere).unwrap().permissions().mode() & 0o7777; + assert_eq!(mode, 0o700, "the symlink target must be untouched"); + assert!(!dir.join("missing").exists()); + std::fs::remove_dir_all(&dir).ok(); + } + + /// A regular file where the dir should be is refused, not bound. + #[test] + fn socket_dir_refuses_non_directory_leaf() { + let dir = scratch(); + std::fs::write(dir.join("probe"), "x").unwrap(); + let err = ensure_socket_dir_in(&dir, "probe").unwrap_err(); + assert!(format!("{err:#}").contains("not a directory"), "{err:#}"); + std::fs::remove_dir_all(&dir).ok(); + } + + /// A symlinked root would place the new dir in a tree of the planter's + /// choosing; the `O_NOFOLLOW` open refuses it before anything is created. + #[test] + fn socket_dir_refuses_symlinked_root() { + let dir = scratch(); + let elsewhere = dir.join("elsewhere"); + std::fs::create_dir(&elsewhere).unwrap(); + let root = dir.join("root"); + std::os::unix::fs::symlink(&elsewhere, &root).unwrap(); + + assert!(ensure_socket_dir_in(&root, "probe").is_err()); + assert!(!elsewhere.join("probe").exists()); + std::fs::remove_dir_all(&dir).ok(); + } + + /// Names that are not a plain agent ident are refused by validation, + /// before anything is created. Runs against a scratch parent, so the + /// refusal cannot come from a missing `/run/hive-agent`; `.`/`..` exist + /// as directories and `Atlas` would be created, so each bad name gets + /// through only if validation is gone. Control: a plain name succeeds + /// in the same parent. + #[test] + fn socket_dir_refuses_bogus_names() { + let dir = scratch(); + for bad in ["", ".", "..", "../etc", "a/b", "Atlas", "a b", "a\0b"] { + let err = ensure_socket_dir_in(&dir, bad).unwrap_err(); + assert!( + format!("{err:#}").contains("invalid name"), + "agent name {bad:?} must be refused by validation, got: {err:#}" + ); + } + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0); + ensure_socket_dir_in(&dir, "atlas").unwrap(); + assert!(dir.join("atlas").is_dir()); + std::fs::remove_dir_all(&dir).ok(); + } + + /// The pause marker round-trips through the same root-only path the + /// credential writes use, and BOTH directions are idempotent — the + /// dashboard toggle and `hivectl pause|resume` fire blind, without + /// reading the current state first. + #[test] + fn pause_marker_create_and_remove_are_idempotent() { + let dir = scratch(); + let path = dir.join(PAUSED_MARKER_FILE); + + for _ in 0..2 { + write_agent_dir_file("a", &dir, PAUSED_MARKER_FILE, "").unwrap(); + assert!(path.exists(), "marker must exist after pause"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), ""); + } + for _ in 0..2 { + remove_marker_in(&dir, PAUSED_MARKER_FILE).unwrap(); + assert!(!path.exists(), "marker must be gone after resume"); + } + std::fs::remove_dir_all(&dir).ok(); + } + + /// A resume must never follow an agent-planted symlink at the marker + /// path: this unlink runs as root, so following it would let an agent + /// delete an arbitrary file on the host. + #[test] + fn resume_unlinks_the_symlink_not_its_target() { + let dir = scratch(); + let target = dir.join("target"); + std::fs::write(&target, "original").unwrap(); + let link = dir.join(PAUSED_MARKER_FILE); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + remove_marker_in(&dir, PAUSED_MARKER_FILE).unwrap(); + // `exists()` follows the link, so it can't tell "link removed" from + // "target removed, dangling link left" — stat the link itself. + assert!( + std::fs::symlink_metadata(&link).is_err(), + "the link itself must be unlinked" + ); + assert_eq!( + std::fs::read_to_string(&target).unwrap(), + "original", + "symlink target must survive the root unlink" + ); + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/hive-priv/src/infra.rs b/hive-priv/src/infra.rs new file mode 100644 index 00000000..d6f5186d --- /dev/null +++ b/hive-priv/src/infra.rs @@ -0,0 +1,447 @@ +//! Host infrastructure ops: `RunForgeAdmin` (`forgejo admin` inside the +//! `hive-forge` container), `ControlInfraContainer` (start/stop of an +//! [`InfraContainer`] unit) and `ReloadGatewayNginx`. None of them takes a +//! unit or container name from the caller. + +use anyhow::{Context as _, Result, bail}; +use hive_priv_sock::{InfraAction, InfraContainer}; +use tokio::process::Command; + +/// `RunForgeAdmin`: every arg must pass [`validate_forge_admin_arg`] before +/// the admin CLI ever sees it. +pub(crate) async fn exec_forge_admin(args: &[String]) -> Result<(String, String)> { + for arg in args { + validate_forge_admin_arg(arg)?; + } + run_forge_admin(args).await +} + +/// Validate a single argument destined for `forgejo admin`. Rejects null +/// bytes, newlines and carriage returns (which could corrupt the +/// subprocess args list or log output). +/// +/// Shell metacharacters are **not** rejected, and do not need to be: the +/// command is spawned directly, with no shell to interpret them. +fn validate_forge_admin_arg(arg: &str) -> Result<()> { + if arg.bytes().any(|b| b == 0 || b == b'\n' || b == b'\r') { + bail!("forge admin arg {arg:?} contains null byte or newline"); + } + Ok(()) +} + +/// Redact a line before it hits the (root-readable, but still +/// unnecessarily exposed) host journal. +/// +/// Two independent rules, because the previous single rule failed open. +/// It matched only the substring "password", chosen to be robust against +/// forgejo *rewording* its password line — and the leak arrived from the +/// other axis entirely: a **different kind of secret** on a differently +/// worded line. `forgejo admin user generate-access-token` prints +/// `Access token was successfully created: <40 hex>`, which contains no +/// "password" and went to the journal verbatim for every agent ever +/// provisioned. +/// +/// So the second rule matches on **shape, not vocabulary**: a long +/// unbroken run of secret-alphabet characters. A new secret type is then +/// caught by default rather than by someone remembering to add a keyword. +/// +/// ⚠️ This deliberately over-matches. A nix store hash is also a long +/// opaque run and will redact its line. That is the correct direction to +/// be wrong in: the cost of a false positive is one less log line, and +/// the cost of a false negative is a live credential sitting in the host +/// journal, readable by anything with access to it. +fn redact_secret_line(line: &str) -> std::borrow::Cow<'_, str> { + if line.to_ascii_lowercase().contains("password") { + return std::borrow::Cow::Borrowed("[redacted: line mentions a password]"); + } + if contains_secret_shaped_run(line) { + return std::borrow::Cow::Borrowed("[redacted: line contains a secret-shaped token]"); + } + std::borrow::Cow::Borrowed(line) +} + +/// True when the line contains an unbroken run of at least 32 characters +/// from the hex / base64url alphabet. 32 sits below forgejo's 40-hex +/// access token and above the ordinary words and path segments that +/// appear in `forgejo admin` output. +/// +/// ⚠️ Scans for a RUN, not for a whitespace-delimited word. An earlier +/// version split on whitespace and required the whole word to match, +/// which a secret with punctuation glued to it defeats: `","` and +/// `"[]"` both fail an all-chars check on the word while still +/// containing the credential in full. Whitespace is not what delimits a +/// secret — the alphabet is (thanks @argus for catching it). +fn contains_secret_shaped_run(line: &str) -> bool { + const MIN: usize = 32; + let mut run = 0usize; + for b in line.bytes() { + if b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=' | b'_' | b'-') { + run += 1; + if run >= MIN { + return true; + } + } else { + run = 0; + } + } + false +} + +/// Name a `forgejo admin` invocation for an error message without +/// reproducing its arguments: keep the leading verb path, stop at the +/// first flag. `["user", "change-password", "--username", "iris", +/// "--password", "…"]` becomes `forgejo admin user change-password`. +/// +/// Same allowlist as hive-c0re's `describe_forge_admin` +/// (`hive-c0re/src/forge/mod.rs`), for the same reason: the verbs are a +/// closed set this crate chooses itself, but hive-c0re passes a live +/// `--password` value as an argument on some calls, and argument values +/// are never safe to assume closed. Redacting the value after +/// `--password` instead would repeat the bug `redact_secret_line`'s own +/// doc comment describes — a denylist of one flag name fails open the +/// moment a differently named secret-bearing flag is added. +fn describe_forge_admin(args: &[String]) -> String { + let verbs: Vec<&str> = args + .iter() + .take_while(|a| !a.starts_with('-')) + .map(String::as_str) + .collect(); + if verbs.is_empty() { + "forgejo admin".to_owned() + } else { + format!("forgejo admin {}", verbs.join(" ")) + } +} + +/// Run `forgejo admin ` inside the `hive-forge` container as the +/// `forgejo` unix user. Requires root (for nsenter into the container's +/// namespaces). Returns `(stdout, stderr)`. +async fn run_forge_admin(args: &[String]) -> Result<(String, String)> { + let mut cmd_args: Vec<&str> = vec![ + "run", + "hive-forge", + "--", + "runuser", + "-u", + "forgejo", + "--", + "forgejo", + "--work-path", + "/var/lib/forgejo", + "admin", + ]; + for a in args { + cmd_args.push(a.as_str()); + } + let out = Command::new("nixos-container") + .args(&cmd_args) + .output() + .await + .context("invoke nixos-container run hive-forge -- forgejo admin")?; + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + // stdout at DEBUG, not INFO: on the success path this stream carries + // the *product* of the command (the freshly minted token, the created + // user's details) and nothing an operator needs at default verbosity. + // Redaction stays on as the second layer — the level decides who sees + // it, the redactor decides what it says, and neither alone is enough. + for line in stdout.lines() { + tracing::debug!(target: "forgejo-admin", "{}", redact_secret_line(line)); + } + for line in stderr.lines() { + tracing::warn!(target: "forgejo-admin", "{}", redact_secret_line(line)); + } + if !out.status.success() { + // Redact here too. The error string is propagated to the caller and + // ends up logged; a partial-failure stderr can carry the same + // material stdout would have. And name the invocation via + // `describe_forge_admin` rather than `args.join(" ")`: some callers + // pass a live `--password` value as an argument, so reproducing the + // raw vector here is the same "two of three sites" gap that makes + // these leaks survive a fix — the log was redacted, but the error + // was not. + let safe_stderr: String = stderr + .lines() + .map(|l| redact_secret_line(l).into_owned()) + .collect::>() + .join("; "); + bail!( + "{} failed ({}): {}", + describe_forge_admin(args), + out.status, + safe_stderr.trim() + ); + } + Ok((stdout, stderr)) +} + +/// `ControlInfraContainer` — start/stop a hive infrastructure service via +/// `systemctl `. The [`InfraContainer`] enum is the allowlist: +/// serde already rejected any unknown / unsafe name (hive-c0re, the forge +/// and the matrix have no variant) at deserialisation, so no root-side +/// `.contains()` check is needed here. Serves the hive-wide +/// `hivectl stop`/`start`/`restart` flow. No agent-facing path exists. +/// +/// ⚠️ The unit is derived from the variant, never sent by the caller — +/// which is what keeps this from being a general `systemctl` pass-through. +pub(crate) async fn control_infra_container( + container: InfraContainer, + action: InfraAction, +) -> Result<(String, String)> { + let verb = action.systemctl_verb(); + let unit = container.service_unit(); + let out = Command::new("systemctl") + .args([verb, unit]) + .output() + .await + .with_context(|| format!("systemctl {verb} {unit}"))?; + if !out.status.success() { + bail!( + "systemctl {verb} {unit} exited {}: {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + tracing::info!(target: "infra-control", "{verb} {unit}"); + Ok(( + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + )) +} + +/// Synchronise the host's nginx unit after an `agents.conf` write. +/// +/// Queries `ActiveState` and dispatches: +/// - `active` → `systemctl reload nginx` (SIGHUP, zero-downtime) +/// - `failed` → `systemctl reset-failed nginx` + `systemctl start nginx` +/// - otherwise → `systemctl start nginx` +/// +/// ⚠️ `nginx` is hard-coded on purpose — see `PrivRequest::ReloadGatewayNginx`. +/// The unit name is the scope of this verb: nginx is a host unit, so no +/// namespace bounds it and the literal is the only thing standing between +/// "reload the gateway" and "reload anything". +/// +/// Returns `(String::new(), String::new())` on success so it fits the +/// `exec` return type directly. +pub(crate) async fn sync_gateway_nginx() -> Result<(String, String)> { + let state_out = Command::new("systemctl") + .args(["show", "--property=ActiveState", "--value", "nginx"]) + .output() + .await + .context("query gateway nginx ActiveState")?; + if !state_out.status.success() { + tracing::warn!( + exit_code = ?state_out.status.code(), + stderr = %String::from_utf8_lossy(&state_out.stderr).trim(), + "systemctl show ActiveState exited non-zero — gateway nginx may be down" + ); + } + let state = String::from_utf8_lossy(&state_out.stdout).trim().to_owned(); + // State-aware dispatch: reload when running; reset+start after + // start-limit failure; plain start when inactive or unknown. + match state.as_str() { + "active" => { + let out = Command::new("systemctl") + .args(["reload", "nginx"]) + .output() + .await + .context("reload gateway nginx")?; + if !out.status.success() { + bail!( + "gateway nginx reload failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + } + "failed" => { + // Clear start-limit so the next start can proceed. + let _ = Command::new("systemctl") + .args(["reset-failed", "nginx"]) + .status() + .await; + let out = Command::new("systemctl") + .args(["start", "nginx"]) + .output() + .await + .context("start gateway nginx after reset-failed")?; + if !out.status.success() { + bail!( + "gateway nginx start (after reset-failed) failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + } + _ => { + // inactive, activating, deactivating, unknown — just start. + let out = Command::new("systemctl") + .args(["start", "nginx"]) + .output() + .await + .context("start gateway nginx")?; + if !out.status.success() { + bail!( + "gateway nginx start failed (state={state:?}) ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + } + } + Ok((String::new(), String::new())) +} +#[cfg(test)] +mod tests { + use super::{ + InfraAction, InfraContainer, contains_secret_shaped_run, describe_forge_admin, + redact_secret_line, + }; + use hive_priv_sock::PrivRequest; + + #[test] + fn redacts_lines_mentioning_password_case_insensitively() { + assert_eq!( + redact_secret_line("New password: hunter2"), + "[redacted: line mentions a password]" + ); + assert_eq!( + redact_secret_line("PASSWORD=hunter2"), + "[redacted: line mentions a password]" + ); + assert_eq!( + redact_secret_line("User \"foo\" was successfully created."), + "User \"foo\" was successfully created." + ); + } + + /// `run_forge_admin`'s `bail!` names the invocation through this + /// function instead of `args.join(" ")` — the regression this test + /// guards. A `--password` value never survives to the error string. + #[test] + fn describe_forge_admin_does_not_reproduce_a_password_argument() { + let described = describe_forge_admin(&[ + "user".to_owned(), + "create".to_owned(), + "--username".to_owned(), + "x".to_owned(), + "--password".to_owned(), + "S3cret".to_owned(), + ]); + assert_eq!(described, "forgejo admin user create"); + assert!(!described.contains("S3cret"), "{described}"); + } + + /// The `--password=value` single-argument form is also excluded: it + /// starts with `-`, so it never enters the kept verb path. + #[test] + fn describe_forge_admin_redacts_the_equals_sign_form_too() { + let described = describe_forge_admin(&[ + "user".to_owned(), + "create".to_owned(), + "--password=S3cret".to_owned(), + ]); + assert!(!described.contains("S3cret"), "{described}"); + } + + /// Control: a non-secret argument (the verb path) still appears, so + /// the message stays useful for diagnosing which call failed. + #[test] + fn describe_forge_admin_keeps_the_verb_path() { + assert_eq!( + describe_forge_admin(&["user".to_owned(), "change-password".to_owned()]), + "forgejo admin user change-password" + ); + assert_eq!( + describe_forge_admin(&["--help".to_owned()]), + "forgejo admin" + ); + assert_eq!(describe_forge_admin(&[]), "forgejo admin"); + } + + /// The regression this function exists for. The keyword rule passes + /// this line straight through — it says nothing about a password — so + /// only the shape rule catches it. + #[test] + fn redacts_access_token_line_which_mentions_no_password() { + let line = + "Access token was successfully created: 0123456789abcdef0123456789abcdef01234567"; + assert!( + !line.to_ascii_lowercase().contains("password"), + "fixture must not contain the keyword, or it proves nothing" + ); + assert_eq!( + redact_secret_line(line), + "[redacted: line contains a secret-shaped token]" + ); + } + + #[test] + fn secret_shape_boundaries() { + // Ordinary forgejo-admin output survives: no run is long enough. + assert_eq!( + redact_secret_line("Command 'user' 'create' finished with no errors."), + "Command 'user' 'create' finished with no errors." + ); + // 31 chars is below the floor, 32 is at it. + assert!(!contains_secret_shaped_run(&"a".repeat(31))); + assert!(contains_secret_shaped_run(&"a".repeat(32))); + // Punctuation breaks the run — a sentence never trips it however long. + assert!(!contains_secret_shaped_run( + "this.is.a.very.long.dotted.identifier.but.not.a.secret" + )); + // base64url and hex alphabets both count. + assert!(contains_secret_shaped_run( + "ZGVhZGJlZWZkZWFkYmVlZmRlYWRiZWVmZGVhZA==" + )); + assert!(contains_secret_shaped_run( + "aG93-dy_there-aG93dy1theresomething" + )); + } + + /// argus on the review: a whitespace-delimited check is defeated by + /// punctuation glued to the secret — the punctuation joins the "word" + /// and fails the alphabet test for the whole run, while the credential + /// sits there in full. Scanning for a RUN rather than a WORD closes it. + /// These are the shapes that used to slip through. + #[test] + fn secret_is_caught_with_punctuation_glued_to_it() { + const TOK: &str = "0123456789abcdef0123456789abcdef01234567"; + for line in [ + format!("token: {TOK},"), + format!("token: {TOK}."), + format!("using [{TOK}] now"), + format!("value=\"{TOK}\""), + format!("(created {TOK})"), + // no whitespace anywhere -- one glued blob + format!("Bearer:{TOK};next"), + ] { + assert_eq!( + redact_secret_line(&line), + "[redacted: line contains a secret-shaped token]", + "leaked through: {line}" + ); + } + } + + /// A start/stop naming the forge or matrix container fails to decode, so + /// `run` answers it with a parse error before anything reaches + /// `systemctl`. Control: the same request naming the gateway decodes. + #[test] + fn infra_control_refuses_swarm_services() { + let req = |c: &str| { + format!(r#"{{"op":"control_infra_container","container":"{c}","action":"stop"}}"#) + }; + for c in ["Forge", "Matrix"] { + let err = serde_json::from_str::(&req(c)).unwrap_err(); + assert!(err.to_string().contains("unknown variant"), "{c}: {err}"); + } + assert!(matches!( + serde_json::from_str::(&req("Gateway")), + Ok(PrivRequest::ControlInfraContainer { + container: InfraContainer::Gateway, + action: InfraAction::Stop, + }) + )); + } +} diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index c000d0e2..1981918a 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -15,24 +15,29 @@ //! **Socket activation**: when systemd passes the listener socket via //! `LISTEN_FDS=1` + `LISTEN_PID=`, the inherited fd 3 is used //! instead of binding a fresh socket. +//! +//! **Layout**: this file holds the socket loop and the one-arm-per-op +//! dispatch table. Each op family's handlers and validation live in their +//! own module: [`container`] (lifecycle), [`container_config`] (nspawn flags, +//! systemd drop-ins), [`storage`] (btrfs subvolumes, quotas, snapshots, disk +//! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles) and +//! [`infra`] (gateway nginx, forge admin, infra containers). Helpers more than +//! one family uses are in [`shared`]. + +mod container; +mod container_config; +mod files; +mod infra; +mod shared; +mod storage; use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd, RawFd}; -use std::path::{Path, PathBuf}; -use anyhow::{Context as _, Result, anyhow, bail}; -use hive_priv_sock::{ - AGENT_PREFIX, AGENT_RUNTIME_ROOT, AGENT_STATE_ROOT, BindMount, CredentialMount, HiveFacts, - InfraAction, InfraContainer, MEASURE_AGENT_DISK_TIMEOUT, META_DIR, MIGRATE_STAGING_ROOT, - NetworkIsolation, PAUSED_MARKER_FILE, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, - PrivStream, PrivStreamLine, -}; -use tokio::io::{AsyncWriteExt, BufReader}; +use anyhow::{Context as _, Result, bail}; +use hive_priv_sock::{PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse}; +use tokio::io::AsyncWriteExt; use tokio::net::unix::OwnedWriteHalf; use tokio::net::{UnixListener, UnixStream}; -use tokio::process::Command; - -/// Root of the per-agent unix-socket dirs on the host. -const SOCKET_DIR_ROOT: &str = "/run/hive-agent"; #[tokio::main] async fn main() -> Result<()> { @@ -47,7 +52,7 @@ async fn main() -> Result<()> { .init(); let listener = socket_listener()?; - if let Err(e) = remove_legacy_tmpfiles() { + if let Err(e) = files::remove_legacy_tmpfiles() { tracing::warn!(error = %format!("{e:#}"), "legacy tmpfiles cleanup failed"); } tracing::info!("hive-priv listening"); @@ -318,21 +323,6 @@ async fn run( exec(req, fd, writer).await } -/// Write one `PrivEvent::Line` to the client. Best-effort: a write -/// failure is logged but doesn't abort the running subprocess. -async fn write_line_event(writer: &mut OwnedWriteHalf, stream: PrivStream, data: &str) { - let event = PrivEvent::Line(PrivStreamLine { - stream, - data: data.to_owned(), - }); - if let Ok(mut json) = serde_json::to_string(&event) { - json.push('\n'); - if let Err(e) = writer.write_all(json.as_bytes()).await { - tracing::warn!(error = %e, "write_line_event: write failed"); - } - } -} - /// Execute a validated `PrivRequest`. Returns `(stdout, stderr)` on success. /// For streaming ops (`CreateContainer`/`UpdateContainer` with `stream: true`) /// output lines are forwarded to `writer` as `PrivEvent::Line` messages and @@ -353,23 +343,23 @@ async fn exec( writer: &mut OwnedWriteHalf, ) -> Result<(String, String)> { match req { - PrivRequest::StartContainer { ref name } => start_container(name).await, + PrivRequest::StartContainer { ref name } => container::start_container(name).await, - PrivRequest::StopContainer { ref name } => stop_container(name).await, + PrivRequest::StopContainer { ref name } => container::stop_container(name).await, - PrivRequest::KillContainer { ref name } => kill_container(name).await, + PrivRequest::KillContainer { ref name } => container::kill_container(name).await, PrivRequest::UpdateContainer { ref name, stream } => { - container_flake_action("update", name, stream, writer).await + container::container_flake_action("update", name, stream, writer).await } PrivRequest::CreateContainer { ref name, stream } => { - container_flake_action("create", name, stream, writer).await + container::container_flake_action("create", name, stream, writer).await } - PrivRequest::DestroyContainer { ref name } => destroy_container(name).await, + PrivRequest::DestroyContainer { ref name } => container::destroy_container(name).await, - PrivRequest::ListContainers => container_run(&["list"]).await, + PrivRequest::ListContainers => container::container_run(&["list"]).await, PrivRequest::WriteNspawnFlags { ref container, @@ -377,9 +367,15 @@ async fn exec( ref isolation, ref load_credentials, ref hive_facts, - } => handle_write_nspawn_flags(container, binds, isolation, load_credentials, hive_facts), + } => container_config::handle_write_nspawn_flags( + container, + binds, + isolation, + load_credentials, + hive_facts, + ), - PrivRequest::EnsureAgentSocketDir { ref name } => ensure_agent_socket_dir(name), + PrivRequest::EnsureAgentSocketDir { ref name } => files::ensure_agent_socket_dir(name), PrivRequest::WriteResourceLimits { ref container, @@ -387,61 +383,65 @@ async fn exec( ref cpu_quota, cpu_weight, io_weight, - } => write_resource_limits(container, memory_max, cpu_quota, cpu_weight, io_weight), + } => container_config::write_resource_limits( + container, memory_max, cpu_quota, cpu_weight, io_weight, + ), - PrivRequest::RemoveServiceDropin { ref container } => remove_service_dropin(container), + PrivRequest::RemoveServiceDropin { ref container } => { + container_config::remove_service_dropin(container) + } - PrivRequest::DaemonReload => daemon_reload().await, + PrivRequest::DaemonReload => container_config::daemon_reload().await, - PrivRequest::ReloadGatewayNginx => sync_gateway_nginx().await, + PrivRequest::ReloadGatewayNginx => infra::sync_gateway_nginx().await, - PrivRequest::RunForgeAdmin { ref args } => exec_forge_admin(args).await, + PrivRequest::RunForgeAdmin { ref args } => infra::exec_forge_admin(args).await, PrivRequest::SetAgentPaused { ref agent_name, paused, - } => exec_set_agent_paused(agent_name, paused), + } => files::exec_set_agent_paused(agent_name, paused), PrivRequest::ControlInfraContainer { container, action } => { - control_infra_container(container, action).await + infra::control_infra_container(container, action).await } PrivRequest::EnsureAgentSubvolume { ref agent_name } => { - exec_ensure_agent_subvolume(agent_name).await + storage::exec_ensure_agent_subvolume(agent_name).await } PrivRequest::DeleteAgentSubvolume { ref agent_name } => { - exec_delete_agent_subvolume(agent_name).await + storage::exec_delete_agent_subvolume(agent_name).await } - PrivRequest::EnsureBtrfsQuota => ensure_btrfs_quota().await, + PrivRequest::EnsureBtrfsQuota => storage::ensure_btrfs_quota().await, PrivRequest::ReadSubvolumeUsage { ref agent_name } => { - exec_read_subvolume_usage(agent_name).await + storage::exec_read_subvolume_usage(agent_name).await } PrivRequest::SetSubvolumeQuota { ref agent_name, limit_bytes, - } => exec_set_subvolume_quota(agent_name, limit_bytes).await, + } => storage::exec_set_subvolume_quota(agent_name, limit_bytes).await, PrivRequest::MeasureAgentDisk { ref agent_name } => { - exec_measure_agent_disk(agent_name).await + storage::exec_measure_agent_disk(agent_name).await } PrivRequest::UpgradeAgentSubvolume { ref agent_name } => { - exec_upgrade_agent_subvolume(agent_name).await + storage::exec_upgrade_agent_subvolume(agent_name).await } PrivRequest::SnapshotAgentSubvolume { ref agent_name, ref snapshot_name, - } => exec_snapshot_agent_subvolume(agent_name, snapshot_name).await, + } => storage::exec_snapshot_agent_subvolume(agent_name, snapshot_name).await, PrivRequest::DeleteAgentSnapshot { ref agent_name, ref snapshot_name, - } => exec_delete_agent_snapshot(agent_name, snapshot_name).await, + } => storage::exec_delete_agent_snapshot(agent_name, snapshot_name).await, PrivRequest::SendAgentSnapshotToFile { ref agent_name, @@ -449,7 +449,7 @@ async fn exec( ref parent_snapshot_name, ref dest_file_name, } => { - exec_send_agent_snapshot_to_file( + storage::exec_send_agent_snapshot_to_file( agent_name, snapshot_name, parent_snapshot_name.as_deref(), @@ -463,7 +463,7 @@ async fn exec( ref snapshot_name, ref parent_snapshot_name, } => { - exec_send_agent_snapshot_to_fd( + storage::exec_send_agent_snapshot_to_fd( agent_name, snapshot_name, parent_snapshot_name.as_deref(), @@ -473,2783 +473,14 @@ async fn exec( } PrivRequest::SyncAgentTmpfiles => { - remove_legacy_tmpfiles()?; + files::remove_legacy_tmpfiles()?; Ok((String::new(), String::new())) } } } - -/// `StartContainer`: clear any start-limit lockout left by earlier failures -/// so a now-correct start isn't blocked (`nixos-container start` does not do -/// this itself). Best-effort — if the unit doesn't exist yet (first-time -/// create), `reset-failed` is a no-op and the start proceeds regardless. -async fn start_container(name: &str) -> Result<(String, String)> { - validate_agent_name(name)?; - let machine = container_system_name(name); - // Silenced: a first-time create means the unit doesn't exist yet, and - // `systemctl`'s own "Unit ... not loaded" complaint for that case would - // otherwise land in the journal at error priority on every such start — - // routine, not a signal, since the result is already discarded below. - let _ = Command::new("systemctl") - .args(["reset-failed", &format!("container@{machine}.service")]) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .status() - .await; - container_run(&["start", &machine]).await -} - -/// `RunForgeAdmin`: every arg must pass [`validate_forge_admin_arg`] before -/// the admin CLI ever sees it. -async fn exec_forge_admin(args: &[String]) -> Result<(String, String)> { - for arg in args { - validate_forge_admin_arg(arg)?; - } - run_forge_admin(args).await -} - -/// `StopContainer`. -async fn stop_container(name: &str) -> Result<(String, String)> { - validate_agent_name(name)?; - stop_and_release(&container_system_name(name)).await -} - -/// `KillContainer`: `nixos-container` has no kill verb, so this uses -/// `machinectl` to send `SIGKILL` to every process in the container — the -/// right semantics for a forced shutdown after a graceful stop has already -/// been attempted. -async fn kill_container(name: &str) -> Result<(String, String)> { - validate_agent_name(name)?; - let machine = container_system_name(name); - machinectl_run(&["kill", &machine, "--signal=SIGKILL"]).await -} - -/// `DestroyContainer`. -async fn destroy_container(name: &str) -> Result<(String, String)> { - validate_agent_name(name)?; - container_run(&["destroy", &container_system_name(name)]).await -} - -/// `SetAgentPaused`. -fn exec_set_agent_paused(agent_name: &str, paused: bool) -> Result<(String, String)> { - validate_agent_name(agent_name)?; - set_agent_paused(agent_name, paused) -} - -/// `EnsureAgentSubvolume`. -async fn exec_ensure_agent_subvolume(agent_name: &str) -> Result<(String, String)> { - validate_agent_name(agent_name)?; - ensure_agent_subvolume(agent_name).await -} - -/// `DeleteAgentSubvolume`. -async fn exec_delete_agent_subvolume(agent_name: &str) -> Result<(String, String)> { - validate_agent_name(agent_name)?; - delete_agent_subvolume(agent_name).await -} - -/// `ReadSubvolumeUsage`. -async fn exec_read_subvolume_usage(agent_name: &str) -> Result<(String, String)> { - validate_agent_name(agent_name)?; - read_subvolume_usage(agent_name).await -} - -/// `MeasureAgentDisk`. -async fn exec_measure_agent_disk(agent_name: &str) -> Result<(String, String)> { - validate_agent_name(agent_name)?; - measure_agent_disk(agent_name).await -} - -/// `SetSubvolumeQuota`. -async fn exec_set_subvolume_quota( - agent_name: &str, - limit_bytes: Option, -) -> Result<(String, String)> { - validate_agent_name(agent_name)?; - set_subvolume_quota(agent_name, limit_bytes).await -} - -/// `UpgradeAgentSubvolume`. -async fn exec_upgrade_agent_subvolume(agent_name: &str) -> Result<(String, String)> { - validate_agent_name(agent_name)?; - upgrade_agent_subvolume(agent_name).await -} - -/// `SnapshotAgentSubvolume`. -async fn exec_snapshot_agent_subvolume( - agent_name: &str, - snapshot_name: &str, -) -> Result<(String, String)> { - validate_agent_name(agent_name)?; - validate_snapshot_name(snapshot_name)?; - snapshot_agent_subvolume(agent_name, snapshot_name).await -} - -/// `DeleteAgentSnapshot`. -async fn exec_delete_agent_snapshot( - agent_name: &str, - snapshot_name: &str, -) -> Result<(String, String)> { - validate_agent_name(agent_name)?; - validate_snapshot_name(snapshot_name)?; - delete_agent_snapshot(agent_name, snapshot_name).await -} - -/// `SendAgentSnapshotToFile`. -async fn exec_send_agent_snapshot_to_file( - agent_name: &str, - snapshot_name: &str, - parent_snapshot_name: Option<&str>, - dest_file_name: &str, -) -> Result<(String, String)> { - validate_agent_name(agent_name)?; - validate_snapshot_name(snapshot_name)?; - if let Some(parent) = parent_snapshot_name { - validate_snapshot_name(parent)?; - } - validate_credential_name(dest_file_name)?; - send_agent_snapshot_to_file( - agent_name, - snapshot_name, - parent_snapshot_name, - dest_file_name, - ) - .await -} - -/// `SendAgentSnapshotToFd`. -async fn exec_send_agent_snapshot_to_fd( - agent_name: &str, - snapshot_name: &str, - parent_snapshot_name: Option<&str>, - fd: Option, -) -> Result<(String, String)> { - validate_agent_name(agent_name)?; - validate_snapshot_name(snapshot_name)?; - if let Some(parent) = parent_snapshot_name { - validate_snapshot_name(parent)?; - } - let dest = fd.context("no descriptor to stream into")?; - send_agent_snapshot_to_fd(agent_name, snapshot_name, parent_snapshot_name, dest).await -} - -/// Shared body for `CreateContainer` / `UpdateContainer`: validate the -/// name, build the toplevel ourselves, and apply it (streaming line -/// events to `writer` when `stream` is set). -/// -/// **Both verbs build explicitly now — not just `update`.** The first cut -/// of this fix only rewrote `update`, reasoning that `create` was already -/// safe: it wraps its whole action in an exclusive `flock` before calling -/// `nixos-container`'s own `buildFlake()`, and once `update` stopped -/// writing to `buildFlake()`'s shared `.tmp` out-link, concurrent -/// `create`s were the only remaining writers — mutually excluded by that -/// lock. True, but it leaves `create`'s safety resting on an internal -/// implementation detail of a script we don't own (its current locking -/// behavior, which could change upstream without notice) instead of on -/// something we control. Building here for both verbs removes `buildFlake()` -/// from the picture entirely — there's no shared `.tmp` left to race on, -/// so there's nothing left to reason about staying in sync with. -/// -/// **`update` no longer calls `nixos-container update` at all.** That -/// action's own version-compat probe runs unconditionally before -/// `--system-path` is ever honored, dying on every agent's update. -/// [`swap_container_profile`] replicates exactly what `update`'s own -/// action does *past* that probe (confirmed against its source): `nix-env -/// --set` the per-container profile, then `systemctl reload` if the -/// container is running. `create` is untouched — it isn't the failing -/// verb. -async fn container_flake_action( - verb: &str, - name: &str, - stream: bool, - writer: &mut OwnedWriteHalf, -) -> Result<(String, String)> { - validate_agent_name(name)?; - // The build is the multi-minute phase of this operation — give it the - // same live-line treatment `container_run_streaming` gives - // `nixos-container` itself when the caller asked for it. Without - // this, moving the build out of the streamed `nixos-container` call - // (which is the whole point of this fix) would silently regress every - // UI that shows build progress: nothing until the longest phase - // finishes, then everything at once. - let toplevel = nix_build_toplevel(name, stream.then_some(&mut *writer)).await?; - let system_name = container_system_name(name); - if verb == "update" { - return swap_container_profile(&system_name, &toplevel, stream, writer).await; - } - let args = [verb, &system_name, "--system-path", &toplevel]; - if stream { - container_run_streaming(&args, writer).await - } else { - container_run(&args).await - } -} - -/// Apply a prebuilt toplevel to an existing container directly, without -/// going through `nixos-container update` (see `container_flake_action`'s -/// doc comment for why). Mirrors `nixos-container.pl`'s own `update` -/// action verbatim, past its version probe: point the per-container `nix- -/// env` profile at the new toplevel, then reload the container unit *if* -/// it's currently running — the container's own next start already reads -/// from the profile, so a stopped container needs nothing further. -/// -/// Both steps are near-instant in the happy case, but still stream- -/// forwarded like the rest of this operation: a failure here (e.g. a -/// wedged `systemctl reload`) is exactly when the caller most wants the -/// live line in the build log, not just a summary error afterward. -async fn swap_container_profile( - system_name: &str, - toplevel: &str, - stream: bool, - writer: &mut OwnedWriteHalf, -) -> Result<(String, String)> { - let profile = format!("/nix/var/nix/profiles/per-container/{system_name}/system"); - let set_out = Command::new("nix-env") - .args(["-p", &profile, "--set", toplevel]) - .output() - .await - .context("invoke nix-env --set")?; - let mut stdout = String::from_utf8_lossy(&set_out.stdout).into_owned(); - let mut stderr = String::from_utf8_lossy(&set_out.stderr).into_owned(); - log_and_forward(&stdout, &stderr, stream, writer).await; - if !set_out.status.success() { - bail!( - "nix-env -p {profile} --set failed ({}): {}", - set_out.status, - stderr.trim() - ); - } - - let unit = format!("container@{system_name}"); - let state_out = Command::new("systemctl") - .args(["show", "--property=ActiveState", "--value", &unit]) - .output() - .await - .context("query container ActiveState")?; - let active = String::from_utf8_lossy(&state_out.stdout).trim() == "active"; - if active { - let reload_out = Command::new("systemctl") - .args(["reload", &unit]) - .output() - .await - .context("invoke systemctl reload")?; - let r_stdout = String::from_utf8_lossy(&reload_out.stdout).into_owned(); - let r_stderr = String::from_utf8_lossy(&reload_out.stderr).into_owned(); - log_and_forward(&r_stdout, &r_stderr, stream, writer).await; - if !reload_out.status.success() { - bail!( - "systemctl reload {unit} failed ({}): {}", - reload_out.status, - r_stderr.trim() - ); - } - stdout.push_str(&r_stdout); - stderr.push_str(&r_stderr); - } - Ok((stdout, stderr)) -} - -/// Log a command's captured stdout/stderr the same way every other -/// `nixos-container`-adjacent shellout in this file does, and — when -/// `stream` is set — forward each line to the client as a live -/// [`PrivEvent::Line`] too, so a caller watching the build log sees these -/// lines exactly like any other step's, not just a summary on failure. -async fn log_and_forward(stdout: &str, stderr: &str, stream: bool, writer: &mut OwnedWriteHalf) { - for line in stdout.lines() { - tracing::info!(target: "nixos-container", "{line}"); - if stream { - write_line_event(writer, PrivStream::Stdout, line).await; - } - } - for line in stderr.lines() { - tracing::warn!(target: "nixos-container", "{line}"); - if stream { - write_line_event(writer, PrivStream::Stderr, line).await; - } - } -} - -/// The explicit `nixosConfigurations..config.system.build.toplevel` -/// flake attr path — same construction `hive-c0re`'s own -/// `lifecycle::prebuild_toplevel` uses, kept here as a pure function so -/// the exact string shape is unit-tested without needing to run `nix`. -fn toplevel_attr(name: &str) -> String { - format!("{META_DIR}#nixosConfigurations.{name}.config.system.build.toplevel") -} - -/// Wall-clock bound on [`nix_build_toplevel`]. Four times CI's observed -/// cold-cache `nix flake check` (~15 min, `.forgejo/workflows/ci.yml`), which -/// compiles this workspace's crates, the part of an agent toplevel no binary -/// cache holds; the rebuild path has usually warmed this exact attr already -/// (`hive-c0re`'s `prebuild_toplevel`). A stalled download is bounded by nix's -/// own `stalled-download-timeout`; this covers the rest, such as a wedged -/// nix-daemon holding the caller's build slot forever. -const NIX_BUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_hours(1); - -/// How a [`run_bounded`] child ended. -enum BoundedRun { - Exited { - status: std::process::ExitStatus, - stdout: String, - stderr: String, - }, - /// Still running at the limit; its whole process group was killed. - TimedOut, -} - -/// Run `cmd` to completion or until `limit`, draining both pipes: stdout is -/// captured silently, stderr is logged and, when `writer` is set, forwarded -/// live as [`PrivEvent::Line`]s. -/// -/// The child leads its own process group and a timeout `SIGKILL`s the whole -/// group, so a grandchild still holding the pipes (or the daemon connection) -/// dies with it instead of outliving the error. -async fn run_bounded( - mut cmd: Command, - limit: std::time::Duration, - mut writer: Option<&mut OwnedWriteHalf>, -) -> std::io::Result { - use tokio::io::AsyncBufReadExt as _; - - let mut child = cmd - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .process_group(0) - .spawn()?; - // Group leader, so its pid is the pgid. - let pgid = child.id().and_then(|pid| i32::try_from(pid).ok()); - - let stdout = child.stdout.take().expect("stdout piped"); - let stderr = child.stderr.take().expect("stderr piped"); - let run = async { - let mut stdout_lines = BufReader::new(stdout).lines(); - let mut stderr_lines = BufReader::new(stderr).lines(); - let mut stdout_buf = String::new(); - let mut stderr_buf = String::new(); - - // ⚠️ Both pipes are drained concurrently even though only one is - // streamed: reading stderr alone would let stdout fill its pipe - // buffer and deadlock the child on a build with enough stdout output - // to fill it. - // - // ⚠️ Each stream's EOF is tracked separately rather than breaking on - // the first `None`: `next_line()` on a closed stream returns - // `Ok(None)` immediately and forever, so a loop that keeps polling a - // finished stream spins hot until the other one ends too. - let mut stdout_done = false; - let mut stderr_done = false; - while !(stdout_done && stderr_done) { - tokio::select! { - line = stdout_lines.next_line(), if !stdout_done => { - match line { - // Captured, never streamed — this is the store path. - Ok(Some(l)) => { - stdout_buf.push_str(&l); - stdout_buf.push('\n'); - } - Ok(None) => stdout_done = true, - Err(e) => { - tracing::warn!(error = %e, "nix build stdout read error"); - stdout_done = true; - } - } - } - line = stderr_lines.next_line(), if !stderr_done => { - match line { - // Streamed as it arrives, so the dashboard and - // `journalctl -f` show build progress live. - Ok(Some(l)) => { - tracing::info!(target: "nix-build-toplevel", "{l}"); - if let Some(w) = writer.as_deref_mut() { - write_line_event(w, PrivStream::Stderr, &l).await; - } - if !stderr_buf.is_empty() { - stderr_buf.push('\n'); - } - stderr_buf.push_str(&l); - } - Ok(None) => stderr_done = true, - Err(e) => { - tracing::warn!(error = %e, "nix build stderr read error"); - stderr_done = true; - } - } - } - } - } - let status = child.wait().await?; - Ok::<_, std::io::Error>(BoundedRun::Exited { - status, - stdout: stdout_buf, - stderr: stderr_buf, - }) - }; - if let Ok(exited) = tokio::time::timeout(limit, run).await { - return exited; - } - if let Some(pgid) = pgid { - // SAFETY: plain `kill(2)`; `-pgid` targets the child's own process - // group, which `process_group(0)` made it lead. - unsafe { - libc::kill(-pgid, libc::SIGKILL); - } - } - child.wait().await?; - Ok(BoundedRun::TimedOut) -} - -/// Build `nixosConfigurations..config.system.build.toplevel` and -/// return the resulting store path, so `create`/`update` can hand -/// `nixos-container` an explicit `--system-path` instead of letting its -/// own `buildFlake()` build to a racy shared out-link. See "Container -/// toplevel builds" in this crate's README for the full story — the -/// concurrency bug this closes (the "agent container gets closure of -/// other agent" mystery bug) and why stdout/stderr are drained -/// concurrently but handled asymmetrically (stderr streamed live, -/// stdout captured and required to be exactly one line). -/// -/// `writer` is `None` for the non-streaming call shape (`stream: false`); -/// stderr still logs to journald either way, just without the -/// `PrivEvent::Line` forwarding. Bounded by [`NIX_BUILD_TIMEOUT`]. -async fn nix_build_toplevel(name: &str, writer: Option<&mut OwnedWriteHalf>) -> Result { - let attr = toplevel_attr(name); - let mut cmd = Command::new("nix"); - cmd.args([ - "--extra-experimental-features", - "nix-command flakes", - "build", - "--no-link", - "--print-out-paths", - &attr, - ]); - let (status, stdout_buf, stderr_buf) = match run_bounded(cmd, NIX_BUILD_TIMEOUT, writer) - .await - .with_context(|| format!("run nix build {attr}"))? - { - BoundedRun::Exited { - status, - stdout, - stderr, - } => (status, stdout, stderr), - BoundedRun::TimedOut => bail!( - "nix build {attr} timed out after {}s; killed it", - NIX_BUILD_TIMEOUT.as_secs() - ), - }; - - // ⚠️ Success is decided by the exit status, not by "we parsed a - // path" — a build can print to stdout and still fail. - if !status.success() { - bail!( - "nix build {attr} failed ({status}): {}", - stderr_buf.lines().last().unwrap_or("").trim() - ); - } - single_output_path(&stdout_buf) - .map(str::to_owned) - .map_err(|count| { - anyhow!("nix build {attr} produced {count} output path(s), expected exactly 1: {stdout_buf:?}") - }) -} - -/// Parse `nix build --print-out-paths`' stdout down to the single output -/// path this function's caller expects. `--print-out-paths` prints one -/// line *per output*, not one line total (`nix build --no-link -/// --print-out-paths nixpkgs#openssl` prints two: `…-bin`, `…-man`); -/// `config.system.build.toplevel` is single-output today, so this is one -/// line in practice — but a bare whole-buffer `.trim()` would silently -/// hand a multi-line string on to `--system-path` the day that ever -/// changes, the same corrupted-argument failure this function exists to -/// avoid. Trims and drops empty lines *before* counting, so a lone -/// `"\n"` (or trailing whitespace on the real line) can't be mistaken -/// for a present-but-blank path — see the unit tests below for the exact -/// table this closes. `Err` carries the surviving line count, for the -/// caller's error message. -fn single_output_path(stdout: &str) -> Result<&str, usize> { - let lines: Vec<&str> = stdout - .lines() - .map(str::trim) - .filter(|l| !l.is_empty()) - .collect(); - match lines[..] { - [path] => Ok(path), - _ => Err(lines.len()), - } -} - -/// `WriteNspawnFlags` — validate the container + every bind path + every -/// credential entry, then write the container's nspawn flag overrides. -fn handle_write_nspawn_flags( - container: &str, - binds: &[BindMount], - isolation: &NetworkIsolation, - load_credentials: &[CredentialMount], - hive_facts: &HiveFacts, -) -> Result<(String, String)> { - validate_container_system_name(container)?; - for bind in binds { - validate_bind_path(&bind.host_path)?; - validate_bind_path(&bind.container_path)?; - } - for cred in load_credentials { - validate_credential_name(&cred.name)?; - // Same path rules as binds (absolute, no colon/newline/quote/null): - // the colon ban is essential since `--load-credential=name:path` - // uses `:` as the name/path separator. - validate_bind_path(&cred.host_path)?; - } - let hive_facts = render_hive_facts(hive_facts)?; - write_nspawn_flags(container, binds, isolation, load_credentials, &hive_facts)?; - Ok((String::new(), String::new())) -} - -/// A btrfs snapshot label must start with `hive-` — this doubles as an -/// allow-list: only names hivectl itself constructs (or an operator who -/// knows the convention) can reach the `btrfs subvolume snapshot`/`delete` -/// shellouts, so an arbitrary caller can't use the snapshot ops to probe or -/// churn unrelated paths under `AGENT_STATE_ROOT`. Beyond the prefix, the -/// same charset restriction as [`validate_credential_name`] applies (it's -/// interpolated straight into a filesystem path). -fn validate_snapshot_name(name: &str) -> Result<()> { - if !name.starts_with("hive-") { - bail!("invalid snapshot label {name:?}: must start with \"hive-\""); - } - validate_credential_name(name) -} - -/// A systemd credential id must be a short token — restrict to -/// `[A-Za-z0-9_-]` (no `.`) so it can't inject extra `--load-credential` -/// argv or break the `name:path` shape. `.` is deliberately excluded, not -/// just a bare `..`: this name gets interpolated into filesystem paths -/// (snapshot labels) and there's no legitimate need for a dot in either a -/// systemd credential id or a `hive-`-prefixed snapshot label — we're -/// defining this token format from scratch, so keep it maximally strict -/// rather than allow-then-patch each traversal-adjacent character -/// (mara: "we are making up the rules here, lets go strict"). -fn validate_credential_name(name: &str) -> Result<()> { - if name.is_empty() - || !name - .bytes() - .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-')) - { - bail!("invalid credential name {name:?}: must be non-empty [A-Za-z0-9_-]"); - } - Ok(()) -} - -/// `RemoveServiceDropin` — remove the container service's drop-in dir -/// if present (idempotent). -fn remove_service_dropin(container: &str) -> Result<(String, String)> { - validate_container_system_name(container)?; - let dir = format!("/run/systemd/system/container@{container}.service.d"); - if Path::new(&dir).exists() { - std::fs::remove_dir_all(&dir).with_context(|| format!("remove {dir}"))?; - } - Ok((String::new(), String::new())) -} - -/// `WriteResourceLimits` — drop the systemd resource settings into the -/// container service's drop-in dir, together with a -/// `ConditionPathIsDirectory=` guard on the agent's MCP runtime dir. -/// -/// Two different kinds of setting land in the same file. `MemoryMax=` / -/// `CPUQuota=` are hard caps that throttle even on an idle host; -/// `CPUWeight=` / `IOWeight=` are cgroup v2 relative shares that only -/// decide who yields *under contention*. A weight of `None` means "not -/// configured" and omits the line, so a hive-c0re built before the weights -/// existed keeps producing the old two-line drop-in. -/// -/// The condition causes systemd to *skip* (not *fail*) the unit when the -/// bind-mount source dir is absent — result is `condition`, which does not -/// increment the start-limit counter. This is belt-and-braces on top of -/// hive-c0re creating the dir in its start preamble: if it is missing at -/// start time, the unit idles rather than restart-looping into -/// `start-limit-hit`. -fn write_resource_limits( - container: &str, - memory_max: &str, - cpu_quota: &str, - cpu_weight: Option, - io_weight: Option, -) -> Result<(String, String)> { - let logical = validate_container_system_name(container)?; - let runtime_dir = format!("{AGENT_RUNTIME_ROOT}/{logical}"); - let dir = format!("/run/systemd/system/container@{container}.service.d"); - std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?; - let path = format!("{dir}/hyperhive-limits.conf"); - let content = limits_dropin_body(&runtime_dir, memory_max, cpu_quota, cpu_weight, io_weight); - // Published whole: a `daemon-reload` between truncate and write would - // otherwise load the unit with its limits and start guard missing. - publish_file(Path::new(&path), content.as_bytes(), 0o644, None)?; - Ok((String::new(), String::new())) -} - -/// How long a window the start-limit counts over, and how many starts it -/// allows inside it. -/// -/// `container@.service` sets `Restart=on-failure` and **no** start limit, so -/// systemd's defaults apply: 5 starts per 10s, `RestartSec` 100ms. That -/// makes the bound depend on *how fast* a container dies — one that fails -/// instantly trips the limit in under a second, one that takes longer than -/// ~2s never trips it and restarts forever. Whether an agent gets bounded -/// is not meant to be a function of its failure speed. -/// -/// The window has to exceed the worst-case time to burn the burst, or the -/// counter ages out between attempts and the limit is again unreachable: -/// `TimeoutStartSec` is 1min, so `BURST` slow failures plus their backoff -/// can span several minutes. 10min covers that with room. -/// -/// Giving up is cheap here **because it is not terminal** — hive-c0re's -/// reconcile sweep retries later, and `reset-failed` (see `StartContainer`) -/// clears the latch first. That is what makes a tight burst safe. -const START_LIMIT_INTERVAL_SEC: u32 = 600; -/// One start plus two retries — the operator's ruling was "retry once or -/// twice", with the reconcile sweep as the slow path after that. -const START_LIMIT_BURST: u32 = 3; -/// Backoff between those retries. The 100ms default is for processes that -/// respawn instantly; a container that just failed to boot gains nothing -/// from being retried a tenth of a second later. -const RESTART_SEC: u32 = 5; - -/// Render the body of `hyperhive-limits.conf`. -/// -/// `[Unit]`: the condition is checked at start time — it skips (not fails) -/// the unit when the MCP socket dir is absent, avoiding restart loops — -/// plus the bounded start limit (see the constants above; `StartLimit*` are -/// `[Unit]` settings since systemd 229 and are silently ignored under -/// `[Service]`). -/// `[Service]`: the restart backoff, the hard caps, then the relative -/// weights. A weight of `None` means "not configured" and omits its line -/// entirely, so a request from a hive-c0re built before the weights -/// existed — or one whose nix option is `null` — renders no weight lines. -fn limits_dropin_body( - runtime_dir: &str, - memory_max: &str, - cpu_quota: &str, - cpu_weight: Option, - io_weight: Option, -) -> String { - // Built as two possibly-empty lines rather than pushed onto the - // string: `format!` appended to a `String` trips clippy::pedantic's - // `format_push_string`, and a `write!` would need an unwrap. - let cpu_weight_line = cpu_weight.map_or_else(String::new, |w| format!("CPUWeight={w}\n")); - let io_weight_line = io_weight.map_or_else(String::new, |w| format!("IOWeight={w}\n")); - format!( - "[Unit]\n\ - ConditionPathIsDirectory={runtime_dir}\n\ - StartLimitIntervalSec={START_LIMIT_INTERVAL_SEC}\n\ - StartLimitBurst={START_LIMIT_BURST}\n\ - \n\ - [Service]\n\ - RestartSec={RESTART_SEC}\n\ - MemoryMax={memory_max}\n\ - CPUQuota={cpu_quota}\n\ - {cpu_weight_line}{io_weight_line}" - ) -} - -/// `DaemonReload` — `systemctl daemon-reload` on the host. -async fn daemon_reload() -> Result<(String, String)> { - let out = Command::new("systemctl") - .arg("daemon-reload") - .output() - .await - .context("invoke systemctl daemon-reload")?; - if !out.status.success() { - bail!( - "systemctl daemon-reload failed ({}): {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ); - } - Ok((String::new(), String::new())) -} - -/// `ControlInfraContainer` — start/stop a hive infrastructure service via -/// `systemctl `. The [`InfraContainer`] enum is the allowlist: -/// serde already rejected any unknown / unsafe name (hive-c0re, the forge -/// and the matrix have no variant) at deserialisation, so no root-side -/// `.contains()` check is needed here. Serves the hive-wide -/// `hivectl stop`/`start`/`restart` flow. No agent-facing path exists. -/// -/// ⚠️ The unit is derived from the variant, never sent by the caller — -/// which is what keeps this from being a general `systemctl` pass-through. -async fn control_infra_container( - container: InfraContainer, - action: InfraAction, -) -> Result<(String, String)> { - let verb = action.systemctl_verb(); - let unit = container.service_unit(); - let out = Command::new("systemctl") - .args([verb, unit]) - .output() - .await - .with_context(|| format!("systemctl {verb} {unit}"))?; - if !out.status.success() { - bail!( - "systemctl {verb} {unit} exited {}: {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ); - } - tracing::info!(target: "infra-control", "{verb} {unit}"); - Ok(( - String::from_utf8_lossy(&out.stdout).into_owned(), - String::from_utf8_lossy(&out.stderr).into_owned(), - )) -} - -/// A leaf safe to join onto an agent-owned directory: one plain component, so -/// it can neither climb out of the directory nor name the directory itself. -fn ensure_plain_filename(who: &str, filename: &str) -> Result<()> { - if filename.is_empty() || filename == "." || filename == ".." || filename.contains('/') { - bail!("{who}: refusing non-plain filename {filename:?}"); - } - Ok(()) -} - -/// Name the content is written under before being renamed onto `filename`, -/// unique per call so two concurrent writes of one file never share a temp. -/// -/// The leading dot and the `.partial` suffix are load-bearing: a reader -/// globbing on a published name's prefix, or systemd reading `*.conf` out of -/// `tmpfiles.d` and drop-in dirs, cannot pick up a half-written temp. -fn partial_name(filename: &str) -> String { - use std::sync::atomic::{AtomicU64, Ordering}; - static SEQ: AtomicU64 = AtomicU64::new(0); - let n = SEQ.fetch_add(1, Ordering::Relaxed); - format!(".{filename}.{}-{n}.partial", std::process::id()) -} - -/// Open `path` as a directory fd for [`StagedFile`] to work relative to. -fn open_dir(path: &Path) -> Result { - use std::os::unix::fs::OpenOptionsExt as _; - std::fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_DIRECTORY) - .open(path) - .with_context(|| format!("open directory {}", path.display())) -} - -/// A file written under a [`partial_name`] in its destination directory and -/// then renamed onto its final name, so a reader sees either the old file or -/// the complete new one, never a partial or wrong-mode one. Mode, and owner -/// when the caller `fchown`s [`file`](Self::file), are on the inode before the -/// rename makes it visible. -/// -/// Every step is relative to the `dir` fd, so it is safe in a directory an -/// agent or container controls: the temp is created `O_EXCL|O_NOFOLLOW`, which -/// refuses any entry already planted at its name, and `rename` replaces -/// whatever sits at the final name without following it. -/// -/// Dropped without [`publish`](Self::publish), it unlinks the temp, so a -/// failure between the two leaves the old file untouched and no residue. -struct StagedFile<'a> { - dir: &'a std::fs::File, - tmp: std::ffi::CString, - dest: std::ffi::CString, - dest_path: PathBuf, - file: std::fs::File, - published: bool, -} - -impl<'a> StagedFile<'a> { - /// Create the temp at `mode` and write `content` to it. The mode is set - /// again on the fd after the write, so the umask cannot change it. - fn write( - dir: &'a std::fs::File, - dir_path: &Path, - filename: &str, - content: &[u8], - mode: u32, - ) -> Result { - use std::io::Write as _; - use std::os::unix::fs::PermissionsExt as _; - - ensure_plain_filename("StagedFile::write", filename)?; - let dest_path = dir_path.join(filename); - let tmp_name = partial_name(filename); - let tmp = std::ffi::CString::new(tmp_name.as_str()) - .with_context(|| format!("NUL in filename {filename:?}"))?; - let dest = std::ffi::CString::new(filename) - .with_context(|| format!("NUL in filename {filename:?}"))?; - // SAFETY: `dir` is an open directory fd and `tmp` is NUL-terminated; - // the result is checked before it is wrapped. - let fd = unsafe { - libc::openat( - dir.as_raw_fd(), - tmp.as_ptr(), - libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_NOFOLLOW | libc::O_CLOEXEC, - mode, - ) - }; - if fd < 0 { - return Err(std::io::Error::last_os_error()) - .with_context(|| format!("create {}", dir_path.join(&tmp_name).display())); - } - // SAFETY: `fd` is a fresh, valid fd that nothing else owns. - let file = std::fs::File::from(unsafe { OwnedFd::from_raw_fd(fd) }); - let mut staged = Self { - dir, - tmp, - dest, - dest_path, - file, - published: false, - }; - staged - .file - .write_all(content) - .with_context(|| format!("write {}", staged.dest_path.display()))?; - staged - .file - .set_permissions(std::fs::Permissions::from_mode(mode)) - .with_context(|| format!("chmod {mode:o} {}", staged.dest_path.display()))?; - Ok(staged) - } - - /// The temp's open fd, for an `fchown` that must land before publishing. - fn file(&self) -> &std::fs::File { - &self.file - } - - /// Flush the content, rename the temp onto the final name, and flush the - /// directory: without the last step a crash can lose the rename itself. - fn publish(mut self) -> Result<()> { - self.file - .sync_all() - .with_context(|| format!("fsync {}", self.dest_path.display()))?; - let fd = self.dir.as_raw_fd(); - // SAFETY: `fd` is an open directory fd; both names are NUL-terminated. - if unsafe { libc::renameat(fd, self.tmp.as_ptr(), fd, self.dest.as_ptr()) } != 0 { - return Err(std::io::Error::last_os_error()) - .with_context(|| format!("publishing {}", self.dest_path.display())); - } - self.published = true; - self.dir - .sync_all() - .with_context(|| format!("fsync directory of {}", self.dest_path.display())) - } -} - -impl Drop for StagedFile<'_> { - fn drop(&mut self) { - if self.published { - return; - } - // SAFETY: `self.dir` is an open directory fd; `self.tmp` is - // NUL-terminated. - if unsafe { libc::unlinkat(self.dir.as_raw_fd(), self.tmp.as_ptr(), 0) } != 0 { - tracing::warn!( - dest = %self.dest_path.display(), - error = %std::io::Error::last_os_error(), - "failed to remove unpublished temp" - ); - } - } -} - -/// Publish `content` at `path` through a [`StagedFile`], at `mode` and, when -/// given, owned by `owner` (`(uid, gid)`). -fn publish_file(path: &Path, content: &[u8], mode: u32, owner: Option<(u32, u32)>) -> Result<()> { - let (Some(dir_path), Some(filename)) = - (path.parent(), path.file_name().and_then(|n| n.to_str())) - else { - bail!( - "publish_file: {} has no directory or file name", - path.display() - ); - }; - let dir = open_dir(dir_path)?; - let staged = StagedFile::write(&dir, dir_path, filename, content, mode)?; - if let Some((uid, gid)) = owner { - std::os::unix::fs::fchown(staged.file(), Some(uid), Some(gid)) - .with_context(|| format!("chown {}", path.display()))?; - } - staged.publish() -} - -/// Create or remove an agent's pause marker under its harness dir. The -/// marker is written empty and chowned to the harness dir's owner (the -/// agent), matching how the harness itself would have created it. -/// -/// Both directions are idempotent: re-pausing truncates the existing empty -/// marker rather than failing, and a `NotFound` on removal is the -/// already-resumed case, not an error. -fn set_agent_paused(agent_name: &str, paused: bool) -> Result<(String, String)> { - let harness_dir = PathBuf::from(AGENT_STATE_ROOT) - .join(agent_name) - .join("harness"); - if paused { - return write_agent_dir_file(agent_name, &harness_dir, PAUSED_MARKER_FILE, ""); - } - remove_marker_in(&harness_dir, PAUSED_MARKER_FILE)?; - tracing::info!(agent = %agent_name, "cleared pause marker"); - Ok((String::new(), String::new())) -} - -/// Unlink `dir/filename`, treating "already gone" as success. -/// -/// `remove_file` unlinks the leaf itself and never follows a symlink, so an -/// agent-planted link at the marker path cannot redirect this root unlink -/// at another file — the same threat [`StagedFile`] closes on the create side. -fn remove_marker_in(dir: &Path, filename: &str) -> Result<()> { - let path = dir.join(filename); - match std::fs::remove_file(&path) { - Ok(()) => Ok(()), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(e) => Err(e).with_context(|| format!("remove {}", path.display())), - } -} - -/// Write `content` to `dir/filename` as root, chowning the result to `dir`'s -/// owner so the agent process can read it back. Its caller is the pause -/// marker, in `harness/`, a directory owned by the agent, which is precisely -/// why it needs hive-priv at all. -/// -/// The file is published through a [`StagedFile`], so a reader woken by its -/// appearance cannot catch it empty or half-written: several of these paths -/// have a `systemd.path` unit watching them, and one of those triggers on the -/// file existing rather than changing. -fn write_agent_dir_file( - agent_name: &str, - dir: &Path, - filename: &str, - content: &str, -) -> Result<(String, String)> { - use std::os::fd::AsRawFd as _; - use std::os::unix::fs::MetadataExt as _; - - // Checked before `create_dir_all`, so a bad leaf does no filesystem work. - ensure_plain_filename("write_agent_dir_file", filename)?; - - let state_dir = dir.to_path_buf(); - // NOTE: `create_dir_all` is normally a no-op — lifecycle creates and chowns - // the state dir during spawn. On the rare edge where the dir doesn't exist - // yet (container being provisioned for the first time), the newly created dir - // is root:root. The `stat state_dir` chown below will then see uid=0 and - // leave the file root-owned (0600). The agent won't be able to read it until - // its lifecycle completes. - std::fs::create_dir_all(&state_dir) - .with_context(|| format!("create state dir {}", state_dir.display()))?; - - // Security-critical: the (state/-owning) agent may plant a symlink or - // hardlink in this dir, and `StagedFile` neither follows nor reuses one, - // so this root-privileged create/write/chmod/chown can't be redirected at - // an arbitrary file. - let path = state_dir.join(filename); - let dir_fd = open_dir(&state_dir)?; - let staged = StagedFile::write(&dir_fd, &state_dir, filename, content.as_bytes(), 0o600)?; - - // Chown to the state dir's owner so the agent process can read the file. - // fchown on the temp's fd — TOCTOU-immune (the inode the write hit, never a - // swapped path). If stat fails (e.g. dir just created, owner is root), the - // file stays root-owned and 0600 — still unreadable by others, just not - // agent-readable. Log a warning so operators can diagnose. - match dir_fd.metadata() { - Ok(meta) => { - // SAFETY: the temp's fd is open and owned by `staged` for the whole - // call; `fchown` only mutates that inode's uid/gid. - let rc = unsafe { libc::fchown(staged.file().as_raw_fd(), meta.uid(), meta.gid()) }; - if rc != 0 { - let e = std::io::Error::last_os_error(); - tracing::warn!( - agent = %agent_name, - path = %path.display(), - error = %e, - "write_agent_dir_file: fchown failed" - ); - } - } - Err(e) => { - tracing::warn!( - agent = %agent_name, - error = %e, - "write_agent_dir_file: stat state_dir failed, leaving file root-owned" - ); - } - } - // After the chown, never before: the file must never be visible under its - // final name while still root-owned. A failed publish drops `staged`, - // which removes the temp and the credential in it. - staged.publish()?; - - tracing::info!( - agent = %agent_name, - dir = %state_dir.display(), - file = %filename, - "wrote agent file" - ); - Ok((String::new(), String::new())) -} - -/// btrfs superblock magic, as reported by `statfs(2)`'s `f_type`. -const BTRFS_SUPER_MAGIC: i64 = 0x9123_683E; - -/// Inode number of a btrfs subvolume root (`BTRFS_FIRST_FREE_OBJECTID`). -/// Every subvolume's top directory has this inode; plain directories do -/// not, so `statfs == btrfs && st_ino == 256` reliably identifies a -/// subvolume root. -const BTRFS_SUBVOL_ROOT_INO: u64 = 256; - -/// Whether `path` lives on a btrfs filesystem (via `statfs(2)`). -fn is_on_btrfs(path: &Path) -> Result { - use std::os::unix::ffi::OsStrExt as _; - let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()) - .with_context(|| format!("path {} has an interior null byte", path.display()))?; - // SAFETY: `c_path` is a valid NUL-terminated C string that outlives the - // call; `statfs` only writes into the zero-initialised `buf`. - let mut buf: libc::statfs = unsafe { std::mem::zeroed() }; - let rc = unsafe { libc::statfs(c_path.as_ptr(), &raw mut buf) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()) - .with_context(|| format!("statfs {}", path.display())); - } - Ok(buf.f_type == BTRFS_SUPER_MAGIC) -} - -/// Whether `path` is the root of a btrfs subvolume (on btrfs and inode 256). -fn is_btrfs_subvolume(path: &Path) -> bool { - use std::os::unix::fs::MetadataExt as _; - let on_btrfs = is_on_btrfs(path).unwrap_or(false); - let ino_match = std::fs::metadata(path).is_ok_and(|m| m.ino() == BTRFS_SUBVOL_ROOT_INO); - on_btrfs && ino_match -} - -/// `EnsureAgentSubvolume` — make the agent's state root a btrfs subvolume -/// when the FS supports it. Idempotent + progressive: no-op when the root -/// already exists or the FS isn't btrfs. See the wire doc on the variant. -async fn ensure_agent_subvolume(agent_name: &str) -> Result<(String, String)> { - use std::os::unix::fs::MetadataExt as _; - - let root = PathBuf::from(AGENT_STATE_ROOT); - let agent_root = root.join(agent_name); - - // Progressive: existing agents (plain dir OR already a subvol) are left - // untouched — never auto-migrated. - if agent_root.exists() { - return Ok((String::new(), String::new())); - } - - // Only btrfs supports subvolumes; on anything else hive-c0re's normal - // `create_dir_all` makes a plain dir (the pre-subvolume behaviour). The - // parent must exist for both statfs and `btrfs subvolume create`. - std::fs::create_dir_all(&root) - .with_context(|| format!("create agents root {}", root.display()))?; - if !is_on_btrfs(&root)? { - return Ok((String::new(), String::new())); - } - - let out = Command::new("btrfs") - .args(["subvolume", "create"]) - .arg(&agent_root) - .output() - .await - .with_context(|| format!("spawn btrfs subvolume create {}", agent_root.display()))?; - if !out.status.success() { - bail!( - "btrfs subvolume create {} failed: {}", - agent_root.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - - // The subvol root is created root-owned, but hive-c0re (the `hive-core` - // user) must be able to mkdir state/ claude/ harness/ inside it — exactly - // as it would in a plain dir. Chown it to AGENT_STATE_ROOT's owner - // (hive-core). This MUST succeed: a root-owned subvol would make the - // downstream dir creation fail with a confusing permission error, and the - // c0re-side exists-check would then skip re-running this op on retry, - // wedging the agent. So on any failure roll the subvol back and bail — the - // create path surfaces a clear error and a retry starts clean. - let chown_result = std::fs::metadata(&root) - .with_context(|| format!("stat agents root {} for ownership", root.display())) - .and_then(|meta| { - std::os::unix::fs::chown(&agent_root, Some(meta.uid()), Some(meta.gid())).with_context( - || format!("chown subvol {} to agents-root owner", agent_root.display()), - ) - }); - if let Err(e) = chown_result { - // Best-effort rollback so we never leave a root-owned subvol behind. - let _ = Command::new("btrfs") - .args(["subvolume", "delete"]) - .arg(&agent_root) - .output() - .await; - return Err(e.context(format!( - "rolled back subvolume {} after chown failed", - agent_root.display() - ))); - } - tracing::info!(agent = %agent_name, path = %agent_root.display(), "created agent state subvolume"); - Ok((String::new(), String::new())) -} - -/// `DeleteAgentSubvolume` — delete the agent's state root iff it's a btrfs -/// subvolume (purge path only). No-op for plain dirs / missing paths; -/// hive-c0re's own `remove_dir_all` covers those. See the wire doc. -async fn delete_agent_subvolume(agent_name: &str) -> Result<(String, String)> { - let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); - if !is_btrfs_subvolume(&agent_root) { - return Ok((String::new(), String::new())); - } - let out = Command::new("btrfs") - .args(["subvolume", "delete"]) - .arg(&agent_root) - .output() - .await - .with_context(|| format!("spawn btrfs subvolume delete {}", agent_root.display()))?; - if !out.status.success() { - bail!( - "btrfs subvolume delete {} failed: {}", - agent_root.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - tracing::info!(agent = %agent_name, path = %agent_root.display(), "deleted agent state subvolume"); - Ok((String::new(), String::new())) -} - -/// `EnsureBtrfsQuota` — enable btrfs qgroup accounting on the filesystem -/// holding `AGENT_STATE_ROOT`. Idempotent + statfs-gated (no-op off btrfs). -/// Operator opt-in only; see the wire doc. -async fn ensure_btrfs_quota() -> Result<(String, String)> { - let root = PathBuf::from(AGENT_STATE_ROOT); - std::fs::create_dir_all(&root) - .with_context(|| format!("create agents root {}", root.display()))?; - if !is_on_btrfs(&root)? { - // Non-btrfs host: quota/qgroups don't apply. No-op success so the - // operator-facing verb degrades cleanly. - return Ok(( - String::new(), - "filesystem is not btrfs — quota not applicable".to_owned(), - )); - } - let out = Command::new("btrfs") - .args(["quota", "enable"]) - .arg(&root) - .output() - .await - .with_context(|| format!("spawn btrfs quota enable {}", root.display()))?; - if !out.status.success() { - bail!( - "btrfs quota enable {} failed: {}", - root.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - tracing::info!(path = %root.display(), "enabled btrfs qgroup accounting"); - Ok((String::new(), String::new())) -} - -/// `ReadSubvolumeUsage` — return an agent subvolume's qgroup rows -/// (`btrfs qgroup show -f --raw <…/agent_name>`) verbatim in stdout for -/// hive-c0re to parse. `-f` lists the qgroups impacting the given path, -/// excluding ancestral qgroups (per btrfs-qgroup-show(8)) — so it scopes -/// to this subvolume and never mixes in other agents'. hive-c0re then -/// selects the level-0 (`0/`) leaf row. See the wire doc. -async fn read_subvolume_usage(agent_name: &str) -> Result<(String, String)> { - let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); - let out = Command::new("btrfs") - .args(["qgroup", "show", "-f", "--raw"]) - .arg(&agent_root) - .output() - .await - .with_context(|| format!("spawn btrfs qgroup show {}", agent_root.display()))?; - if !out.status.success() { - // The common failure is "quota not enabled" — pass the stderr - // through so hive-c0re can surface it gracefully. - bail!( - "btrfs qgroup show {} failed: {}", - agent_root.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - Ok(( - String::from_utf8_lossy(&out.stdout).into_owned(), - String::new(), - )) -} - -/// The host paths `MeasureAgentDisk` sums for `agent_name`: its `state/` -/// and `work/` dirs under `AGENT_STATE_ROOT` and its container rootfs. -/// These are the only paths that op ever walks. -// `state` / `work` must match hive-c0re's `agent_notes_dir` and -// `paths::WORK_SUBDIR`. -fn agent_disk_paths(agent_name: &str) -> [PathBuf; 3] { - let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); - [ - agent_root.join("state"), - agent_root.join("work"), - PathBuf::from("/var/lib/nixos-containers").join(container_system_name(agent_name)), - ] -} - -/// `MeasureAgentDisk` — one `du` over the [`agent_disk_paths`] that exist, -/// returning the grand total in bytes. An agent with none of them has no -/// measurable size, which is an error rather than a total of 0. -async fn measure_agent_disk(agent_name: &str) -> Result<(String, String)> { - let mut paths = Vec::new(); - for path in agent_disk_paths(agent_name) { - match std::fs::symlink_metadata(&path) { - Ok(_) => paths.push(path), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => return Err(e).with_context(|| format!("stat {}", path.display())), - } - } - let total = du_total(agent_name, du_command(&paths)?, MEASURE_AGENT_DISK_TIMEOUT).await?; - Ok((total.to_string(), String::new())) -} - -/// `du -s -x -b -c -P -- `. `-x` keeps each walk on its operand's -/// filesystem (so the bind-mounted nix store is never counted), `-P` never -/// follows a symlink, `-b` counts apparent size, `-c` adds the `total` line -/// [`parse_du_total`] reads. Errors on no paths: `du` with no operand would -/// measure its working directory instead. -fn du_command(paths: &[PathBuf]) -> Result { - if paths.is_empty() { - bail!("none of the agent's state, work or rootfs paths exist"); - } - let mut cmd = Command::new("du"); - cmd.args(["-s", "-x", "-b", "-c", "-P", "--"]).args(paths); - Ok(cmd) -} - -/// Run `cmd` (a [`du_command`]) for at most `limit` and parse its total. -/// The child is killed on timeout. A timeout, a spawn failure, a non-zero -/// exit or output without a total is an error, never a partial total. -async fn du_total(agent_name: &str, mut cmd: Command, limit: std::time::Duration) -> Result { - cmd.kill_on_drop(true); - let Ok(out) = tokio::time::timeout(limit, cmd.output()).await else { - bail!( - "du for {agent_name} timed out after {}s; killed it", - limit.as_secs() - ); - }; - let out = out.with_context(|| format!("spawn du for {agent_name}"))?; - if !out.status.success() { - bail!( - "du for {agent_name} failed ({}): {}", - out.status, - String::from_utf8_lossy(&out.stderr) - .lines() - .last() - .unwrap_or("") - .trim() - ); - } - let stdout = String::from_utf8_lossy(&out.stdout); - parse_du_total(&stdout) - .ok_or_else(|| anyhow!("no `total` line in du output for {agent_name}: {stdout:?}")) -} - -/// The byte count on `du -c`'s final `\ttotal` line. `None` when the -/// last line isn't that shape, so truncated or unexpected output is never -/// read as a smaller total. -fn parse_du_total(stdout: &str) -> Option { - let last = stdout.lines().rev().find(|l| !l.trim().is_empty())?; - let (bytes, label) = last.split_once('\t')?; - if label != "total" { - return None; - } - bytes.parse().ok() -} - -/// Best-effort removal of a leftover migration path from a prior aborted -/// upgrade: try `btrfs subvolume delete` (in case it's a half-created -/// subvolume) then a plain recursive remove. Both failures are ignored — -/// the path may simply not exist. -async fn cleanup_stale_path(path: &Path) { - if path.exists() { - let _ = Command::new("btrfs") - .args(["subvolume", "delete"]) - .arg(path) - .output() - .await; - let _ = std::fs::remove_dir_all(path); - } -} - -/// Stage a populated subvolume at `tmp` mirroring `agent_root`: create the -/// subvolume, copy `agent_root`'s contents into it preserving -/// ownership/permissions/xattrs, then match the subvolume root's owner + mode -/// to the original. On any failure the partially-staged `tmp` is cleaned up -/// (so the caller can bail with the original dir still untouched). -async fn stage_upgrade_subvolume(agent_root: &Path, tmp: &Path) -> Result<()> { - use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; - - // Fresh subvolume to receive the copy. - let out = Command::new("btrfs") - .args(["subvolume", "create"]) - .arg(tmp) - .output() - .await - .with_context(|| format!("spawn btrfs subvolume create {}", tmp.display()))?; - if !out.status.success() { - bail!( - "btrfs subvolume create {} failed: {}", - tmp.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - - // Copy contents preserving everything (`-a` = --preserve=all → mode, - // ownership, timestamps, links, xattrs); reflink for fast CoW clones on the - // same btrfs. `/.` copies the directory's contents (incl. dotfiles) - // into the subvolume rather than nesting it. - let copy = Command::new("cp") - .arg("-a") - .arg("--reflink=auto") - .arg(format!("{}/.", agent_root.display())) - .arg(tmp) - .output() - .await - .with_context(|| format!("spawn cp into {}", tmp.display()))?; - if !copy.status.success() { - cleanup_stale_path(tmp).await; - bail!( - "copy {} -> {} failed (original left untouched): {}", - agent_root.display(), - tmp.display(), - String::from_utf8_lossy(©.stderr).trim() - ); - } - - // Match the new subvolume root's ownership + mode to the original dir. - // `cp -a /.` copies the *contents* but the subvolume root keeps its - // create-time root ownership, so set it explicitly — the swapped-in - // subvolume must be indistinguishable from the original to hive-c0re. - let apply = std::fs::metadata(agent_root) - .with_context(|| format!("stat {} for ownership", agent_root.display())) - .and_then(|m| { - std::os::unix::fs::chown(tmp, Some(m.uid()), Some(m.gid())) - .with_context(|| format!("chown {} to match original", tmp.display()))?; - std::fs::set_permissions(tmp, std::fs::Permissions::from_mode(m.mode())) - .with_context(|| format!("chmod {} to match original", tmp.display()))?; - Ok(()) - }); - if let Err(e) = apply { - cleanup_stale_path(tmp).await; - return Err(e.context("upgrade aborted before swap; original left untouched")); - } - Ok(()) -} - -/// `UpgradeAgentSubvolume` — convert an existing plain-dir agent state root -/// into a btrfs subvolume in place. Operator opt-in; the caller (hivectl) -/// stops the agent first and restarts it after. See the wire doc. -/// -/// Migration: stage a sibling subvolume mirroring the dir -/// ([`stage_upgrade_subvolume`]), then rename the original aside and the -/// subvolume into place, then remove the original. Any failure before the -/// rename-swap leaves the original dir untouched. -async fn upgrade_agent_subvolume(agent_name: &str) -> Result<(String, String)> { - let root = PathBuf::from(AGENT_STATE_ROOT); - let agent_root = root.join(agent_name); - - if !agent_root.exists() { - // A crash between the two swap renames (original → `..old` - // succeeded, `..migrating` → agent_root did not) leaves the - // agent root missing but the original data intact under `..old`. - // Point at the recovery rather than a bare "nothing to do" so the - // operator isn't left guessing where the data went. - let old = root.join(format!(".{agent_name}.old")); - if old.exists() { - bail!( - "no state dir at {agent} — but {old} holds the original data from an \ - interrupted upgrade (host crashed mid-swap). Restore it with \ - `mv {old} {agent}`, then re-run the upgrade.", - agent = agent_root.display(), - old = old.display(), - ); - } - bail!( - "no state dir to upgrade at {} — nothing to do", - agent_root.display() - ); - } - // Idempotent: already a subvolume → nothing to do. - if is_btrfs_subvolume(&agent_root) { - return Ok((String::new(), String::new())); - } - if !is_on_btrfs(&root)? { - bail!( - "{} is not on btrfs — subvolumes are unsupported, cannot upgrade", - root.display() - ); - } - - // Sibling temp paths on the same filesystem (so the copy can reflink and - // the swap renames are atomic). Leading dots keep them out of the agent - // namespace (`validate_agent_name` rejects dot-prefixed names). - let tmp = root.join(format!(".{agent_name}.migrating")); - let old = root.join(format!(".{agent_name}.old")); - // Clear any debris from a previously interrupted run before starting. - cleanup_stale_path(&tmp).await; - cleanup_stale_path(&old).await; - - stage_upgrade_subvolume(&agent_root, &tmp).await?; - - // Swap. `rename` is atomic within a filesystem. The window between the - // two renames is the only unsafe point: a crash there leaves the agent - // root missing but both `.old` (original) and the new subvolume present - // — recoverable by hand, hence the loud logging. - if let Err(e) = std::fs::rename(&agent_root, &old) { - cleanup_stale_path(&tmp).await; - return Err(anyhow::Error::new(e).context(format!( - "rename {} -> {} failed; original left untouched", - agent_root.display(), - old.display() - ))); - } - if let Err(e) = std::fs::rename(&tmp, &agent_root) { - // Restore the original from its renamed-aside copy. - let restored = std::fs::rename(&old, &agent_root).is_ok(); - cleanup_stale_path(&tmp).await; - return Err(anyhow::Error::new(e).context(format!( - "rename {} -> {} failed; original {}", - tmp.display(), - agent_root.display(), - if restored { - "restored" - } else { - "COULD NOT BE RESTORED — manual recovery needed" - } - ))); - } - - // 5. Success: drop the original (a plain dir) and report. - if let Err(e) = std::fs::remove_dir_all(&old) { - // The migration succeeded; a leftover `.old` is cosmetic. Warn only. - tracing::warn!( - agent = %agent_name, path = %old.display(), - "upgraded subvolume but failed to remove old dir: {e}" - ); - } - tracing::info!( - agent = %agent_name, path = %agent_root.display(), - "upgraded agent state dir to btrfs subvolume" - ); - Ok(( - format!("upgraded {} to a btrfs subvolume", agent_root.display()), - String::new(), - )) -} - -/// Derive a snapshot's path from the agent name + label: a dot-prefixed -/// sibling of the agent's state root so it can never collide with a real -/// agent directory (`validate_agent_name` rejects dot-prefixed names). -fn snapshot_path(agent_name: &str, snapshot_name: &str) -> PathBuf { - PathBuf::from(AGENT_STATE_ROOT).join(format!(".{agent_name}.snapshot.{snapshot_name}")) -} - -/// `SnapshotAgentSubvolume` — create a read-only btrfs snapshot of an -/// agent's state subvolume, for `btrfs send` to stream from during -/// inter-hive migration. See the wire doc. -async fn snapshot_agent_subvolume( - agent_name: &str, - snapshot_name: &str, -) -> Result<(String, String)> { - let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); - if !is_btrfs_subvolume(&agent_root) { - bail!( - "{} is not a btrfs subvolume — nothing to snapshot (run `hivectl agent subvol upgrade` first)", - agent_root.display() - ); - } - let snap = snapshot_path(agent_name, snapshot_name); - if snap.exists() { - bail!( - "snapshot {} already exists — delete it first or pick a different name", - snap.display() - ); - } - let out = Command::new("btrfs") - .args(["subvolume", "snapshot", "-r"]) - .arg(&agent_root) - .arg(&snap) - .output() - .await - .with_context(|| { - format!( - "spawn btrfs subvolume snapshot -r {} {}", - agent_root.display(), - snap.display() - ) - })?; - if !out.status.success() { - bail!( - "btrfs subvolume snapshot -r {} {} failed: {}", - agent_root.display(), - snap.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - tracing::info!( - agent = %agent_name, snapshot = %snap.display(), - "created read-only agent state snapshot" - ); - Ok((snap.display().to_string(), String::new())) -} - -/// `DeleteAgentSnapshot` — delete a previously-created read-only snapshot. -/// No-op if the path doesn't exist. See the wire doc. -async fn delete_agent_snapshot(agent_name: &str, snapshot_name: &str) -> Result<(String, String)> { - let snap = snapshot_path(agent_name, snapshot_name); - if !snap.exists() { - return Ok((String::new(), String::new())); - } - let out = Command::new("btrfs") - .args(["subvolume", "delete"]) - .arg(&snap) - .output() - .await - .with_context(|| format!("spawn btrfs subvolume delete {}", snap.display()))?; - if !out.status.success() { - bail!( - "btrfs subvolume delete {} failed: {}", - snap.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - tracing::info!(agent = %agent_name, snapshot = %snap.display(), "deleted agent state snapshot"); - Ok((String::new(), String::new())) -} - -/// Both paths a `SendAgentSnapshotToFile` request reads from — the -/// snapshot itself, and its optional incremental parent — must exist -/// before we touch `dest`. Pulled out on its own so that invariant is -/// checkable without a `dest` path, a `MIGRATE_STAGING_ROOT`, or `btrfs` -/// at all: a caller cannot create the output file by accident while -/// still holding a reference to this check. -fn check_send_snapshot_preconditions(snap: &Path, parent_path: Option<&Path>) -> Result<()> { - if !snap.exists() { - bail!( - "snapshot {} does not exist — create it with `subvol snapshot create` first", - snap.display() - ); - } - if let Some(parent_path) = parent_path - && !parent_path.exists() - { - bail!( - "parent snapshot {} does not exist — pick an existing parent or omit it for a full send", - parent_path.display() - ); - } - Ok(()) -} - -/// Validate preconditions, then atomically create `dest` for a snapshot -/// export. The two are combined in one function on purpose: keeping -/// `check_send_snapshot_preconditions` ahead of the `create_new` call is -/// the entire fix, and folding them together here means that ordering -/// can't drift back apart in a later edit to `send_agent_snapshot_to_file`. -/// No `btrfs` involved, so this is unit-testable against plain files. -fn open_export_dest(snap: &Path, parent_path: Option<&Path>, dest: &Path) -> Result { - check_send_snapshot_preconditions(snap, parent_path)?; - // `create_new` (O_CREAT|O_EXCL) makes the no-overwrite guarantee atomic - // instead of a check-then-create race against a concurrent request. - match std::fs::File::options() - .write(true) - .create_new(true) - .open(dest) - { - Ok(f) => Ok(f), - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => bail!( - "{} already exists — pick a different destination or remove it first \ - (send never overwrites an existing export)", - dest.display() - ), - Err(e) => Err(e).with_context(|| format!("create {}", dest.display())), - } -} - -/// Deletes `path` when dropped, unless [`disarm`](Self::disarm)ed first. -/// Guards `dest` between the moment `send_agent_snapshot_to_file` creates -/// it and the moment the export fully succeeds, so any `?`/`bail!` on -/// that path — including ones a future edit adds — removes the partial -/// file instead of leaving a zero-byte export that then makes every -/// retry against the same `--dest` fail with "already exists". -struct PartialExportGuard<'a> { - path: &'a Path, - armed: bool, -} - -impl<'a> PartialExportGuard<'a> { - fn new(path: &'a Path) -> Self { - Self { path, armed: true } - } - - fn disarm(mut self) { - self.armed = false; - } -} - -impl Drop for PartialExportGuard<'_> { - fn drop(&mut self) { - if !self.armed { - return; - } - // Best-effort: warn (don't fail the whole call over it) if removal - // itself fails, so a stuck partial file that later masquerades as a - // completed export is at least visible in the log. - if let Err(rm_err) = std::fs::remove_file(self.path) { - tracing::warn!( - dest = %self.path.display(), error = %rm_err, - "failed to remove partial export after failure — \ - next attempt at this dest will hit the already-exists guard" - ); - } - } -} - -/// `SendAgentSnapshotToFile` — stream a read-only snapshot (optionally -/// incremental against `parent_name`) to a file under -/// `MIGRATE_STAGING_ROOT` via `btrfs send`. Local-file half of the -/// inter-hive migration transport; see `PrivRequest::SendAgentSnapshotToFile` -/// for the cross-hive follow-up. -async fn send_agent_snapshot_to_file( - agent_name: &str, - snapshot_name: &str, - parent_name: Option<&str>, - dest_file_name: &str, -) -> Result<(String, String)> { - let snap = snapshot_path(agent_name, snapshot_name); - let parent_path = parent_name.map(|parent| snapshot_path(agent_name, parent)); - - std::fs::create_dir_all(MIGRATE_STAGING_ROOT) - .with_context(|| format!("create {MIGRATE_STAGING_ROOT}"))?; - let dest = Path::new(MIGRATE_STAGING_ROOT).join(dest_file_name); - // Preconditions are checked *inside* this call, before `dest` is - // created — see `open_export_dest`'s doc comment for why that ordering - // matters and is kept in one place. - let dest_file = open_export_dest(&snap, parent_path.as_deref(), &dest)?; - let cleanup = PartialExportGuard::new(&dest); - - let mut cmd = Command::new("btrfs"); - cmd.arg("send"); - if let Some(parent_path) = &parent_path { - cmd.arg("-p").arg(parent_path); - } - cmd.arg(&snap); - cmd.stdout(std::process::Stdio::from(dest_file)); - cmd.stderr(std::process::Stdio::piped()); - - let out = cmd - .spawn() - .with_context(|| format!("spawn btrfs send {}", snap.display()))? - .wait_with_output() - .await - .with_context(|| format!("wait on btrfs send {}", snap.display()))?; - if !out.status.success() { - bail!( - "btrfs send {} failed: {}", - snap.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - cleanup.disarm(); - tracing::info!( - agent = %agent_name, snapshot = %snap.display(), dest = %dest.display(), - parent = ?parent_name, "exported agent snapshot to file" - ); - Ok((dest.display().to_string(), String::new())) -} - -/// `SendAgentSnapshotToFd` — stream a read-only snapshot (optionally -/// incremental against `parent_name`) straight into a descriptor the -/// caller passed us. -/// -/// The network half of the inter-hive migration transport, arranged so -/// this helper never learns there *is* a network: hive-c0re connects to -/// the peer's snapshot store, writes the header itself, and hands the -/// connected socket over. We only ever see "a thing to write bytes into", -/// which keeps a root process out of any address, protocol or trust -/// decision — and keeps everyone out of the data path once `btrfs send` -/// starts, which matters at multi-gigabyte sizes. -async fn send_agent_snapshot_to_fd( - agent_name: &str, - snapshot_name: &str, - parent_name: Option<&str>, - dest: OwnedFd, -) -> Result<(String, String)> { - let snap = snapshot_path(agent_name, snapshot_name); - if !snap.exists() { - bail!( - "snapshot {} does not exist — create it with `subvol snapshot create` first", - snap.display() - ); - } - - let mut cmd = Command::new("btrfs"); - cmd.arg("send"); - if let Some(parent) = parent_name { - let parent_path = snapshot_path(agent_name, parent); - if !parent_path.exists() { - bail!( - "parent snapshot {} does not exist — pick an existing parent or omit it for a full send", - parent_path.display() - ); - } - cmd.arg("-p").arg(&parent_path); - } - cmd.arg(&snap); - cmd.stdout(std::process::Stdio::from(dest)); - cmd.stderr(std::process::Stdio::piped()); - - let out = cmd - .spawn() - .with_context(|| format!("spawn btrfs send {}", snap.display()))? - .wait_with_output() - .await - .with_context(|| format!("wait on btrfs send {}", snap.display()))?; - if !out.status.success() { - // Nothing to clean up: the destination isn't ours. A partial - // stream is the receiving end's problem, and `btrfs receive` - // refuses to commit an incomplete subvolume anyway. - bail!( - "btrfs send {} failed: {}", - snap.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - tracing::info!( - agent = %agent_name, snapshot = %snap.display(), parent = ?parent_name, - "streamed agent snapshot into a passed descriptor" - ); - Ok((String::new(), String::new())) -} - -/// `SetSubvolumeQuota` — set or clear a qgroup size limit on an agent -/// subvolume (`btrfs qgroup limit <…/agent_name>`). See the -/// wire doc. -async fn set_subvolume_quota( - agent_name: &str, - limit_bytes: Option, -) -> Result<(String, String)> { - let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); - let limit = limit_bytes.map_or_else(|| "none".to_owned(), |n| n.to_string()); - let out = Command::new("btrfs") - .args(["qgroup", "limit", &limit]) - .arg(&agent_root) - .output() - .await - .with_context(|| format!("spawn btrfs qgroup limit {}", agent_root.display()))?; - if !out.status.success() { - bail!( - "btrfs qgroup limit {limit} {} failed: {}", - agent_root.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - tracing::info!(agent = %agent_name, %limit, "set agent subvolume quota"); - Ok((String::new(), String::new())) -} - -/// Validate a single argument destined for `forgejo admin`. Rejects null -/// bytes, newlines and carriage returns (which could corrupt the -/// subprocess args list or log output). -/// -/// Shell metacharacters are **not** rejected, and do not need to be: the -/// command is spawned directly, with no shell to interpret them. -fn validate_forge_admin_arg(arg: &str) -> Result<()> { - if arg.bytes().any(|b| b == 0 || b == b'\n' || b == b'\r') { - bail!("forge admin arg {arg:?} contains null byte or newline"); - } - Ok(()) -} - -/// Redact a line before it hits the (root-readable, but still -/// unnecessarily exposed) host journal. -/// -/// Two independent rules, because the previous single rule failed open. -/// It matched only the substring "password", chosen to be robust against -/// forgejo *rewording* its password line — and the leak arrived from the -/// other axis entirely: a **different kind of secret** on a differently -/// worded line. `forgejo admin user generate-access-token` prints -/// `Access token was successfully created: <40 hex>`, which contains no -/// "password" and went to the journal verbatim for every agent ever -/// provisioned. -/// -/// So the second rule matches on **shape, not vocabulary**: a long -/// unbroken run of secret-alphabet characters. A new secret type is then -/// caught by default rather than by someone remembering to add a keyword. -/// -/// ⚠️ This deliberately over-matches. A nix store hash is also a long -/// opaque run and will redact its line. That is the correct direction to -/// be wrong in: the cost of a false positive is one less log line, and -/// the cost of a false negative is a live credential sitting in the host -/// journal, readable by anything with access to it. -fn redact_secret_line(line: &str) -> std::borrow::Cow<'_, str> { - if line.to_ascii_lowercase().contains("password") { - return std::borrow::Cow::Borrowed("[redacted: line mentions a password]"); - } - if contains_secret_shaped_run(line) { - return std::borrow::Cow::Borrowed("[redacted: line contains a secret-shaped token]"); - } - std::borrow::Cow::Borrowed(line) -} - -/// True when the line contains an unbroken run of at least 32 characters -/// from the hex / base64url alphabet. 32 sits below forgejo's 40-hex -/// access token and above the ordinary words and path segments that -/// appear in `forgejo admin` output. -/// -/// ⚠️ Scans for a RUN, not for a whitespace-delimited word. An earlier -/// version split on whitespace and required the whole word to match, -/// which a secret with punctuation glued to it defeats: `","` and -/// `"[]"` both fail an all-chars check on the word while still -/// containing the credential in full. Whitespace is not what delimits a -/// secret — the alphabet is (thanks @argus for catching it). -fn contains_secret_shaped_run(line: &str) -> bool { - const MIN: usize = 32; - let mut run = 0usize; - for b in line.bytes() { - if b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=' | b'_' | b'-') { - run += 1; - if run >= MIN { - return true; - } - } else { - run = 0; - } - } - false -} - -/// Name a `forgejo admin` invocation for an error message without -/// reproducing its arguments: keep the leading verb path, stop at the -/// first flag. `["user", "change-password", "--username", "iris", -/// "--password", "…"]` becomes `forgejo admin user change-password`. -/// -/// Same allowlist as hive-c0re's `describe_forge_admin` -/// (`hive-c0re/src/forge/mod.rs`), for the same reason: the verbs are a -/// closed set this crate chooses itself, but hive-c0re passes a live -/// `--password` value as an argument on some calls, and argument values -/// are never safe to assume closed. Redacting the value after -/// `--password` instead would repeat the bug `redact_secret_line`'s own -/// doc comment describes — a denylist of one flag name fails open the -/// moment a differently named secret-bearing flag is added. -fn describe_forge_admin(args: &[String]) -> String { - let verbs: Vec<&str> = args - .iter() - .take_while(|a| !a.starts_with('-')) - .map(String::as_str) - .collect(); - if verbs.is_empty() { - "forgejo admin".to_owned() - } else { - format!("forgejo admin {}", verbs.join(" ")) - } -} - -/// Run `forgejo admin ` inside the `hive-forge` container as the -/// `forgejo` unix user. Requires root (for nsenter into the container's -/// namespaces). Returns `(stdout, stderr)`. -async fn run_forge_admin(args: &[String]) -> Result<(String, String)> { - let mut cmd_args: Vec<&str> = vec![ - "run", - "hive-forge", - "--", - "runuser", - "-u", - "forgejo", - "--", - "forgejo", - "--work-path", - "/var/lib/forgejo", - "admin", - ]; - for a in args { - cmd_args.push(a.as_str()); - } - let out = Command::new("nixos-container") - .args(&cmd_args) - .output() - .await - .context("invoke nixos-container run hive-forge -- forgejo admin")?; - let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); - let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); - // stdout at DEBUG, not INFO: on the success path this stream carries - // the *product* of the command (the freshly minted token, the created - // user's details) and nothing an operator needs at default verbosity. - // Redaction stays on as the second layer — the level decides who sees - // it, the redactor decides what it says, and neither alone is enough. - for line in stdout.lines() { - tracing::debug!(target: "forgejo-admin", "{}", redact_secret_line(line)); - } - for line in stderr.lines() { - tracing::warn!(target: "forgejo-admin", "{}", redact_secret_line(line)); - } - if !out.status.success() { - // Redact here too. The error string is propagated to the caller and - // ends up logged; a partial-failure stderr can carry the same - // material stdout would have. And name the invocation via - // `describe_forge_admin` rather than `args.join(" ")`: some callers - // pass a live `--password` value as an argument, so reproducing the - // raw vector here is the same "two of three sites" gap that makes - // these leaks survive a fix — the log was redacted, but the error - // was not. - let safe_stderr: String = stderr - .lines() - .map(|l| redact_secret_line(l).into_owned()) - .collect::>() - .join("; "); - bail!( - "{} failed ({}): {}", - describe_forge_admin(args), - out.status, - safe_stderr.trim() - ); - } - Ok((stdout, stderr)) -} - -/// Invoke `nixos-container` with the given args, log output to journald. -async fn container_run(args: &[&str]) -> Result<(String, String)> { - let out = Command::new("nixos-container") - .args(args) - .output() - .await - .context("invoke nixos-container")?; - let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); - let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); - // `list` is a read-only enumeration called on the hot path (dashboard - // rescan, forge + boot sweeps) — its stdout is the return value, not - // progress, so logging every container name on every call floods the - // journal. Log stdout only for the mutating ops, where each line is - // genuine progress. stderr is always logged (errors matter regardless). - if args.first() != Some(&"list") { - for line in stdout.lines() { - tracing::info!(target: "nixos-container", "{line}"); - } - } - for line in stderr.lines() { - tracing::warn!(target: "nixos-container", "{line}"); - } - if !out.status.success() { - bail!( - "nixos-container {} failed ({}): {}", - args.join(" "), - out.status, - stderr.trim() - ); - } - Ok((stdout, stderr)) -} - -/// Invoke `machinectl` with the given args, log output to journald. -/// Used for operations that nixos-container doesn't expose (e.g. sending -/// signals to running containers). -async fn machinectl_run(args: &[&str]) -> Result<(String, String)> { - let out = Command::new("machinectl") - .args(args) - .output() - .await - .context("invoke machinectl")?; - let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); - let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); - for line in stdout.lines() { - tracing::info!(target: "machinectl", "{line}"); - } - for line in stderr.lines() { - tracing::warn!(target: "machinectl", "{line}"); - } - if !out.status.success() { - bail!( - "machinectl {} failed ({}): {}", - args.join(" "), - out.status, - stderr.trim() - ); - } - Ok((stdout, stderr)) -} - -/// How long to wait for machined to drop a machine's registration after a -/// shutdown has been asked for. Generous: a container with slow-stopping -/// units legitimately takes a while, and escalating early would SIGKILL a -/// shutdown that was going to finish on its own. -const NAME_RELEASE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); - -/// Poll cadence while waiting for the registration to go away. -const NAME_RELEASE_POLL: std::time::Duration = std::time::Duration::from_millis(500); - -/// Cap on a single registration probe. The probe is a D-Bus round trip to -/// machined; if machined itself is wedged the call would otherwise sit on -/// the D-Bus method timeout, which is far longer than the whole stop -/// sequence should take. -const MACHINE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - -/// True when machined still holds a registration for `machine`. -/// -/// This asks machined the same question the registration itself answers — -/// `machinectl show` resolves the name through `GetMachine`, the very lookup -/// that makes a later boot fail with `Failed to register machine: already -/// exists`. So a positive answer here is exactly the condition that breaks -/// the next start, not a proxy for it. -/// -/// Deliberately NOT the container's systemd unit state: the unit can be -/// `inactive` while the registration is still held, and that gap is the -/// whole bug this probe exists to catch. -/// -/// Errors and timeouts answer "still registered". Being wrong that way costs -/// a redundant SIGKILL to something already gone; being wrong the other way -/// hands back a stop that silently leaked the name. -async fn machine_registered(machine: &str) -> bool { - let probe = Command::new("machinectl") - .args(["show", machine, "--property=Name"]) - // Don't leave a probe behind when the timeout below fires. - .kill_on_drop(true) - .output(); - match tokio::time::timeout(MACHINE_PROBE_TIMEOUT, probe).await { - Ok(Ok(out)) => out.status.success(), - Ok(Err(e)) => { - tracing::warn!(%machine, error = %e, "machinectl show failed to run; assuming still registered"); - true - } - Err(_) => { - tracing::warn!(%machine, "machinectl show timed out; assuming still registered"); - true - } - } -} - -/// Poll [`machine_registered`] until the name is free or the timeout expires. -/// Returns true once the registration is gone. -async fn wait_name_released(machine: &str) -> bool { - let deadline = tokio::time::Instant::now() + NAME_RELEASE_TIMEOUT; - loop { - if !machine_registered(machine).await { - return true; - } - if tokio::time::Instant::now() >= deadline { - return false; - } - tokio::time::sleep(NAME_RELEASE_POLL).await; - } -} - -/// Stop `machine` and only report success once machined has actually released -/// the name. -/// -/// `nixos-container stop` exiting 0 does not mean the machine is gone. A -/// process that sits in the machine's cgroup without being a child of the -/// container's init — a shell exec'd in from outside, say — never receives -/// the shutdown's SIGTERM if it has been stopped with SIGSTOP, so the -/// registration outlives the "successful" stop. Every later start of that -/// container then fails with `Failed to register machine: already exists`, -/// and machined re-persists the stale record across its own restart, so -/// there is no cleaning it up after the fact. The only place to catch it is -/// here, in the stop. -/// -/// So: ask for the stop, wait for the name, and fail loudly if it is still -/// held — a caller that is told the stop worked will go on to start the -/// container and hit the confusing registration error instead of this one. -/// -/// This used to escalate to `machinectl kill --signal=SIGKILL` here. Dropped -/// per real incident data: the case this guards is a container genuinely -/// wedged (e.g. a root-login process survived the stop), and SIGKILL -/// doesn't recover that in practice — only a host-level reboot has. -/// Pretending a kill attempt handled it hides a condition that needs a human -/// to look at the host, so this now just reports the failure instead of -/// quietly (and ineffectually) trying to force it. -/// -/// The verify-then-fail lives in the helper rather than at a call site so -/// that every stop gets it: dashboard, reconcile, destroy, cold-start -/// fallback. The start path already distrusts its own exit code the same way; -/// this is the missing half of that pair. -async fn stop_and_release(machine: &str) -> Result<(String, String)> { - let stop = container_run(&["stop", machine]).await; - if wait_name_released(machine).await { - // Happy path, and also the path where a stop that reported failure - // nonetheless brought the machine down. Either way the caller gets - // the original result untouched. - return stop; - } - - tracing::error!( - %machine, - "stop finished but machined still holds the registration — the \ - container is likely wedged (a process outside its init tree \ - survived the stop); this needs a host-level look, not another \ - stop attempt" - ); - bail!( - "stop {machine}: machined still holds the machine name after {}s. \ - This container is likely wedged and starting it again will fail to \ - register — needs host-level intervention (a reboot has been the \ - only reliable fix in practice).", - NAME_RELEASE_TIMEOUT.as_secs() - ) -} - -/// Invoke `nixos-container` with the given args and forward output lines -/// to the caller as `PrivEvent::Line` messages in real time, logging each -/// line to journald as it arrives. Returns `(String::new(), String::new())` -/// on success (all output was streamed); the error string includes stderr -/// tail on failure. -async fn container_run_streaming( - args: &[&str], - writer: &mut OwnedWriteHalf, -) -> Result<(String, String)> { - use tokio::io::AsyncBufReadExt as _; - use tokio::process::Command; - - let mut child = Command::new("nixos-container") - .args(args) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("invoke nixos-container (streaming)")?; - - let stdout = child.stdout.take().expect("stdout piped"); - let stderr = child.stderr.take().expect("stderr piped"); - - let mut stdout_lines = BufReader::new(stdout).lines(); - let mut stderr_lines = BufReader::new(stderr).lines(); - - // Collect stderr for the error message; stream both to the client. - let mut stderr_buf = String::new(); - - // Drive stdout and stderr concurrently. `tokio::select!` interleaves - // them without bias — both streams drain at roughly the same rate - // as the subprocess produces output. - loop { - tokio::select! { - line = stdout_lines.next_line() => { - match line { - Ok(Some(l)) => { - tracing::info!(target: "nixos-container", "{l}"); - write_line_event(writer, PrivStream::Stdout, &l).await; - } - Ok(None) => break, - Err(e) => { - tracing::warn!(error = %e, "nixos-container stdout read error"); - break; - } - } - } - line = stderr_lines.next_line() => { - match line { - Ok(Some(l)) => { - tracing::warn!(target: "nixos-container", "{l}"); - write_line_event(writer, PrivStream::Stderr, &l).await; - if !stderr_buf.is_empty() { - stderr_buf.push('\n'); - } - stderr_buf.push_str(&l); - } - Ok(None) => {} - Err(e) => { - tracing::warn!(error = %e, "nixos-container stderr read error"); - } - } - } - } - } - - // Drain any remaining stderr after stdout closed. - while let Ok(Some(l)) = stderr_lines.next_line().await { - tracing::warn!(target: "nixos-container", "{l}"); - write_line_event(writer, PrivStream::Stderr, &l).await; - if !stderr_buf.is_empty() { - stderr_buf.push('\n'); - } - stderr_buf.push_str(&l); - } - - let status = child.wait().await.context("wait nixos-container")?; - if !status.success() { - // Only the last stderr line is embedded — the full stderr was - // already forwarded line-by-line as PrivEvent::Line messages and - // is captured in build_logs.sqlite by the caller. Keeping the - // error message short avoids bloating the anyhow chain. - bail!( - "nixos-container {} failed ({}): {}", - args.join(" "), - status, - stderr_buf.lines().last().unwrap_or("").trim() - ); - } - Ok((String::new(), String::new())) -} - -/// Synchronise the host's nginx unit after an `agents.conf` write. -/// -/// Queries `ActiveState` and dispatches: -/// - `active` → `systemctl reload nginx` (SIGHUP, zero-downtime) -/// - `failed` → `systemctl reset-failed nginx` + `systemctl start nginx` -/// - otherwise → `systemctl start nginx` -/// -/// ⚠️ `nginx` is hard-coded on purpose — see `PrivRequest::ReloadGatewayNginx`. -/// The unit name is the scope of this verb: nginx is a host unit, so no -/// namespace bounds it and the literal is the only thing standing between -/// "reload the gateway" and "reload anything". -/// -/// Returns `(String::new(), String::new())` on success so it fits the -/// `exec` return type directly. -async fn sync_gateway_nginx() -> Result<(String, String)> { - let state_out = Command::new("systemctl") - .args(["show", "--property=ActiveState", "--value", "nginx"]) - .output() - .await - .context("query gateway nginx ActiveState")?; - if !state_out.status.success() { - tracing::warn!( - exit_code = ?state_out.status.code(), - stderr = %String::from_utf8_lossy(&state_out.stderr).trim(), - "systemctl show ActiveState exited non-zero — gateway nginx may be down" - ); - } - let state = String::from_utf8_lossy(&state_out.stdout).trim().to_owned(); - // State-aware dispatch: reload when running; reset+start after - // start-limit failure; plain start when inactive or unknown. - match state.as_str() { - "active" => { - let out = Command::new("systemctl") - .args(["reload", "nginx"]) - .output() - .await - .context("reload gateway nginx")?; - if !out.status.success() { - bail!( - "gateway nginx reload failed ({}): {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ); - } - } - "failed" => { - // Clear start-limit so the next start can proceed. - let _ = Command::new("systemctl") - .args(["reset-failed", "nginx"]) - .status() - .await; - let out = Command::new("systemctl") - .args(["start", "nginx"]) - .output() - .await - .context("start gateway nginx after reset-failed")?; - if !out.status.success() { - bail!( - "gateway nginx start (after reset-failed) failed ({}): {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ); - } - } - _ => { - // inactive, activating, deactivating, unknown — just start. - let out = Command::new("systemctl") - .args(["start", "nginx"]) - .output() - .await - .context("start gateway nginx")?; - if !out.status.success() { - bail!( - "gateway nginx start failed (state={state:?}) ({}): {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ); - } - } - } - Ok((String::new(), String::new())) -} - -/// Return the system container name for a logical agent name. -/// All agents (including the manager) use the `h-` prefix. -/// -/// **This prefix is what confines the lifecycle verbs to agent containers** — -/// not their name validation, which only checks characters. A caller cannot -/// name `hive-forge` and reach it: it becomes `h-hive-forge`. No lifecycle -/// verb takes a sibling name: the forge and matrix are swarm services a hive -/// does not start or stop, and the gateway is reached through -/// [`PrivRequest::ControlInfraContainer`] and the [`InfraContainer`] enum. -fn container_system_name(name: &str) -> String { - format!("{AGENT_PREFIX}{name}") -} - -/// Validate a logical agent name (the name hive-c0re uses internally, -/// before the `h-` container prefix is applied). -fn validate_agent_name(name: &str) -> Result<()> { - validate_name_chars(name)?; - Ok(()) -} - -/// Validate a system-level container name (`h-` prefix for all agents, -/// the manager included) and return the logical agent name after it. -/// Anything else is refused, the swarm's forge and matrix containers -/// included: they belong to the host that runs them, not to a hive. -fn validate_container_system_name(name: &str) -> Result<&str> { - let Some(suffix) = name.strip_prefix(AGENT_PREFIX) else { - bail!("container name {name:?} is not managed by hive"); - }; - validate_name_chars(suffix)?; - Ok(suffix) -} - -fn validate_name_chars(name: &str) -> Result<()> { - if name.is_empty() - || !name - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') - { - bail!("invalid name {name:?}: must be non-empty lowercase ascii + digits + hyphens"); - } - Ok(()) -} - -/// Validate a bind-mount path: must be absolute, non-empty, and contain -/// no newlines, null bytes, double-quotes, or colons. -/// -/// The first three would break the `EXTRA_NSPAWN_FLAGS="..."` conf line -/// format. The colon is the one that matters most and had been left out -/// of this list: `--bind=SRC:DST` is colon-separated, so a path carrying -/// one does not corrupt the line — it silently becomes a *different -/// mount* than the caller asked for. -fn validate_bind_path(path: &str) -> Result<()> { - if path.is_empty() - || !path.starts_with('/') - || path - .bytes() - .any(|b| b == 0 || b == b'\n' || b == b'"' || b == b':') - { - bail!( - "invalid bind path {path:?}: must be an absolute path with no colons, newlines, null bytes, or double-quotes" - ); - } - Ok(()) -} - -/// `--tmpfs=/.git` for every bound git repo, hiding its metadata -/// from inside the container. -/// -/// Two kinds of mount qualify, for one reason: **the agent is given a -/// working tree, never a repository.** `/knowledge` is the hive's shared -/// docs — whose `.git/config` has held a credential the host-side worker -/// embedded — and `/agents//config` is a config repo, an agent's own -/// or a parent's read-only view of a child's. In both cases `.git` carries -/// every branch and the full history of a document whose *currently -/// deployed* value is the only thing a reader may act on, and an abandoned -/// branch is indistinguishable from a live one. -/// -/// An overlay rather than an exported copy: there is no second tree to -/// keep in sync, so nothing can go stale, and no code path has to remember -/// to refresh it. -/// -/// ⚠️ Ordering matters — these must be appended **after** the `--bind` -/// flags so nspawn mounts them on top of the already-mounted trees. -/// Config mounts are matched by shape, not by a name list: the set is -/// dynamic, growing with each child bound into a parent. -fn git_overlay_flags(binds: &[BindMount]) -> Vec { - binds - .iter() - .map(|b| b.container_path.as_str()) - .filter(|p| *p == "/knowledge" || (p.starts_with("/agents/") && p.ends_with("/config"))) - .map(|p| format!("--tmpfs={p}/.git")) - .collect() -} - -/// Update `/etc/nixos-containers/.conf`: strip old network vars -/// (`PRIVATE_NETWORK`, `HOST_ADDRESS*`, `LOCAL_ADDRESS*`, `HOST_BRIDGE`), -/// write the current network-isolation settings, then append -/// `EXTRA_NSPAWN_FLAGS`. Always writes `PRIVATE_NETWORK=1` + veth -/// wiring — isolation is the only mode, so there is no branch that -/// leaves a container on the host's network namespace. -fn write_nspawn_flags( - container: &str, - binds: &[BindMount], - isolation: &NetworkIsolation, - load_credentials: &[CredentialMount], - hive_facts: &str, -) -> Result<()> { - use std::fmt::Write as _; - use std::os::unix::fs::MetadataExt as _; - let path = format!("/etc/nixos-containers/{container}.conf"); - let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?; - let lines: Vec<&str> = original - .lines() - .filter(|line| { - let t = line.trim_start(); - !t.starts_with("EXTRA_NSPAWN_FLAGS=") - && !t.starts_with("PRIVATE_NETWORK=") - && !t.starts_with("HOST_ADDRESS=") - && !t.starts_with("LOCAL_ADDRESS=") - && !t.starts_with("HOST_ADDRESS6=") - && !t.starts_with("LOCAL_ADDRESS6=") - && !t.starts_with("HOST_BRIDGE=") - }) - .collect(); - let mut out = lines.join("\n"); - if !out.is_empty() { - out.push('\n'); - } - { - let iso = isolation; - out.push_str("PRIVATE_NETWORK=1\n"); - // HOST_ADDRESS = the bridge gateway IP. nixos-container's - // container-side setup only installs a default route - // (`ip route add default via $HOST_ADDRESS`) when HOST_ADDRESS is - // non-empty; leaving it blank gave the container an address but no - // route off the bridge subnet (no internet, no api.anthropic.com). - // In bridge mode (HOST_BRIDGE set) the host-side address/route - // setup is skipped, so this only affects the container's route — - // exactly what we want. - let _ = writeln!(out, "HOST_ADDRESS={}", iso.gateway_ip); - // LOCAL_ADDRESS is intentionally empty: containers take their IP by - // DHCP from the bridge dnsmasq pool. (Why HOST_ADDRESS is still - // written: the comment directly above.) - out.push_str("LOCAL_ADDRESS=\n"); - out.push_str("HOST_ADDRESS6=\n"); - out.push_str("LOCAL_ADDRESS6=\n"); - let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge); - } - let mut flags: Vec = binds - .iter() - .map(|b| { - let flag = if b.read_only { "--bind-ro" } else { "--bind" }; - format!("{flag}={}:{}", b.host_path, b.container_path) - }) - .collect(); - flags.extend(git_overlay_flags(binds)); - // Credential forwarding: nspawn loads each host secret into the - // container's credential store under ``; inner units inherit it - // via `LoadCredential=`. Validated (name charset + bind-path - // rules) in handle_write_nspawn_flags above. - for cred in load_credentials { - flags.push(format!( - "--load-credential={}:{}", - cred.name, cred.host_path - )); - } - let flags_joined = flags.join(" "); - let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\""); - // Keeps the owner and mode `nixos-container` created the file with; this - // helper only rewrites its lines. - let meta = std::fs::metadata(&path).with_context(|| format!("stat {path}"))?; - publish_file( - Path::new(&path), - out.as_bytes(), - meta.mode() & 0o7777, - Some((meta.uid(), meta.gid())), - )?; - - // DNS marker for the in-container resolver oneshot: the oneshot only - // rewrites the container's resolv.conf when this marker exists, and the - // marker carries the gateway IP so the container need not re-derive it. - // Always written — every container is isolated, so there is no mode in - // which the marker should be absent. Why the rewrite is needed at all, - // and which unit does it: `docs/networking/network.md` § *How the isolated - // container gets its resolver*. - write_bridge_dns_marker(container, isolation)?; - // Read by the agent's environment generator at boot - // (`nix/agent-modules/hive-facts.nix`). - write_container_etc_file_in(&container_rootfs(container), HIVE_FACTS_FILE, hive_facts)?; - Ok(()) -} - -/// Filename of the in-container DNS marker, in the container's own `/etc`. -const BRIDGE_DNS_MARKER: &str = "hyperhive-bridge-dns"; - -/// Filename of the hive-facts environment file, in the container's own `/etc`. -const HIVE_FACTS_FILE: &str = "hyperhive-hive-facts"; - -/// The container's `/` on the host. -fn container_rootfs(container: &str) -> PathBuf { - PathBuf::from(format!("/var/lib/nixos-containers/{container}")) -} - -/// Render [`HiveFacts`] as the systemd environment file the agent loads, -/// one `KEY=value` line per present fact. -/// -/// A value is refused unless it is DNS-name characters only, so it can carry -/// neither a newline (a second assignment) nor anything an environment file -/// or a shell would interpret. -fn render_hive_facts(facts: &HiveFacts) -> Result { - use std::fmt::Write as _; - let mut out = String::new(); - for (key, value) in [ - ("HYPERHIVE_HIVE_NAME", &facts.name), - ("HYPERHIVE_HIVE_DOMAIN", &facts.domain), - ] { - let Some(value) = value else { continue }; - if value.is_empty() - || value.len() > 253 - || !value - .bytes() - .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'_')) - { - bail!("invalid {key} value {value:?}: must be 1-253 chars of [A-Za-z0-9._-]"); - } - let _ = writeln!(out, "{key}={value}"); - } - Ok(out) -} - -/// Write the bridge-DNS marker the `hyperhive-isolated-dns` oneshot keys -/// off. The marker file contains just the gateway IP. Always written: -/// every container is isolated, so there is no host-netns case that -/// wants the marker absent. -fn write_bridge_dns_marker(container: &str, isolation: &NetworkIsolation) -> Result<()> { - write_bridge_dns_marker_in(&container_rootfs(container), &isolation.gateway_ip) -} - -/// Write `/etc/hyperhive-bridge-dns`; see [`write_container_etc_file_in`]. -fn write_bridge_dns_marker_in(rootfs: &Path, gateway_ip: &str) -> Result<()> { - write_container_etc_file_in(rootfs, BRIDGE_DNS_MARKER, &format!("{gateway_ip}\n")) -} - -/// Write `/etc/` without following a symlink. -/// -/// Root inside the container owns its `/etc`, and this write runs as host -/// root, where an absolute symlink planted in the container resolves against -/// the host's `/`. So this opens `etc` with `O_DIRECTORY|O_NOFOLLOW` (a -/// symlinked `etc` fails with `ELOOP`) and publishes the file through a -/// [`StagedFile`] relative to that fd: whatever the container planted at the -/// leaf, symlink or FIFO or device node, is replaced by the rename, never -/// written through or opened. -/// -/// On a fresh install the container's `/etc` may not exist yet (nixos-container -/// materialises the rootfs on the first start), so this creates it first; the -/// dir and the file persist through that start. Mode 0644: nothing written -/// here is a secret. -fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &str) -> Result<()> { - use std::os::unix::fs::OpenOptionsExt as _; - - let etc_path = rootfs.join("etc"); - // The rootfs dir is the container's `/`, which the container cannot - // replace, so creating it by path is safe. - std::fs::create_dir_all(rootfs) - .with_context(|| format!("create container rootfs {}", rootfs.display()))?; - let root = std::fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) - .open(rootfs) - .with_context(|| format!("open container rootfs {}", rootfs.display()))?; - // SAFETY: `root` is an open directory fd and the name a NUL-terminated - // literal. - if unsafe { libc::mkdirat(root.as_raw_fd(), c"etc".as_ptr(), 0o755) } != 0 { - let e = std::io::Error::last_os_error(); - if e.kind() != std::io::ErrorKind::AlreadyExists { - return Err(e).with_context(|| format!("create {}", etc_path.display())); - } - } - // SAFETY: as above; this checks the result before wrapping it. - let etc_fd = unsafe { - libc::openat( - root.as_raw_fd(), - c"etc".as_ptr(), - libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC, - ) - }; - if etc_fd < 0 { - return Err(std::io::Error::last_os_error()) - .with_context(|| format!("open (no-follow) {}", etc_path.display())); - } - // SAFETY: `etc_fd` is a fresh, valid fd that nothing else owns. - let etc = std::fs::File::from(unsafe { OwnedFd::from_raw_fd(etc_fd) }); - StagedFile::write(&etc, &etc_path, name, contents.as_bytes(), 0o644)?.publish() -} - -/// Written by older hive-priv builds. Nothing writes it now, but left on disk -/// systemd-tmpfiles would still apply it at every boot. -const LEGACY_TMPFILES_PATH: &str = "/etc/tmpfiles.d/hyperhive-agents.conf"; - -/// Unlink [`LEGACY_TMPFILES_PATH`]; already absent is success. Run at every -/// hive-priv start and by the legacy `SyncAgentTmpfiles` request. -fn remove_legacy_tmpfiles() -> Result<()> { - match std::fs::remove_file(LEGACY_TMPFILES_PATH) { - Ok(()) => { - tracing::info!("removed legacy {LEGACY_TMPFILES_PATH}"); - Ok(()) - } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(e) => Err(e).with_context(|| format!("remove {LEGACY_TMPFILES_PATH}")), - } -} - -/// `EnsureAgentSocketDir` — create `/run/hive-agent/` if missing. -fn ensure_agent_socket_dir(name: &str) -> Result<(String, String)> { - ensure_socket_dir_in(Path::new(SOCKET_DIR_ROOT), name)?; - Ok((String::new(), String::new())) -} - -/// Create `/` as a `0751` directory owned by the caller (root in -/// production), or accept an existing directory untouched. -/// `name` must be an agent ident; it is validated here, before any path is -/// built from it. -/// -/// The result is an nspawn bind source, so a symlink here would bind a host -/// path of the planter's choosing into the container. Every step is relative -/// to an `O_DIRECTORY|O_NOFOLLOW` fd for `root`, and an existing entry is -/// checked with `AT_SYMLINK_NOFOLLOW` and refused unless it is a directory. -/// -/// An existing directory keeps its owner and mode: the container's activation -/// hands it to the agent user, and re-asserting root here would undo that on -/// every start. -fn ensure_socket_dir_in(root: &Path, name: &str) -> Result<()> { - use std::os::unix::fs::OpenOptionsExt as _; - - validate_agent_name(name)?; - let path = root.join(name); - let c_name = std::ffi::CString::new(name) - .with_context(|| format!("agent name {name:?} contains a NUL byte"))?; - let root_dir = std::fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) - .open(root) - .with_context(|| format!("open (no-follow) {}", root.display()))?; - // SAFETY: `root_dir` is an open directory fd and `c_name` NUL-terminated. - if unsafe { libc::mkdirat(root_dir.as_raw_fd(), c_name.as_ptr(), 0o751) } == 0 { - // mkdirat's mode is masked by the umask; the mode is part of the - // contract, so it is set explicitly. - // SAFETY: as above. - let rc = unsafe { - libc::fchmodat( - root_dir.as_raw_fd(), - c_name.as_ptr(), - 0o751, - libc::AT_SYMLINK_NOFOLLOW, - ) - }; - if rc != 0 { - return Err(std::io::Error::last_os_error()) - .with_context(|| format!("chmod 0751 {}", path.display())); - } - tracing::info!(path = %path.display(), "created agent socket dir"); - return Ok(()); - } - let err = std::io::Error::last_os_error(); - if err.kind() != std::io::ErrorKind::AlreadyExists { - return Err(err).with_context(|| format!("create {}", path.display())); - } - // SAFETY: `stat` is plain old data; the zeroed value is only read after - // `fstatat` has filled it in. - let mut st: libc::stat = unsafe { std::mem::zeroed() }; - // SAFETY: as for `mkdirat`; `st` is a valid, writable `stat`. - let rc = unsafe { - libc::fstatat( - root_dir.as_raw_fd(), - c_name.as_ptr(), - &raw mut st, - libc::AT_SYMLINK_NOFOLLOW, - ) - }; - if rc != 0 { - return Err(std::io::Error::last_os_error()) - .with_context(|| format!("stat (no-follow) {}", path.display())); - } - if st.st_mode & libc::S_IFMT != libc::S_IFDIR { - bail!( - "{} exists but is not a directory; refusing it as a bind source", - path.display() - ); - } - Ok(()) -} - #[cfg(test)] mod tests { - use super::{ - BindMount, BoundedRun, Command, HiveFacts, InfraAction, InfraContainer, NetworkIsolation, - OwnedFd, PAUSED_MARKER_FILE, PrivRequest, StagedFile, agent_disk_paths, check_fd_agreement, - contains_secret_shaped_run, describe_forge_admin, du_command, du_total, - ensure_plain_filename, ensure_socket_dir_in, exec_measure_agent_disk, git_overlay_flags, - handle_write_nspawn_flags, limits_dropin_body, open_dir, open_export_dest, parse_du_total, - partial_name, publish_file, redact_secret_line, remove_marker_in, remove_service_dropin, - render_hive_facts, run_bounded, single_output_path, toplevel_attr, - validate_container_system_name, validate_credential_name, validate_snapshot_name, - write_agent_dir_file, write_bridge_dns_marker_in, write_resource_limits, - }; - use std::path::PathBuf; - use std::sync::atomic::{AtomicU32, Ordering}; - use std::time::Duration; - - fn bind(container_path: &str) -> BindMount { - BindMount { - host_path: "/var/lib/hyperhive/whatever".to_owned(), - container_path: container_path.to_owned(), - read_only: true, - } - } - - /// Pins the exact attr path we hand to `nix build` — it has to match - /// `hive-c0re`'s own `lifecycle::prebuild_toplevel` construction, since - /// that step's whole point is warming the store for this later build. - /// A drifted attr path defeats the cache-warming silently — no error, - /// just a slower `update`. - #[test] - fn toplevel_attr_matches_prebuild_toplevels_construction() { - assert_eq!( - toplevel_attr("atlas"), - "/var/lib/hyperhive/meta#nixosConfigurations.atlas.config.system.build.toplevel" - ); - } - - /// `--print-out-paths`' exact per-shape table (measured against real - /// `rustc` semantics, not just reasoned about) — a lone `"\n"` and a - /// trailing-space path are the two shapes a bare `.lines().collect()` - /// gets wrong, both catchable only by trimming + dropping empties - /// before counting rather than after. - #[test] - fn single_output_path_rejects_blank_and_trims_whitespace() { - assert_eq!(single_output_path(""), Err(0)); - assert_eq!(single_output_path("\n"), Err(0)); - assert_eq!(single_output_path("\n\n"), Err(0)); - assert_eq!(single_output_path("/nix/store/abc\n"), Ok("/nix/store/abc")); - assert_eq!( - single_output_path("/nix/store/abc \n"), - Ok("/nix/store/abc") - ); - assert_eq!( - single_output_path("/nix/store/abc\r\n"), - Ok("/nix/store/abc") - ); - assert_eq!( - single_output_path("/nix/store/abc\n/nix/store/def\n"), - Err(2) - ); - } - - /// Every bound git repo gets its `.git` overlaid — the knowledge tree - /// and *each* config mount, an agent's own plus every child's. - /// - /// The child case is the one worth pinning: that set grows at runtime - /// as agents gain children, so a rule written as a list of names would - /// silently stop covering new ones. - #[test] - fn every_bound_git_repo_gets_its_dot_git_hidden() { - let flags = git_overlay_flags(&[ - bind("/knowledge"), - bind("/agents/atlas/config"), - bind("/agents/kiddo/config"), - ]); - assert_eq!( - flags, - [ - "--tmpfs=/knowledge/.git", - "--tmpfs=/agents/atlas/config/.git", - "--tmpfs=/agents/kiddo/config/.git", - ] - ); - } - - /// ...and nothing else does. A blanket "overlay .git on every bind" - /// would mask a real `.git` under `state/`, where an agent legitimately - /// keeps working clones of its own. - #[test] - fn non_repo_mounts_are_left_alone() { - let flags = git_overlay_flags(&[ - bind("/agents/atlas/state"), - bind("/shared"), - bind("/applied"), - bind("/agents/atlas/config-notes"), - ]); - assert!(flags.is_empty(), "overlaid a non-repo mount: {flags:?}"); - } + use super::{OwnedFd, PrivRequest, check_fd_agreement}; /// A request that streams into a caller-supplied descriptor. fn fd_taking_request() -> PrivRequest { @@ -3307,721 +538,6 @@ mod tests { .expect("every ordinary request arrives without a descriptor"); } - /// The whole rendered body, pinned literally — including the values, - /// so changing the restart policy is a visible test edit rather than a - /// silent one. - /// - /// Placement is the part most worth pinning: `StartLimit*` are `[Unit]` - /// settings and systemd **silently ignores** them under `[Service]`, so - /// a bound that moved sections would look configured and do nothing. - /// An unset weight still emits no line at all, which is what keeps a - /// hive-c0re older than that field from changing what lands on disk. - #[test] - fn dropin_body_is_pinned_exactly() { - assert_eq!( - limits_dropin_body("/run/hyperhive/agents/iris", "4G", "200%", None, None), - "[Unit]\n\ - ConditionPathIsDirectory=/run/hyperhive/agents/iris\n\ - StartLimitIntervalSec=600\n\ - StartLimitBurst=3\n\ - \n\ - [Service]\n\ - RestartSec=5\n\ - MemoryMax=4G\n\ - CPUQuota=200%\n" - ); - } - - /// The bound is hive-wide **policy**, not a per-agent parameter: it is - /// rendered from constants and no caller-supplied value can omit or - /// alter it. This is the property that justifies keeping it out of the - /// wire protocol — if it ever varies by request, that argument is gone. - #[test] - fn the_start_limit_is_present_whatever_the_caller_passes() { - for (mem, cpu, cw, iw) in [ - ("4G", "200%", None, None), - ("512M", "50%", Some(10), Some(10)), - ("infinity", "infinity", Some(10_000), None), - ] { - let body = limits_dropin_body("/rt/x", mem, cpu, cw, iw); - let unit = body - .split("[Service]") - .next() - .expect("the body always has a [Unit] section before [Service]"); - assert!( - unit.contains("StartLimitIntervalSec=600") && unit.contains("StartLimitBurst=3"), - "start limit missing from [Unit] for ({mem}, {cpu}): {body}" - ); - assert!( - body.contains("\nRestartSec=5\n"), - "restart backoff missing for ({mem}, {cpu}): {body}" - ); - } - } - - /// Weights are appended to the `[Service]` section, each omitted - /// independently when `None`. - #[test] - fn weights_are_emitted_only_when_set() { - let both = limits_dropin_body("/rt/x", "4G", "200%", Some(80), Some(80)); - assert!( - both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"), - "{both}" - ); - - let cpu_only = limits_dropin_body("/rt/x", "4G", "200%", Some(80), None); - assert!( - cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"), - "{cpu_only}" - ); - assert!(!cpu_only.contains("IOWeight"), "{cpu_only}"); - - let io_only = limits_dropin_body("/rt/x", "4G", "200%", None, Some(80)); - assert!( - io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"), - "{io_only}" - ); - assert!(!io_only.contains("CPUWeight"), "{io_only}"); - } - - #[test] - fn redacts_lines_mentioning_password_case_insensitively() { - assert_eq!( - redact_secret_line("New password: hunter2"), - "[redacted: line mentions a password]" - ); - assert_eq!( - redact_secret_line("PASSWORD=hunter2"), - "[redacted: line mentions a password]" - ); - assert_eq!( - redact_secret_line("User \"foo\" was successfully created."), - "User \"foo\" was successfully created." - ); - } - - /// `run_forge_admin`'s `bail!` names the invocation through this - /// function instead of `args.join(" ")` — the regression this test - /// guards. A `--password` value never survives to the error string. - #[test] - fn describe_forge_admin_does_not_reproduce_a_password_argument() { - let described = describe_forge_admin(&[ - "user".to_owned(), - "create".to_owned(), - "--username".to_owned(), - "x".to_owned(), - "--password".to_owned(), - "S3cret".to_owned(), - ]); - assert_eq!(described, "forgejo admin user create"); - assert!(!described.contains("S3cret"), "{described}"); - } - - /// The `--password=value` single-argument form is also excluded: it - /// starts with `-`, so it never enters the kept verb path. - #[test] - fn describe_forge_admin_redacts_the_equals_sign_form_too() { - let described = describe_forge_admin(&[ - "user".to_owned(), - "create".to_owned(), - "--password=S3cret".to_owned(), - ]); - assert!(!described.contains("S3cret"), "{described}"); - } - - /// Control: a non-secret argument (the verb path) still appears, so - /// the message stays useful for diagnosing which call failed. - #[test] - fn describe_forge_admin_keeps_the_verb_path() { - assert_eq!( - describe_forge_admin(&["user".to_owned(), "change-password".to_owned()]), - "forgejo admin user change-password" - ); - assert_eq!( - describe_forge_admin(&["--help".to_owned()]), - "forgejo admin" - ); - assert_eq!(describe_forge_admin(&[]), "forgejo admin"); - } - - /// The regression this function exists for. The keyword rule passes - /// this line straight through — it says nothing about a password — so - /// only the shape rule catches it. - #[test] - fn redacts_access_token_line_which_mentions_no_password() { - let line = - "Access token was successfully created: 0123456789abcdef0123456789abcdef01234567"; - assert!( - !line.to_ascii_lowercase().contains("password"), - "fixture must not contain the keyword, or it proves nothing" - ); - assert_eq!( - redact_secret_line(line), - "[redacted: line contains a secret-shaped token]" - ); - } - - #[test] - fn secret_shape_boundaries() { - // Ordinary forgejo-admin output survives: no run is long enough. - assert_eq!( - redact_secret_line("Command 'user' 'create' finished with no errors."), - "Command 'user' 'create' finished with no errors." - ); - // 31 chars is below the floor, 32 is at it. - assert!(!contains_secret_shaped_run(&"a".repeat(31))); - assert!(contains_secret_shaped_run(&"a".repeat(32))); - // Punctuation breaks the run — a sentence never trips it however long. - assert!(!contains_secret_shaped_run( - "this.is.a.very.long.dotted.identifier.but.not.a.secret" - )); - // base64url and hex alphabets both count. - assert!(contains_secret_shaped_run( - "ZGVhZGJlZWZkZWFkYmVlZmRlYWRiZWVmZGVhZA==" - )); - assert!(contains_secret_shaped_run( - "aG93-dy_there-aG93dy1theresomething" - )); - } - - /// argus on the review: a whitespace-delimited check is defeated by - /// punctuation glued to the secret — the punctuation joins the "word" - /// and fails the alphabet test for the whole run, while the credential - /// sits there in full. Scanning for a RUN rather than a WORD closes it. - /// These are the shapes that used to slip through. - #[test] - fn secret_is_caught_with_punctuation_glued_to_it() { - const TOK: &str = "0123456789abcdef0123456789abcdef01234567"; - for line in [ - format!("token: {TOK},"), - format!("token: {TOK}."), - format!("using [{TOK}] now"), - format!("value=\"{TOK}\""), - format!("(created {TOK})"), - // no whitespace anywhere -- one glued blob - format!("Bearer:{TOK};next"), - ] { - assert_eq!( - redact_secret_line(&line), - "[redacted: line contains a secret-shaped token]", - "leaked through: {line}" - ); - } - } - - /// Unique scratch dir per test, no external tempfile dep. - fn scratch() -> PathBuf { - static CTR: AtomicU32 = AtomicU32::new(0); - let n = CTR.fetch_add(1, Ordering::Relaxed); - let dir = std::env::temp_dir().join(format!( - "hive-priv-nofollow-test-{}-{n}", - std::process::id() - )); - std::fs::create_dir_all(&dir).unwrap(); - dir - } - - #[test] - fn rejects_non_plain_filenames() { - let dir = scratch(); - let dir_fd = open_dir(&dir).unwrap(); - for bad in ["", ".", "..", "a/b", "/etc/passwd", "../escape", "sub/tok"] { - assert!( - StagedFile::write(&dir_fd, &dir, bad, b"x", 0o600).is_err(), - "must reject filename {bad:?}" - ); - // The wrapper's own check runs before its `create_dir_all`. - // Pointing it at a directory that does not exist yet is what makes - // this arm bite: the property worth pinning is that a bad leaf - // fails BEFORE any root-privileged filesystem work. - let absent = dir.join("never-created"); - assert!( - write_agent_dir_file("a", &absent, bad, "x").is_err(), - "wrapper must reject filename {bad:?}" - ); - assert!( - !absent.exists(), - "rejected write created a directory for {bad:?}" - ); - } - std::fs::remove_dir_all(&dir).ok(); - } - - /// The temp name is the whole reason the rename is safe to watch: a - /// `systemd.path` unit globbing `matrix-token*` would fire on a temp that - /// merely suffixed the real name, on exactly the empty file the rename - /// exists to hide. - #[test] - fn a_temp_cannot_match_a_glob_on_the_name_it_publishes() { - for real in ["matrix-token", "matrix-token-alice", "forge-token"] { - let tmp = partial_name(real); - assert!( - !tmp.starts_with(real), - "{tmp:?} matches a `{real}*` path unit while half-written" - ); - assert!(!tmp.contains('/'), "temp must stay in the same directory"); - } - } - - /// A published file arrives complete and alone. The residue arm is the - /// load-bearing one: a stranded temp holds the same secret, is owned by - /// root, and nothing else in the system would ever remove it. - #[test] - fn publishing_leaves_only_the_finished_file() { - use std::os::unix::fs::PermissionsExt as _; - let dir = scratch(); - let names = |d: &PathBuf| { - let mut v: Vec = std::fs::read_dir(d) - .unwrap() - .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - v.sort(); - v - }; - assert!(names(&dir).is_empty(), "control: scratch starts empty"); - - let path = dir.join("matrix-token-alice"); - for content in ["SECRET", "ROTATED"] { - write_agent_dir_file("a", &dir, "matrix-token-alice", content).unwrap(); - assert_eq!(names(&dir), ["matrix-token-alice"], "temp left behind"); - assert_eq!(std::fs::read_to_string(&path).unwrap(), content); - } - let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; - assert_eq!(mode, 0o600, "credential must be 0600 once published"); - - std::fs::remove_dir_all(&dir).ok(); - } - - /// A failed publish must not strand the temp. - #[test] - fn a_failed_publish_removes_the_temp() { - let dir = scratch(); - // A directory at the destination fails the `rename` (EISDIR) without - // needing to drop privileges; the write and chown ahead of it still run. - std::fs::create_dir(dir.join("forge-token")).unwrap(); - - let err = write_agent_dir_file("a", &dir, "forge-token", "SECRET") - .expect_err("publishing onto a directory must fail"); - assert!( - format!("{err:#}").contains("publishing"), - "must fail AT the publish -- failing earlier leaves no temp to \ - clean up and makes the next assertion vacuous. got: {err:#}" - ); - assert_eq!( - entries(&dir), - ["forge-token"], - "temp still holds the secret after a failed publish" - ); - - std::fs::remove_dir_all(&dir).ok(); - } - - /// Names in `dir`, sorted — the residue check every publish test ends on. - fn entries(dir: &PathBuf) -> Vec { - let mut v: Vec = std::fs::read_dir(dir) - .unwrap() - .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - v.sort(); - v - } - - /// The agent plants a symlink where the token goes. The rename replaces - /// the link; the root-privileged write never reaches its target. - #[test] - fn a_symlink_at_the_name_is_replaced_not_followed() { - let dir = scratch(); - let target = dir.join("target"); - std::fs::write(&target, "original").unwrap(); - std::os::unix::fs::symlink(&target, dir.join("forge-token")).unwrap(); - - write_agent_dir_file("a", &dir, "forge-token", "SECRET").unwrap(); - assert_eq!( - std::fs::read_to_string(&target).unwrap(), - "original", - "symlink target must be untouched" - ); - let meta = std::fs::symlink_metadata(dir.join("forge-token")).unwrap(); - assert!(meta.file_type().is_file(), "the link must be replaced"); - assert_eq!( - std::fs::read_to_string(dir.join("forge-token")).unwrap(), - "SECRET" - ); - std::fs::remove_dir_all(&dir).ok(); - } - - /// Any failure between staging and publishing drops the `StagedFile`. - /// Until the rename a reader sees only the old file, and afterwards the - /// old file is still whole and the temp is gone. - #[test] - fn a_failed_write_leaves_the_old_file_intact() { - let dir = scratch(); - let dest = dir.join("hyperhive-limits.conf"); - std::fs::write(&dest, "OLD").unwrap(); - let dir_fd = open_dir(&dir).unwrap(); - - let staged = - StagedFile::write(&dir_fd, &dir, "hyperhive-limits.conf", b"NEW", 0o644).unwrap(); - assert_eq!(std::fs::read_to_string(&dest).unwrap(), "OLD"); - assert_eq!( - entries(&dir).len(), - 2, - "control: the temp exists while staged" - ); - drop(staged); - - assert_eq!(std::fs::read_to_string(&dest).unwrap(), "OLD"); - assert_eq!(entries(&dir), ["hyperhive-limits.conf"], "temp left behind"); - std::fs::remove_dir_all(&dir).ok(); - } - - /// Two overlapping writes of one file each own a temp, so whichever - /// publishes last lands whole — never the other's bytes, never a mix. - #[test] - fn overlapping_writes_of_one_file_never_share_a_temp() { - let dir = scratch(); - let dir_fd = open_dir(&dir).unwrap(); - let name = "hyperhive-agents.conf"; - - let first = StagedFile::write(&dir_fd, &dir, name, b"roster A\n", 0o644).unwrap(); - let second = StagedFile::write(&dir_fd, &dir, name, b"roster B, longer\n", 0o644).unwrap(); - second.publish().unwrap(); - first.publish().unwrap(); - - assert_eq!( - std::fs::read_to_string(dir.join(name)).unwrap(), - "roster A\n" - ); - assert_eq!(entries(&dir), [name], "temp left behind"); - std::fs::remove_dir_all(&dir).ok(); - } - - /// The mode lands exactly as asked, whatever the umask would have made - /// of it (0666 under the usual 022 would otherwise come out 0644), and - /// the requested owner is on the file. - #[test] - fn publish_file_applies_mode_and_owner() { - use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; - let dir = scratch(); - let path = dir.join("c.conf"); - let owner = std::fs::metadata(&dir).unwrap(); - - publish_file(&path, b"x\n", 0o666, Some((owner.uid(), owner.gid()))).unwrap(); - let meta = std::fs::metadata(&path).unwrap(); - assert_eq!(meta.permissions().mode() & 0o7777, 0o666); - assert_eq!((meta.uid(), meta.gid()), (owner.uid(), owner.gid())); - - publish_file(&path, b"y\n", 0o600, None).unwrap(); - let meta = std::fs::metadata(&path).unwrap(); - assert_eq!(meta.permissions().mode() & 0o7777, 0o600); - assert_eq!(std::fs::read_to_string(&path).unwrap(), "y\n"); - assert_eq!(entries(&dir), ["c.conf"], "temp left behind"); - std::fs::remove_dir_all(&dir).ok(); - } - - /// The container plants a symlink where the bridge-DNS marker goes; the - /// rename replaces it without writing through it. - #[test] - fn bridge_dns_marker_replaces_symlink_leaf() { - let dir = scratch(); - let rootfs = dir.join("rootfs"); - std::fs::create_dir_all(rootfs.join("etc")).unwrap(); - let target = dir.join("target"); - std::fs::write(&target, "original").unwrap(); - let leaf = rootfs.join("etc/hyperhive-bridge-dns"); - std::os::unix::fs::symlink(&target, &leaf).unwrap(); - - write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap(); - assert_eq!( - std::fs::read_to_string(&target).unwrap(), - "original", - "symlink target must be untouched" - ); - assert!( - std::fs::symlink_metadata(&leaf) - .unwrap() - .file_type() - .is_file() - ); - assert_eq!(std::fs::read_to_string(&leaf).unwrap(), "10.0.0.1\n"); - std::fs::remove_dir_all(&dir).ok(); - } - - /// The container replaces its whole `/etc` with a symlink, so the leaf - /// name resolves inside a directory of its choosing. - #[test] - fn bridge_dns_marker_refuses_symlink_etc() { - let dir = scratch(); - let rootfs = dir.join("rootfs"); - std::fs::create_dir_all(&rootfs).unwrap(); - let elsewhere = dir.join("elsewhere"); - std::fs::create_dir_all(&elsewhere).unwrap(); - let target = elsewhere.join("hyperhive-bridge-dns"); - std::fs::write(&target, "original").unwrap(); - std::os::unix::fs::symlink(&elsewhere, rootfs.join("etc")).unwrap(); - - let res = write_bridge_dns_marker_in(&rootfs, "10.0.0.1"); - assert!(res.is_err(), "O_NOFOLLOW must refuse a symlinked etc"); - assert_eq!( - std::fs::read_to_string(&target).unwrap(), - "original", - "file behind the symlinked etc must be untouched" - ); - std::fs::remove_dir_all(&dir).ok(); - } - - /// A FIFO at the leaf is replaced, never opened, so it cannot block the - /// helper. - #[test] - fn bridge_dns_marker_replaces_fifo_leaf() { - use std::os::unix::ffi::OsStrExt as _; - let dir = scratch(); - let rootfs = dir.join("rootfs"); - std::fs::create_dir_all(rootfs.join("etc")).unwrap(); - let fifo = rootfs.join("etc/hyperhive-bridge-dns"); - let c_fifo = std::ffi::CString::new(fifo.as_os_str().as_bytes()).unwrap(); - // SAFETY: `c_fifo` is a valid NUL-terminated path. - assert_eq!(unsafe { libc::mkfifo(c_fifo.as_ptr(), 0o644) }, 0); - - write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap(); - assert!( - std::fs::symlink_metadata(&fifo) - .unwrap() - .file_type() - .is_file() - ); - assert_eq!(std::fs::read_to_string(&fifo).unwrap(), "10.0.0.1\n"); - std::fs::remove_dir_all(&dir).ok(); - } - - /// The control: a plain rootfs with no `etc` yet (a fresh install) gets - /// the dir and a 0644 marker, and a second write replaces it. - #[test] - fn bridge_dns_marker_written_into_plain_etc() { - use std::os::unix::fs::PermissionsExt as _; - let dir = scratch(); - let rootfs = dir.join("rootfs"); - std::fs::create_dir_all(&rootfs).unwrap(); - - write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap(); - let path = rootfs.join("etc/hyperhive-bridge-dns"); - assert_eq!(std::fs::read_to_string(&path).unwrap(), "10.0.0.1\n"); - let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; - assert_eq!(mode, 0o644, "marker must stay readable in the container"); - - write_bridge_dns_marker_in(&rootfs, "10.9.8.7").unwrap(); - assert_eq!(std::fs::read_to_string(&path).unwrap(), "10.9.8.7\n"); - std::fs::remove_dir_all(&dir).ok(); - } - - /// Each present fact is one environment-file line; an absent one writes - /// nothing, so the agent sees the variable unset rather than empty. - #[test] - fn hive_facts_render_one_line_per_present_fact() { - let both = HiveFacts { - name: Some("pr1ma".to_owned()), - domain: Some("pr1ma.darkest.space".to_owned()), - }; - assert_eq!( - render_hive_facts(&both).unwrap(), - "HYPERHIVE_HIVE_NAME=pr1ma\nHYPERHIVE_HIVE_DOMAIN=pr1ma.darkest.space\n" - ); - let domain_only = HiveFacts { - name: None, - domain: Some("h1.example.test".to_owned()), - }; - assert_eq!( - render_hive_facts(&domain_only).unwrap(), - "HYPERHIVE_HIVE_DOMAIN=h1.example.test\n" - ); - assert_eq!(render_hive_facts(&HiveFacts::default()).unwrap(), ""); - } - - /// A value that could end its line or be read as syntax is refused, not - /// escaped: a newline would add an assignment of the caller's choosing. - #[test] - fn hive_facts_refuse_values_outside_dns_characters() { - for bad in ["", "pr1ma\nLD_PRELOAD=/x", "a b", "a\"b", "$(x)"] { - let facts = HiveFacts { - name: Some(bad.to_owned()), - domain: None, - }; - assert!( - render_hive_facts(&facts).is_err(), - "{bad:?} must be refused" - ); - } - } - - /// Whether `pid` is gone, or only a zombie awaiting its reaper: either - /// way it is no longer running. - fn is_dead(pid: &str) -> bool { - match std::fs::read_to_string(format!("/proc/{pid}/stat")) { - Err(_) => true, - // The state letter follows the parenthesised command name. - Ok(stat) => stat - .rsplit_once(") ") - .is_some_and(|(_, rest)| rest.starts_with('Z')), - } - } - - /// A child still running at the limit comes back `TimedOut`, and the kill - /// reaches its whole process group: the backgrounded grandchild dies too, - /// not just the shell that started it. - #[tokio::test] - async fn a_build_past_its_limit_is_killed_with_its_group() { - use std::time::{Duration, Instant}; - let dir = scratch(); - let pidfile = dir.join("grandchild"); - let mut cmd = tokio::process::Command::new("sh"); - cmd.arg("-c") - .arg(format!("sleep 600 & echo $! > {}; wait", pidfile.display())); - - let started = Instant::now(); - let run = run_bounded(cmd, Duration::from_secs(2), None) - .await - .unwrap(); - assert!(matches!(run, BoundedRun::TimedOut), "must time out"); - assert!( - started.elapsed() < Duration::from_mins(1), - "the limit did not bound the run" - ); - - let pid = std::fs::read_to_string(&pidfile).unwrap(); - let pid = pid.trim(); - let deadline = Instant::now() + Duration::from_secs(5); - while !is_dead(pid) && Instant::now() < deadline { - tokio::time::sleep(Duration::from_millis(50)).await; - } - assert!(is_dead(pid), "grandchild {pid} outlived the timeout"); - std::fs::remove_dir_all(&dir).ok(); - } - - /// The control: a child that exits inside its limit is `Exited`, with its - /// status and both streams kept apart. - #[tokio::test] - async fn a_build_inside_its_limit_returns_its_output() { - let mut cmd = tokio::process::Command::new("sh"); - cmd.arg("-c") - .arg("echo progress >&2; echo /nix/store/abc-toplevel"); - match run_bounded(cmd, std::time::Duration::from_mins(1), None) - .await - .unwrap() - { - BoundedRun::Exited { - status, - stdout, - stderr, - } => { - assert!(status.success()); - assert_eq!(stdout, "/nix/store/abc-toplevel\n"); - assert_eq!(stderr, "progress"); - } - BoundedRun::TimedOut => panic!("a fast child must not time out"), - } - } - - /// A missing socket dir is created `0751`, owned by the caller (root in - /// production), which is the mode the container's activation expects to - /// hand over and the one dialers traverse. - #[test] - fn socket_dir_created_0751_when_absent() { - use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; - let dir = scratch(); - ensure_socket_dir_in(&dir, "probe").unwrap(); - let meta = std::fs::symlink_metadata(dir.join("probe")).unwrap(); - assert!(meta.is_dir()); - assert_eq!(meta.permissions().mode() & 0o7777, 0o751); - // SAFETY: `geteuid` has no preconditions. - assert_eq!(meta.uid(), unsafe { libc::geteuid() }); - std::fs::remove_dir_all(&dir).ok(); - } - - /// An existing directory keeps its mode and contents: by the second start - /// it belongs to the agent user, and a live socket may sit inside it. - #[test] - fn socket_dir_existing_directory_left_alone() { - use std::os::unix::fs::PermissionsExt as _; - let dir = scratch(); - let sub = dir.join("probe"); - std::fs::create_dir(&sub).unwrap(); - std::fs::set_permissions(&sub, std::fs::Permissions::from_mode(0o700)).unwrap(); - std::fs::write(sub.join("web.sock"), "").unwrap(); - - ensure_socket_dir_in(&dir, "probe").unwrap(); - let mode = std::fs::metadata(&sub).unwrap().permissions().mode() & 0o7777; - assert_eq!(mode, 0o700, "an existing dir's mode must not be reset"); - assert!(sub.join("web.sock").exists()); - std::fs::remove_dir_all(&dir).ok(); - } - - /// A symlink at the leaf would make nspawn bind wherever it points; it is - /// refused, whether it points at a directory or nowhere. - #[test] - fn socket_dir_refuses_symlink_leaf() { - use std::os::unix::fs::PermissionsExt as _; - let dir = scratch(); - let elsewhere = dir.join("elsewhere"); - std::fs::create_dir(&elsewhere).unwrap(); - std::fs::set_permissions(&elsewhere, std::fs::Permissions::from_mode(0o700)).unwrap(); - std::os::unix::fs::symlink(&elsewhere, dir.join("probe")).unwrap(); - std::os::unix::fs::symlink(dir.join("missing"), dir.join("dangling")).unwrap(); - - assert!(ensure_socket_dir_in(&dir, "probe").is_err()); - assert!(ensure_socket_dir_in(&dir, "dangling").is_err()); - let mode = std::fs::metadata(&elsewhere).unwrap().permissions().mode() & 0o7777; - assert_eq!(mode, 0o700, "the symlink target must be untouched"); - assert!(!dir.join("missing").exists()); - std::fs::remove_dir_all(&dir).ok(); - } - - /// A regular file where the dir should be is refused, not bound. - #[test] - fn socket_dir_refuses_non_directory_leaf() { - let dir = scratch(); - std::fs::write(dir.join("probe"), "x").unwrap(); - let err = ensure_socket_dir_in(&dir, "probe").unwrap_err(); - assert!(format!("{err:#}").contains("not a directory"), "{err:#}"); - std::fs::remove_dir_all(&dir).ok(); - } - - /// A symlinked root would place the new dir in a tree of the planter's - /// choosing; the `O_NOFOLLOW` open refuses it before anything is created. - #[test] - fn socket_dir_refuses_symlinked_root() { - let dir = scratch(); - let elsewhere = dir.join("elsewhere"); - std::fs::create_dir(&elsewhere).unwrap(); - let root = dir.join("root"); - std::os::unix::fs::symlink(&elsewhere, &root).unwrap(); - - assert!(ensure_socket_dir_in(&root, "probe").is_err()); - assert!(!elsewhere.join("probe").exists()); - std::fs::remove_dir_all(&dir).ok(); - } - - /// Names that are not a plain agent ident are refused by validation, - /// before anything is created. Runs against a scratch parent, so the - /// refusal cannot come from a missing `/run/hive-agent`; `.`/`..` exist - /// as directories and `Atlas` would be created, so each bad name gets - /// through only if validation is gone. Control: a plain name succeeds - /// in the same parent. - #[test] - fn socket_dir_refuses_bogus_names() { - let dir = scratch(); - for bad in ["", ".", "..", "../etc", "a/b", "Atlas", "a b", "a\0b"] { - let err = ensure_socket_dir_in(&dir, bad).unwrap_err(); - assert!( - format!("{err:#}").contains("invalid name"), - "agent name {bad:?} must be refused by validation, got: {err:#}" - ); - } - assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0); - ensure_socket_dir_in(&dir, "atlas").unwrap(); - assert!(dir.join("atlas").is_dir()); - std::fs::remove_dir_all(&dir).ok(); - } - /// An older hive-c0re's `SyncAgentTmpfiles` still decodes, payload and /// all, so it gets the legacy cleanup rather than a parse error. Control: /// an unknown op does not decode. @@ -4035,314 +551,4 @@ mod tests { )); assert!(serde_json::from_str::(r#"{"op":"sync_agent_tmpfilez"}"#).is_err()); } - - /// A start/stop naming the forge or matrix container fails to decode, so - /// `run` answers it with a parse error before anything reaches - /// `systemctl`. Control: the same request naming the gateway decodes. - #[test] - fn infra_control_refuses_swarm_services() { - let req = |c: &str| { - format!(r#"{{"op":"control_infra_container","container":"{c}","action":"stop"}}"#) - }; - for c in ["Forge", "Matrix"] { - let err = serde_json::from_str::(&req(c)).unwrap_err(); - assert!(err.to_string().contains("unknown variant"), "{c}: {err}"); - } - assert!(matches!( - serde_json::from_str::(&req("Gateway")), - Ok(PrivRequest::ControlInfraContainer { - container: InfraContainer::Gateway, - action: InfraAction::Stop, - }) - )); - } - - /// The pause marker round-trips through the same root-only path the - /// credential writes use, and BOTH directions are idempotent — the - /// dashboard toggle and `hivectl pause|resume` fire blind, without - /// reading the current state first. - #[test] - fn pause_marker_create_and_remove_are_idempotent() { - let dir = scratch(); - let path = dir.join(PAUSED_MARKER_FILE); - - for _ in 0..2 { - write_agent_dir_file("a", &dir, PAUSED_MARKER_FILE, "").unwrap(); - assert!(path.exists(), "marker must exist after pause"); - assert_eq!(std::fs::read_to_string(&path).unwrap(), ""); - } - for _ in 0..2 { - remove_marker_in(&dir, PAUSED_MARKER_FILE).unwrap(); - assert!(!path.exists(), "marker must be gone after resume"); - } - std::fs::remove_dir_all(&dir).ok(); - } - - /// A resume must never follow an agent-planted symlink at the marker - /// path: this unlink runs as root, so following it would let an agent - /// delete an arbitrary file on the host. - #[test] - fn resume_unlinks_the_symlink_not_its_target() { - let dir = scratch(); - let target = dir.join("target"); - std::fs::write(&target, "original").unwrap(); - let link = dir.join(PAUSED_MARKER_FILE); - std::os::unix::fs::symlink(&target, &link).unwrap(); - - remove_marker_in(&dir, PAUSED_MARKER_FILE).unwrap(); - // `exists()` follows the link, so it can't tell "link removed" from - // "target removed, dangling link left" — stat the link itself. - assert!( - std::fs::symlink_metadata(&link).is_err(), - "the link itself must be unlinked" - ); - assert_eq!( - std::fs::read_to_string(&target).unwrap(), - "original", - "symlink target must survive the root unlink" - ); - std::fs::remove_dir_all(&dir).ok(); - } - - /// The only gate on the root-privileged socket for a `--load-credential` - /// name. The doc comment is explicit that `.` is excluded on purpose - /// ("this name gets interpolated into filesystem paths") — a future - /// "let's allow dots for version numbers" loosening must fail this, - /// not just the `/` case below. - #[test] - fn validate_credential_name_rejects_dot_slash_and_empty_but_allows_the_charset() { - assert!( - validate_credential_name("").is_err(), - "empty must be rejected" - ); - assert!(validate_credential_name("valid-name_123").is_ok()); - assert!( - validate_credential_name("bad.name").is_err(), - "dot must be rejected — see the fn's doc comment" - ); - assert!(validate_credential_name("bad/name").is_err()); - } - - /// A snapshot label must additionally carry the `hive-` prefix (it - /// doubles as the allow-list gating the btrfs snapshot/delete - /// shellouts) on top of [`validate_credential_name`]'s charset rule. - #[test] - fn validate_snapshot_name_requires_hive_prefix() { - assert!(validate_snapshot_name("not-hive-prefixed").is_err()); - assert!(validate_snapshot_name("hive-valid").is_ok()); - } - - /// The swarm's forge and matrix containers are not the hive's: every - /// request that names a container as a string refuses them before - /// touching the filesystem. Control: an agent container name passes. - #[test] - fn container_name_requests_refuse_swarm_service_containers() { - let isolation = NetworkIsolation { - bridge: "hive0".to_owned(), - gateway_ip: "10.42.0.1".to_owned(), - }; - let facts = HiveFacts { - name: None, - domain: None, - }; - for name in ["hive-forge", "hive-matrix"] { - let errs = [ - handle_write_nspawn_flags(name, &[], &isolation, &[], &facts).unwrap_err(), - write_resource_limits(name, "4G", "200%", None, None).unwrap_err(), - remove_service_dropin(name).unwrap_err(), - ]; - for e in errs { - assert!( - e.to_string().contains("is not managed by hive"), - "{name}: {e:#}" - ); - } - } - assert_eq!(validate_container_system_name("h-atlas").unwrap(), "atlas"); - } - - /// `ensure_plain_filename` is the gate between an agent-writable - /// directory and a root-privileged file write. `.`/`..` would resolve - /// to the directory itself or its parent; any `/` climbs into a path - /// component the caller never named. - #[test] - fn ensure_plain_filename_rejects_dot_dotdot_and_any_slash() { - assert!(ensure_plain_filename("test", ".").is_err()); - assert!(ensure_plain_filename("test", "..").is_err()); - assert!(ensure_plain_filename("test", "sub/dir").is_err()); - assert!(ensure_plain_filename("test", "matrix-token").is_ok()); - } - - /// Unique scratch dir per test, no external tempfile dep. Same pattern - /// as `scratch()` above, kept separate so a rename of one doesn't - /// collide with the other's directory names under parallel test runs. - fn snapshot_export_scratch() -> PathBuf { - static CTR: AtomicU32 = AtomicU32::new(0); - let n = CTR.fetch_add(1, Ordering::Relaxed); - let dir = std::env::temp_dir().join(format!( - "hive-priv-snapshot-export-test-{}-{n}", - std::process::id() - )); - std::fs::create_dir_all(&dir).unwrap(); - dir - } - - /// The fix: a missing parent must be caught before `dest` is created, - /// not after. Fails against the pre-fix ordering (`dest` created via - /// `create_new` first, parent checked second) — see the issue for the - /// inverted run. - #[test] - fn missing_parent_leaves_no_file_at_dest() { - let dir = snapshot_export_scratch(); - let snap = dir.join("snap"); - std::fs::write(&snap, b"").unwrap(); - let parent = dir.join("parent-that-does-not-exist"); - let dest = dir.join("dest"); - - let err = open_export_dest(&snap, Some(&parent), &dest).unwrap_err(); - assert!( - err.to_string().contains("does not exist"), - "wrong error: {err}" - ); - assert!( - !dest.exists(), - "a failed export must not leave a file at dest" - ); - } - - /// A retry against the same `dest` after a precondition failure must - /// not trip the no-overwrite guard — there is nothing there to - /// overwrite, since the failed attempt never created `dest` (the test - /// above). This is the behavior a "stale parent falls back to a full - /// send, retried without -p" caller needs against a fixed `--dest`. - #[test] - fn retry_after_precondition_failure_succeeds() { - let dir = snapshot_export_scratch(); - let snap = dir.join("snap"); - std::fs::write(&snap, b"").unwrap(); - let parent = dir.join("parent-that-does-not-exist"); - let dest = dir.join("dest"); - - assert!(open_export_dest(&snap, Some(&parent), &dest).is_err()); - assert!(!dest.exists()); - - // Retry without the missing parent, same dest. - let file = open_export_dest(&snap, None, &dest); - assert!( - file.is_ok(), - "retry against a dest no previous attempt touched must succeed: {:?}", - file.err() - ); - assert!(dest.exists()); - } - - /// Control for the two tests above: an export that actually landed at - /// `dest` must still refuse a second one. Without this, a bug that - /// simply stopped creating `dest` at all (rather than fixing the - /// ordering) would also make the retry test above pass. - #[test] - fn existing_dest_still_refuses_overwrite() { - let dir = snapshot_export_scratch(); - let snap = dir.join("snap"); - std::fs::write(&snap, b"").unwrap(); - let dest = dir.join("dest"); - std::fs::write(&dest, b"already here").unwrap(); - - let err = open_export_dest(&snap, None, &dest).unwrap_err(); - assert!( - err.to_string().contains("already exists"), - "wrong error: {err}" - ); - assert_eq!( - std::fs::read(&dest).unwrap(), - b"already here", - "must not have touched the existing file" - ); - } - - #[test] - fn disk_paths_are_derived_from_the_name_only() { - assert_eq!( - agent_disk_paths("iris"), - [ - PathBuf::from("/var/lib/hyperhive/agents/iris/state"), - PathBuf::from("/var/lib/hyperhive/agents/iris/work"), - PathBuf::from("/var/lib/nixos-containers/h-iris"), - ] - ); - } - - /// Every name that could steer the walk out of the agent's own paths is - /// refused before any path is built, let alone `du` spawned. - #[tokio::test] - async fn measure_rejects_names_that_are_not_agent_names() { - for bad in [ - "", "..", "../etc", "a/b", "/etc", "Iris", "a b", "a\0b", "foo_bar", - ] { - let err = exec_measure_agent_disk(bad) - .await - .expect_err(&format!("{bad:?} must be rejected")); - assert!( - err.to_string().contains("invalid name"), - "{bad:?}: wrong error: {err}" - ); - } - } - - #[test] - fn du_total_reads_the_total_line() { - let out = "100\t/var/lib/hyperhive/agents/iris/state\n\ - 2000\t/var/lib/nixos-containers/h-iris\n\ - 2100\ttotal\n"; - assert_eq!(parse_du_total(out), Some(2100)); - assert_eq!(parse_du_total("0\ttotal"), Some(0)); - } - - #[test] - fn du_total_rejects_output_without_a_total_line() { - assert_eq!(parse_du_total(""), None); - assert_eq!(parse_du_total("\n"), None); - // A per-path line is not the total, even when it is the last one. - assert_eq!(parse_du_total("100\t/var/lib/x\n"), None); - assert_eq!(parse_du_total("abc\ttotal\n"), None); - assert_eq!(parse_du_total("-1\ttotal\n"), None); - assert_eq!(parse_du_total("100 total\n"), None); - } - - #[test] - fn no_existing_paths_is_an_error_not_zero() { - let err = du_command(&[]).expect_err("no operands must not run du"); - assert!(err.to_string().contains("none of"), "{err}"); - } - - #[tokio::test] - async fn non_zero_du_exit_is_an_error() { - let err = du_total("iris", Command::new("false"), Duration::from_secs(5)) - .await - .expect_err("a failed du must not yield a total"); - assert!(err.to_string().contains("failed"), "{err}"); - } - - #[tokio::test] - async fn a_walk_past_the_limit_is_killed_and_errors() { - let mut cmd = Command::new("sleep"); - cmd.arg("30"); - let started = std::time::Instant::now(); - let err = du_total("iris", cmd, Duration::from_millis(50)) - .await - .expect_err("a walk past the limit must fail"); - assert!(err.to_string().contains("timed out"), "{err}"); - assert!(started.elapsed() < Duration::from_secs(10)); - } - - #[tokio::test] - async fn du_total_sums_a_real_walk() { - let dir = std::env::temp_dir().join(format!("hive-priv-du-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - std::fs::write(dir.join("f"), b"hello").unwrap(); - let cmd = du_command(std::slice::from_ref(&dir)).unwrap(); - let total = du_total("iris", cmd, Duration::from_secs(30)).await; - std::fs::remove_dir_all(&dir).unwrap(); - assert!(total.unwrap() >= 5); - } } diff --git a/hive-priv/src/shared.rs b/hive-priv/src/shared.rs new file mode 100644 index 00000000..0c06f35e --- /dev/null +++ b/hive-priv/src/shared.rs @@ -0,0 +1,342 @@ +//! Helpers more than one op family uses: agent-name validation and the +//! `h-` container prefix, the credential-name charset, and [`StagedFile`], +//! the no-follow write-then-rename every root-side file write goes through. + +use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, Result, bail}; +use hive_priv_sock::AGENT_PREFIX; + +/// Return the system container name for a logical agent name. +/// All agents (including the manager) use the `h-` prefix. +/// +/// **This prefix is what confines the lifecycle verbs to agent containers** — +/// not their name validation, which only checks characters. A caller cannot +/// name `hive-forge` and reach it: it becomes `h-hive-forge`. No lifecycle +/// verb takes a sibling name: the forge and matrix are swarm services a hive +/// does not start or stop, and the gateway is reached through +/// [`hive_priv_sock::PrivRequest::ControlInfraContainer`] and the +/// [`hive_priv_sock::InfraContainer`] enum. +pub(crate) fn container_system_name(name: &str) -> String { + format!("{AGENT_PREFIX}{name}") +} + +/// Validate a logical agent name (the name hive-c0re uses internally, +/// before the `h-` container prefix is applied). +pub(crate) fn validate_agent_name(name: &str) -> Result<()> { + validate_name_chars(name)?; + Ok(()) +} + +pub(crate) fn validate_name_chars(name: &str) -> Result<()> { + if name.is_empty() + || !name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') + { + bail!("invalid name {name:?}: must be non-empty lowercase ascii + digits + hyphens"); + } + Ok(()) +} + +/// A systemd credential id must be a short token — restrict to +/// `[A-Za-z0-9_-]` (no `.`) so it can't inject extra `--load-credential` +/// argv or break the `name:path` shape. `.` is deliberately excluded, not +/// just a bare `..`: this name gets interpolated into filesystem paths +/// (snapshot labels) and there's no legitimate need for a dot in either a +/// systemd credential id or a `hive-`-prefixed snapshot label — we're +/// defining this token format from scratch, so keep it maximally strict +/// rather than allow-then-patch each traversal-adjacent character +/// (mara: "we are making up the rules here, lets go strict"). +pub(crate) fn validate_credential_name(name: &str) -> Result<()> { + if name.is_empty() + || !name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-')) + { + bail!("invalid credential name {name:?}: must be non-empty [A-Za-z0-9_-]"); + } + Ok(()) +} + +/// A leaf safe to join onto an agent-owned directory: one plain component, so +/// it can neither climb out of the directory nor name the directory itself. +pub(crate) fn ensure_plain_filename(who: &str, filename: &str) -> Result<()> { + if filename.is_empty() || filename == "." || filename == ".." || filename.contains('/') { + bail!("{who}: refusing non-plain filename {filename:?}"); + } + Ok(()) +} + +/// Name the content is written under before being renamed onto `filename`, +/// unique per call so two concurrent writes of one file never share a temp. +/// +/// The leading dot and the `.partial` suffix are load-bearing: a reader +/// globbing on a published name's prefix, or systemd reading `*.conf` out of +/// `tmpfiles.d` and drop-in dirs, cannot pick up a half-written temp. +fn partial_name(filename: &str) -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static SEQ: AtomicU64 = AtomicU64::new(0); + let n = SEQ.fetch_add(1, Ordering::Relaxed); + format!(".{filename}.{}-{n}.partial", std::process::id()) +} + +/// Open `path` as a directory fd for [`StagedFile`] to work relative to. +pub(crate) fn open_dir(path: &Path) -> Result { + use std::os::unix::fs::OpenOptionsExt as _; + std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY) + .open(path) + .with_context(|| format!("open directory {}", path.display())) +} + +/// A file written under a [`partial_name`] in its destination directory and +/// then renamed onto its final name, so a reader sees either the old file or +/// the complete new one, never a partial or wrong-mode one. Mode, and owner +/// when the caller `fchown`s [`file`](Self::file), are on the inode before the +/// rename makes it visible. +/// +/// Every step is relative to the `dir` fd, so it is safe in a directory an +/// agent or container controls: the temp is created `O_EXCL|O_NOFOLLOW`, which +/// refuses any entry already planted at its name, and `rename` replaces +/// whatever sits at the final name without following it. +/// +/// Dropped without [`publish`](Self::publish), it unlinks the temp, so a +/// failure between the two leaves the old file untouched and no residue. +pub(crate) struct StagedFile<'a> { + dir: &'a std::fs::File, + tmp: std::ffi::CString, + dest: std::ffi::CString, + dest_path: PathBuf, + file: std::fs::File, + published: bool, +} + +impl<'a> StagedFile<'a> { + /// Create the temp at `mode` and write `content` to it. The mode is set + /// again on the fd after the write, so the umask cannot change it. + pub(crate) fn write( + dir: &'a std::fs::File, + dir_path: &Path, + filename: &str, + content: &[u8], + mode: u32, + ) -> Result { + use std::io::Write as _; + use std::os::unix::fs::PermissionsExt as _; + + ensure_plain_filename("StagedFile::write", filename)?; + let dest_path = dir_path.join(filename); + let tmp_name = partial_name(filename); + let tmp = std::ffi::CString::new(tmp_name.as_str()) + .with_context(|| format!("NUL in filename {filename:?}"))?; + let dest = std::ffi::CString::new(filename) + .with_context(|| format!("NUL in filename {filename:?}"))?; + // SAFETY: `dir` is an open directory fd and `tmp` is NUL-terminated; + // the result is checked before it is wrapped. + let fd = unsafe { + libc::openat( + dir.as_raw_fd(), + tmp.as_ptr(), + libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_NOFOLLOW | libc::O_CLOEXEC, + mode, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("create {}", dir_path.join(&tmp_name).display())); + } + // SAFETY: `fd` is a fresh, valid fd that nothing else owns. + let file = std::fs::File::from(unsafe { OwnedFd::from_raw_fd(fd) }); + let mut staged = Self { + dir, + tmp, + dest, + dest_path, + file, + published: false, + }; + staged + .file + .write_all(content) + .with_context(|| format!("write {}", staged.dest_path.display()))?; + staged + .file + .set_permissions(std::fs::Permissions::from_mode(mode)) + .with_context(|| format!("chmod {mode:o} {}", staged.dest_path.display()))?; + Ok(staged) + } + + /// The temp's open fd, for an `fchown` that must land before publishing. + pub(crate) fn file(&self) -> &std::fs::File { + &self.file + } + + /// Flush the content, rename the temp onto the final name, and flush the + /// directory: without the last step a crash can lose the rename itself. + pub(crate) fn publish(mut self) -> Result<()> { + self.file + .sync_all() + .with_context(|| format!("fsync {}", self.dest_path.display()))?; + let fd = self.dir.as_raw_fd(); + // SAFETY: `fd` is an open directory fd; both names are NUL-terminated. + if unsafe { libc::renameat(fd, self.tmp.as_ptr(), fd, self.dest.as_ptr()) } != 0 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("publishing {}", self.dest_path.display())); + } + self.published = true; + self.dir + .sync_all() + .with_context(|| format!("fsync directory of {}", self.dest_path.display())) + } +} + +impl Drop for StagedFile<'_> { + fn drop(&mut self) { + if self.published { + return; + } + // SAFETY: `self.dir` is an open directory fd; `self.tmp` is + // NUL-terminated. + if unsafe { libc::unlinkat(self.dir.as_raw_fd(), self.tmp.as_ptr(), 0) } != 0 { + tracing::warn!( + dest = %self.dest_path.display(), + error = %std::io::Error::last_os_error(), + "failed to remove unpublished temp" + ); + } + } +} + +/// Scratch-directory helpers shared by every module's tests. One counter for +/// the whole crate, so no two tests get the same directory. +#[cfg(test)] +pub(crate) mod test_support { + use std::path::PathBuf; + use std::sync::atomic::{AtomicU32, Ordering}; + + /// Unique scratch dir per test, no external tempfile dep. + pub(crate) fn scratch() -> PathBuf { + static CTR: AtomicU32 = AtomicU32::new(0); + let n = CTR.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!( + "hive-priv-nofollow-test-{}-{n}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + /// Names in `dir`, sorted — the residue check every publish test ends on. + pub(crate) fn entries(dir: &PathBuf) -> Vec { + let mut v: Vec = std::fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + v.sort(); + v + } +} + +#[cfg(test)] +mod tests { + use super::test_support::{entries, scratch}; + use super::{ + StagedFile, ensure_plain_filename, open_dir, partial_name, validate_credential_name, + }; + + /// The temp name is the whole reason the rename is safe to watch: a + /// `systemd.path` unit globbing `matrix-token*` would fire on a temp that + /// merely suffixed the real name, on exactly the empty file the rename + /// exists to hide. + #[test] + fn a_temp_cannot_match_a_glob_on_the_name_it_publishes() { + for real in ["matrix-token", "matrix-token-alice", "forge-token"] { + let tmp = partial_name(real); + assert!( + !tmp.starts_with(real), + "{tmp:?} matches a `{real}*` path unit while half-written" + ); + assert!(!tmp.contains('/'), "temp must stay in the same directory"); + } + } + + /// Any failure between staging and publishing drops the `StagedFile`. + /// Until the rename a reader sees only the old file, and afterwards the + /// old file is still whole and the temp is gone. + #[test] + fn a_failed_write_leaves_the_old_file_intact() { + let dir = scratch(); + let dest = dir.join("hyperhive-limits.conf"); + std::fs::write(&dest, "OLD").unwrap(); + let dir_fd = open_dir(&dir).unwrap(); + + let staged = + StagedFile::write(&dir_fd, &dir, "hyperhive-limits.conf", b"NEW", 0o644).unwrap(); + assert_eq!(std::fs::read_to_string(&dest).unwrap(), "OLD"); + assert_eq!( + entries(&dir).len(), + 2, + "control: the temp exists while staged" + ); + drop(staged); + + assert_eq!(std::fs::read_to_string(&dest).unwrap(), "OLD"); + assert_eq!(entries(&dir), ["hyperhive-limits.conf"], "temp left behind"); + std::fs::remove_dir_all(&dir).ok(); + } + + /// Two overlapping writes of one file each own a temp, so whichever + /// publishes last lands whole — never the other's bytes, never a mix. + #[test] + fn overlapping_writes_of_one_file_never_share_a_temp() { + let dir = scratch(); + let dir_fd = open_dir(&dir).unwrap(); + let name = "hyperhive-agents.conf"; + + let first = StagedFile::write(&dir_fd, &dir, name, b"roster A\n", 0o644).unwrap(); + let second = StagedFile::write(&dir_fd, &dir, name, b"roster B, longer\n", 0o644).unwrap(); + second.publish().unwrap(); + first.publish().unwrap(); + + assert_eq!( + std::fs::read_to_string(dir.join(name)).unwrap(), + "roster A\n" + ); + assert_eq!(entries(&dir), [name], "temp left behind"); + std::fs::remove_dir_all(&dir).ok(); + } + + /// The only gate on the root-privileged socket for a `--load-credential` + /// name. The doc comment is explicit that `.` is excluded on purpose + /// ("this name gets interpolated into filesystem paths") — a future + /// "let's allow dots for version numbers" loosening must fail this, + /// not just the `/` case below. + #[test] + fn validate_credential_name_rejects_dot_slash_and_empty_but_allows_the_charset() { + assert!( + validate_credential_name("").is_err(), + "empty must be rejected" + ); + assert!(validate_credential_name("valid-name_123").is_ok()); + assert!( + validate_credential_name("bad.name").is_err(), + "dot must be rejected — see the fn's doc comment" + ); + assert!(validate_credential_name("bad/name").is_err()); + } + + /// `ensure_plain_filename` is the gate between an agent-writable + /// directory and a root-privileged file write. `.`/`..` would resolve + /// to the directory itself or its parent; any `/` climbs into a path + /// component the caller never named. + #[test] + fn ensure_plain_filename_rejects_dot_dotdot_and_any_slash() { + assert!(ensure_plain_filename("test", ".").is_err()); + assert!(ensure_plain_filename("test", "..").is_err()); + assert!(ensure_plain_filename("test", "sub/dir").is_err()); + assert!(ensure_plain_filename("test", "matrix-token").is_ok()); + } +} diff --git a/hive-priv/src/storage.rs b/hive-priv/src/storage.rs new file mode 100644 index 00000000..da20acf9 --- /dev/null +++ b/hive-priv/src/storage.rs @@ -0,0 +1,1085 @@ +//! Agent state storage ops: btrfs subvolumes (`EnsureAgentSubvolume`, +//! `DeleteAgentSubvolume`, `UpgradeAgentSubvolume`), qgroup accounting and +//! quotas (`EnsureBtrfsQuota`, `ReadSubvolumeUsage`, `SetSubvolumeQuota`), +//! read-only snapshots and their export (`SnapshotAgentSubvolume`, +//! `DeleteAgentSnapshot`, `SendAgentSnapshotToFile`, `SendAgentSnapshotToFd`) +//! and the `du`-based `MeasureAgentDisk`. +//! +//! Every path is derived from a validated agent name (and, for snapshots, a +//! `hive-`-prefixed label) under `AGENT_STATE_ROOT`. + +use std::os::fd::OwnedFd; +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, Result, anyhow, bail}; +use hive_priv_sock::{AGENT_STATE_ROOT, MEASURE_AGENT_DISK_TIMEOUT, MIGRATE_STAGING_ROOT}; +use tokio::process::Command; + +use crate::shared::{container_system_name, validate_agent_name, validate_credential_name}; + +/// `EnsureAgentSubvolume`. +pub(crate) async fn exec_ensure_agent_subvolume(agent_name: &str) -> Result<(String, String)> { + validate_agent_name(agent_name)?; + ensure_agent_subvolume(agent_name).await +} + +/// `DeleteAgentSubvolume`. +pub(crate) async fn exec_delete_agent_subvolume(agent_name: &str) -> Result<(String, String)> { + validate_agent_name(agent_name)?; + delete_agent_subvolume(agent_name).await +} + +/// `ReadSubvolumeUsage`. +pub(crate) async fn exec_read_subvolume_usage(agent_name: &str) -> Result<(String, String)> { + validate_agent_name(agent_name)?; + read_subvolume_usage(agent_name).await +} + +/// `MeasureAgentDisk`. +pub(crate) async fn exec_measure_agent_disk(agent_name: &str) -> Result<(String, String)> { + validate_agent_name(agent_name)?; + measure_agent_disk(agent_name).await +} + +/// `SetSubvolumeQuota`. +pub(crate) async fn exec_set_subvolume_quota( + agent_name: &str, + limit_bytes: Option, +) -> Result<(String, String)> { + validate_agent_name(agent_name)?; + set_subvolume_quota(agent_name, limit_bytes).await +} + +/// `UpgradeAgentSubvolume`. +pub(crate) async fn exec_upgrade_agent_subvolume(agent_name: &str) -> Result<(String, String)> { + validate_agent_name(agent_name)?; + upgrade_agent_subvolume(agent_name).await +} + +/// `SnapshotAgentSubvolume`. +pub(crate) async fn exec_snapshot_agent_subvolume( + agent_name: &str, + snapshot_name: &str, +) -> Result<(String, String)> { + validate_agent_name(agent_name)?; + validate_snapshot_name(snapshot_name)?; + snapshot_agent_subvolume(agent_name, snapshot_name).await +} + +/// `DeleteAgentSnapshot`. +pub(crate) async fn exec_delete_agent_snapshot( + agent_name: &str, + snapshot_name: &str, +) -> Result<(String, String)> { + validate_agent_name(agent_name)?; + validate_snapshot_name(snapshot_name)?; + delete_agent_snapshot(agent_name, snapshot_name).await +} + +/// `SendAgentSnapshotToFile`. +pub(crate) async fn exec_send_agent_snapshot_to_file( + agent_name: &str, + snapshot_name: &str, + parent_snapshot_name: Option<&str>, + dest_file_name: &str, +) -> Result<(String, String)> { + validate_agent_name(agent_name)?; + validate_snapshot_name(snapshot_name)?; + if let Some(parent) = parent_snapshot_name { + validate_snapshot_name(parent)?; + } + validate_credential_name(dest_file_name)?; + send_agent_snapshot_to_file( + agent_name, + snapshot_name, + parent_snapshot_name, + dest_file_name, + ) + .await +} + +/// `SendAgentSnapshotToFd`. +pub(crate) async fn exec_send_agent_snapshot_to_fd( + agent_name: &str, + snapshot_name: &str, + parent_snapshot_name: Option<&str>, + fd: Option, +) -> Result<(String, String)> { + validate_agent_name(agent_name)?; + validate_snapshot_name(snapshot_name)?; + if let Some(parent) = parent_snapshot_name { + validate_snapshot_name(parent)?; + } + let dest = fd.context("no descriptor to stream into")?; + send_agent_snapshot_to_fd(agent_name, snapshot_name, parent_snapshot_name, dest).await +} + +/// A btrfs snapshot label must start with `hive-` — this doubles as an +/// allow-list: only names hivectl itself constructs (or an operator who +/// knows the convention) can reach the `btrfs subvolume snapshot`/`delete` +/// shellouts, so an arbitrary caller can't use the snapshot ops to probe or +/// churn unrelated paths under `AGENT_STATE_ROOT`. Beyond the prefix, the +/// same charset restriction as [`validate_credential_name`] applies (it's +/// interpolated straight into a filesystem path). +fn validate_snapshot_name(name: &str) -> Result<()> { + if !name.starts_with("hive-") { + bail!("invalid snapshot label {name:?}: must start with \"hive-\""); + } + validate_credential_name(name) +} + +/// btrfs superblock magic, as reported by `statfs(2)`'s `f_type`. +const BTRFS_SUPER_MAGIC: i64 = 0x9123_683E; + +/// Inode number of a btrfs subvolume root (`BTRFS_FIRST_FREE_OBJECTID`). +/// Every subvolume's top directory has this inode; plain directories do +/// not, so `statfs == btrfs && st_ino == 256` reliably identifies a +/// subvolume root. +const BTRFS_SUBVOL_ROOT_INO: u64 = 256; + +/// Whether `path` lives on a btrfs filesystem (via `statfs(2)`). +fn is_on_btrfs(path: &Path) -> Result { + use std::os::unix::ffi::OsStrExt as _; + let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()) + .with_context(|| format!("path {} has an interior null byte", path.display()))?; + // SAFETY: `c_path` is a valid NUL-terminated C string that outlives the + // call; `statfs` only writes into the zero-initialised `buf`. + let mut buf: libc::statfs = unsafe { std::mem::zeroed() }; + let rc = unsafe { libc::statfs(c_path.as_ptr(), &raw mut buf) }; + if rc != 0 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("statfs {}", path.display())); + } + Ok(buf.f_type == BTRFS_SUPER_MAGIC) +} + +/// Whether `path` is the root of a btrfs subvolume (on btrfs and inode 256). +fn is_btrfs_subvolume(path: &Path) -> bool { + use std::os::unix::fs::MetadataExt as _; + let on_btrfs = is_on_btrfs(path).unwrap_or(false); + let ino_match = std::fs::metadata(path).is_ok_and(|m| m.ino() == BTRFS_SUBVOL_ROOT_INO); + on_btrfs && ino_match +} + +/// `EnsureAgentSubvolume` — make the agent's state root a btrfs subvolume +/// when the FS supports it. Idempotent + progressive: no-op when the root +/// already exists or the FS isn't btrfs. See the wire doc on the variant. +async fn ensure_agent_subvolume(agent_name: &str) -> Result<(String, String)> { + use std::os::unix::fs::MetadataExt as _; + + let root = PathBuf::from(AGENT_STATE_ROOT); + let agent_root = root.join(agent_name); + + // Progressive: existing agents (plain dir OR already a subvol) are left + // untouched — never auto-migrated. + if agent_root.exists() { + return Ok((String::new(), String::new())); + } + + // Only btrfs supports subvolumes; on anything else hive-c0re's normal + // `create_dir_all` makes a plain dir (the pre-subvolume behaviour). The + // parent must exist for both statfs and `btrfs subvolume create`. + std::fs::create_dir_all(&root) + .with_context(|| format!("create agents root {}", root.display()))?; + if !is_on_btrfs(&root)? { + return Ok((String::new(), String::new())); + } + + let out = Command::new("btrfs") + .args(["subvolume", "create"]) + .arg(&agent_root) + .output() + .await + .with_context(|| format!("spawn btrfs subvolume create {}", agent_root.display()))?; + if !out.status.success() { + bail!( + "btrfs subvolume create {} failed: {}", + agent_root.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + + // The subvol root is created root-owned, but hive-c0re (the `hive-core` + // user) must be able to mkdir state/ claude/ harness/ inside it — exactly + // as it would in a plain dir. Chown it to AGENT_STATE_ROOT's owner + // (hive-core). This MUST succeed: a root-owned subvol would make the + // downstream dir creation fail with a confusing permission error, and the + // c0re-side exists-check would then skip re-running this op on retry, + // wedging the agent. So on any failure roll the subvol back and bail — the + // create path surfaces a clear error and a retry starts clean. + let chown_result = std::fs::metadata(&root) + .with_context(|| format!("stat agents root {} for ownership", root.display())) + .and_then(|meta| { + std::os::unix::fs::chown(&agent_root, Some(meta.uid()), Some(meta.gid())).with_context( + || format!("chown subvol {} to agents-root owner", agent_root.display()), + ) + }); + if let Err(e) = chown_result { + // Best-effort rollback so we never leave a root-owned subvol behind. + let _ = Command::new("btrfs") + .args(["subvolume", "delete"]) + .arg(&agent_root) + .output() + .await; + return Err(e.context(format!( + "rolled back subvolume {} after chown failed", + agent_root.display() + ))); + } + tracing::info!(agent = %agent_name, path = %agent_root.display(), "created agent state subvolume"); + Ok((String::new(), String::new())) +} + +/// `DeleteAgentSubvolume` — delete the agent's state root iff it's a btrfs +/// subvolume (purge path only). No-op for plain dirs / missing paths; +/// hive-c0re's own `remove_dir_all` covers those. See the wire doc. +async fn delete_agent_subvolume(agent_name: &str) -> Result<(String, String)> { + let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); + if !is_btrfs_subvolume(&agent_root) { + return Ok((String::new(), String::new())); + } + let out = Command::new("btrfs") + .args(["subvolume", "delete"]) + .arg(&agent_root) + .output() + .await + .with_context(|| format!("spawn btrfs subvolume delete {}", agent_root.display()))?; + if !out.status.success() { + bail!( + "btrfs subvolume delete {} failed: {}", + agent_root.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + tracing::info!(agent = %agent_name, path = %agent_root.display(), "deleted agent state subvolume"); + Ok((String::new(), String::new())) +} + +/// `EnsureBtrfsQuota` — enable btrfs qgroup accounting on the filesystem +/// holding `AGENT_STATE_ROOT`. Idempotent + statfs-gated (no-op off btrfs). +/// Operator opt-in only; see the wire doc. +pub(crate) async fn ensure_btrfs_quota() -> Result<(String, String)> { + let root = PathBuf::from(AGENT_STATE_ROOT); + std::fs::create_dir_all(&root) + .with_context(|| format!("create agents root {}", root.display()))?; + if !is_on_btrfs(&root)? { + // Non-btrfs host: quota/qgroups don't apply. No-op success so the + // operator-facing verb degrades cleanly. + return Ok(( + String::new(), + "filesystem is not btrfs — quota not applicable".to_owned(), + )); + } + let out = Command::new("btrfs") + .args(["quota", "enable"]) + .arg(&root) + .output() + .await + .with_context(|| format!("spawn btrfs quota enable {}", root.display()))?; + if !out.status.success() { + bail!( + "btrfs quota enable {} failed: {}", + root.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + tracing::info!(path = %root.display(), "enabled btrfs qgroup accounting"); + Ok((String::new(), String::new())) +} + +/// `ReadSubvolumeUsage` — return an agent subvolume's qgroup rows +/// (`btrfs qgroup show -f --raw <…/agent_name>`) verbatim in stdout for +/// hive-c0re to parse. `-f` lists the qgroups impacting the given path, +/// excluding ancestral qgroups (per btrfs-qgroup-show(8)) — so it scopes +/// to this subvolume and never mixes in other agents'. hive-c0re then +/// selects the level-0 (`0/`) leaf row. See the wire doc. +async fn read_subvolume_usage(agent_name: &str) -> Result<(String, String)> { + let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); + let out = Command::new("btrfs") + .args(["qgroup", "show", "-f", "--raw"]) + .arg(&agent_root) + .output() + .await + .with_context(|| format!("spawn btrfs qgroup show {}", agent_root.display()))?; + if !out.status.success() { + // The common failure is "quota not enabled" — pass the stderr + // through so hive-c0re can surface it gracefully. + bail!( + "btrfs qgroup show {} failed: {}", + agent_root.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + Ok(( + String::from_utf8_lossy(&out.stdout).into_owned(), + String::new(), + )) +} + +/// The host paths `MeasureAgentDisk` sums for `agent_name`: its `state/` +/// and `work/` dirs under `AGENT_STATE_ROOT` and its container rootfs. +/// These are the only paths that op ever walks. +// `state` / `work` must match hive-c0re's `agent_notes_dir` and +// `paths::WORK_SUBDIR`. +fn agent_disk_paths(agent_name: &str) -> [PathBuf; 3] { + let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); + [ + agent_root.join("state"), + agent_root.join("work"), + PathBuf::from("/var/lib/nixos-containers").join(container_system_name(agent_name)), + ] +} + +/// `MeasureAgentDisk` — one `du` over the [`agent_disk_paths`] that exist, +/// returning the grand total in bytes. An agent with none of them has no +/// measurable size, which is an error rather than a total of 0. +async fn measure_agent_disk(agent_name: &str) -> Result<(String, String)> { + let mut paths = Vec::new(); + for path in agent_disk_paths(agent_name) { + match std::fs::symlink_metadata(&path) { + Ok(_) => paths.push(path), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e).with_context(|| format!("stat {}", path.display())), + } + } + let total = du_total(agent_name, du_command(&paths)?, MEASURE_AGENT_DISK_TIMEOUT).await?; + Ok((total.to_string(), String::new())) +} + +/// `du -s -x -b -c -P -- `. `-x` keeps each walk on its operand's +/// filesystem (so the bind-mounted nix store is never counted), `-P` never +/// follows a symlink, `-b` counts apparent size, `-c` adds the `total` line +/// [`parse_du_total`] reads. Errors on no paths: `du` with no operand would +/// measure its working directory instead. +fn du_command(paths: &[PathBuf]) -> Result { + if paths.is_empty() { + bail!("none of the agent's state, work or rootfs paths exist"); + } + let mut cmd = Command::new("du"); + cmd.args(["-s", "-x", "-b", "-c", "-P", "--"]).args(paths); + Ok(cmd) +} + +/// Run `cmd` (a [`du_command`]) for at most `limit` and parse its total. +/// The child is killed on timeout. A timeout, a spawn failure, a non-zero +/// exit or output without a total is an error, never a partial total. +async fn du_total(agent_name: &str, mut cmd: Command, limit: std::time::Duration) -> Result { + cmd.kill_on_drop(true); + let Ok(out) = tokio::time::timeout(limit, cmd.output()).await else { + bail!( + "du for {agent_name} timed out after {}s; killed it", + limit.as_secs() + ); + }; + let out = out.with_context(|| format!("spawn du for {agent_name}"))?; + if !out.status.success() { + bail!( + "du for {agent_name} failed ({}): {}", + out.status, + String::from_utf8_lossy(&out.stderr) + .lines() + .last() + .unwrap_or("") + .trim() + ); + } + let stdout = String::from_utf8_lossy(&out.stdout); + parse_du_total(&stdout) + .ok_or_else(|| anyhow!("no `total` line in du output for {agent_name}: {stdout:?}")) +} + +/// The byte count on `du -c`'s final `\ttotal` line. `None` when the +/// last line isn't that shape, so truncated or unexpected output is never +/// read as a smaller total. +fn parse_du_total(stdout: &str) -> Option { + let last = stdout.lines().rev().find(|l| !l.trim().is_empty())?; + let (bytes, label) = last.split_once('\t')?; + if label != "total" { + return None; + } + bytes.parse().ok() +} + +/// Best-effort removal of a leftover migration path from a prior aborted +/// upgrade: try `btrfs subvolume delete` (in case it's a half-created +/// subvolume) then a plain recursive remove. Both failures are ignored — +/// the path may simply not exist. +async fn cleanup_stale_path(path: &Path) { + if path.exists() { + let _ = Command::new("btrfs") + .args(["subvolume", "delete"]) + .arg(path) + .output() + .await; + let _ = std::fs::remove_dir_all(path); + } +} + +/// Stage a populated subvolume at `tmp` mirroring `agent_root`: create the +/// subvolume, copy `agent_root`'s contents into it preserving +/// ownership/permissions/xattrs, then match the subvolume root's owner + mode +/// to the original. On any failure the partially-staged `tmp` is cleaned up +/// (so the caller can bail with the original dir still untouched). +async fn stage_upgrade_subvolume(agent_root: &Path, tmp: &Path) -> Result<()> { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + + // Fresh subvolume to receive the copy. + let out = Command::new("btrfs") + .args(["subvolume", "create"]) + .arg(tmp) + .output() + .await + .with_context(|| format!("spawn btrfs subvolume create {}", tmp.display()))?; + if !out.status.success() { + bail!( + "btrfs subvolume create {} failed: {}", + tmp.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + + // Copy contents preserving everything (`-a` = --preserve=all → mode, + // ownership, timestamps, links, xattrs); reflink for fast CoW clones on the + // same btrfs. `/.` copies the directory's contents (incl. dotfiles) + // into the subvolume rather than nesting it. + let copy = Command::new("cp") + .arg("-a") + .arg("--reflink=auto") + .arg(format!("{}/.", agent_root.display())) + .arg(tmp) + .output() + .await + .with_context(|| format!("spawn cp into {}", tmp.display()))?; + if !copy.status.success() { + cleanup_stale_path(tmp).await; + bail!( + "copy {} -> {} failed (original left untouched): {}", + agent_root.display(), + tmp.display(), + String::from_utf8_lossy(©.stderr).trim() + ); + } + + // Match the new subvolume root's ownership + mode to the original dir. + // `cp -a /.` copies the *contents* but the subvolume root keeps its + // create-time root ownership, so set it explicitly — the swapped-in + // subvolume must be indistinguishable from the original to hive-c0re. + let apply = std::fs::metadata(agent_root) + .with_context(|| format!("stat {} for ownership", agent_root.display())) + .and_then(|m| { + std::os::unix::fs::chown(tmp, Some(m.uid()), Some(m.gid())) + .with_context(|| format!("chown {} to match original", tmp.display()))?; + std::fs::set_permissions(tmp, std::fs::Permissions::from_mode(m.mode())) + .with_context(|| format!("chmod {} to match original", tmp.display()))?; + Ok(()) + }); + if let Err(e) = apply { + cleanup_stale_path(tmp).await; + return Err(e.context("upgrade aborted before swap; original left untouched")); + } + Ok(()) +} + +/// `UpgradeAgentSubvolume` — convert an existing plain-dir agent state root +/// into a btrfs subvolume in place. Operator opt-in; the caller (hivectl) +/// stops the agent first and restarts it after. See the wire doc. +/// +/// Migration: stage a sibling subvolume mirroring the dir +/// ([`stage_upgrade_subvolume`]), then rename the original aside and the +/// subvolume into place, then remove the original. Any failure before the +/// rename-swap leaves the original dir untouched. +async fn upgrade_agent_subvolume(agent_name: &str) -> Result<(String, String)> { + let root = PathBuf::from(AGENT_STATE_ROOT); + let agent_root = root.join(agent_name); + + if !agent_root.exists() { + // A crash between the two swap renames (original → `..old` + // succeeded, `..migrating` → agent_root did not) leaves the + // agent root missing but the original data intact under `..old`. + // Point at the recovery rather than a bare "nothing to do" so the + // operator isn't left guessing where the data went. + let old = root.join(format!(".{agent_name}.old")); + if old.exists() { + bail!( + "no state dir at {agent} — but {old} holds the original data from an \ + interrupted upgrade (host crashed mid-swap). Restore it with \ + `mv {old} {agent}`, then re-run the upgrade.", + agent = agent_root.display(), + old = old.display(), + ); + } + bail!( + "no state dir to upgrade at {} — nothing to do", + agent_root.display() + ); + } + // Idempotent: already a subvolume → nothing to do. + if is_btrfs_subvolume(&agent_root) { + return Ok((String::new(), String::new())); + } + if !is_on_btrfs(&root)? { + bail!( + "{} is not on btrfs — subvolumes are unsupported, cannot upgrade", + root.display() + ); + } + + // Sibling temp paths on the same filesystem (so the copy can reflink and + // the swap renames are atomic). Leading dots keep them out of the agent + // namespace (`validate_agent_name` rejects dot-prefixed names). + let tmp = root.join(format!(".{agent_name}.migrating")); + let old = root.join(format!(".{agent_name}.old")); + // Clear any debris from a previously interrupted run before starting. + cleanup_stale_path(&tmp).await; + cleanup_stale_path(&old).await; + + stage_upgrade_subvolume(&agent_root, &tmp).await?; + + // Swap. `rename` is atomic within a filesystem. The window between the + // two renames is the only unsafe point: a crash there leaves the agent + // root missing but both `.old` (original) and the new subvolume present + // — recoverable by hand, hence the loud logging. + if let Err(e) = std::fs::rename(&agent_root, &old) { + cleanup_stale_path(&tmp).await; + return Err(anyhow::Error::new(e).context(format!( + "rename {} -> {} failed; original left untouched", + agent_root.display(), + old.display() + ))); + } + if let Err(e) = std::fs::rename(&tmp, &agent_root) { + // Restore the original from its renamed-aside copy. + let restored = std::fs::rename(&old, &agent_root).is_ok(); + cleanup_stale_path(&tmp).await; + return Err(anyhow::Error::new(e).context(format!( + "rename {} -> {} failed; original {}", + tmp.display(), + agent_root.display(), + if restored { + "restored" + } else { + "COULD NOT BE RESTORED — manual recovery needed" + } + ))); + } + + // 5. Success: drop the original (a plain dir) and report. + if let Err(e) = std::fs::remove_dir_all(&old) { + // The migration succeeded; a leftover `.old` is cosmetic. Warn only. + tracing::warn!( + agent = %agent_name, path = %old.display(), + "upgraded subvolume but failed to remove old dir: {e}" + ); + } + tracing::info!( + agent = %agent_name, path = %agent_root.display(), + "upgraded agent state dir to btrfs subvolume" + ); + Ok(( + format!("upgraded {} to a btrfs subvolume", agent_root.display()), + String::new(), + )) +} + +/// Derive a snapshot's path from the agent name + label: a dot-prefixed +/// sibling of the agent's state root so it can never collide with a real +/// agent directory (`validate_agent_name` rejects dot-prefixed names). +fn snapshot_path(agent_name: &str, snapshot_name: &str) -> PathBuf { + PathBuf::from(AGENT_STATE_ROOT).join(format!(".{agent_name}.snapshot.{snapshot_name}")) +} + +/// `SnapshotAgentSubvolume` — create a read-only btrfs snapshot of an +/// agent's state subvolume, for `btrfs send` to stream from during +/// inter-hive migration. See the wire doc. +async fn snapshot_agent_subvolume( + agent_name: &str, + snapshot_name: &str, +) -> Result<(String, String)> { + let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); + if !is_btrfs_subvolume(&agent_root) { + bail!( + "{} is not a btrfs subvolume — nothing to snapshot (run `hivectl agent subvol upgrade` first)", + agent_root.display() + ); + } + let snap = snapshot_path(agent_name, snapshot_name); + if snap.exists() { + bail!( + "snapshot {} already exists — delete it first or pick a different name", + snap.display() + ); + } + let out = Command::new("btrfs") + .args(["subvolume", "snapshot", "-r"]) + .arg(&agent_root) + .arg(&snap) + .output() + .await + .with_context(|| { + format!( + "spawn btrfs subvolume snapshot -r {} {}", + agent_root.display(), + snap.display() + ) + })?; + if !out.status.success() { + bail!( + "btrfs subvolume snapshot -r {} {} failed: {}", + agent_root.display(), + snap.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + tracing::info!( + agent = %agent_name, snapshot = %snap.display(), + "created read-only agent state snapshot" + ); + Ok((snap.display().to_string(), String::new())) +} + +/// `DeleteAgentSnapshot` — delete a previously-created read-only snapshot. +/// No-op if the path doesn't exist. See the wire doc. +async fn delete_agent_snapshot(agent_name: &str, snapshot_name: &str) -> Result<(String, String)> { + let snap = snapshot_path(agent_name, snapshot_name); + if !snap.exists() { + return Ok((String::new(), String::new())); + } + let out = Command::new("btrfs") + .args(["subvolume", "delete"]) + .arg(&snap) + .output() + .await + .with_context(|| format!("spawn btrfs subvolume delete {}", snap.display()))?; + if !out.status.success() { + bail!( + "btrfs subvolume delete {} failed: {}", + snap.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + tracing::info!(agent = %agent_name, snapshot = %snap.display(), "deleted agent state snapshot"); + Ok((String::new(), String::new())) +} + +/// Both paths a `SendAgentSnapshotToFile` request reads from — the +/// snapshot itself, and its optional incremental parent — must exist +/// before we touch `dest`. Pulled out on its own so that invariant is +/// checkable without a `dest` path, a `MIGRATE_STAGING_ROOT`, or `btrfs` +/// at all: a caller cannot create the output file by accident while +/// still holding a reference to this check. +fn check_send_snapshot_preconditions(snap: &Path, parent_path: Option<&Path>) -> Result<()> { + if !snap.exists() { + bail!( + "snapshot {} does not exist — create it with `subvol snapshot create` first", + snap.display() + ); + } + if let Some(parent_path) = parent_path + && !parent_path.exists() + { + bail!( + "parent snapshot {} does not exist — pick an existing parent or omit it for a full send", + parent_path.display() + ); + } + Ok(()) +} + +/// Validate preconditions, then atomically create `dest` for a snapshot +/// export. The two are combined in one function on purpose: keeping +/// `check_send_snapshot_preconditions` ahead of the `create_new` call is +/// the entire fix, and folding them together here means that ordering +/// can't drift back apart in a later edit to `send_agent_snapshot_to_file`. +/// No `btrfs` involved, so this is unit-testable against plain files. +fn open_export_dest(snap: &Path, parent_path: Option<&Path>, dest: &Path) -> Result { + check_send_snapshot_preconditions(snap, parent_path)?; + // `create_new` (O_CREAT|O_EXCL) makes the no-overwrite guarantee atomic + // instead of a check-then-create race against a concurrent request. + match std::fs::File::options() + .write(true) + .create_new(true) + .open(dest) + { + Ok(f) => Ok(f), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => bail!( + "{} already exists — pick a different destination or remove it first \ + (send never overwrites an existing export)", + dest.display() + ), + Err(e) => Err(e).with_context(|| format!("create {}", dest.display())), + } +} + +/// Deletes `path` when dropped, unless [`disarm`](Self::disarm)ed first. +/// Guards `dest` between the moment `send_agent_snapshot_to_file` creates +/// it and the moment the export fully succeeds, so any `?`/`bail!` on +/// that path — including ones a future edit adds — removes the partial +/// file instead of leaving a zero-byte export that then makes every +/// retry against the same `--dest` fail with "already exists". +struct PartialExportGuard<'a> { + path: &'a Path, + armed: bool, +} + +impl<'a> PartialExportGuard<'a> { + fn new(path: &'a Path) -> Self { + Self { path, armed: true } + } + + fn disarm(mut self) { + self.armed = false; + } +} + +impl Drop for PartialExportGuard<'_> { + fn drop(&mut self) { + if !self.armed { + return; + } + // Best-effort: warn (don't fail the whole call over it) if removal + // itself fails, so a stuck partial file that later masquerades as a + // completed export is at least visible in the log. + if let Err(rm_err) = std::fs::remove_file(self.path) { + tracing::warn!( + dest = %self.path.display(), error = %rm_err, + "failed to remove partial export after failure — \ + next attempt at this dest will hit the already-exists guard" + ); + } + } +} + +/// `SendAgentSnapshotToFile` — stream a read-only snapshot (optionally +/// incremental against `parent_name`) to a file under +/// `MIGRATE_STAGING_ROOT` via `btrfs send`. Local-file half of the +/// inter-hive migration transport; see `PrivRequest::SendAgentSnapshotToFile` +/// for the cross-hive follow-up. +async fn send_agent_snapshot_to_file( + agent_name: &str, + snapshot_name: &str, + parent_name: Option<&str>, + dest_file_name: &str, +) -> Result<(String, String)> { + let snap = snapshot_path(agent_name, snapshot_name); + let parent_path = parent_name.map(|parent| snapshot_path(agent_name, parent)); + + std::fs::create_dir_all(MIGRATE_STAGING_ROOT) + .with_context(|| format!("create {MIGRATE_STAGING_ROOT}"))?; + let dest = Path::new(MIGRATE_STAGING_ROOT).join(dest_file_name); + // Preconditions are checked *inside* this call, before `dest` is + // created — see `open_export_dest`'s doc comment for why that ordering + // matters and is kept in one place. + let dest_file = open_export_dest(&snap, parent_path.as_deref(), &dest)?; + let cleanup = PartialExportGuard::new(&dest); + + let mut cmd = Command::new("btrfs"); + cmd.arg("send"); + if let Some(parent_path) = &parent_path { + cmd.arg("-p").arg(parent_path); + } + cmd.arg(&snap); + cmd.stdout(std::process::Stdio::from(dest_file)); + cmd.stderr(std::process::Stdio::piped()); + + let out = cmd + .spawn() + .with_context(|| format!("spawn btrfs send {}", snap.display()))? + .wait_with_output() + .await + .with_context(|| format!("wait on btrfs send {}", snap.display()))?; + if !out.status.success() { + bail!( + "btrfs send {} failed: {}", + snap.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + cleanup.disarm(); + tracing::info!( + agent = %agent_name, snapshot = %snap.display(), dest = %dest.display(), + parent = ?parent_name, "exported agent snapshot to file" + ); + Ok((dest.display().to_string(), String::new())) +} + +/// `SendAgentSnapshotToFd` — stream a read-only snapshot (optionally +/// incremental against `parent_name`) straight into a descriptor the +/// caller passed us. +/// +/// The network half of the inter-hive migration transport, arranged so +/// this helper never learns there *is* a network: hive-c0re connects to +/// the peer's snapshot store, writes the header itself, and hands the +/// connected socket over. We only ever see "a thing to write bytes into", +/// which keeps a root process out of any address, protocol or trust +/// decision — and keeps everyone out of the data path once `btrfs send` +/// starts, which matters at multi-gigabyte sizes. +async fn send_agent_snapshot_to_fd( + agent_name: &str, + snapshot_name: &str, + parent_name: Option<&str>, + dest: OwnedFd, +) -> Result<(String, String)> { + let snap = snapshot_path(agent_name, snapshot_name); + if !snap.exists() { + bail!( + "snapshot {} does not exist — create it with `subvol snapshot create` first", + snap.display() + ); + } + + let mut cmd = Command::new("btrfs"); + cmd.arg("send"); + if let Some(parent) = parent_name { + let parent_path = snapshot_path(agent_name, parent); + if !parent_path.exists() { + bail!( + "parent snapshot {} does not exist — pick an existing parent or omit it for a full send", + parent_path.display() + ); + } + cmd.arg("-p").arg(&parent_path); + } + cmd.arg(&snap); + cmd.stdout(std::process::Stdio::from(dest)); + cmd.stderr(std::process::Stdio::piped()); + + let out = cmd + .spawn() + .with_context(|| format!("spawn btrfs send {}", snap.display()))? + .wait_with_output() + .await + .with_context(|| format!("wait on btrfs send {}", snap.display()))?; + if !out.status.success() { + // Nothing to clean up: the destination isn't ours. A partial + // stream is the receiving end's problem, and `btrfs receive` + // refuses to commit an incomplete subvolume anyway. + bail!( + "btrfs send {} failed: {}", + snap.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + tracing::info!( + agent = %agent_name, snapshot = %snap.display(), parent = ?parent_name, + "streamed agent snapshot into a passed descriptor" + ); + Ok((String::new(), String::new())) +} + +/// `SetSubvolumeQuota` — set or clear a qgroup size limit on an agent +/// subvolume (`btrfs qgroup limit <…/agent_name>`). See the +/// wire doc. +async fn set_subvolume_quota( + agent_name: &str, + limit_bytes: Option, +) -> Result<(String, String)> { + let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); + let limit = limit_bytes.map_or_else(|| "none".to_owned(), |n| n.to_string()); + let out = Command::new("btrfs") + .args(["qgroup", "limit", &limit]) + .arg(&agent_root) + .output() + .await + .with_context(|| format!("spawn btrfs qgroup limit {}", agent_root.display()))?; + if !out.status.success() { + bail!( + "btrfs qgroup limit {limit} {} failed: {}", + agent_root.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + tracing::info!(agent = %agent_name, %limit, "set agent subvolume quota"); + Ok((String::new(), String::new())) +} +#[cfg(test)] +mod tests { + use super::{ + Command, agent_disk_paths, du_command, du_total, exec_measure_agent_disk, open_export_dest, + parse_du_total, validate_snapshot_name, + }; + use std::path::PathBuf; + use std::sync::atomic::{AtomicU32, Ordering}; + use std::time::Duration; + + /// A snapshot label must additionally carry the `hive-` prefix (it + /// doubles as the allow-list gating the btrfs snapshot/delete + /// shellouts) on top of [`validate_credential_name`]'s charset rule. + #[test] + fn validate_snapshot_name_requires_hive_prefix() { + assert!(validate_snapshot_name("not-hive-prefixed").is_err()); + assert!(validate_snapshot_name("hive-valid").is_ok()); + } + + /// Unique scratch dir per test, no external tempfile dep. Same pattern + /// as `shared::test_support::scratch()`, kept separate so a rename of one doesn't + /// collide with the other's directory names under parallel test runs. + fn snapshot_export_scratch() -> PathBuf { + static CTR: AtomicU32 = AtomicU32::new(0); + let n = CTR.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!( + "hive-priv-snapshot-export-test-{}-{n}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + /// The fix: a missing parent must be caught before `dest` is created, + /// not after. Fails against the pre-fix ordering (`dest` created via + /// `create_new` first, parent checked second) — see the issue for the + /// inverted run. + #[test] + fn missing_parent_leaves_no_file_at_dest() { + let dir = snapshot_export_scratch(); + let snap = dir.join("snap"); + std::fs::write(&snap, b"").unwrap(); + let parent = dir.join("parent-that-does-not-exist"); + let dest = dir.join("dest"); + + let err = open_export_dest(&snap, Some(&parent), &dest).unwrap_err(); + assert!( + err.to_string().contains("does not exist"), + "wrong error: {err}" + ); + assert!( + !dest.exists(), + "a failed export must not leave a file at dest" + ); + } + + /// A retry against the same `dest` after a precondition failure must + /// not trip the no-overwrite guard — there is nothing there to + /// overwrite, since the failed attempt never created `dest` (the test + /// above). This is the behavior a "stale parent falls back to a full + /// send, retried without -p" caller needs against a fixed `--dest`. + #[test] + fn retry_after_precondition_failure_succeeds() { + let dir = snapshot_export_scratch(); + let snap = dir.join("snap"); + std::fs::write(&snap, b"").unwrap(); + let parent = dir.join("parent-that-does-not-exist"); + let dest = dir.join("dest"); + + assert!(open_export_dest(&snap, Some(&parent), &dest).is_err()); + assert!(!dest.exists()); + + // Retry without the missing parent, same dest. + let file = open_export_dest(&snap, None, &dest); + assert!( + file.is_ok(), + "retry against a dest no previous attempt touched must succeed: {:?}", + file.err() + ); + assert!(dest.exists()); + } + + /// Control for the two tests above: an export that actually landed at + /// `dest` must still refuse a second one. Without this, a bug that + /// simply stopped creating `dest` at all (rather than fixing the + /// ordering) would also make the retry test above pass. + #[test] + fn existing_dest_still_refuses_overwrite() { + let dir = snapshot_export_scratch(); + let snap = dir.join("snap"); + std::fs::write(&snap, b"").unwrap(); + let dest = dir.join("dest"); + std::fs::write(&dest, b"already here").unwrap(); + + let err = open_export_dest(&snap, None, &dest).unwrap_err(); + assert!( + err.to_string().contains("already exists"), + "wrong error: {err}" + ); + assert_eq!( + std::fs::read(&dest).unwrap(), + b"already here", + "must not have touched the existing file" + ); + } + + #[test] + fn disk_paths_are_derived_from_the_name_only() { + assert_eq!( + agent_disk_paths("iris"), + [ + PathBuf::from("/var/lib/hyperhive/agents/iris/state"), + PathBuf::from("/var/lib/hyperhive/agents/iris/work"), + PathBuf::from("/var/lib/nixos-containers/h-iris"), + ] + ); + } + + /// Every name that could steer the walk out of the agent's own paths is + /// refused before any path is built, let alone `du` spawned. + #[tokio::test] + async fn measure_rejects_names_that_are_not_agent_names() { + for bad in [ + "", "..", "../etc", "a/b", "/etc", "Iris", "a b", "a\0b", "foo_bar", + ] { + let err = exec_measure_agent_disk(bad) + .await + .expect_err(&format!("{bad:?} must be rejected")); + assert!( + err.to_string().contains("invalid name"), + "{bad:?}: wrong error: {err}" + ); + } + } + + #[test] + fn du_total_reads_the_total_line() { + let out = "100\t/var/lib/hyperhive/agents/iris/state\n\ + 2000\t/var/lib/nixos-containers/h-iris\n\ + 2100\ttotal\n"; + assert_eq!(parse_du_total(out), Some(2100)); + assert_eq!(parse_du_total("0\ttotal"), Some(0)); + } + + #[test] + fn du_total_rejects_output_without_a_total_line() { + assert_eq!(parse_du_total(""), None); + assert_eq!(parse_du_total("\n"), None); + // A per-path line is not the total, even when it is the last one. + assert_eq!(parse_du_total("100\t/var/lib/x\n"), None); + assert_eq!(parse_du_total("abc\ttotal\n"), None); + assert_eq!(parse_du_total("-1\ttotal\n"), None); + assert_eq!(parse_du_total("100 total\n"), None); + } + + #[test] + fn no_existing_paths_is_an_error_not_zero() { + let err = du_command(&[]).expect_err("no operands must not run du"); + assert!(err.to_string().contains("none of"), "{err}"); + } + + #[tokio::test] + async fn non_zero_du_exit_is_an_error() { + let err = du_total("iris", Command::new("false"), Duration::from_secs(5)) + .await + .expect_err("a failed du must not yield a total"); + assert!(err.to_string().contains("failed"), "{err}"); + } + + #[tokio::test] + async fn a_walk_past_the_limit_is_killed_and_errors() { + let mut cmd = Command::new("sleep"); + cmd.arg("30"); + let started = std::time::Instant::now(); + let err = du_total("iris", cmd, Duration::from_millis(50)) + .await + .expect_err("a walk past the limit must fail"); + assert!(err.to_string().contains("timed out"), "{err}"); + assert!(started.elapsed() < Duration::from_secs(10)); + } + + #[tokio::test] + async fn du_total_sums_a_real_walk() { + let dir = std::env::temp_dir().join(format!("hive-priv-du-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("f"), b"hello").unwrap(); + let cmd = du_command(std::slice::from_ref(&dir)).unwrap(); + let total = du_total("iris", cmd, Duration::from_secs(30)).await; + std::fs::remove_dir_all(&dir).unwrap(); + assert!(total.unwrap() >= 5); + } +}