97fb76ce99
hive-matrix-daemon now learns which external matrix accounts it has from the swarm secret store, under the agent's own certificate, and the hive push chain for matrix is gone. The daemon lists swarm/agents/<agent>/matrix/ (the `list` its policy grants on its own metadata subtree), reads each account's homeserver from its credential, and brings the accounts up with their tokens from the store. Every two minutes it lists again and exits with 75 when the set of linked accounts changed; the unit restarts on 75 without counting a failure. A listed name whose credential reads as absent is skipped and logged once. At start it removes the matrix-token-<a> / matrix-account-<a>.json pairs a hive delivered (a sidecar marks a pair as delivered; a declared tokenFile keeps its token). Removed: CredentialNotice and the $SWARM.credential.* subject and NATS grant, the controller's publish and its queue precondition on the PUT route, hive-c0re's credential subscription arm and workers/credential.rs, priv_client::write_agent_matrix_token, hive-priv's WriteAgentMatrixToken and its helpers, and the daemon's state-dir account discovery. Kept: WriteAgentGithubToken and the external-forge path (WriteAgentExtraForgeAccount, extra_forges.rs) are untouched, and a declared matrixAccounts tokenFile is still read when the store has no token for that account. Refs #4348
4373 lines
177 KiB
Rust
4373 lines
177 KiB
Rust
//! Minimal privileged helper for hive-c0re.
|
|
//!
|
|
//! Runs as root. Exposes a narrow unix socket at `/run/hive/priv.sock`
|
|
//! that accepts `PrivRequest` JSON lines and executes only the
|
|
//! operations that genuinely require root. All coordination logic,
|
|
//! broker, HTTP, and scheduling stay in the unprivileged hive-c0re
|
|
//! process.
|
|
//!
|
|
//! **Security model**: every request is validated against a strict
|
|
//! container-name allowlist before any filesystem or process operation.
|
|
//! Only containers whose names match the hive convention (`h-*`,
|
|
//! the manager container, or known sibling service containers) are
|
|
//! accepted. Every variant maps to a single known operation — no
|
|
//! arbitrary command pass-through.
|
|
//!
|
|
//! **Socket activation**: when systemd passes the listener socket via
|
|
//! `LISTEN_FDS=1` + `LISTEN_PID=<self>`, the inherited fd 3 is used
|
|
//! instead of binding a fresh socket.
|
|
|
|
use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd, RawFd};
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use anyhow::{Context as _, Result, anyhow, bail};
|
|
use hive_priv_sock::{
|
|
AGENT_PREFIX, AGENT_RUNTIME_ROOT, AGENT_STATE_ROOT, BindMount, CredentialMount, InfraAction,
|
|
InfraContainer, JournalQuery, META_DIR, MIGRATE_STAGING_ROOT, NetworkIsolation,
|
|
PAUSED_MARKER_FILE, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream,
|
|
PrivStreamLine, SIBLING_CONTAINERS,
|
|
};
|
|
use serde::Serialize;
|
|
use tokio::io::{AsyncWriteExt, BufReader};
|
|
use tokio::net::unix::OwnedWriteHalf;
|
|
use tokio::net::{UnixListener, UnixStream};
|
|
use tokio::process::Command;
|
|
|
|
/// Root of the per-agent unix-socket dirs on the host.
|
|
const SOCKET_DIR_ROOT: &str = "/run/hive-agent";
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
tracing_subscriber::fmt()
|
|
.with_env_filter(
|
|
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
|
|
)
|
|
// This is a systemd-managed daemon — stdout always goes to journald,
|
|
// never a human terminal, and journald doesn't strip ANSI escapes:
|
|
// they land in victorialogs as raw byte-array spam otherwise.
|
|
.with_ansi(false)
|
|
.init();
|
|
|
|
let listener = socket_listener()?;
|
|
if let Err(e) = remove_legacy_tmpfiles() {
|
|
tracing::warn!(error = %format!("{e:#}"), "legacy tmpfiles cleanup failed");
|
|
}
|
|
tracing::info!("hive-priv listening");
|
|
|
|
loop {
|
|
match listener.accept().await {
|
|
Ok((stream, _)) => {
|
|
tokio::spawn(handle(stream));
|
|
}
|
|
Err(e) => {
|
|
tracing::error!(error = %e, "accept failed");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn socket_listener() -> Result<UnixListener> {
|
|
// hive-priv is ALWAYS socket-activated by the `hive-priv.socket` unit
|
|
// (fd 3 via LISTEN_FDS). There is intentionally no self-bind fallback,
|
|
// so dev and prod take the same path; see docs/trust-boundary/boundary.md.
|
|
let listen_fds: Option<i32> = std::env::var("LISTEN_FDS")
|
|
.ok()
|
|
.and_then(|s| s.parse().ok());
|
|
let listen_pid: Option<u32> = std::env::var("LISTEN_PID")
|
|
.ok()
|
|
.and_then(|s| s.parse().ok());
|
|
|
|
let activated =
|
|
matches!(listen_fds, Some(n) if n >= 1) && listen_pid == Some(std::process::id());
|
|
if !activated {
|
|
bail!(
|
|
"hive-priv requires systemd socket activation (expected LISTEN_FDS>=1 + \
|
|
LISTEN_PID=<self> for {PRIV_SOCK}); run it via the hive-priv.socket unit, \
|
|
not directly"
|
|
);
|
|
}
|
|
|
|
// SAFETY: systemd has passed us a ready UnixListener on fd 3.
|
|
let std_listener = unsafe {
|
|
use std::os::unix::io::FromRawFd;
|
|
std::os::unix::net::UnixListener::from_raw_fd(3)
|
|
};
|
|
std_listener
|
|
.set_nonblocking(true)
|
|
.context("set socket non-blocking")?;
|
|
let listener =
|
|
tokio::net::UnixListener::from_std(std_listener).context("wrap systemd socket")?;
|
|
tracing::info!("using systemd-activated socket");
|
|
Ok(listener)
|
|
}
|
|
|
|
/// Ancillary-data buffer sized and aligned for one `SCM_RIGHTS` message.
|
|
///
|
|
/// `CMSG_SPACE` is not a `const fn`, so the size is a literal with room
|
|
/// to spare (24 bytes are needed for a single descriptor on x86-64). The
|
|
/// union member gives the `cmsghdr` alignment `CMSG_FIRSTHDR` requires —
|
|
/// a bare `[u8; N]` is only byte-aligned and would be undefined behaviour
|
|
/// to walk.
|
|
#[repr(C)]
|
|
union CmsgSpace {
|
|
_align: libc::cmsghdr,
|
|
bytes: [u8; 32],
|
|
}
|
|
|
|
/// One `recvmsg` into `buf`, returning the bytes read plus any file
|
|
/// descriptors that rode along as `SCM_RIGHTS`.
|
|
///
|
|
/// Why not a plain read: ancillary data is attached to a *specific*
|
|
/// `recvmsg` call, so a buffered line reader cannot surface it — it
|
|
/// reads the bytes and silently drops the descriptor.
|
|
///
|
|
/// `MSG_CMSG_CLOEXEC` is not optional: without it a received descriptor
|
|
/// is inherited by every `btrfs` / `nixos-container` child this helper
|
|
/// later spawns.
|
|
fn recv_with_fds(sock: RawFd, buf: &mut [u8]) -> std::io::Result<(usize, Vec<OwnedFd>)> {
|
|
const FD_SIZE: usize = std::mem::size_of::<RawFd>();
|
|
|
|
let mut iov = libc::iovec {
|
|
iov_base: buf.as_mut_ptr().cast(),
|
|
iov_len: buf.len(),
|
|
};
|
|
let mut cmsg = CmsgSpace { bytes: [0; 32] };
|
|
// SAFETY: msghdr is a plain C struct with no invalid bit patterns;
|
|
// every field we care about is set immediately below.
|
|
let mut msg: libc::msghdr = unsafe { std::mem::zeroed() };
|
|
msg.msg_iov = &raw mut iov;
|
|
msg.msg_iovlen = 1;
|
|
msg.msg_control = std::ptr::addr_of_mut!(cmsg.bytes).cast();
|
|
msg.msg_controllen = 32;
|
|
|
|
// SAFETY: `msg` points at a live iovec covering `buf` and a live,
|
|
// correctly aligned control buffer of the length we just declared.
|
|
let n = unsafe { libc::recvmsg(sock, &raw mut msg, libc::MSG_CMSG_CLOEXEC) };
|
|
if n < 0 {
|
|
return Err(std::io::Error::last_os_error());
|
|
}
|
|
|
|
// Take ownership of every descriptor the kernel attached, even ones
|
|
// this protocol never expects: an `OwnedFd` we drop is closed, an
|
|
// fd we fail to claim is leaked for the lifetime of the process.
|
|
let mut fds = Vec::new();
|
|
// SAFETY: `msg` was just filled in by a successful `recvmsg`.
|
|
let mut cmsgp = unsafe { libc::CMSG_FIRSTHDR(&raw const msg) };
|
|
while !cmsgp.is_null() {
|
|
// SAFETY: CMSG_FIRSTHDR / CMSG_NXTHDR only ever return a pointer
|
|
// to a complete header inside the control buffer.
|
|
let hdr = unsafe { std::ptr::read_unaligned(cmsgp) };
|
|
if hdr.cmsg_level == libc::SOL_SOCKET && hdr.cmsg_type == libc::SCM_RIGHTS {
|
|
// SAFETY: same, and CMSG_LEN(0) is the header's own length.
|
|
let payload = hdr.cmsg_len as usize - unsafe { libc::CMSG_LEN(0) } as usize;
|
|
let count = payload / FD_SIZE;
|
|
// SAFETY: CMSG_DATA points at `payload` bytes of descriptors.
|
|
let data = unsafe { libc::CMSG_DATA(cmsgp) };
|
|
for i in 0..count {
|
|
// Copied out byte-wise rather than read through a
|
|
// `*const RawFd`: the control buffer is only guaranteed
|
|
// `cmsghdr`-aligned, so casting to a more strictly
|
|
// aligned pointer would be unsound even where it happens
|
|
// to work.
|
|
let mut raw = [0u8; FD_SIZE];
|
|
// SAFETY: i < count, so this reads inside the payload.
|
|
unsafe {
|
|
std::ptr::copy_nonoverlapping(data.add(i * FD_SIZE), raw.as_mut_ptr(), FD_SIZE);
|
|
}
|
|
// SAFETY: the kernel just created this descriptor for
|
|
// us — we are its only owner.
|
|
fds.push(unsafe { OwnedFd::from_raw_fd(RawFd::from_ne_bytes(raw)) });
|
|
}
|
|
}
|
|
// SAFETY: `cmsgp` came from this same message.
|
|
cmsgp = unsafe { libc::CMSG_NXTHDR(&raw const msg, cmsgp) };
|
|
}
|
|
|
|
// `n >= 0` was checked above, so the conversion cannot fail; going
|
|
// through `try_from` keeps it a cast-free, lint-clean widening.
|
|
let read = usize::try_from(n).unwrap_or_default();
|
|
Ok((read, fds))
|
|
}
|
|
|
|
/// Reads newline-delimited requests off one connection, pairing each
|
|
/// with the descriptor that arrived with it.
|
|
///
|
|
/// The pairing is deliberately trivial, because the protocol is:
|
|
/// `hive-sock-client` connects per request, so a connection carries one
|
|
/// line and at most one descriptor. The loop below still handles several
|
|
/// sequential requests (the server always has), but it refuses to guess
|
|
/// — a second descriptor arriving before its line is a protocol error,
|
|
/// not something to queue and hope about.
|
|
struct Requests<'a> {
|
|
sock: &'a UnixStream,
|
|
buf: Vec<u8>,
|
|
fd: Option<OwnedFd>,
|
|
}
|
|
|
|
impl Requests<'_> {
|
|
/// Next complete request line and its descriptor, or `None` at EOF.
|
|
async fn next(&mut self) -> Result<Option<(String, Option<OwnedFd>)>> {
|
|
loop {
|
|
if let Some(nl) = self.buf.iter().position(|&b| b == b'\n') {
|
|
let line: Vec<u8> = self.buf.drain(..=nl).take(nl).collect();
|
|
let line = String::from_utf8(line).context("request line was not valid UTF-8")?;
|
|
return Ok(Some((line, self.fd.take())));
|
|
}
|
|
|
|
let mut chunk = [0u8; 8192];
|
|
let raw = self.sock.as_raw_fd();
|
|
let (n, fds) = self
|
|
.sock
|
|
.async_io(tokio::io::Interest::READABLE, || {
|
|
recv_with_fds(raw, &mut chunk)
|
|
})
|
|
.await
|
|
.context("recvmsg on the priv socket")?;
|
|
|
|
for fd in fds {
|
|
if self.fd.replace(fd).is_some() {
|
|
bail!("more than one file descriptor passed for a single request");
|
|
}
|
|
}
|
|
if n == 0 {
|
|
if !self.buf.is_empty() {
|
|
bail!("connection closed mid-request ({} bytes)", self.buf.len());
|
|
}
|
|
return Ok(None);
|
|
}
|
|
self.buf.extend_from_slice(&chunk[..n]);
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn handle(stream: UnixStream) {
|
|
let (reader, mut writer) = stream.into_split();
|
|
let mut requests = Requests {
|
|
sock: reader.as_ref(),
|
|
buf: Vec::new(),
|
|
fd: None,
|
|
};
|
|
loop {
|
|
let (line, fd) = match requests.next().await {
|
|
Ok(Some(req)) => req,
|
|
Ok(None) => break,
|
|
Err(e) => {
|
|
tracing::warn!(error = %format!("{e:#}"), "reading request failed");
|
|
break;
|
|
}
|
|
};
|
|
let resp = dispatch(&line, fd, &mut writer).await;
|
|
// Write the terminal PrivResponse as a PrivEvent::Done. Wire-identical
|
|
// to a bare PrivResponse (untagged), so old hive-c0re callers that
|
|
// deserialise directly to PrivResponse continue to work.
|
|
let event = PrivEvent::Done(resp);
|
|
let mut json = serde_json::to_string(&event).unwrap_or_else(|e| {
|
|
format!("{{\"ok\":false,\"stdout\":\"\",\"stderr\":\"\",\"error\":\"serialise failed: {e}\"}}")
|
|
});
|
|
json.push('\n');
|
|
if let Err(e) = writer.write_all(json.as_bytes()).await {
|
|
tracing::warn!(error = %e, "write response failed");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Reject a request whose descriptor and operation disagree, in either
|
|
/// direction.
|
|
///
|
|
/// No guessing when the caller didn't say: an op that streams into a
|
|
/// passed descriptor cannot invent one, and an op that takes none must
|
|
/// not silently accept one. Returning the `Err` here drops the
|
|
/// `OwnedFd`, which closes it.
|
|
fn check_fd_agreement(req: &PrivRequest, fd: Option<&OwnedFd>) -> Result<()> {
|
|
let wants_fd = matches!(req, PrivRequest::SendAgentSnapshotToFd { .. });
|
|
match (wants_fd, fd.is_some()) {
|
|
(true, false) => bail!("this operation requires a passed file descriptor, none arrived"),
|
|
(false, true) => bail!("this operation does not take a passed file descriptor"),
|
|
_ => Ok(()),
|
|
}
|
|
}
|
|
|
|
async fn dispatch(line: &str, fd: Option<OwnedFd>, writer: &mut OwnedWriteHalf) -> PrivResponse {
|
|
match run(line, fd, writer).await {
|
|
Ok((stdout, stderr)) => PrivResponse {
|
|
ok: true,
|
|
stdout,
|
|
stderr,
|
|
error: None,
|
|
},
|
|
Err(e) => PrivResponse {
|
|
ok: false,
|
|
stdout: String::new(),
|
|
stderr: String::new(),
|
|
error: Some(format!("{e:#}")),
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Parse one request line, check it agrees with the descriptor that
|
|
/// arrived with it, and execute it.
|
|
///
|
|
/// Split out of [`dispatch`] so the three failure modes collapse into one
|
|
/// `Result` instead of three nested matches building the same struct.
|
|
async fn run(
|
|
line: &str,
|
|
fd: Option<OwnedFd>,
|
|
writer: &mut OwnedWriteHalf,
|
|
) -> Result<(String, String)> {
|
|
let req = serde_json::from_str::<PrivRequest>(line).context("parse request")?;
|
|
check_fd_agreement(&req, fd.as_ref())?;
|
|
exec(req, fd, writer).await
|
|
}
|
|
|
|
/// Write one `PrivEvent::Line` to the client. Best-effort: a write
|
|
/// failure is logged but doesn't abort the running subprocess.
|
|
async fn write_line_event(writer: &mut OwnedWriteHalf, stream: PrivStream, data: &str) {
|
|
let event = PrivEvent::Line(PrivStreamLine {
|
|
stream,
|
|
data: data.to_owned(),
|
|
});
|
|
if let Ok(mut json) = serde_json::to_string(&event) {
|
|
json.push('\n');
|
|
if let Err(e) = writer.write_all(json.as_bytes()).await {
|
|
tracing::warn!(error = %e, "write_line_event: write failed");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Execute a validated `PrivRequest`. Returns `(stdout, stderr)` on success.
|
|
/// For streaming ops (`CreateContainer`/`UpdateContainer` with `stream: true`)
|
|
/// output lines are forwarded to `writer` as `PrivEvent::Line` messages and
|
|
/// the returned strings are empty.
|
|
///
|
|
/// `fd` is the descriptor that arrived with this request, already checked
|
|
/// against the operation by [`check_fd_agreement`]: `Some` exactly for
|
|
/// the variants that stream into a caller-supplied descriptor, `None`
|
|
/// for every other operation.
|
|
// One match arm per priv op — a flat 1:1 dispatch table. Every arm either
|
|
// delegates directly or validates then delegates; an op whose handling is
|
|
// more than that gets its own named function instead, so the match's length
|
|
// tracks the op count, not complexity.
|
|
#[allow(clippy::too_many_lines)]
|
|
async fn exec(
|
|
req: PrivRequest,
|
|
fd: Option<OwnedFd>,
|
|
writer: &mut OwnedWriteHalf,
|
|
) -> Result<(String, String)> {
|
|
match req {
|
|
PrivRequest::StartContainer { ref name } => start_container(name).await,
|
|
|
|
PrivRequest::StopContainer { ref name } => stop_container(name).await,
|
|
|
|
PrivRequest::KillContainer { ref name } => kill_container(name).await,
|
|
|
|
PrivRequest::UpdateContainer { ref name, stream } => {
|
|
container_flake_action("update", name, stream, writer).await
|
|
}
|
|
|
|
PrivRequest::CreateContainer { ref name, stream } => {
|
|
container_flake_action("create", name, stream, writer).await
|
|
}
|
|
|
|
PrivRequest::DestroyContainer { ref name } => destroy_container(name).await,
|
|
|
|
PrivRequest::ListContainers => container_run(&["list"]).await,
|
|
|
|
PrivRequest::ReadContainerJournal {
|
|
ref container,
|
|
ref query,
|
|
} => exec_read_container_journal(container, query).await,
|
|
|
|
PrivRequest::WriteNspawnFlags {
|
|
ref container,
|
|
ref binds,
|
|
ref isolation,
|
|
ref load_credentials,
|
|
} => handle_write_nspawn_flags(container, binds, isolation, load_credentials),
|
|
|
|
PrivRequest::EnsureAgentSocketDir { ref name } => ensure_agent_socket_dir(name),
|
|
|
|
PrivRequest::WriteResourceLimits {
|
|
ref container,
|
|
ref memory_max,
|
|
ref cpu_quota,
|
|
cpu_weight,
|
|
io_weight,
|
|
} => write_resource_limits(container, memory_max, cpu_quota, cpu_weight, io_weight),
|
|
|
|
PrivRequest::RemoveServiceDropin { ref container } => remove_service_dropin(container),
|
|
|
|
PrivRequest::DaemonReload => daemon_reload().await,
|
|
|
|
PrivRequest::ReloadGatewayNginx => sync_gateway_nginx().await,
|
|
|
|
PrivRequest::RunForgeAdmin { ref args } => exec_forge_admin(args).await,
|
|
|
|
PrivRequest::SetAgentPaused {
|
|
ref agent_name,
|
|
paused,
|
|
} => exec_set_agent_paused(agent_name, paused),
|
|
|
|
PrivRequest::WriteAgentGithubToken {
|
|
ref agent_name,
|
|
ref token,
|
|
} => write_github_token(agent_name, token),
|
|
|
|
PrivRequest::WriteAgentExtraForgeAccount {
|
|
ref agent_name,
|
|
ref label,
|
|
ref base_url,
|
|
ref token,
|
|
} => write_extra_forge_account(agent_name, label, base_url, token),
|
|
|
|
PrivRequest::DeleteAgentExtraForgeAccount {
|
|
ref agent_name,
|
|
ref label,
|
|
} => delete_extra_forge_account(agent_name, label),
|
|
|
|
PrivRequest::RegisterCiRunner { ref token } => register_ci_runner(token).await,
|
|
|
|
PrivRequest::ControlInfraContainer { container, action } => {
|
|
control_infra_container(container, action).await
|
|
}
|
|
|
|
PrivRequest::EnsureAgentSubvolume { ref agent_name } => {
|
|
exec_ensure_agent_subvolume(agent_name).await
|
|
}
|
|
|
|
PrivRequest::DeleteAgentSubvolume { ref agent_name } => {
|
|
exec_delete_agent_subvolume(agent_name).await
|
|
}
|
|
|
|
PrivRequest::EnsureBtrfsQuota => ensure_btrfs_quota().await,
|
|
|
|
PrivRequest::ReadSubvolumeUsage { ref agent_name } => {
|
|
exec_read_subvolume_usage(agent_name).await
|
|
}
|
|
|
|
PrivRequest::SetSubvolumeQuota {
|
|
ref agent_name,
|
|
limit_bytes,
|
|
} => exec_set_subvolume_quota(agent_name, limit_bytes).await,
|
|
|
|
PrivRequest::UpgradeAgentSubvolume { ref agent_name } => {
|
|
exec_upgrade_agent_subvolume(agent_name).await
|
|
}
|
|
|
|
PrivRequest::SnapshotAgentSubvolume {
|
|
ref agent_name,
|
|
ref snapshot_name,
|
|
} => exec_snapshot_agent_subvolume(agent_name, snapshot_name).await,
|
|
|
|
PrivRequest::DeleteAgentSnapshot {
|
|
ref agent_name,
|
|
ref snapshot_name,
|
|
} => exec_delete_agent_snapshot(agent_name, snapshot_name).await,
|
|
|
|
PrivRequest::SendAgentSnapshotToFile {
|
|
ref agent_name,
|
|
ref snapshot_name,
|
|
ref parent_snapshot_name,
|
|
ref dest_file_name,
|
|
} => {
|
|
exec_send_agent_snapshot_to_file(
|
|
agent_name,
|
|
snapshot_name,
|
|
parent_snapshot_name.as_deref(),
|
|
dest_file_name,
|
|
)
|
|
.await
|
|
}
|
|
|
|
PrivRequest::SendAgentSnapshotToFd {
|
|
ref agent_name,
|
|
ref snapshot_name,
|
|
ref parent_snapshot_name,
|
|
} => {
|
|
exec_send_agent_snapshot_to_fd(
|
|
agent_name,
|
|
snapshot_name,
|
|
parent_snapshot_name.as_deref(),
|
|
fd,
|
|
)
|
|
.await
|
|
}
|
|
|
|
PrivRequest::SyncAgentTmpfiles => {
|
|
remove_legacy_tmpfiles()?;
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
}
|
|
}
|
|
|
|
/// `StartContainer`: clear any start-limit lockout left by earlier failures
|
|
/// so a now-correct start isn't blocked (`nixos-container start` does not do
|
|
/// this itself). Best-effort — if the unit doesn't exist yet (first-time
|
|
/// create), `reset-failed` is a no-op and the start proceeds regardless.
|
|
async fn start_container(name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(name)?;
|
|
let machine = container_system_name(name);
|
|
let _ = Command::new("systemctl")
|
|
.args(["reset-failed", &format!("container@{machine}.service")])
|
|
.status()
|
|
.await;
|
|
container_run(&["start", &machine]).await
|
|
}
|
|
|
|
/// `RunForgeAdmin`: every arg must pass [`validate_forge_admin_arg`] before
|
|
/// the admin CLI ever sees it.
|
|
async fn exec_forge_admin(args: &[String]) -> Result<(String, String)> {
|
|
for arg in args {
|
|
validate_forge_admin_arg(arg)?;
|
|
}
|
|
run_forge_admin(args).await
|
|
}
|
|
|
|
/// `WriteAgentExtraForgeAccount`: writes the token, then a
|
|
/// `forge-<label>.json` sidecar carrying the base URL — there's no
|
|
/// host-side nix config for extra forges, so this is the only place it's
|
|
/// persisted.
|
|
fn write_extra_forge_account(
|
|
agent_name: &str,
|
|
label: &str,
|
|
base_url: &str,
|
|
token: &str,
|
|
) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
validate_name_chars(label)?;
|
|
let res = write_agent_state_file(
|
|
agent_name,
|
|
&format!("forge-{label}-token"),
|
|
&format!("{token}\n"),
|
|
)?;
|
|
let meta = serde_json::to_string(&ForgeSidecar { base_url })
|
|
.context("serialize forge account sidecar")?;
|
|
write_agent_state_file(agent_name, &format!("forge-{label}.json"), &meta)?;
|
|
Ok(res)
|
|
}
|
|
|
|
/// `StopContainer`.
|
|
async fn stop_container(name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(name)?;
|
|
stop_and_release(&container_system_name(name)).await
|
|
}
|
|
|
|
/// `KillContainer`: `nixos-container` has no kill verb, so this uses
|
|
/// `machinectl` to send `SIGKILL` to every process in the container — the
|
|
/// right semantics for a forced shutdown after a graceful stop has already
|
|
/// been attempted.
|
|
async fn kill_container(name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(name)?;
|
|
let machine = container_system_name(name);
|
|
machinectl_run(&["kill", &machine, "--signal=SIGKILL"]).await
|
|
}
|
|
|
|
/// `DestroyContainer`.
|
|
async fn destroy_container(name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(name)?;
|
|
container_run(&["destroy", &container_system_name(name)]).await
|
|
}
|
|
|
|
/// `ReadContainerJournal`.
|
|
async fn exec_read_container_journal(
|
|
container: &str,
|
|
query: &JournalQuery,
|
|
) -> Result<(String, String)> {
|
|
validate_container_system_name(container)?;
|
|
read_container_journal(container, query).await
|
|
}
|
|
|
|
/// `SetAgentPaused`.
|
|
fn exec_set_agent_paused(agent_name: &str, paused: bool) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
set_agent_paused(agent_name, paused)
|
|
}
|
|
|
|
/// `WriteAgentGithubToken`.
|
|
fn write_github_token(agent_name: &str, token: &str) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
write_agent_state_file(agent_name, "github-token", &format!("{token}\n"))
|
|
}
|
|
|
|
/// `DeleteAgentExtraForgeAccount`. Missing files are not an error
|
|
/// (idempotent revoke).
|
|
fn delete_extra_forge_account(agent_name: &str, label: &str) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
validate_name_chars(label)?;
|
|
delete_agent_state_file(agent_name, &format!("forge-{label}-token"))?;
|
|
delete_agent_state_file(agent_name, &format!("forge-{label}.json"))
|
|
}
|
|
|
|
/// `EnsureAgentSubvolume`.
|
|
async fn exec_ensure_agent_subvolume(agent_name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
ensure_agent_subvolume(agent_name).await
|
|
}
|
|
|
|
/// `DeleteAgentSubvolume`.
|
|
async fn exec_delete_agent_subvolume(agent_name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
delete_agent_subvolume(agent_name).await
|
|
}
|
|
|
|
/// `ReadSubvolumeUsage`.
|
|
async fn exec_read_subvolume_usage(agent_name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
read_subvolume_usage(agent_name).await
|
|
}
|
|
|
|
/// `SetSubvolumeQuota`.
|
|
async fn exec_set_subvolume_quota(
|
|
agent_name: &str,
|
|
limit_bytes: Option<u64>,
|
|
) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
set_subvolume_quota(agent_name, limit_bytes).await
|
|
}
|
|
|
|
/// `UpgradeAgentSubvolume`.
|
|
async fn exec_upgrade_agent_subvolume(agent_name: &str) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
upgrade_agent_subvolume(agent_name).await
|
|
}
|
|
|
|
/// `SnapshotAgentSubvolume`.
|
|
async fn exec_snapshot_agent_subvolume(
|
|
agent_name: &str,
|
|
snapshot_name: &str,
|
|
) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
validate_snapshot_name(snapshot_name)?;
|
|
snapshot_agent_subvolume(agent_name, snapshot_name).await
|
|
}
|
|
|
|
/// `DeleteAgentSnapshot`.
|
|
async fn exec_delete_agent_snapshot(
|
|
agent_name: &str,
|
|
snapshot_name: &str,
|
|
) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
validate_snapshot_name(snapshot_name)?;
|
|
delete_agent_snapshot(agent_name, snapshot_name).await
|
|
}
|
|
|
|
/// `SendAgentSnapshotToFile`.
|
|
async fn exec_send_agent_snapshot_to_file(
|
|
agent_name: &str,
|
|
snapshot_name: &str,
|
|
parent_snapshot_name: Option<&str>,
|
|
dest_file_name: &str,
|
|
) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
validate_snapshot_name(snapshot_name)?;
|
|
if let Some(parent) = parent_snapshot_name {
|
|
validate_snapshot_name(parent)?;
|
|
}
|
|
validate_credential_name(dest_file_name)?;
|
|
send_agent_snapshot_to_file(
|
|
agent_name,
|
|
snapshot_name,
|
|
parent_snapshot_name,
|
|
dest_file_name,
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// `SendAgentSnapshotToFd`.
|
|
async fn exec_send_agent_snapshot_to_fd(
|
|
agent_name: &str,
|
|
snapshot_name: &str,
|
|
parent_snapshot_name: Option<&str>,
|
|
fd: Option<OwnedFd>,
|
|
) -> Result<(String, String)> {
|
|
validate_agent_name(agent_name)?;
|
|
validate_snapshot_name(snapshot_name)?;
|
|
if let Some(parent) = parent_snapshot_name {
|
|
validate_snapshot_name(parent)?;
|
|
}
|
|
let dest = fd.context("no descriptor to stream into")?;
|
|
send_agent_snapshot_to_fd(agent_name, snapshot_name, parent_snapshot_name, dest).await
|
|
}
|
|
|
|
/// Shared body for `CreateContainer` / `UpdateContainer`: validate the
|
|
/// name, build the toplevel ourselves, and apply it (streaming line
|
|
/// events to `writer` when `stream` is set).
|
|
///
|
|
/// **Both verbs build explicitly now — not just `update`.** The first cut
|
|
/// of this fix only rewrote `update`, reasoning that `create` was already
|
|
/// safe: it wraps its whole action in an exclusive `flock` before calling
|
|
/// `nixos-container`'s own `buildFlake()`, and once `update` stopped
|
|
/// writing to `buildFlake()`'s shared `.tmp` out-link, concurrent
|
|
/// `create`s were the only remaining writers — mutually excluded by that
|
|
/// lock. True, but it leaves `create`'s safety resting on an internal
|
|
/// implementation detail of a script we don't own (its current locking
|
|
/// behavior, which could change upstream without notice) instead of on
|
|
/// something we control. Building here for both verbs removes `buildFlake()`
|
|
/// from the picture entirely — there's no shared `.tmp` left to race on,
|
|
/// so there's nothing left to reason about staying in sync with.
|
|
///
|
|
/// **`update` no longer calls `nixos-container update` at all.** That
|
|
/// action's own version-compat probe runs unconditionally before
|
|
/// `--system-path` is ever honored, dying on every agent's update.
|
|
/// [`swap_container_profile`] replicates exactly what `update`'s own
|
|
/// action does *past* that probe (confirmed against its source): `nix-env
|
|
/// --set` the per-container profile, then `systemctl reload` if the
|
|
/// container is running. `create` is untouched — it isn't the failing
|
|
/// verb.
|
|
async fn container_flake_action(
|
|
verb: &str,
|
|
name: &str,
|
|
stream: bool,
|
|
writer: &mut OwnedWriteHalf,
|
|
) -> Result<(String, String)> {
|
|
validate_agent_name(name)?;
|
|
// The build is the multi-minute phase of this operation — give it the
|
|
// same live-line treatment `container_run_streaming` gives
|
|
// `nixos-container` itself when the caller asked for it. Without
|
|
// this, moving the build out of the streamed `nixos-container` call
|
|
// (which is the whole point of this fix) would silently regress every
|
|
// UI that shows build progress: nothing until the longest phase
|
|
// finishes, then everything at once.
|
|
let toplevel = nix_build_toplevel(name, stream.then_some(&mut *writer)).await?;
|
|
let system_name = container_system_name(name);
|
|
if verb == "update" {
|
|
return swap_container_profile(&system_name, &toplevel, stream, writer).await;
|
|
}
|
|
let args = [verb, &system_name, "--system-path", &toplevel];
|
|
if stream {
|
|
container_run_streaming(&args, writer).await
|
|
} else {
|
|
container_run(&args).await
|
|
}
|
|
}
|
|
|
|
/// Apply a prebuilt toplevel to an existing container directly, without
|
|
/// going through `nixos-container update` (see `container_flake_action`'s
|
|
/// doc comment for why). Mirrors `nixos-container.pl`'s own `update`
|
|
/// action verbatim, past its version probe: point the per-container `nix-
|
|
/// env` profile at the new toplevel, then reload the container unit *if*
|
|
/// it's currently running — the container's own next start already reads
|
|
/// from the profile, so a stopped container needs nothing further.
|
|
///
|
|
/// Both steps are near-instant in the happy case, but still stream-
|
|
/// forwarded like the rest of this operation: a failure here (e.g. a
|
|
/// wedged `systemctl reload`) is exactly when the caller most wants the
|
|
/// live line in the build log, not just a summary error afterward.
|
|
async fn swap_container_profile(
|
|
system_name: &str,
|
|
toplevel: &str,
|
|
stream: bool,
|
|
writer: &mut OwnedWriteHalf,
|
|
) -> Result<(String, String)> {
|
|
let profile = format!("/nix/var/nix/profiles/per-container/{system_name}/system");
|
|
let set_out = Command::new("nix-env")
|
|
.args(["-p", &profile, "--set", toplevel])
|
|
.output()
|
|
.await
|
|
.context("invoke nix-env --set")?;
|
|
let mut stdout = String::from_utf8_lossy(&set_out.stdout).into_owned();
|
|
let mut stderr = String::from_utf8_lossy(&set_out.stderr).into_owned();
|
|
log_and_forward(&stdout, &stderr, stream, writer).await;
|
|
if !set_out.status.success() {
|
|
bail!(
|
|
"nix-env -p {profile} --set failed ({}): {}",
|
|
set_out.status,
|
|
stderr.trim()
|
|
);
|
|
}
|
|
|
|
let unit = format!("container@{system_name}");
|
|
let state_out = Command::new("systemctl")
|
|
.args(["show", "--property=ActiveState", "--value", &unit])
|
|
.output()
|
|
.await
|
|
.context("query container ActiveState")?;
|
|
let active = String::from_utf8_lossy(&state_out.stdout).trim() == "active";
|
|
if active {
|
|
let reload_out = Command::new("systemctl")
|
|
.args(["reload", &unit])
|
|
.output()
|
|
.await
|
|
.context("invoke systemctl reload")?;
|
|
let r_stdout = String::from_utf8_lossy(&reload_out.stdout).into_owned();
|
|
let r_stderr = String::from_utf8_lossy(&reload_out.stderr).into_owned();
|
|
log_and_forward(&r_stdout, &r_stderr, stream, writer).await;
|
|
if !reload_out.status.success() {
|
|
bail!(
|
|
"systemctl reload {unit} failed ({}): {}",
|
|
reload_out.status,
|
|
r_stderr.trim()
|
|
);
|
|
}
|
|
stdout.push_str(&r_stdout);
|
|
stderr.push_str(&r_stderr);
|
|
}
|
|
Ok((stdout, stderr))
|
|
}
|
|
|
|
/// Log a command's captured stdout/stderr the same way every other
|
|
/// `nixos-container`-adjacent shellout in this file does, and — when
|
|
/// `stream` is set — forward each line to the client as a live
|
|
/// [`PrivEvent::Line`] too, so a caller watching the build log sees these
|
|
/// lines exactly like any other step's, not just a summary on failure.
|
|
async fn log_and_forward(stdout: &str, stderr: &str, stream: bool, writer: &mut OwnedWriteHalf) {
|
|
for line in stdout.lines() {
|
|
tracing::info!(target: "nixos-container", "{line}");
|
|
if stream {
|
|
write_line_event(writer, PrivStream::Stdout, line).await;
|
|
}
|
|
}
|
|
for line in stderr.lines() {
|
|
tracing::warn!(target: "nixos-container", "{line}");
|
|
if stream {
|
|
write_line_event(writer, PrivStream::Stderr, line).await;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The explicit `nixosConfigurations.<name>.config.system.build.toplevel`
|
|
/// flake attr path — same construction `hive-c0re`'s own
|
|
/// `lifecycle::prebuild_toplevel` uses, kept here as a pure function so
|
|
/// the exact string shape is unit-tested without needing to run `nix`.
|
|
fn toplevel_attr(name: &str) -> String {
|
|
format!("{META_DIR}#nixosConfigurations.{name}.config.system.build.toplevel")
|
|
}
|
|
|
|
/// Wall-clock bound on [`nix_build_toplevel`]. Four times CI's observed
|
|
/// cold-cache `nix flake check` (~15 min, `.forgejo/workflows/ci.yml`), which
|
|
/// compiles this workspace's crates, the part of an agent toplevel no binary
|
|
/// cache holds; the rebuild path has usually warmed this exact attr already
|
|
/// (`hive-c0re`'s `prebuild_toplevel`). A stalled download is bounded by nix's
|
|
/// own `stalled-download-timeout`; this covers the rest, such as a wedged
|
|
/// nix-daemon holding the caller's build slot forever.
|
|
const NIX_BUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_hours(1);
|
|
|
|
/// How a [`run_bounded`] child ended.
|
|
enum BoundedRun {
|
|
Exited {
|
|
status: std::process::ExitStatus,
|
|
stdout: String,
|
|
stderr: String,
|
|
},
|
|
/// Still running at the limit; its whole process group was killed.
|
|
TimedOut,
|
|
}
|
|
|
|
/// Run `cmd` to completion or until `limit`, draining both pipes: stdout is
|
|
/// captured silently, stderr is logged and, when `writer` is set, forwarded
|
|
/// live as [`PrivEvent::Line`]s.
|
|
///
|
|
/// The child leads its own process group and a timeout `SIGKILL`s the whole
|
|
/// group, so a grandchild still holding the pipes (or the daemon connection)
|
|
/// dies with it instead of outliving the error.
|
|
async fn run_bounded(
|
|
mut cmd: Command,
|
|
limit: std::time::Duration,
|
|
mut writer: Option<&mut OwnedWriteHalf>,
|
|
) -> std::io::Result<BoundedRun> {
|
|
use tokio::io::AsyncBufReadExt as _;
|
|
|
|
let mut child = cmd
|
|
.stdout(std::process::Stdio::piped())
|
|
.stderr(std::process::Stdio::piped())
|
|
.process_group(0)
|
|
.spawn()?;
|
|
// Group leader, so its pid is the pgid.
|
|
let pgid = child.id().and_then(|pid| i32::try_from(pid).ok());
|
|
|
|
let stdout = child.stdout.take().expect("stdout piped");
|
|
let stderr = child.stderr.take().expect("stderr piped");
|
|
let run = async {
|
|
let mut stdout_lines = BufReader::new(stdout).lines();
|
|
let mut stderr_lines = BufReader::new(stderr).lines();
|
|
let mut stdout_buf = String::new();
|
|
let mut stderr_buf = String::new();
|
|
|
|
// ⚠️ Both pipes are drained concurrently even though only one is
|
|
// streamed: reading stderr alone would let stdout fill its pipe
|
|
// buffer and deadlock the child on a build with enough stdout output
|
|
// to fill it.
|
|
//
|
|
// ⚠️ Each stream's EOF is tracked separately rather than breaking on
|
|
// the first `None`: `next_line()` on a closed stream returns
|
|
// `Ok(None)` immediately and forever, so a loop that keeps polling a
|
|
// finished stream spins hot until the other one ends too.
|
|
let mut stdout_done = false;
|
|
let mut stderr_done = false;
|
|
while !(stdout_done && stderr_done) {
|
|
tokio::select! {
|
|
line = stdout_lines.next_line(), if !stdout_done => {
|
|
match line {
|
|
// Captured, never streamed — this is the store path.
|
|
Ok(Some(l)) => {
|
|
stdout_buf.push_str(&l);
|
|
stdout_buf.push('\n');
|
|
}
|
|
Ok(None) => stdout_done = true,
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "nix build stdout read error");
|
|
stdout_done = true;
|
|
}
|
|
}
|
|
}
|
|
line = stderr_lines.next_line(), if !stderr_done => {
|
|
match line {
|
|
// Streamed as it arrives, so the dashboard and
|
|
// `journalctl -f` show build progress live.
|
|
Ok(Some(l)) => {
|
|
tracing::info!(target: "nix-build-toplevel", "{l}");
|
|
if let Some(w) = writer.as_deref_mut() {
|
|
write_line_event(w, PrivStream::Stderr, &l).await;
|
|
}
|
|
if !stderr_buf.is_empty() {
|
|
stderr_buf.push('\n');
|
|
}
|
|
stderr_buf.push_str(&l);
|
|
}
|
|
Ok(None) => stderr_done = true,
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "nix build stderr read error");
|
|
stderr_done = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
let status = child.wait().await?;
|
|
Ok::<_, std::io::Error>(BoundedRun::Exited {
|
|
status,
|
|
stdout: stdout_buf,
|
|
stderr: stderr_buf,
|
|
})
|
|
};
|
|
if let Ok(exited) = tokio::time::timeout(limit, run).await {
|
|
return exited;
|
|
}
|
|
if let Some(pgid) = pgid {
|
|
// SAFETY: plain `kill(2)`; `-pgid` targets the child's own process
|
|
// group, which `process_group(0)` made it lead.
|
|
unsafe {
|
|
libc::kill(-pgid, libc::SIGKILL);
|
|
}
|
|
}
|
|
child.wait().await?;
|
|
Ok(BoundedRun::TimedOut)
|
|
}
|
|
|
|
/// Build `nixosConfigurations.<name>.config.system.build.toplevel` and
|
|
/// return the resulting store path, so `create`/`update` can hand
|
|
/// `nixos-container` an explicit `--system-path` instead of letting its
|
|
/// own `buildFlake()` build to a racy shared out-link. See "Container
|
|
/// toplevel builds" in this crate's README for the full story — the
|
|
/// concurrency bug this closes (the "agent container gets closure of
|
|
/// other agent" mystery bug) and why stdout/stderr are drained
|
|
/// concurrently but handled asymmetrically (stderr streamed live,
|
|
/// stdout captured and required to be exactly one line).
|
|
///
|
|
/// `writer` is `None` for the non-streaming call shape (`stream: false`);
|
|
/// stderr still logs to journald either way, just without the
|
|
/// `PrivEvent::Line` forwarding. Bounded by [`NIX_BUILD_TIMEOUT`].
|
|
async fn nix_build_toplevel(name: &str, writer: Option<&mut OwnedWriteHalf>) -> Result<String> {
|
|
let attr = toplevel_attr(name);
|
|
let mut cmd = Command::new("nix");
|
|
cmd.args([
|
|
"--extra-experimental-features",
|
|
"nix-command flakes",
|
|
"build",
|
|
"--no-link",
|
|
"--print-out-paths",
|
|
&attr,
|
|
]);
|
|
let (status, stdout_buf, stderr_buf) = match run_bounded(cmd, NIX_BUILD_TIMEOUT, writer)
|
|
.await
|
|
.with_context(|| format!("run nix build {attr}"))?
|
|
{
|
|
BoundedRun::Exited {
|
|
status,
|
|
stdout,
|
|
stderr,
|
|
} => (status, stdout, stderr),
|
|
BoundedRun::TimedOut => bail!(
|
|
"nix build {attr} timed out after {}s; killed it",
|
|
NIX_BUILD_TIMEOUT.as_secs()
|
|
),
|
|
};
|
|
|
|
// ⚠️ Success is decided by the exit status, not by "we parsed a
|
|
// path" — a build can print to stdout and still fail.
|
|
if !status.success() {
|
|
bail!(
|
|
"nix build {attr} failed ({status}): {}",
|
|
stderr_buf.lines().last().unwrap_or("").trim()
|
|
);
|
|
}
|
|
single_output_path(&stdout_buf)
|
|
.map(str::to_owned)
|
|
.map_err(|count| {
|
|
anyhow!("nix build {attr} produced {count} output path(s), expected exactly 1: {stdout_buf:?}")
|
|
})
|
|
}
|
|
|
|
/// Parse `nix build --print-out-paths`' stdout down to the single output
|
|
/// path this function's caller expects. `--print-out-paths` prints one
|
|
/// line *per output*, not one line total (`nix build --no-link
|
|
/// --print-out-paths nixpkgs#openssl` prints two: `…-bin`, `…-man`);
|
|
/// `config.system.build.toplevel` is single-output today, so this is one
|
|
/// line in practice — but a bare whole-buffer `.trim()` would silently
|
|
/// hand a multi-line string on to `--system-path` the day that ever
|
|
/// changes, the same corrupted-argument failure this function exists to
|
|
/// avoid. Trims and drops empty lines *before* counting, so a lone
|
|
/// `"\n"` (or trailing whitespace on the real line) can't be mistaken
|
|
/// for a present-but-blank path — see the unit tests below for the exact
|
|
/// table this closes. `Err` carries the surviving line count, for the
|
|
/// caller's error message.
|
|
fn single_output_path(stdout: &str) -> Result<&str, usize> {
|
|
let lines: Vec<&str> = stdout
|
|
.lines()
|
|
.map(str::trim)
|
|
.filter(|l| !l.is_empty())
|
|
.collect();
|
|
match lines[..] {
|
|
[path] => Ok(path),
|
|
_ => Err(lines.len()),
|
|
}
|
|
}
|
|
|
|
/// `WriteNspawnFlags` — validate the container + every bind path + every
|
|
/// credential entry, then write the container's nspawn flag overrides.
|
|
fn handle_write_nspawn_flags(
|
|
container: &str,
|
|
binds: &[BindMount],
|
|
isolation: &NetworkIsolation,
|
|
load_credentials: &[CredentialMount],
|
|
) -> Result<(String, String)> {
|
|
validate_container_system_name(container)?;
|
|
for bind in binds {
|
|
validate_bind_path(&bind.host_path)?;
|
|
validate_bind_path(&bind.container_path)?;
|
|
}
|
|
for cred in load_credentials {
|
|
validate_credential_name(&cred.name)?;
|
|
// Same path rules as binds (absolute, no colon/newline/quote/null):
|
|
// the colon ban is essential since `--load-credential=name:path`
|
|
// uses `:` as the name/path separator.
|
|
validate_bind_path(&cred.host_path)?;
|
|
}
|
|
write_nspawn_flags(container, binds, isolation, load_credentials)?;
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// A btrfs snapshot label must start with `hive-` — this doubles as an
|
|
/// allow-list: only names hivectl itself constructs (or an operator who
|
|
/// knows the convention) can reach the `btrfs subvolume snapshot`/`delete`
|
|
/// shellouts, so an arbitrary caller can't use the snapshot ops to probe or
|
|
/// churn unrelated paths under `AGENT_STATE_ROOT`. Beyond the prefix, the
|
|
/// same charset restriction as [`validate_credential_name`] applies (it's
|
|
/// interpolated straight into a filesystem path).
|
|
fn validate_snapshot_name(name: &str) -> Result<()> {
|
|
if !name.starts_with("hive-") {
|
|
bail!("invalid snapshot label {name:?}: must start with \"hive-\"");
|
|
}
|
|
validate_credential_name(name)
|
|
}
|
|
|
|
/// A systemd credential id must be a short token — restrict to
|
|
/// `[A-Za-z0-9_-]` (no `.`) so it can't inject extra `--load-credential`
|
|
/// argv or break the `name:path` shape. `.` is deliberately excluded, not
|
|
/// just a bare `..`: this name gets interpolated into filesystem paths
|
|
/// (snapshot labels) and there's no legitimate need for a dot in either a
|
|
/// systemd credential id or a `hive-`-prefixed snapshot label — we're
|
|
/// defining this token format from scratch, so keep it maximally strict
|
|
/// rather than allow-then-patch each traversal-adjacent character
|
|
/// (mara: "we are making up the rules here, lets go strict").
|
|
fn validate_credential_name(name: &str) -> Result<()> {
|
|
if name.is_empty()
|
|
|| !name
|
|
.bytes()
|
|
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-'))
|
|
{
|
|
bail!("invalid credential name {name:?}: must be non-empty [A-Za-z0-9_-]");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// `RemoveServiceDropin` — remove the container service's drop-in dir
|
|
/// if present (idempotent).
|
|
fn remove_service_dropin(container: &str) -> Result<(String, String)> {
|
|
validate_container_system_name(container)?;
|
|
let dir = format!("/run/systemd/system/container@{container}.service.d");
|
|
if Path::new(&dir).exists() {
|
|
std::fs::remove_dir_all(&dir).with_context(|| format!("remove {dir}"))?;
|
|
}
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// `WriteResourceLimits` — drop the systemd resource settings into the
|
|
/// container service's drop-in dir, together with a
|
|
/// `ConditionPathIsDirectory=` guard on the agent's MCP runtime dir.
|
|
///
|
|
/// Two different kinds of setting land in the same file. `MemoryMax=` /
|
|
/// `CPUQuota=` are hard caps that throttle even on an idle host;
|
|
/// `CPUWeight=` / `IOWeight=` are cgroup v2 relative shares that only
|
|
/// decide who yields *under contention*. A weight of `None` means "not
|
|
/// configured" and omits the line, so a hive-c0re built before the weights
|
|
/// existed keeps producing the old two-line drop-in.
|
|
///
|
|
/// The condition causes systemd to *skip* (not *fail*) the unit when the
|
|
/// bind-mount source dir is absent — result is `condition`, which does not
|
|
/// increment the start-limit counter. This is belt-and-braces on top of
|
|
/// hive-c0re creating the dir in its start preamble: if it is missing at
|
|
/// start time, the unit idles rather than restart-looping into
|
|
/// `start-limit-hit`.
|
|
fn write_resource_limits(
|
|
container: &str,
|
|
memory_max: &str,
|
|
cpu_quota: &str,
|
|
cpu_weight: Option<u32>,
|
|
io_weight: Option<u32>,
|
|
) -> Result<(String, String)> {
|
|
validate_container_system_name(container)?;
|
|
// Derive the logical agent name (strip h- prefix) to form the runtime
|
|
// dir path. Falls back to the full container name for infra containers
|
|
// that don't use the h- prefix.
|
|
let logical = container.strip_prefix(AGENT_PREFIX).unwrap_or(container);
|
|
let runtime_dir = format!("{AGENT_RUNTIME_ROOT}/{logical}");
|
|
let dir = format!("/run/systemd/system/container@{container}.service.d");
|
|
std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?;
|
|
let path = format!("{dir}/hyperhive-limits.conf");
|
|
let content = limits_dropin_body(&runtime_dir, memory_max, cpu_quota, cpu_weight, io_weight);
|
|
// Published whole: a `daemon-reload` between truncate and write would
|
|
// otherwise load the unit with its limits and start guard missing.
|
|
publish_file(Path::new(&path), content.as_bytes(), 0o644, None)?;
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// How long a window the start-limit counts over, and how many starts it
|
|
/// allows inside it.
|
|
///
|
|
/// `container@.service` sets `Restart=on-failure` and **no** start limit, so
|
|
/// systemd's defaults apply: 5 starts per 10s, `RestartSec` 100ms. That
|
|
/// makes the bound depend on *how fast* a container dies — one that fails
|
|
/// instantly trips the limit in under a second, one that takes longer than
|
|
/// ~2s never trips it and restarts forever. Whether an agent gets bounded
|
|
/// is not meant to be a function of its failure speed.
|
|
///
|
|
/// The window has to exceed the worst-case time to burn the burst, or the
|
|
/// counter ages out between attempts and the limit is again unreachable:
|
|
/// `TimeoutStartSec` is 1min, so `BURST` slow failures plus their backoff
|
|
/// can span several minutes. 10min covers that with room.
|
|
///
|
|
/// Giving up is cheap here **because it is not terminal** — hive-c0re's
|
|
/// reconcile sweep retries later, and `reset-failed` (see `StartContainer`)
|
|
/// clears the latch first. That is what makes a tight burst safe.
|
|
const START_LIMIT_INTERVAL_SEC: u32 = 600;
|
|
/// One start plus two retries — the operator's ruling was "retry once or
|
|
/// twice", with the reconcile sweep as the slow path after that.
|
|
const START_LIMIT_BURST: u32 = 3;
|
|
/// Backoff between those retries. The 100ms default is for processes that
|
|
/// respawn instantly; a container that just failed to boot gains nothing
|
|
/// from being retried a tenth of a second later.
|
|
const RESTART_SEC: u32 = 5;
|
|
|
|
/// Render the body of `hyperhive-limits.conf`.
|
|
///
|
|
/// `[Unit]`: the condition is checked at start time — it skips (not fails)
|
|
/// the unit when the MCP socket dir is absent, avoiding restart loops —
|
|
/// plus the bounded start limit (see the constants above; `StartLimit*` are
|
|
/// `[Unit]` settings since systemd 229 and are silently ignored under
|
|
/// `[Service]`).
|
|
/// `[Service]`: the restart backoff, the hard caps, then the relative
|
|
/// weights. A weight of `None` means "not configured" and omits its line
|
|
/// entirely, so a request from a hive-c0re built before the weights
|
|
/// existed — or one whose nix option is `null` — renders no weight lines.
|
|
fn limits_dropin_body(
|
|
runtime_dir: &str,
|
|
memory_max: &str,
|
|
cpu_quota: &str,
|
|
cpu_weight: Option<u32>,
|
|
io_weight: Option<u32>,
|
|
) -> String {
|
|
// Built as two possibly-empty lines rather than pushed onto the
|
|
// string: `format!` appended to a `String` trips clippy::pedantic's
|
|
// `format_push_string`, and a `write!` would need an unwrap.
|
|
let cpu_weight_line = cpu_weight.map_or_else(String::new, |w| format!("CPUWeight={w}\n"));
|
|
let io_weight_line = io_weight.map_or_else(String::new, |w| format!("IOWeight={w}\n"));
|
|
format!(
|
|
"[Unit]\n\
|
|
ConditionPathIsDirectory={runtime_dir}\n\
|
|
StartLimitIntervalSec={START_LIMIT_INTERVAL_SEC}\n\
|
|
StartLimitBurst={START_LIMIT_BURST}\n\
|
|
\n\
|
|
[Service]\n\
|
|
RestartSec={RESTART_SEC}\n\
|
|
MemoryMax={memory_max}\n\
|
|
CPUQuota={cpu_quota}\n\
|
|
{cpu_weight_line}{io_weight_line}"
|
|
)
|
|
}
|
|
|
|
/// `DaemonReload` — `systemctl daemon-reload` on the host.
|
|
async fn daemon_reload() -> Result<(String, String)> {
|
|
let out = Command::new("systemctl")
|
|
.arg("daemon-reload")
|
|
.output()
|
|
.await
|
|
.context("invoke systemctl daemon-reload")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"systemctl daemon-reload failed ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// Host path to the hive-ci runner's persisted registration credentials.
|
|
///
|
|
/// Paired with `hive-c0re`'s `forge::ci_runner::RUNNER_FILE`, which reads the
|
|
/// same file to decide whether a runner is registered and whether it still
|
|
/// names the configured forge host. Deliberately duplicated rather than shared:
|
|
/// `hive-priv` is the minimal root helper and does not depend on `hive-c0re`.
|
|
const RUNNER_CREDENTIALS: &str =
|
|
"/var/lib/nixos-containers/hive-ci/var/lib/gitea-runner/hive/.runner";
|
|
|
|
/// Delete the runner's persisted credentials so upstream's `ExecStartPre` takes
|
|
/// its **absence** branch on the next start.
|
|
///
|
|
/// Absence is the state we want, so `NotFound` is success. Anything else — a
|
|
/// permission error above all — is NOT swallowed: it means the file is still
|
|
/// there, the restart will take upstream's already-registered branch, and the
|
|
/// caller would return `Ok` for a registration that never happened. That is the
|
|
/// same shape as a precondition that "passes" because it could not read the file
|
|
/// it was checking, and it is worth failing loudly to avoid.
|
|
///
|
|
/// Split from [`register_ci_runner`] purely so this rule is testable without a
|
|
/// container or a `systemctl`.
|
|
fn clear_runner_credentials(path: &str) -> Result<()> {
|
|
match std::fs::remove_file(path) {
|
|
Ok(()) => Ok(()),
|
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
|
Err(e) => {
|
|
Err(anyhow::Error::new(e).context(format!("remove stale runner credentials {path}")))
|
|
}
|
|
}
|
|
}
|
|
|
|
/// `RegisterCiRunner` — write the runner registration token to the host-side
|
|
/// `/run/hive-ci/runner-token` env-file, then restart the in-container runner
|
|
/// so it re-registers. The forge admin token never enters the container; only
|
|
/// the registration token c0re passes here is written, and it lands on a host
|
|
/// path bind-mounted read-only into hive-ci.
|
|
async fn register_ci_runner(token: &str) -> Result<(String, String)> {
|
|
use std::io::Write as _;
|
|
use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _};
|
|
// Reject anything that could corrupt the `KEY=VALUE` env-file or smuggle a
|
|
// second line — a forge registration token is an opaque single-line string.
|
|
if token.is_empty() || token.contains(['\n', '\r', '\0']) {
|
|
bail!("ci runner registration token empty or contains control characters");
|
|
}
|
|
let token_path = "/run/hive-ci/runner-token";
|
|
// In-place truncate+write of the existing inode (mirrors the prefetch's
|
|
// `echo > $FILE`), NOT a temp+rename: nspawn pins this file's inode into
|
|
// hive-ci at container start, so a rename would leave the running runner
|
|
// reading the old content. Format + perms match the tmpfiles seed and the
|
|
// prefetch: `TOKEN=<tok>`, mode 0600, root-owned. Created 0600 when the
|
|
// seed is missing, so the token is never on disk at a wider mode.
|
|
std::fs::OpenOptions::new()
|
|
.write(true)
|
|
.create(true)
|
|
.truncate(true)
|
|
.mode(0o600)
|
|
.open(token_path)
|
|
.and_then(|mut f| f.write_all(format!("TOKEN={token}\n").as_bytes()))
|
|
.with_context(|| format!("write {token_path}"))?;
|
|
std::fs::set_permissions(token_path, std::fs::Permissions::from_mode(0o600))
|
|
.with_context(|| format!("chmod {token_path}"))?;
|
|
// Remove the persisted credentials BEFORE restarting, or the restart is a
|
|
// no-op as far as registration goes.
|
|
//
|
|
// Upstream's `ExecStartPre` only re-registers when `.runner` is absent, the
|
|
// labels changed, or the *registration token hash* changed — never when the
|
|
// instance URL changed. c0re only calls this helper once it has already
|
|
// decided the existing credentials are absent or stale
|
|
// (`forge::ci_runner::ensure_ci_runner_registered` returns early otherwise),
|
|
// so by the time we are here a re-registration is exactly what is wanted and
|
|
// deleting the file is the narrow way to guarantee it happens.
|
|
//
|
|
// Writing a fresh token is NOT sufficient on its own: whether the hash
|
|
// changes depends on whether the forge mints a new registration token per
|
|
// request or hands back a stable one, which is Forgejo's behaviour to
|
|
// choose and change. Gating our remediation on the absence branch — the one
|
|
// upstream evaluates unconditionally — makes that question moot instead of
|
|
// load-bearing.
|
|
//
|
|
// See [`clear_runner_credentials`] for why absence is the branch we aim at
|
|
// and why only `NotFound` counts as success.
|
|
clear_runner_credentials(RUNNER_CREDENTIALS)?;
|
|
// Restart the in-container runner so it reads the new token and registers.
|
|
let out = Command::new("systemctl")
|
|
.args(["--machine=hive-ci", "restart", "gitea-runner-hive.service"])
|
|
.output()
|
|
.await
|
|
.context("systemctl restart gitea-runner-hive.service in hive-ci")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"systemctl restart gitea-runner-hive.service in hive-ci exited {}: {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok((
|
|
String::from_utf8_lossy(&out.stdout).into_owned(),
|
|
String::from_utf8_lossy(&out.stderr).into_owned(),
|
|
))
|
|
}
|
|
|
|
/// `ControlInfraContainer` — start/stop a hive infrastructure service via
|
|
/// `systemctl <verb> <unit>`. The [`InfraContainer`] enum is the allowlist:
|
|
/// serde already rejected any unknown / unsafe name (hive-c0re has no
|
|
/// variant, so a stop can't sever the daemon socket) at deserialisation, so
|
|
/// no root-side `.contains()` check is needed here. Serves the hive-wide
|
|
/// `hivectl stop`/`start` flow and the dashboard's operator-only infra
|
|
/// panel. No agent-facing path exists.
|
|
///
|
|
/// ⚠️ The unit is derived from the variant, never sent by the caller —
|
|
/// which is what keeps this from being a general `systemctl` pass-through.
|
|
/// It is not always `container@<name>.service`: the gateway resolves to the
|
|
/// host's `nginx.service`.
|
|
async fn control_infra_container(
|
|
container: InfraContainer,
|
|
action: InfraAction,
|
|
) -> Result<(String, String)> {
|
|
let verb = action.systemctl_verb();
|
|
let unit = container.service_unit();
|
|
let out = Command::new("systemctl")
|
|
.args([verb, &unit])
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("systemctl {verb} {unit}"))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"systemctl {verb} {unit} exited {}: {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
tracing::info!(target: "infra-control", "{verb} {unit}");
|
|
Ok((
|
|
String::from_utf8_lossy(&out.stdout).into_owned(),
|
|
String::from_utf8_lossy(&out.stderr).into_owned(),
|
|
))
|
|
}
|
|
|
|
/// A leaf safe to join onto an agent-owned directory: one plain component, so
|
|
/// it can neither climb out of the directory nor name the directory itself.
|
|
fn ensure_plain_filename(who: &str, filename: &str) -> Result<()> {
|
|
if filename.is_empty() || filename == "." || filename == ".." || filename.contains('/') {
|
|
bail!("{who}: refusing non-plain filename {filename:?}");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Name the content is written under before being renamed onto `filename`,
|
|
/// unique per call so two concurrent writes of one file never share a temp.
|
|
///
|
|
/// The leading dot and the `.partial` suffix are load-bearing: a reader
|
|
/// globbing on a published name's prefix, or systemd reading `*.conf` out of
|
|
/// `tmpfiles.d` and drop-in dirs, cannot pick up a half-written temp.
|
|
fn partial_name(filename: &str) -> String {
|
|
use std::sync::atomic::{AtomicU64, Ordering};
|
|
static SEQ: AtomicU64 = AtomicU64::new(0);
|
|
let n = SEQ.fetch_add(1, Ordering::Relaxed);
|
|
format!(".{filename}.{}-{n}.partial", std::process::id())
|
|
}
|
|
|
|
/// Open `path` as a directory fd for [`StagedFile`] to work relative to.
|
|
fn open_dir(path: &Path) -> Result<std::fs::File> {
|
|
use std::os::unix::fs::OpenOptionsExt as _;
|
|
std::fs::OpenOptions::new()
|
|
.read(true)
|
|
.custom_flags(libc::O_DIRECTORY)
|
|
.open(path)
|
|
.with_context(|| format!("open directory {}", path.display()))
|
|
}
|
|
|
|
/// A file written under a [`partial_name`] in its destination directory and
|
|
/// then renamed onto its final name, so a reader sees either the old file or
|
|
/// the complete new one, never a partial or wrong-mode one. Mode, and owner
|
|
/// when the caller `fchown`s [`file`](Self::file), are on the inode before the
|
|
/// rename makes it visible.
|
|
///
|
|
/// Every step is relative to the `dir` fd, so it is safe in a directory an
|
|
/// agent or container controls: the temp is created `O_EXCL|O_NOFOLLOW`, which
|
|
/// refuses any entry already planted at its name, and `rename` replaces
|
|
/// whatever sits at the final name without following it.
|
|
///
|
|
/// Dropped without [`publish`](Self::publish), it unlinks the temp, so a
|
|
/// failure between the two leaves the old file untouched and no residue.
|
|
struct StagedFile<'a> {
|
|
dir: &'a std::fs::File,
|
|
tmp: std::ffi::CString,
|
|
dest: std::ffi::CString,
|
|
dest_path: PathBuf,
|
|
file: std::fs::File,
|
|
published: bool,
|
|
}
|
|
|
|
impl<'a> StagedFile<'a> {
|
|
/// Create the temp at `mode` and write `content` to it. The mode is set
|
|
/// again on the fd after the write, so the umask cannot change it.
|
|
fn write(
|
|
dir: &'a std::fs::File,
|
|
dir_path: &Path,
|
|
filename: &str,
|
|
content: &[u8],
|
|
mode: u32,
|
|
) -> Result<Self> {
|
|
use std::io::Write as _;
|
|
use std::os::unix::fs::PermissionsExt as _;
|
|
|
|
ensure_plain_filename("StagedFile::write", filename)?;
|
|
let dest_path = dir_path.join(filename);
|
|
let tmp_name = partial_name(filename);
|
|
let tmp = std::ffi::CString::new(tmp_name.as_str())
|
|
.with_context(|| format!("NUL in filename {filename:?}"))?;
|
|
let dest = std::ffi::CString::new(filename)
|
|
.with_context(|| format!("NUL in filename {filename:?}"))?;
|
|
// SAFETY: `dir` is an open directory fd and `tmp` is NUL-terminated;
|
|
// the result is checked before it is wrapped.
|
|
let fd = unsafe {
|
|
libc::openat(
|
|
dir.as_raw_fd(),
|
|
tmp.as_ptr(),
|
|
libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_NOFOLLOW | libc::O_CLOEXEC,
|
|
mode,
|
|
)
|
|
};
|
|
if fd < 0 {
|
|
return Err(std::io::Error::last_os_error())
|
|
.with_context(|| format!("create {}", dir_path.join(&tmp_name).display()));
|
|
}
|
|
// SAFETY: `fd` is a fresh, valid fd that nothing else owns.
|
|
let file = std::fs::File::from(unsafe { OwnedFd::from_raw_fd(fd) });
|
|
let mut staged = Self {
|
|
dir,
|
|
tmp,
|
|
dest,
|
|
dest_path,
|
|
file,
|
|
published: false,
|
|
};
|
|
staged
|
|
.file
|
|
.write_all(content)
|
|
.with_context(|| format!("write {}", staged.dest_path.display()))?;
|
|
staged
|
|
.file
|
|
.set_permissions(std::fs::Permissions::from_mode(mode))
|
|
.with_context(|| format!("chmod {mode:o} {}", staged.dest_path.display()))?;
|
|
Ok(staged)
|
|
}
|
|
|
|
/// The temp's open fd, for an `fchown` that must land before publishing.
|
|
fn file(&self) -> &std::fs::File {
|
|
&self.file
|
|
}
|
|
|
|
/// Flush the content, rename the temp onto the final name, and flush the
|
|
/// directory: without the last step a crash can lose the rename itself.
|
|
fn publish(mut self) -> Result<()> {
|
|
self.file
|
|
.sync_all()
|
|
.with_context(|| format!("fsync {}", self.dest_path.display()))?;
|
|
let fd = self.dir.as_raw_fd();
|
|
// SAFETY: `fd` is an open directory fd; both names are NUL-terminated.
|
|
if unsafe { libc::renameat(fd, self.tmp.as_ptr(), fd, self.dest.as_ptr()) } != 0 {
|
|
return Err(std::io::Error::last_os_error())
|
|
.with_context(|| format!("publishing {}", self.dest_path.display()));
|
|
}
|
|
self.published = true;
|
|
self.dir
|
|
.sync_all()
|
|
.with_context(|| format!("fsync directory of {}", self.dest_path.display()))
|
|
}
|
|
}
|
|
|
|
impl Drop for StagedFile<'_> {
|
|
fn drop(&mut self) {
|
|
if self.published {
|
|
return;
|
|
}
|
|
// SAFETY: `self.dir` is an open directory fd; `self.tmp` is
|
|
// NUL-terminated.
|
|
if unsafe { libc::unlinkat(self.dir.as_raw_fd(), self.tmp.as_ptr(), 0) } != 0 {
|
|
tracing::warn!(
|
|
dest = %self.dest_path.display(),
|
|
error = %std::io::Error::last_os_error(),
|
|
"failed to remove unpublished temp"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Publish `content` at `path` through a [`StagedFile`], at `mode` and, when
|
|
/// given, owned by `owner` (`(uid, gid)`).
|
|
fn publish_file(path: &Path, content: &[u8], mode: u32, owner: Option<(u32, u32)>) -> Result<()> {
|
|
let (Some(dir_path), Some(filename)) =
|
|
(path.parent(), path.file_name().and_then(|n| n.to_str()))
|
|
else {
|
|
bail!(
|
|
"publish_file: {} has no directory or file name",
|
|
path.display()
|
|
);
|
|
};
|
|
let dir = open_dir(dir_path)?;
|
|
let staged = StagedFile::write(&dir, dir_path, filename, content, mode)?;
|
|
if let Some((uid, gid)) = owner {
|
|
std::os::unix::fs::fchown(staged.file(), Some(uid), Some(gid))
|
|
.with_context(|| format!("chown {}", path.display()))?;
|
|
}
|
|
staged.publish()
|
|
}
|
|
|
|
/// Sidecar written alongside a dashboard-provisioned extra forge
|
|
/// account's token (`forge-<label>.json`) so `hive-forge` can resolve
|
|
/// the account's base URL. Read side: `hive-forge/src/client.rs`'s own
|
|
/// (separately defined, deserialize-only) `ForgeSidecar` — same field
|
|
/// name (`base_url`), no shared crate between `hive-priv` and
|
|
/// `hive-forge` to hang a common type off, so the two structs are
|
|
/// pinned to the same JSON key by convention, not by the compiler.
|
|
#[derive(Serialize)]
|
|
struct ForgeSidecar<'a> {
|
|
base_url: &'a str,
|
|
}
|
|
|
|
/// Shared helper for the `WriteAgent*Token` requests.
|
|
/// Writes `content` to `AGENT_STATE_ROOT/<agent_name>/state/<filename>`,
|
|
/// chowns to the agent user (derived from the state dir's existing owner),
|
|
/// and chmods 0600. Running as root (hive-priv), so this succeeds
|
|
/// regardless of the file's prior owner/permissions.
|
|
fn write_agent_state_file(
|
|
agent_name: &str,
|
|
filename: &str,
|
|
content: &str,
|
|
) -> Result<(String, String)> {
|
|
let state_dir = PathBuf::from(AGENT_STATE_ROOT)
|
|
.join(agent_name)
|
|
.join("state");
|
|
write_agent_dir_file(agent_name, &state_dir, filename, content)
|
|
}
|
|
|
|
/// Create or remove an agent's pause marker under its harness dir. The
|
|
/// marker is written empty and chowned to the harness dir's owner (the
|
|
/// agent), matching how the harness itself would have created it.
|
|
///
|
|
/// Both directions are idempotent: re-pausing truncates the existing empty
|
|
/// marker rather than failing, and a `NotFound` on removal is the
|
|
/// already-resumed case, not an error.
|
|
fn set_agent_paused(agent_name: &str, paused: bool) -> Result<(String, String)> {
|
|
let harness_dir = PathBuf::from(AGENT_STATE_ROOT)
|
|
.join(agent_name)
|
|
.join("harness");
|
|
if paused {
|
|
return write_agent_dir_file(agent_name, &harness_dir, PAUSED_MARKER_FILE, "");
|
|
}
|
|
remove_marker_in(&harness_dir, PAUSED_MARKER_FILE)?;
|
|
tracing::info!(agent = %agent_name, "cleared pause marker");
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// Unlink `dir/filename`, treating "already gone" as success.
|
|
///
|
|
/// `remove_file` unlinks the leaf itself and never follows a symlink, so an
|
|
/// agent-planted link at the marker path cannot redirect this root unlink
|
|
/// at another file — the same threat [`StagedFile`] closes on the create side.
|
|
fn remove_marker_in(dir: &Path, filename: &str) -> Result<()> {
|
|
let path = dir.join(filename);
|
|
match std::fs::remove_file(&path) {
|
|
Ok(()) => Ok(()),
|
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
|
Err(e) => Err(e).with_context(|| format!("remove {}", path.display())),
|
|
}
|
|
}
|
|
|
|
/// Write `content` to `dir/filename` as root, chowning the result to `dir`'s
|
|
/// owner so the agent process can read it back. Shared by the credential
|
|
/// writes (which target `state/`) and the pause marker (which targets
|
|
/// `harness/`) — both write into a directory owned by the agent, which is
|
|
/// precisely why they need hive-priv at all.
|
|
///
|
|
/// The file is published through a [`StagedFile`], so a reader woken by its
|
|
/// appearance cannot catch it empty or half-written: several of these paths
|
|
/// have a `systemd.path` unit watching them, and one of those triggers on the
|
|
/// file existing rather than changing.
|
|
fn write_agent_dir_file(
|
|
agent_name: &str,
|
|
dir: &Path,
|
|
filename: &str,
|
|
content: &str,
|
|
) -> Result<(String, String)> {
|
|
use std::os::fd::AsRawFd as _;
|
|
use std::os::unix::fs::MetadataExt as _;
|
|
|
|
// Checked before `create_dir_all`, so a bad leaf does no filesystem work.
|
|
ensure_plain_filename("write_agent_dir_file", filename)?;
|
|
|
|
let state_dir = dir.to_path_buf();
|
|
// NOTE: `create_dir_all` is normally a no-op — lifecycle creates and chowns
|
|
// the state dir during spawn. On the rare edge where the dir doesn't exist
|
|
// yet (container being provisioned for the first time), the newly created dir
|
|
// is root:root. The `stat state_dir` chown below will then see uid=0 and
|
|
// leave the file root-owned (0600). The agent won't be able to read it until
|
|
// its lifecycle completes. If that happens, a `systemctl restart hive-c0re`
|
|
// after provisioning will re-mint and re-write the token correctly.
|
|
std::fs::create_dir_all(&state_dir)
|
|
.with_context(|| format!("create state dir {}", state_dir.display()))?;
|
|
|
|
// Security-critical: the (state/-owning) agent may plant a symlink or
|
|
// hardlink in this dir, and `StagedFile` neither follows nor reuses one,
|
|
// so this root-privileged create/write/chmod/chown can't be redirected at
|
|
// an arbitrary file.
|
|
let path = state_dir.join(filename);
|
|
let dir_fd = open_dir(&state_dir)?;
|
|
let staged = StagedFile::write(&dir_fd, &state_dir, filename, content.as_bytes(), 0o600)?;
|
|
|
|
// Chown to the state dir's owner so the agent process can read the file.
|
|
// fchown on the temp's fd — TOCTOU-immune (the inode the write hit, never a
|
|
// swapped path). If stat fails (e.g. dir just created, owner is root), the
|
|
// file stays root-owned and 0600 — still unreadable by others, just not
|
|
// agent-readable. Log a warning so operators can diagnose.
|
|
match dir_fd.metadata() {
|
|
Ok(meta) => {
|
|
// SAFETY: the temp's fd is open and owned by `staged` for the whole
|
|
// call; `fchown` only mutates that inode's uid/gid.
|
|
let rc = unsafe { libc::fchown(staged.file().as_raw_fd(), meta.uid(), meta.gid()) };
|
|
if rc != 0 {
|
|
let e = std::io::Error::last_os_error();
|
|
tracing::warn!(
|
|
agent = %agent_name,
|
|
path = %path.display(),
|
|
error = %e,
|
|
"write_agent_state_file: fchown failed"
|
|
);
|
|
}
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(
|
|
agent = %agent_name,
|
|
error = %e,
|
|
"write_agent_state_file: stat state_dir failed, leaving file root-owned"
|
|
);
|
|
}
|
|
}
|
|
// After the chown, never before: the file must never be visible under its
|
|
// final name while still root-owned. A failed publish drops `staged`,
|
|
// which removes the temp and the credential in it.
|
|
staged.publish()?;
|
|
|
|
tracing::info!(
|
|
agent = %agent_name,
|
|
dir = %state_dir.display(),
|
|
file = %filename,
|
|
"wrote agent file"
|
|
);
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// Remove `AGENT_STATE_ROOT/<agent_name>/state/<filename>` if present.
|
|
/// Idempotent revoke counterpart to [`write_agent_state_file`] — a
|
|
/// missing file is success, not an error. `filename` must be a single
|
|
/// plain component (no `/`, `.`, `..`); callers pass a pre-validated
|
|
/// label into a fixed `forge-<label>-token` shape, same as the write
|
|
/// side.
|
|
fn delete_agent_state_file(agent_name: &str, filename: &str) -> Result<(String, String)> {
|
|
ensure_plain_filename("delete_agent_state_file", filename)?;
|
|
let path = PathBuf::from(AGENT_STATE_ROOT)
|
|
.join(agent_name)
|
|
.join("state")
|
|
.join(filename);
|
|
match std::fs::remove_file(&path) {
|
|
Ok(()) => {
|
|
tracing::info!(agent = %agent_name, file = %filename, "removed agent state file");
|
|
}
|
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
|
Err(e) => return Err(e).with_context(|| format!("remove {}", path.display())),
|
|
}
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// btrfs superblock magic, as reported by `statfs(2)`'s `f_type`.
|
|
const BTRFS_SUPER_MAGIC: i64 = 0x9123_683E;
|
|
|
|
/// Inode number of a btrfs subvolume root (`BTRFS_FIRST_FREE_OBJECTID`).
|
|
/// Every subvolume's top directory has this inode; plain directories do
|
|
/// not, so `statfs == btrfs && st_ino == 256` reliably identifies a
|
|
/// subvolume root.
|
|
const BTRFS_SUBVOL_ROOT_INO: u64 = 256;
|
|
|
|
/// Whether `path` lives on a btrfs filesystem (via `statfs(2)`).
|
|
fn is_on_btrfs(path: &Path) -> Result<bool> {
|
|
use std::os::unix::ffi::OsStrExt as _;
|
|
let c_path = std::ffi::CString::new(path.as_os_str().as_bytes())
|
|
.with_context(|| format!("path {} has an interior null byte", path.display()))?;
|
|
// SAFETY: `c_path` is a valid NUL-terminated C string that outlives the
|
|
// call; `statfs` only writes into the zero-initialised `buf`.
|
|
let mut buf: libc::statfs = unsafe { std::mem::zeroed() };
|
|
let rc = unsafe { libc::statfs(c_path.as_ptr(), &raw mut buf) };
|
|
if rc != 0 {
|
|
return Err(std::io::Error::last_os_error())
|
|
.with_context(|| format!("statfs {}", path.display()));
|
|
}
|
|
Ok(buf.f_type == BTRFS_SUPER_MAGIC)
|
|
}
|
|
|
|
/// Whether `path` is the root of a btrfs subvolume (on btrfs and inode 256).
|
|
fn is_btrfs_subvolume(path: &Path) -> bool {
|
|
use std::os::unix::fs::MetadataExt as _;
|
|
let on_btrfs = is_on_btrfs(path).unwrap_or(false);
|
|
let ino_match = std::fs::metadata(path).is_ok_and(|m| m.ino() == BTRFS_SUBVOL_ROOT_INO);
|
|
on_btrfs && ino_match
|
|
}
|
|
|
|
/// `EnsureAgentSubvolume` — make the agent's state root a btrfs subvolume
|
|
/// when the FS supports it. Idempotent + progressive: no-op when the root
|
|
/// already exists or the FS isn't btrfs. See the wire doc on the variant.
|
|
async fn ensure_agent_subvolume(agent_name: &str) -> Result<(String, String)> {
|
|
use std::os::unix::fs::MetadataExt as _;
|
|
|
|
let root = PathBuf::from(AGENT_STATE_ROOT);
|
|
let agent_root = root.join(agent_name);
|
|
|
|
// Progressive: existing agents (plain dir OR already a subvol) are left
|
|
// untouched — never auto-migrated.
|
|
if agent_root.exists() {
|
|
return Ok((String::new(), String::new()));
|
|
}
|
|
|
|
// Only btrfs supports subvolumes; on anything else hive-c0re's normal
|
|
// `create_dir_all` makes a plain dir (the pre-subvolume behaviour). The
|
|
// parent must exist for both statfs and `btrfs subvolume create`.
|
|
std::fs::create_dir_all(&root)
|
|
.with_context(|| format!("create agents root {}", root.display()))?;
|
|
if !is_on_btrfs(&root)? {
|
|
return Ok((String::new(), String::new()));
|
|
}
|
|
|
|
let out = Command::new("btrfs")
|
|
.args(["subvolume", "create"])
|
|
.arg(&agent_root)
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("spawn btrfs subvolume create {}", agent_root.display()))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"btrfs subvolume create {} failed: {}",
|
|
agent_root.display(),
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
|
|
// The subvol root is created root-owned, but hive-c0re (the `hive-core`
|
|
// user) must be able to mkdir state/ claude/ harness/ inside it — exactly
|
|
// as it would in a plain dir. Chown it to AGENT_STATE_ROOT's owner
|
|
// (hive-core). This MUST succeed: a root-owned subvol would make the
|
|
// downstream dir creation fail with a confusing permission error, and the
|
|
// c0re-side exists-check would then skip re-running this op on retry,
|
|
// wedging the agent. So on any failure roll the subvol back and bail — the
|
|
// create path surfaces a clear error and a retry starts clean.
|
|
let chown_result = std::fs::metadata(&root)
|
|
.with_context(|| format!("stat agents root {} for ownership", root.display()))
|
|
.and_then(|meta| {
|
|
std::os::unix::fs::chown(&agent_root, Some(meta.uid()), Some(meta.gid())).with_context(
|
|
|| format!("chown subvol {} to agents-root owner", agent_root.display()),
|
|
)
|
|
});
|
|
if let Err(e) = chown_result {
|
|
// Best-effort rollback so we never leave a root-owned subvol behind.
|
|
let _ = Command::new("btrfs")
|
|
.args(["subvolume", "delete"])
|
|
.arg(&agent_root)
|
|
.output()
|
|
.await;
|
|
return Err(e.context(format!(
|
|
"rolled back subvolume {} after chown failed",
|
|
agent_root.display()
|
|
)));
|
|
}
|
|
tracing::info!(agent = %agent_name, path = %agent_root.display(), "created agent state subvolume");
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// `DeleteAgentSubvolume` — delete the agent's state root iff it's a btrfs
|
|
/// subvolume (purge path only). No-op for plain dirs / missing paths;
|
|
/// hive-c0re's own `remove_dir_all` covers those. See the wire doc.
|
|
async fn delete_agent_subvolume(agent_name: &str) -> Result<(String, String)> {
|
|
let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name);
|
|
if !is_btrfs_subvolume(&agent_root) {
|
|
return Ok((String::new(), String::new()));
|
|
}
|
|
let out = Command::new("btrfs")
|
|
.args(["subvolume", "delete"])
|
|
.arg(&agent_root)
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("spawn btrfs subvolume delete {}", agent_root.display()))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"btrfs subvolume delete {} failed: {}",
|
|
agent_root.display(),
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
tracing::info!(agent = %agent_name, path = %agent_root.display(), "deleted agent state subvolume");
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// `EnsureBtrfsQuota` — enable btrfs qgroup accounting on the filesystem
|
|
/// holding `AGENT_STATE_ROOT`. Idempotent + statfs-gated (no-op off btrfs).
|
|
/// Operator opt-in only; see the wire doc.
|
|
async fn ensure_btrfs_quota() -> Result<(String, String)> {
|
|
let root = PathBuf::from(AGENT_STATE_ROOT);
|
|
std::fs::create_dir_all(&root)
|
|
.with_context(|| format!("create agents root {}", root.display()))?;
|
|
if !is_on_btrfs(&root)? {
|
|
// Non-btrfs host: quota/qgroups don't apply. No-op success so the
|
|
// operator-facing verb degrades cleanly.
|
|
return Ok((
|
|
String::new(),
|
|
"filesystem is not btrfs — quota not applicable".to_owned(),
|
|
));
|
|
}
|
|
let out = Command::new("btrfs")
|
|
.args(["quota", "enable"])
|
|
.arg(&root)
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("spawn btrfs quota enable {}", root.display()))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"btrfs quota enable {} failed: {}",
|
|
root.display(),
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
tracing::info!(path = %root.display(), "enabled btrfs qgroup accounting");
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// `ReadSubvolumeUsage` — return an agent subvolume's qgroup rows
|
|
/// (`btrfs qgroup show -f --raw <…/agent_name>`) verbatim in stdout for
|
|
/// hive-c0re to parse. `-f` lists the qgroups impacting the given path,
|
|
/// excluding ancestral qgroups (per btrfs-qgroup-show(8)) — so it scopes
|
|
/// to this subvolume and never mixes in other agents'. hive-c0re then
|
|
/// selects the level-0 (`0/<subvolid>`) leaf row. See the wire doc.
|
|
async fn read_subvolume_usage(agent_name: &str) -> Result<(String, String)> {
|
|
let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name);
|
|
let out = Command::new("btrfs")
|
|
.args(["qgroup", "show", "-f", "--raw"])
|
|
.arg(&agent_root)
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("spawn btrfs qgroup show {}", agent_root.display()))?;
|
|
if !out.status.success() {
|
|
// The common failure is "quota not enabled" — pass the stderr
|
|
// through so hive-c0re can surface it gracefully.
|
|
bail!(
|
|
"btrfs qgroup show {} failed: {}",
|
|
agent_root.display(),
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok((
|
|
String::from_utf8_lossy(&out.stdout).into_owned(),
|
|
String::new(),
|
|
))
|
|
}
|
|
|
|
/// Best-effort removal of a leftover migration path from a prior aborted
|
|
/// upgrade: try `btrfs subvolume delete` (in case it's a half-created
|
|
/// subvolume) then a plain recursive remove. Both failures are ignored —
|
|
/// the path may simply not exist.
|
|
async fn cleanup_stale_path(path: &Path) {
|
|
if path.exists() {
|
|
let _ = Command::new("btrfs")
|
|
.args(["subvolume", "delete"])
|
|
.arg(path)
|
|
.output()
|
|
.await;
|
|
let _ = std::fs::remove_dir_all(path);
|
|
}
|
|
}
|
|
|
|
/// Stage a populated subvolume at `tmp` mirroring `agent_root`: create the
|
|
/// subvolume, copy `agent_root`'s contents into it preserving
|
|
/// ownership/permissions/xattrs, then match the subvolume root's owner + mode
|
|
/// to the original. On any failure the partially-staged `tmp` is cleaned up
|
|
/// (so the caller can bail with the original dir still untouched).
|
|
async fn stage_upgrade_subvolume(agent_root: &Path, tmp: &Path) -> Result<()> {
|
|
use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _};
|
|
|
|
// Fresh subvolume to receive the copy.
|
|
let out = Command::new("btrfs")
|
|
.args(["subvolume", "create"])
|
|
.arg(tmp)
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("spawn btrfs subvolume create {}", tmp.display()))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"btrfs subvolume create {} failed: {}",
|
|
tmp.display(),
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
|
|
// Copy contents preserving everything (`-a` = --preserve=all → mode,
|
|
// ownership, timestamps, links, xattrs); reflink for fast CoW clones on the
|
|
// same btrfs. `<src>/.` copies the directory's contents (incl. dotfiles)
|
|
// into the subvolume rather than nesting it.
|
|
let copy = Command::new("cp")
|
|
.arg("-a")
|
|
.arg("--reflink=auto")
|
|
.arg(format!("{}/.", agent_root.display()))
|
|
.arg(tmp)
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("spawn cp into {}", tmp.display()))?;
|
|
if !copy.status.success() {
|
|
cleanup_stale_path(tmp).await;
|
|
bail!(
|
|
"copy {} -> {} failed (original left untouched): {}",
|
|
agent_root.display(),
|
|
tmp.display(),
|
|
String::from_utf8_lossy(©.stderr).trim()
|
|
);
|
|
}
|
|
|
|
// Match the new subvolume root's ownership + mode to the original dir.
|
|
// `cp -a <src>/.` copies the *contents* but the subvolume root keeps its
|
|
// create-time root ownership, so set it explicitly — the swapped-in
|
|
// subvolume must be indistinguishable from the original to hive-c0re.
|
|
let apply = std::fs::metadata(agent_root)
|
|
.with_context(|| format!("stat {} for ownership", agent_root.display()))
|
|
.and_then(|m| {
|
|
std::os::unix::fs::chown(tmp, Some(m.uid()), Some(m.gid()))
|
|
.with_context(|| format!("chown {} to match original", tmp.display()))?;
|
|
std::fs::set_permissions(tmp, std::fs::Permissions::from_mode(m.mode()))
|
|
.with_context(|| format!("chmod {} to match original", tmp.display()))?;
|
|
Ok(())
|
|
});
|
|
if let Err(e) = apply {
|
|
cleanup_stale_path(tmp).await;
|
|
return Err(e.context("upgrade aborted before swap; original left untouched"));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// `UpgradeAgentSubvolume` — convert an existing plain-dir agent state root
|
|
/// into a btrfs subvolume in place. Operator opt-in; the caller (hivectl)
|
|
/// stops the agent first and restarts it after. See the wire doc.
|
|
///
|
|
/// Migration: stage a sibling subvolume mirroring the dir
|
|
/// ([`stage_upgrade_subvolume`]), then rename the original aside and the
|
|
/// subvolume into place, then remove the original. Any failure before the
|
|
/// rename-swap leaves the original dir untouched.
|
|
async fn upgrade_agent_subvolume(agent_name: &str) -> Result<(String, String)> {
|
|
let root = PathBuf::from(AGENT_STATE_ROOT);
|
|
let agent_root = root.join(agent_name);
|
|
|
|
if !agent_root.exists() {
|
|
// A crash between the two swap renames (original → `.<name>.old`
|
|
// succeeded, `.<name>.migrating` → agent_root did not) leaves the
|
|
// agent root missing but the original data intact under `.<name>.old`.
|
|
// Point at the recovery rather than a bare "nothing to do" so the
|
|
// operator isn't left guessing where the data went.
|
|
let old = root.join(format!(".{agent_name}.old"));
|
|
if old.exists() {
|
|
bail!(
|
|
"no state dir at {agent} — but {old} holds the original data from an \
|
|
interrupted upgrade (host crashed mid-swap). Restore it with \
|
|
`mv {old} {agent}`, then re-run the upgrade.",
|
|
agent = agent_root.display(),
|
|
old = old.display(),
|
|
);
|
|
}
|
|
bail!(
|
|
"no state dir to upgrade at {} — nothing to do",
|
|
agent_root.display()
|
|
);
|
|
}
|
|
// Idempotent: already a subvolume → nothing to do.
|
|
if is_btrfs_subvolume(&agent_root) {
|
|
return Ok((String::new(), String::new()));
|
|
}
|
|
if !is_on_btrfs(&root)? {
|
|
bail!(
|
|
"{} is not on btrfs — subvolumes are unsupported, cannot upgrade",
|
|
root.display()
|
|
);
|
|
}
|
|
|
|
// Sibling temp paths on the same filesystem (so the copy can reflink and
|
|
// the swap renames are atomic). Leading dots keep them out of the agent
|
|
// namespace (`validate_agent_name` rejects dot-prefixed names).
|
|
let tmp = root.join(format!(".{agent_name}.migrating"));
|
|
let old = root.join(format!(".{agent_name}.old"));
|
|
// Clear any debris from a previously interrupted run before starting.
|
|
cleanup_stale_path(&tmp).await;
|
|
cleanup_stale_path(&old).await;
|
|
|
|
stage_upgrade_subvolume(&agent_root, &tmp).await?;
|
|
|
|
// Swap. `rename` is atomic within a filesystem. The window between the
|
|
// two renames is the only unsafe point: a crash there leaves the agent
|
|
// root missing but both `.old` (original) and the new subvolume present
|
|
// — recoverable by hand, hence the loud logging.
|
|
if let Err(e) = std::fs::rename(&agent_root, &old) {
|
|
cleanup_stale_path(&tmp).await;
|
|
return Err(anyhow::Error::new(e).context(format!(
|
|
"rename {} -> {} failed; original left untouched",
|
|
agent_root.display(),
|
|
old.display()
|
|
)));
|
|
}
|
|
if let Err(e) = std::fs::rename(&tmp, &agent_root) {
|
|
// Restore the original from its renamed-aside copy.
|
|
let restored = std::fs::rename(&old, &agent_root).is_ok();
|
|
cleanup_stale_path(&tmp).await;
|
|
return Err(anyhow::Error::new(e).context(format!(
|
|
"rename {} -> {} failed; original {}",
|
|
tmp.display(),
|
|
agent_root.display(),
|
|
if restored {
|
|
"restored"
|
|
} else {
|
|
"COULD NOT BE RESTORED — manual recovery needed"
|
|
}
|
|
)));
|
|
}
|
|
|
|
// 5. Success: drop the original (a plain dir) and report.
|
|
if let Err(e) = std::fs::remove_dir_all(&old) {
|
|
// The migration succeeded; a leftover `.old` is cosmetic. Warn only.
|
|
tracing::warn!(
|
|
agent = %agent_name, path = %old.display(),
|
|
"upgraded subvolume but failed to remove old dir: {e}"
|
|
);
|
|
}
|
|
tracing::info!(
|
|
agent = %agent_name, path = %agent_root.display(),
|
|
"upgraded agent state dir to btrfs subvolume"
|
|
);
|
|
Ok((
|
|
format!("upgraded {} to a btrfs subvolume", agent_root.display()),
|
|
String::new(),
|
|
))
|
|
}
|
|
|
|
/// Derive a snapshot's path from the agent name + label: a dot-prefixed
|
|
/// sibling of the agent's state root so it can never collide with a real
|
|
/// agent directory (`validate_agent_name` rejects dot-prefixed names).
|
|
fn snapshot_path(agent_name: &str, snapshot_name: &str) -> PathBuf {
|
|
PathBuf::from(AGENT_STATE_ROOT).join(format!(".{agent_name}.snapshot.{snapshot_name}"))
|
|
}
|
|
|
|
/// `SnapshotAgentSubvolume` — create a read-only btrfs snapshot of an
|
|
/// agent's state subvolume, for `btrfs send` to stream from during
|
|
/// inter-hive migration. See the wire doc.
|
|
async fn snapshot_agent_subvolume(
|
|
agent_name: &str,
|
|
snapshot_name: &str,
|
|
) -> Result<(String, String)> {
|
|
let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name);
|
|
if !is_btrfs_subvolume(&agent_root) {
|
|
bail!(
|
|
"{} is not a btrfs subvolume — nothing to snapshot (run `hivectl agent <name> subvol upgrade` first)",
|
|
agent_root.display()
|
|
);
|
|
}
|
|
let snap = snapshot_path(agent_name, snapshot_name);
|
|
if snap.exists() {
|
|
bail!(
|
|
"snapshot {} already exists — delete it first or pick a different name",
|
|
snap.display()
|
|
);
|
|
}
|
|
let out = Command::new("btrfs")
|
|
.args(["subvolume", "snapshot", "-r"])
|
|
.arg(&agent_root)
|
|
.arg(&snap)
|
|
.output()
|
|
.await
|
|
.with_context(|| {
|
|
format!(
|
|
"spawn btrfs subvolume snapshot -r {} {}",
|
|
agent_root.display(),
|
|
snap.display()
|
|
)
|
|
})?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"btrfs subvolume snapshot -r {} {} failed: {}",
|
|
agent_root.display(),
|
|
snap.display(),
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
tracing::info!(
|
|
agent = %agent_name, snapshot = %snap.display(),
|
|
"created read-only agent state snapshot"
|
|
);
|
|
Ok((snap.display().to_string(), String::new()))
|
|
}
|
|
|
|
/// `DeleteAgentSnapshot` — delete a previously-created read-only snapshot.
|
|
/// No-op if the path doesn't exist. See the wire doc.
|
|
async fn delete_agent_snapshot(agent_name: &str, snapshot_name: &str) -> Result<(String, String)> {
|
|
let snap = snapshot_path(agent_name, snapshot_name);
|
|
if !snap.exists() {
|
|
return Ok((String::new(), String::new()));
|
|
}
|
|
let out = Command::new("btrfs")
|
|
.args(["subvolume", "delete"])
|
|
.arg(&snap)
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("spawn btrfs subvolume delete {}", snap.display()))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"btrfs subvolume delete {} failed: {}",
|
|
snap.display(),
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
tracing::info!(agent = %agent_name, snapshot = %snap.display(), "deleted agent state snapshot");
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// Both paths a `SendAgentSnapshotToFile` request reads from — the
|
|
/// snapshot itself, and its optional incremental parent — must exist
|
|
/// before we touch `dest`. Pulled out on its own so that invariant is
|
|
/// checkable without a `dest` path, a `MIGRATE_STAGING_ROOT`, or `btrfs`
|
|
/// at all: a caller cannot create the output file by accident while
|
|
/// still holding a reference to this check.
|
|
fn check_send_snapshot_preconditions(snap: &Path, parent_path: Option<&Path>) -> Result<()> {
|
|
if !snap.exists() {
|
|
bail!(
|
|
"snapshot {} does not exist — create it with `subvol snapshot create` first",
|
|
snap.display()
|
|
);
|
|
}
|
|
if let Some(parent_path) = parent_path
|
|
&& !parent_path.exists()
|
|
{
|
|
bail!(
|
|
"parent snapshot {} does not exist — pick an existing parent or omit it for a full send",
|
|
parent_path.display()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Validate preconditions, then atomically create `dest` for a snapshot
|
|
/// export. The two are combined in one function on purpose: keeping
|
|
/// `check_send_snapshot_preconditions` ahead of the `create_new` call is
|
|
/// the entire fix, and folding them together here means that ordering
|
|
/// can't drift back apart in a later edit to `send_agent_snapshot_to_file`.
|
|
/// No `btrfs` involved, so this is unit-testable against plain files.
|
|
fn open_export_dest(snap: &Path, parent_path: Option<&Path>, dest: &Path) -> Result<std::fs::File> {
|
|
check_send_snapshot_preconditions(snap, parent_path)?;
|
|
// `create_new` (O_CREAT|O_EXCL) makes the no-overwrite guarantee atomic
|
|
// instead of a check-then-create race against a concurrent request.
|
|
match std::fs::File::options()
|
|
.write(true)
|
|
.create_new(true)
|
|
.open(dest)
|
|
{
|
|
Ok(f) => Ok(f),
|
|
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => bail!(
|
|
"{} already exists — pick a different destination or remove it first \
|
|
(send never overwrites an existing export)",
|
|
dest.display()
|
|
),
|
|
Err(e) => Err(e).with_context(|| format!("create {}", dest.display())),
|
|
}
|
|
}
|
|
|
|
/// Deletes `path` when dropped, unless [`disarm`](Self::disarm)ed first.
|
|
/// Guards `dest` between the moment `send_agent_snapshot_to_file` creates
|
|
/// it and the moment the export fully succeeds, so any `?`/`bail!` on
|
|
/// that path — including ones a future edit adds — removes the partial
|
|
/// file instead of leaving a zero-byte export that then makes every
|
|
/// retry against the same `--dest` fail with "already exists".
|
|
struct PartialExportGuard<'a> {
|
|
path: &'a Path,
|
|
armed: bool,
|
|
}
|
|
|
|
impl<'a> PartialExportGuard<'a> {
|
|
fn new(path: &'a Path) -> Self {
|
|
Self { path, armed: true }
|
|
}
|
|
|
|
fn disarm(mut self) {
|
|
self.armed = false;
|
|
}
|
|
}
|
|
|
|
impl Drop for PartialExportGuard<'_> {
|
|
fn drop(&mut self) {
|
|
if !self.armed {
|
|
return;
|
|
}
|
|
// Best-effort: warn (don't fail the whole call over it) if removal
|
|
// itself fails, so a stuck partial file that later masquerades as a
|
|
// completed export is at least visible in the log.
|
|
if let Err(rm_err) = std::fs::remove_file(self.path) {
|
|
tracing::warn!(
|
|
dest = %self.path.display(), error = %rm_err,
|
|
"failed to remove partial export after failure — \
|
|
next attempt at this dest will hit the already-exists guard"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// `SendAgentSnapshotToFile` — stream a read-only snapshot (optionally
|
|
/// incremental against `parent_name`) to a file under
|
|
/// `MIGRATE_STAGING_ROOT` via `btrfs send`. Local-file half of the
|
|
/// inter-hive migration transport; see `PrivRequest::SendAgentSnapshotToFile`
|
|
/// for the cross-hive follow-up.
|
|
async fn send_agent_snapshot_to_file(
|
|
agent_name: &str,
|
|
snapshot_name: &str,
|
|
parent_name: Option<&str>,
|
|
dest_file_name: &str,
|
|
) -> Result<(String, String)> {
|
|
let snap = snapshot_path(agent_name, snapshot_name);
|
|
let parent_path = parent_name.map(|parent| snapshot_path(agent_name, parent));
|
|
|
|
std::fs::create_dir_all(MIGRATE_STAGING_ROOT)
|
|
.with_context(|| format!("create {MIGRATE_STAGING_ROOT}"))?;
|
|
let dest = Path::new(MIGRATE_STAGING_ROOT).join(dest_file_name);
|
|
// Preconditions are checked *inside* this call, before `dest` is
|
|
// created — see `open_export_dest`'s doc comment for why that ordering
|
|
// matters and is kept in one place.
|
|
let dest_file = open_export_dest(&snap, parent_path.as_deref(), &dest)?;
|
|
let cleanup = PartialExportGuard::new(&dest);
|
|
|
|
let mut cmd = Command::new("btrfs");
|
|
cmd.arg("send");
|
|
if let Some(parent_path) = &parent_path {
|
|
cmd.arg("-p").arg(parent_path);
|
|
}
|
|
cmd.arg(&snap);
|
|
cmd.stdout(std::process::Stdio::from(dest_file));
|
|
cmd.stderr(std::process::Stdio::piped());
|
|
|
|
let out = cmd
|
|
.spawn()
|
|
.with_context(|| format!("spawn btrfs send {}", snap.display()))?
|
|
.wait_with_output()
|
|
.await
|
|
.with_context(|| format!("wait on btrfs send {}", snap.display()))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"btrfs send {} failed: {}",
|
|
snap.display(),
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
cleanup.disarm();
|
|
tracing::info!(
|
|
agent = %agent_name, snapshot = %snap.display(), dest = %dest.display(),
|
|
parent = ?parent_name, "exported agent snapshot to file"
|
|
);
|
|
Ok((dest.display().to_string(), String::new()))
|
|
}
|
|
|
|
/// `SendAgentSnapshotToFd` — stream a read-only snapshot (optionally
|
|
/// incremental against `parent_name`) straight into a descriptor the
|
|
/// caller passed us.
|
|
///
|
|
/// The network half of the inter-hive migration transport, arranged so
|
|
/// this helper never learns there *is* a network: hive-c0re connects to
|
|
/// the peer's snapshot store, writes the header itself, and hands the
|
|
/// connected socket over. We only ever see "a thing to write bytes into",
|
|
/// which keeps a root process out of any address, protocol or trust
|
|
/// decision — and keeps everyone out of the data path once `btrfs send`
|
|
/// starts, which matters at multi-gigabyte sizes.
|
|
async fn send_agent_snapshot_to_fd(
|
|
agent_name: &str,
|
|
snapshot_name: &str,
|
|
parent_name: Option<&str>,
|
|
dest: OwnedFd,
|
|
) -> Result<(String, String)> {
|
|
let snap = snapshot_path(agent_name, snapshot_name);
|
|
if !snap.exists() {
|
|
bail!(
|
|
"snapshot {} does not exist — create it with `subvol snapshot create` first",
|
|
snap.display()
|
|
);
|
|
}
|
|
|
|
let mut cmd = Command::new("btrfs");
|
|
cmd.arg("send");
|
|
if let Some(parent) = parent_name {
|
|
let parent_path = snapshot_path(agent_name, parent);
|
|
if !parent_path.exists() {
|
|
bail!(
|
|
"parent snapshot {} does not exist — pick an existing parent or omit it for a full send",
|
|
parent_path.display()
|
|
);
|
|
}
|
|
cmd.arg("-p").arg(&parent_path);
|
|
}
|
|
cmd.arg(&snap);
|
|
cmd.stdout(std::process::Stdio::from(dest));
|
|
cmd.stderr(std::process::Stdio::piped());
|
|
|
|
let out = cmd
|
|
.spawn()
|
|
.with_context(|| format!("spawn btrfs send {}", snap.display()))?
|
|
.wait_with_output()
|
|
.await
|
|
.with_context(|| format!("wait on btrfs send {}", snap.display()))?;
|
|
if !out.status.success() {
|
|
// Nothing to clean up: the destination isn't ours. A partial
|
|
// stream is the receiving end's problem, and `btrfs receive`
|
|
// refuses to commit an incomplete subvolume anyway.
|
|
bail!(
|
|
"btrfs send {} failed: {}",
|
|
snap.display(),
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
tracing::info!(
|
|
agent = %agent_name, snapshot = %snap.display(), parent = ?parent_name,
|
|
"streamed agent snapshot into a passed descriptor"
|
|
);
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// `SetSubvolumeQuota` — set or clear a qgroup size limit on an agent
|
|
/// subvolume (`btrfs qgroup limit <bytes|none> <…/agent_name>`). See the
|
|
/// wire doc.
|
|
async fn set_subvolume_quota(
|
|
agent_name: &str,
|
|
limit_bytes: Option<u64>,
|
|
) -> Result<(String, String)> {
|
|
let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name);
|
|
let limit = limit_bytes.map_or_else(|| "none".to_owned(), |n| n.to_string());
|
|
let out = Command::new("btrfs")
|
|
.args(["qgroup", "limit", &limit])
|
|
.arg(&agent_root)
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("spawn btrfs qgroup limit {}", agent_root.display()))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"btrfs qgroup limit {limit} {} failed: {}",
|
|
agent_root.display(),
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
tracing::info!(agent = %agent_name, %limit, "set agent subvolume quota");
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// Validate a single argument destined for `forgejo admin`. Rejects null
|
|
/// bytes, newlines and carriage returns (which could corrupt the
|
|
/// subprocess args list or log output).
|
|
///
|
|
/// Shell metacharacters are **not** rejected, and do not need to be: the
|
|
/// command is spawned directly, with no shell to interpret them.
|
|
fn validate_forge_admin_arg(arg: &str) -> Result<()> {
|
|
if arg.bytes().any(|b| b == 0 || b == b'\n' || b == b'\r') {
|
|
bail!("forge admin arg {arg:?} contains null byte or newline");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Redact a line before it hits the (root-readable, but still
|
|
/// unnecessarily exposed) host journal.
|
|
///
|
|
/// Two independent rules, because the previous single rule failed open.
|
|
/// It matched only the substring "password", chosen to be robust against
|
|
/// forgejo *rewording* its password line — and the leak arrived from the
|
|
/// other axis entirely: a **different kind of secret** on a differently
|
|
/// worded line. `forgejo admin user generate-access-token` prints
|
|
/// `Access token was successfully created: <40 hex>`, which contains no
|
|
/// "password" and went to the journal verbatim for every agent ever
|
|
/// provisioned.
|
|
///
|
|
/// So the second rule matches on **shape, not vocabulary**: a long
|
|
/// unbroken run of secret-alphabet characters. A new secret type is then
|
|
/// caught by default rather than by someone remembering to add a keyword.
|
|
///
|
|
/// ⚠️ This deliberately over-matches. A nix store hash is also a long
|
|
/// opaque run and will redact its line. That is the correct direction to
|
|
/// be wrong in: the cost of a false positive is one less log line, and
|
|
/// the cost of a false negative is a live credential sitting in the host
|
|
/// journal, readable by anything with access to it.
|
|
fn redact_secret_line(line: &str) -> std::borrow::Cow<'_, str> {
|
|
if line.to_ascii_lowercase().contains("password") {
|
|
return std::borrow::Cow::Borrowed("[redacted: line mentions a password]");
|
|
}
|
|
if contains_secret_shaped_run(line) {
|
|
return std::borrow::Cow::Borrowed("[redacted: line contains a secret-shaped token]");
|
|
}
|
|
std::borrow::Cow::Borrowed(line)
|
|
}
|
|
|
|
/// True when the line contains an unbroken run of at least 32 characters
|
|
/// from the hex / base64url alphabet. 32 sits below forgejo's 40-hex
|
|
/// access token and above the ordinary words and path segments that
|
|
/// appear in `forgejo admin` output.
|
|
///
|
|
/// ⚠️ Scans for a RUN, not for a whitespace-delimited word. An earlier
|
|
/// version split on whitespace and required the whole word to match,
|
|
/// which a secret with punctuation glued to it defeats: `"<token>,"` and
|
|
/// `"[<token>]"` both fail an all-chars check on the word while still
|
|
/// containing the credential in full. Whitespace is not what delimits a
|
|
/// secret — the alphabet is (thanks @argus for catching it).
|
|
fn contains_secret_shaped_run(line: &str) -> bool {
|
|
const MIN: usize = 32;
|
|
let mut run = 0usize;
|
|
for b in line.bytes() {
|
|
if b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=' | b'_' | b'-') {
|
|
run += 1;
|
|
if run >= MIN {
|
|
return true;
|
|
}
|
|
} else {
|
|
run = 0;
|
|
}
|
|
}
|
|
false
|
|
}
|
|
|
|
/// Name a `forgejo admin` invocation for an error message without
|
|
/// reproducing its arguments: keep the leading verb path, stop at the
|
|
/// first flag. `["user", "change-password", "--username", "iris",
|
|
/// "--password", "…"]` becomes `forgejo admin user change-password`.
|
|
///
|
|
/// Same allowlist as hive-c0re's `describe_forge_admin`
|
|
/// (`hive-c0re/src/forge/mod.rs`), for the same reason: the verbs are a
|
|
/// closed set this crate chooses itself, but hive-c0re passes a live
|
|
/// `--password` value as an argument on some calls, and argument values
|
|
/// are never safe to assume closed. Redacting the value after
|
|
/// `--password` instead would repeat the bug `redact_secret_line`'s own
|
|
/// doc comment describes — a denylist of one flag name fails open the
|
|
/// moment a differently named secret-bearing flag is added.
|
|
fn describe_forge_admin(args: &[String]) -> String {
|
|
let verbs: Vec<&str> = args
|
|
.iter()
|
|
.take_while(|a| !a.starts_with('-'))
|
|
.map(String::as_str)
|
|
.collect();
|
|
if verbs.is_empty() {
|
|
"forgejo admin".to_owned()
|
|
} else {
|
|
format!("forgejo admin {}", verbs.join(" "))
|
|
}
|
|
}
|
|
|
|
/// Run `forgejo admin <args>` inside the `hive-forge` container as the
|
|
/// `forgejo` unix user. Requires root (for nsenter into the container's
|
|
/// namespaces). Returns `(stdout, stderr)`.
|
|
async fn run_forge_admin(args: &[String]) -> Result<(String, String)> {
|
|
let mut cmd_args: Vec<&str> = vec![
|
|
"run",
|
|
"hive-forge",
|
|
"--",
|
|
"runuser",
|
|
"-u",
|
|
"forgejo",
|
|
"--",
|
|
"forgejo",
|
|
"--work-path",
|
|
"/var/lib/forgejo",
|
|
"admin",
|
|
];
|
|
for a in args {
|
|
cmd_args.push(a.as_str());
|
|
}
|
|
let out = Command::new("nixos-container")
|
|
.args(&cmd_args)
|
|
.output()
|
|
.await
|
|
.context("invoke nixos-container run hive-forge -- forgejo admin")?;
|
|
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
|
|
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
|
// stdout at DEBUG, not INFO: on the success path this stream carries
|
|
// the *product* of the command (the freshly minted token, the created
|
|
// user's details) and nothing an operator needs at default verbosity.
|
|
// Redaction stays on as the second layer — the level decides who sees
|
|
// it, the redactor decides what it says, and neither alone is enough.
|
|
for line in stdout.lines() {
|
|
tracing::debug!(target: "forgejo-admin", "{}", redact_secret_line(line));
|
|
}
|
|
for line in stderr.lines() {
|
|
tracing::warn!(target: "forgejo-admin", "{}", redact_secret_line(line));
|
|
}
|
|
if !out.status.success() {
|
|
// Redact here too. The error string is propagated to the caller and
|
|
// ends up logged; a partial-failure stderr can carry the same
|
|
// material stdout would have. And name the invocation via
|
|
// `describe_forge_admin` rather than `args.join(" ")`: some callers
|
|
// pass a live `--password` value as an argument, so reproducing the
|
|
// raw vector here is the same "two of three sites" gap that makes
|
|
// these leaks survive a fix — the log was redacted, but the error
|
|
// was not.
|
|
let safe_stderr: String = stderr
|
|
.lines()
|
|
.map(|l| redact_secret_line(l).into_owned())
|
|
.collect::<Vec<_>>()
|
|
.join("; ");
|
|
bail!(
|
|
"{} failed ({}): {}",
|
|
describe_forge_admin(args),
|
|
out.status,
|
|
safe_stderr.trim()
|
|
);
|
|
}
|
|
Ok((stdout, stderr))
|
|
}
|
|
|
|
/// Invoke `nixos-container` with the given args, log output to journald.
|
|
async fn container_run(args: &[&str]) -> Result<(String, String)> {
|
|
let out = Command::new("nixos-container")
|
|
.args(args)
|
|
.output()
|
|
.await
|
|
.context("invoke nixos-container")?;
|
|
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
|
|
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
|
// `list` is a read-only enumeration called on the hot path (dashboard
|
|
// rescan, forge + boot sweeps) — its stdout is the return value, not
|
|
// progress, so logging every container name on every call floods the
|
|
// journal. Log stdout only for the mutating ops, where each line is
|
|
// genuine progress. stderr is always logged (errors matter regardless).
|
|
if args.first() != Some(&"list") {
|
|
for line in stdout.lines() {
|
|
tracing::info!(target: "nixos-container", "{line}");
|
|
}
|
|
}
|
|
for line in stderr.lines() {
|
|
tracing::warn!(target: "nixos-container", "{line}");
|
|
}
|
|
if !out.status.success() {
|
|
bail!(
|
|
"nixos-container {} failed ({}): {}",
|
|
args.join(" "),
|
|
out.status,
|
|
stderr.trim()
|
|
);
|
|
}
|
|
Ok((stdout, stderr))
|
|
}
|
|
|
|
/// Invoke `machinectl` with the given args, log output to journald.
|
|
/// Used for operations that nixos-container doesn't expose (e.g. sending
|
|
/// signals to running containers).
|
|
async fn machinectl_run(args: &[&str]) -> Result<(String, String)> {
|
|
let out = Command::new("machinectl")
|
|
.args(args)
|
|
.output()
|
|
.await
|
|
.context("invoke machinectl")?;
|
|
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
|
|
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
|
for line in stdout.lines() {
|
|
tracing::info!(target: "machinectl", "{line}");
|
|
}
|
|
for line in stderr.lines() {
|
|
tracing::warn!(target: "machinectl", "{line}");
|
|
}
|
|
if !out.status.success() {
|
|
bail!(
|
|
"machinectl {} failed ({}): {}",
|
|
args.join(" "),
|
|
out.status,
|
|
stderr.trim()
|
|
);
|
|
}
|
|
Ok((stdout, stderr))
|
|
}
|
|
|
|
/// How long to wait for machined to drop a machine's registration after a
|
|
/// shutdown has been asked for. Generous: a container with slow-stopping
|
|
/// units legitimately takes a while, and escalating early would SIGKILL a
|
|
/// shutdown that was going to finish on its own.
|
|
const NAME_RELEASE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
|
|
|
|
/// Poll cadence while waiting for the registration to go away.
|
|
const NAME_RELEASE_POLL: std::time::Duration = std::time::Duration::from_millis(500);
|
|
|
|
/// Cap on a single registration probe. The probe is a D-Bus round trip to
|
|
/// machined; if machined itself is wedged the call would otherwise sit on
|
|
/// the D-Bus method timeout, which is far longer than the whole stop
|
|
/// sequence should take.
|
|
const MACHINE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
|
|
|
|
/// True when machined still holds a registration for `machine`.
|
|
///
|
|
/// This asks machined the same question the registration itself answers —
|
|
/// `machinectl show` resolves the name through `GetMachine`, the very lookup
|
|
/// that makes a later boot fail with `Failed to register machine: already
|
|
/// exists`. So a positive answer here is exactly the condition that breaks
|
|
/// the next start, not a proxy for it.
|
|
///
|
|
/// Deliberately NOT the container's systemd unit state: the unit can be
|
|
/// `inactive` while the registration is still held, and that gap is the
|
|
/// whole bug this probe exists to catch.
|
|
///
|
|
/// Errors and timeouts answer "still registered". Being wrong that way costs
|
|
/// a redundant SIGKILL to something already gone; being wrong the other way
|
|
/// hands back a stop that silently leaked the name.
|
|
async fn machine_registered(machine: &str) -> bool {
|
|
let probe = Command::new("machinectl")
|
|
.args(["show", machine, "--property=Name"])
|
|
// Don't leave a probe behind when the timeout below fires.
|
|
.kill_on_drop(true)
|
|
.output();
|
|
match tokio::time::timeout(MACHINE_PROBE_TIMEOUT, probe).await {
|
|
Ok(Ok(out)) => out.status.success(),
|
|
Ok(Err(e)) => {
|
|
tracing::warn!(%machine, error = %e, "machinectl show failed to run; assuming still registered");
|
|
true
|
|
}
|
|
Err(_) => {
|
|
tracing::warn!(%machine, "machinectl show timed out; assuming still registered");
|
|
true
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Poll [`machine_registered`] until the name is free or the timeout expires.
|
|
/// Returns true once the registration is gone.
|
|
async fn wait_name_released(machine: &str) -> bool {
|
|
let deadline = tokio::time::Instant::now() + NAME_RELEASE_TIMEOUT;
|
|
loop {
|
|
if !machine_registered(machine).await {
|
|
return true;
|
|
}
|
|
if tokio::time::Instant::now() >= deadline {
|
|
return false;
|
|
}
|
|
tokio::time::sleep(NAME_RELEASE_POLL).await;
|
|
}
|
|
}
|
|
|
|
/// Stop `machine` and only report success once machined has actually released
|
|
/// the name.
|
|
///
|
|
/// `nixos-container stop` exiting 0 does not mean the machine is gone. A
|
|
/// process that sits in the machine's cgroup without being a child of the
|
|
/// container's init — a shell exec'd in from outside, say — never receives
|
|
/// the shutdown's SIGTERM if it has been stopped with SIGSTOP, so the
|
|
/// registration outlives the "successful" stop. Every later start of that
|
|
/// container then fails with `Failed to register machine: already exists`,
|
|
/// and machined re-persists the stale record across its own restart, so
|
|
/// there is no cleaning it up after the fact. The only place to catch it is
|
|
/// here, in the stop.
|
|
///
|
|
/// So: ask for the stop, wait for the name, and fail loudly if it is still
|
|
/// held — a caller that is told the stop worked will go on to start the
|
|
/// container and hit the confusing registration error instead of this one.
|
|
///
|
|
/// This used to escalate to `machinectl kill --signal=SIGKILL` here. Dropped
|
|
/// per real incident data: the case this guards is a container genuinely
|
|
/// wedged (e.g. a root-login process survived the stop), and SIGKILL
|
|
/// doesn't recover that in practice — only a host-level reboot has.
|
|
/// Pretending a kill attempt handled it hides a condition that needs a human
|
|
/// to look at the host, so this now just reports the failure instead of
|
|
/// quietly (and ineffectually) trying to force it.
|
|
///
|
|
/// The verify-then-fail lives in the helper rather than at a call site so
|
|
/// that every stop gets it: dashboard, reconcile, destroy, cold-start
|
|
/// fallback. The start path already distrusts its own exit code the same way;
|
|
/// this is the missing half of that pair.
|
|
async fn stop_and_release(machine: &str) -> Result<(String, String)> {
|
|
let stop = container_run(&["stop", machine]).await;
|
|
if wait_name_released(machine).await {
|
|
// Happy path, and also the path where a stop that reported failure
|
|
// nonetheless brought the machine down. Either way the caller gets
|
|
// the original result untouched.
|
|
return stop;
|
|
}
|
|
|
|
tracing::error!(
|
|
%machine,
|
|
"stop finished but machined still holds the registration — the \
|
|
container is likely wedged (a process outside its init tree \
|
|
survived the stop); this needs a host-level look, not another \
|
|
stop attempt"
|
|
);
|
|
bail!(
|
|
"stop {machine}: machined still holds the machine name after {}s. \
|
|
This container is likely wedged and starting it again will fail to \
|
|
register — needs host-level intervention (a reboot has been the \
|
|
only reliable fix in practice).",
|
|
NAME_RELEASE_TIMEOUT.as_secs()
|
|
)
|
|
}
|
|
|
|
/// Invoke `nixos-container` with the given args and forward output lines
|
|
/// to the caller as `PrivEvent::Line` messages in real time, logging each
|
|
/// line to journald as it arrives. Returns `(String::new(), String::new())`
|
|
/// on success (all output was streamed); the error string includes stderr
|
|
/// tail on failure.
|
|
async fn container_run_streaming(
|
|
args: &[&str],
|
|
writer: &mut OwnedWriteHalf,
|
|
) -> Result<(String, String)> {
|
|
use tokio::io::AsyncBufReadExt as _;
|
|
use tokio::process::Command;
|
|
|
|
let mut child = Command::new("nixos-container")
|
|
.args(args)
|
|
.stdout(std::process::Stdio::piped())
|
|
.stderr(std::process::Stdio::piped())
|
|
.spawn()
|
|
.context("invoke nixos-container (streaming)")?;
|
|
|
|
let stdout = child.stdout.take().expect("stdout piped");
|
|
let stderr = child.stderr.take().expect("stderr piped");
|
|
|
|
let mut stdout_lines = BufReader::new(stdout).lines();
|
|
let mut stderr_lines = BufReader::new(stderr).lines();
|
|
|
|
// Collect stderr for the error message; stream both to the client.
|
|
let mut stderr_buf = String::new();
|
|
|
|
// Drive stdout and stderr concurrently. `tokio::select!` interleaves
|
|
// them without bias — both streams drain at roughly the same rate
|
|
// as the subprocess produces output.
|
|
loop {
|
|
tokio::select! {
|
|
line = stdout_lines.next_line() => {
|
|
match line {
|
|
Ok(Some(l)) => {
|
|
tracing::info!(target: "nixos-container", "{l}");
|
|
write_line_event(writer, PrivStream::Stdout, &l).await;
|
|
}
|
|
Ok(None) => break,
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "nixos-container stdout read error");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
line = stderr_lines.next_line() => {
|
|
match line {
|
|
Ok(Some(l)) => {
|
|
tracing::warn!(target: "nixos-container", "{l}");
|
|
write_line_event(writer, PrivStream::Stderr, &l).await;
|
|
if !stderr_buf.is_empty() {
|
|
stderr_buf.push('\n');
|
|
}
|
|
stderr_buf.push_str(&l);
|
|
}
|
|
Ok(None) => {}
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "nixos-container stderr read error");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Drain any remaining stderr after stdout closed.
|
|
while let Ok(Some(l)) = stderr_lines.next_line().await {
|
|
tracing::warn!(target: "nixos-container", "{l}");
|
|
write_line_event(writer, PrivStream::Stderr, &l).await;
|
|
if !stderr_buf.is_empty() {
|
|
stderr_buf.push('\n');
|
|
}
|
|
stderr_buf.push_str(&l);
|
|
}
|
|
|
|
let status = child.wait().await.context("wait nixos-container")?;
|
|
if !status.success() {
|
|
// Only the last stderr line is embedded — the full stderr was
|
|
// already forwarded line-by-line as PrivEvent::Line messages and
|
|
// is captured in build_logs.sqlite by the caller. Keeping the
|
|
// error message short avoids bloating the anyhow chain.
|
|
bail!(
|
|
"nixos-container {} failed ({}): {}",
|
|
args.join(" "),
|
|
status,
|
|
stderr_buf.lines().last().unwrap_or("").trim()
|
|
);
|
|
}
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// Read a container's journal as root via `journalctl -M`. Returns
|
|
/// `(stdout, stderr)`. Unlike `container_run` a non-zero exit is *not* a
|
|
/// hard error — journalctl's own diagnostic (folded into `stderr` with
|
|
/// the exit status) is what the caller surfaces to the operator, so the
|
|
/// helper never bails.
|
|
async fn read_container_journal(container: &str, query: &JournalQuery) -> Result<(String, String)> {
|
|
let mut args: Vec<String> = vec![
|
|
"-M".to_owned(),
|
|
container.to_owned(),
|
|
"--no-pager".to_owned(),
|
|
format!("--output={}", query.output.as_journalctl()),
|
|
"-n".to_owned(),
|
|
query.lines.to_string(),
|
|
];
|
|
if query.boot {
|
|
args.push("-b".to_owned());
|
|
}
|
|
if let Some(u) = &query.unit {
|
|
args.push("-u".to_owned());
|
|
args.push(u.clone());
|
|
}
|
|
if let Some(p) = &query.priority {
|
|
args.push("-p".to_owned());
|
|
args.push(p.clone());
|
|
}
|
|
// `--grep=`/`--since=`/`--until=` use the `=`-joined form so a value
|
|
// can never be parsed as a separate journalctl flag.
|
|
if let Some(g) = &query.grep {
|
|
args.push(format!("--grep={g}"));
|
|
}
|
|
if let Some(s) = &query.since {
|
|
args.push(format!("--since={s}"));
|
|
}
|
|
if let Some(u) = &query.until {
|
|
args.push(format!("--until={u}"));
|
|
}
|
|
let out = Command::new("journalctl")
|
|
.args(&args)
|
|
.output()
|
|
.await
|
|
.context("invoke journalctl -M")?;
|
|
let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
|
|
let stderr = if out.status.success() {
|
|
String::from_utf8_lossy(&out.stderr).into_owned()
|
|
} else {
|
|
format!(
|
|
"journalctl -M {container} exited {}: {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
)
|
|
};
|
|
Ok((stdout, stderr))
|
|
}
|
|
|
|
/// Synchronise the host's nginx unit after an `agents.conf` write.
|
|
///
|
|
/// Queries `ActiveState` and dispatches:
|
|
/// - `active` → `systemctl reload nginx` (SIGHUP, zero-downtime)
|
|
/// - `failed` → `systemctl reset-failed nginx` + `systemctl start nginx`
|
|
/// - otherwise → `systemctl start nginx`
|
|
///
|
|
/// ⚠️ `nginx` is hard-coded on purpose — see `PrivRequest::ReloadGatewayNginx`.
|
|
/// The unit name is the scope of this verb: nginx is a host unit, so no
|
|
/// namespace bounds it and the literal is the only thing standing between
|
|
/// "reload the gateway" and "reload anything".
|
|
///
|
|
/// Returns `(String::new(), String::new())` on success so it fits the
|
|
/// `exec` return type directly.
|
|
async fn sync_gateway_nginx() -> Result<(String, String)> {
|
|
let state_out = Command::new("systemctl")
|
|
.args(["show", "--property=ActiveState", "--value", "nginx"])
|
|
.output()
|
|
.await
|
|
.context("query gateway nginx ActiveState")?;
|
|
if !state_out.status.success() {
|
|
tracing::warn!(
|
|
exit_code = ?state_out.status.code(),
|
|
stderr = %String::from_utf8_lossy(&state_out.stderr).trim(),
|
|
"systemctl show ActiveState exited non-zero — gateway nginx may be down"
|
|
);
|
|
}
|
|
let state = String::from_utf8_lossy(&state_out.stdout).trim().to_owned();
|
|
// State-aware dispatch: reload when running; reset+start after
|
|
// start-limit failure; plain start when inactive or unknown.
|
|
match state.as_str() {
|
|
"active" => {
|
|
let out = Command::new("systemctl")
|
|
.args(["reload", "nginx"])
|
|
.output()
|
|
.await
|
|
.context("reload gateway nginx")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"gateway nginx reload failed ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
}
|
|
"failed" => {
|
|
// Clear start-limit so the next start can proceed.
|
|
let _ = Command::new("systemctl")
|
|
.args(["reset-failed", "nginx"])
|
|
.status()
|
|
.await;
|
|
let out = Command::new("systemctl")
|
|
.args(["start", "nginx"])
|
|
.output()
|
|
.await
|
|
.context("start gateway nginx after reset-failed")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"gateway nginx start (after reset-failed) failed ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
}
|
|
_ => {
|
|
// inactive, activating, deactivating, unknown — just start.
|
|
let out = Command::new("systemctl")
|
|
.args(["start", "nginx"])
|
|
.output()
|
|
.await
|
|
.context("start gateway nginx")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"gateway nginx start failed (state={state:?}) ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
}
|
|
}
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// Return the system container name for a logical agent name.
|
|
/// All agents (including the manager) use the `h-` prefix.
|
|
///
|
|
/// **This prefix is what confines the lifecycle verbs to agent containers** —
|
|
/// not their name validation, which only checks characters. A caller cannot
|
|
/// name `hive-forge` and reach it: it becomes `h-hive-forge`. Infra containers
|
|
/// are reached through [`PrivRequest::ControlInfraContainer`] and the
|
|
/// [`InfraContainer`] enum instead, which is why no lifecycle verb here takes
|
|
/// a sibling name.
|
|
fn container_system_name(name: &str) -> String {
|
|
format!("{AGENT_PREFIX}{name}")
|
|
}
|
|
|
|
/// Validate a logical agent name (the name hive-c0re uses internally,
|
|
/// before the `h-` container prefix is applied).
|
|
fn validate_agent_name(name: &str) -> Result<()> {
|
|
validate_name_chars(name)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Validate a system-level container name (already has `h-` prefix for
|
|
/// all agents including the manager, or is a sibling service name).
|
|
fn validate_container_system_name(name: &str) -> Result<()> {
|
|
if SIBLING_CONTAINERS.contains(&name) {
|
|
return Ok(());
|
|
}
|
|
if let Some(suffix) = name.strip_prefix(AGENT_PREFIX) {
|
|
validate_name_chars(suffix)?;
|
|
return Ok(());
|
|
}
|
|
bail!("container name {name:?} is not managed by hive");
|
|
}
|
|
|
|
fn validate_name_chars(name: &str) -> Result<()> {
|
|
if name.is_empty()
|
|
|| !name
|
|
.chars()
|
|
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
|
|
{
|
|
bail!("invalid name {name:?}: must be non-empty lowercase ascii + digits + hyphens");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Validate a bind-mount path: must be absolute, non-empty, and contain
|
|
/// no newlines, null bytes, double-quotes, or colons.
|
|
///
|
|
/// The first three would break the `EXTRA_NSPAWN_FLAGS="..."` conf line
|
|
/// format. The colon is the one that matters most and had been left out
|
|
/// of this list: `--bind=SRC:DST` is colon-separated, so a path carrying
|
|
/// one does not corrupt the line — it silently becomes a *different
|
|
/// mount* than the caller asked for.
|
|
fn validate_bind_path(path: &str) -> Result<()> {
|
|
if path.is_empty()
|
|
|| !path.starts_with('/')
|
|
|| path
|
|
.bytes()
|
|
.any(|b| b == 0 || b == b'\n' || b == b'"' || b == b':')
|
|
{
|
|
bail!(
|
|
"invalid bind path {path:?}: must be an absolute path with no colons, newlines, null bytes, or double-quotes"
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// `--tmpfs=<mount>/.git` for every bound git repo, hiding its metadata
|
|
/// from inside the container.
|
|
///
|
|
/// Two kinds of mount qualify, for one reason: **the agent is given a
|
|
/// working tree, never a repository.** `/knowledge` is the hive's shared
|
|
/// docs — whose `.git/config` has held a credential the host-side worker
|
|
/// embedded — and `/agents/<name>/config` is a config repo, an agent's own
|
|
/// or a parent's read-only view of a child's. In both cases `.git` carries
|
|
/// every branch and the full history of a document whose *currently
|
|
/// deployed* value is the only thing a reader may act on, and an abandoned
|
|
/// branch is indistinguishable from a live one.
|
|
///
|
|
/// An overlay rather than an exported copy: there is no second tree to
|
|
/// keep in sync, so nothing can go stale, and no code path has to remember
|
|
/// to refresh it.
|
|
///
|
|
/// ⚠️ Ordering matters — these must be appended **after** the `--bind`
|
|
/// flags so nspawn mounts them on top of the already-mounted trees.
|
|
/// Config mounts are matched by shape, not by a name list: the set is
|
|
/// dynamic, growing with each child bound into a parent.
|
|
fn git_overlay_flags(binds: &[BindMount]) -> Vec<String> {
|
|
binds
|
|
.iter()
|
|
.map(|b| b.container_path.as_str())
|
|
.filter(|p| *p == "/knowledge" || (p.starts_with("/agents/") && p.ends_with("/config")))
|
|
.map(|p| format!("--tmpfs={p}/.git"))
|
|
.collect()
|
|
}
|
|
|
|
/// Update `/etc/nixos-containers/<container>.conf`: strip old network vars
|
|
/// (`PRIVATE_NETWORK`, `HOST_ADDRESS*`, `LOCAL_ADDRESS*`, `HOST_BRIDGE`),
|
|
/// write the current network-isolation settings, then append
|
|
/// `EXTRA_NSPAWN_FLAGS`. Always writes `PRIVATE_NETWORK=1` + veth
|
|
/// wiring — isolation is the only mode, so there is no branch that
|
|
/// leaves a container on the host's network namespace.
|
|
fn write_nspawn_flags(
|
|
container: &str,
|
|
binds: &[BindMount],
|
|
isolation: &NetworkIsolation,
|
|
load_credentials: &[CredentialMount],
|
|
) -> Result<()> {
|
|
use std::fmt::Write as _;
|
|
use std::os::unix::fs::MetadataExt as _;
|
|
let path = format!("/etc/nixos-containers/{container}.conf");
|
|
let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?;
|
|
let lines: Vec<&str> = original
|
|
.lines()
|
|
.filter(|line| {
|
|
let t = line.trim_start();
|
|
!t.starts_with("EXTRA_NSPAWN_FLAGS=")
|
|
&& !t.starts_with("PRIVATE_NETWORK=")
|
|
&& !t.starts_with("HOST_ADDRESS=")
|
|
&& !t.starts_with("LOCAL_ADDRESS=")
|
|
&& !t.starts_with("HOST_ADDRESS6=")
|
|
&& !t.starts_with("LOCAL_ADDRESS6=")
|
|
&& !t.starts_with("HOST_BRIDGE=")
|
|
})
|
|
.collect();
|
|
let mut out = lines.join("\n");
|
|
if !out.is_empty() {
|
|
out.push('\n');
|
|
}
|
|
{
|
|
let iso = isolation;
|
|
out.push_str("PRIVATE_NETWORK=1\n");
|
|
// HOST_ADDRESS = the bridge gateway IP. nixos-container's
|
|
// container-side setup only installs a default route
|
|
// (`ip route add default via $HOST_ADDRESS`) when HOST_ADDRESS is
|
|
// non-empty; leaving it blank gave the container an address but no
|
|
// route off the bridge subnet (no internet, no api.anthropic.com).
|
|
// In bridge mode (HOST_BRIDGE set) the host-side address/route
|
|
// setup is skipped, so this only affects the container's route —
|
|
// exactly what we want.
|
|
let _ = writeln!(out, "HOST_ADDRESS={}", iso.gateway_ip);
|
|
// LOCAL_ADDRESS is intentionally empty: containers take their IP by
|
|
// DHCP from the bridge dnsmasq pool. (Why HOST_ADDRESS is still
|
|
// written: the comment directly above.)
|
|
out.push_str("LOCAL_ADDRESS=\n");
|
|
out.push_str("HOST_ADDRESS6=\n");
|
|
out.push_str("LOCAL_ADDRESS6=\n");
|
|
let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge);
|
|
}
|
|
let mut flags: Vec<String> = binds
|
|
.iter()
|
|
.map(|b| {
|
|
let flag = if b.read_only { "--bind-ro" } else { "--bind" };
|
|
format!("{flag}={}:{}", b.host_path, b.container_path)
|
|
})
|
|
.collect();
|
|
flags.extend(git_overlay_flags(binds));
|
|
// Credential forwarding: nspawn loads each host secret into the
|
|
// container's credential store under `<name>`; inner units inherit it
|
|
// via `LoadCredential=<name>`. Validated (name charset + bind-path
|
|
// rules) in handle_write_nspawn_flags above.
|
|
for cred in load_credentials {
|
|
flags.push(format!(
|
|
"--load-credential={}:{}",
|
|
cred.name, cred.host_path
|
|
));
|
|
}
|
|
let flags_joined = flags.join(" ");
|
|
let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\"");
|
|
// Keeps the owner and mode `nixos-container` created the file with; this
|
|
// helper only rewrites its lines.
|
|
let meta = std::fs::metadata(&path).with_context(|| format!("stat {path}"))?;
|
|
publish_file(
|
|
Path::new(&path),
|
|
out.as_bytes(),
|
|
meta.mode() & 0o7777,
|
|
Some((meta.uid(), meta.gid())),
|
|
)?;
|
|
|
|
// DNS marker for the in-container resolver oneshot: the oneshot only
|
|
// rewrites the container's resolv.conf when this marker exists, and the
|
|
// marker carries the gateway IP so the container need not re-derive it.
|
|
// Always written — every container is isolated, so there is no mode in
|
|
// which the marker should be absent. Why the rewrite is needed at all,
|
|
// and which unit does it: `docs/networking/network.md` § *How the isolated
|
|
// container gets its resolver*.
|
|
write_bridge_dns_marker(container, isolation)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Filename of the in-container DNS marker, in the container's own `/etc`.
|
|
const BRIDGE_DNS_MARKER: &str = "hyperhive-bridge-dns";
|
|
|
|
/// Write the bridge-DNS marker the `hyperhive-isolated-dns` oneshot keys
|
|
/// off. The marker file contains just the gateway IP. Always written:
|
|
/// every container is isolated, so there is no host-netns case that
|
|
/// wants the marker absent.
|
|
fn write_bridge_dns_marker(container: &str, isolation: &NetworkIsolation) -> Result<()> {
|
|
let rootfs = PathBuf::from(format!("/var/lib/nixos-containers/{container}"));
|
|
write_bridge_dns_marker_in(&rootfs, &isolation.gateway_ip)
|
|
}
|
|
|
|
/// Write `<rootfs>/etc/hyperhive-bridge-dns` without following a symlink.
|
|
///
|
|
/// Root inside the container owns its `/etc`, and this write runs as host
|
|
/// root, where an absolute symlink planted in the container resolves against
|
|
/// the host's `/`. So this opens `etc` with `O_DIRECTORY|O_NOFOLLOW` (a
|
|
/// symlinked `etc` fails with `ELOOP`) and publishes the marker through a
|
|
/// [`StagedFile`] relative to that fd: whatever the container planted at the
|
|
/// leaf, symlink or FIFO or device node, is replaced by the rename, never
|
|
/// written through or opened.
|
|
///
|
|
/// On a fresh install the container's `/etc` may not exist yet (nixos-container
|
|
/// materialises the rootfs on the first start), so this creates it first; the
|
|
/// dir and the marker persist through that start. Mode 0644: the gateway IP
|
|
/// is not a secret.
|
|
fn write_bridge_dns_marker_in(rootfs: &Path, gateway_ip: &str) -> Result<()> {
|
|
use std::os::unix::fs::OpenOptionsExt as _;
|
|
|
|
let etc_path = rootfs.join("etc");
|
|
// The rootfs dir is the container's `/`, which the container cannot
|
|
// replace, so creating it by path is safe.
|
|
std::fs::create_dir_all(rootfs)
|
|
.with_context(|| format!("create container rootfs {}", rootfs.display()))?;
|
|
let root = std::fs::OpenOptions::new()
|
|
.read(true)
|
|
.custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW)
|
|
.open(rootfs)
|
|
.with_context(|| format!("open container rootfs {}", rootfs.display()))?;
|
|
// SAFETY: `root` is an open directory fd and the name a NUL-terminated
|
|
// literal.
|
|
if unsafe { libc::mkdirat(root.as_raw_fd(), c"etc".as_ptr(), 0o755) } != 0 {
|
|
let e = std::io::Error::last_os_error();
|
|
if e.kind() != std::io::ErrorKind::AlreadyExists {
|
|
return Err(e).with_context(|| format!("create {}", etc_path.display()));
|
|
}
|
|
}
|
|
// SAFETY: as above; this checks the result before wrapping it.
|
|
let etc_fd = unsafe {
|
|
libc::openat(
|
|
root.as_raw_fd(),
|
|
c"etc".as_ptr(),
|
|
libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC,
|
|
)
|
|
};
|
|
if etc_fd < 0 {
|
|
return Err(std::io::Error::last_os_error())
|
|
.with_context(|| format!("open (no-follow) {}", etc_path.display()));
|
|
}
|
|
// SAFETY: `etc_fd` is a fresh, valid fd that nothing else owns.
|
|
let etc = std::fs::File::from(unsafe { OwnedFd::from_raw_fd(etc_fd) });
|
|
StagedFile::write(
|
|
&etc,
|
|
&etc_path,
|
|
BRIDGE_DNS_MARKER,
|
|
format!("{gateway_ip}\n").as_bytes(),
|
|
0o644,
|
|
)?
|
|
.publish()
|
|
}
|
|
|
|
/// Written by older hive-priv builds. Nothing writes it now, but left on disk
|
|
/// systemd-tmpfiles would still apply it at every boot.
|
|
const LEGACY_TMPFILES_PATH: &str = "/etc/tmpfiles.d/hyperhive-agents.conf";
|
|
|
|
/// Unlink [`LEGACY_TMPFILES_PATH`]; already absent is success. Run at every
|
|
/// hive-priv start and by the legacy `SyncAgentTmpfiles` request.
|
|
fn remove_legacy_tmpfiles() -> Result<()> {
|
|
match std::fs::remove_file(LEGACY_TMPFILES_PATH) {
|
|
Ok(()) => {
|
|
tracing::info!("removed legacy {LEGACY_TMPFILES_PATH}");
|
|
Ok(())
|
|
}
|
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
|
Err(e) => Err(e).with_context(|| format!("remove {LEGACY_TMPFILES_PATH}")),
|
|
}
|
|
}
|
|
|
|
/// `EnsureAgentSocketDir` — create `/run/hive-agent/<name>` if missing.
|
|
fn ensure_agent_socket_dir(name: &str) -> Result<(String, String)> {
|
|
ensure_socket_dir_in(Path::new(SOCKET_DIR_ROOT), name)?;
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
/// Create `<root>/<name>` as a `0751` directory owned by the caller (root in
|
|
/// production), or accept an existing directory untouched.
|
|
/// `name` must be an agent ident; it is validated here, before any path is
|
|
/// built from it.
|
|
///
|
|
/// The result is an nspawn bind source, so a symlink here would bind a host
|
|
/// path of the planter's choosing into the container. Every step is relative
|
|
/// to an `O_DIRECTORY|O_NOFOLLOW` fd for `root`, and an existing entry is
|
|
/// checked with `AT_SYMLINK_NOFOLLOW` and refused unless it is a directory.
|
|
///
|
|
/// An existing directory keeps its owner and mode: the container's activation
|
|
/// hands it to the agent user, and re-asserting root here would undo that on
|
|
/// every start.
|
|
fn ensure_socket_dir_in(root: &Path, name: &str) -> Result<()> {
|
|
use std::os::unix::fs::OpenOptionsExt as _;
|
|
|
|
validate_agent_name(name)?;
|
|
let path = root.join(name);
|
|
let c_name = std::ffi::CString::new(name)
|
|
.with_context(|| format!("agent name {name:?} contains a NUL byte"))?;
|
|
let root_dir = std::fs::OpenOptions::new()
|
|
.read(true)
|
|
.custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC)
|
|
.open(root)
|
|
.with_context(|| format!("open (no-follow) {}", root.display()))?;
|
|
// SAFETY: `root_dir` is an open directory fd and `c_name` NUL-terminated.
|
|
if unsafe { libc::mkdirat(root_dir.as_raw_fd(), c_name.as_ptr(), 0o751) } == 0 {
|
|
// mkdirat's mode is masked by the umask; the mode is part of the
|
|
// contract, so it is set explicitly.
|
|
// SAFETY: as above.
|
|
let rc = unsafe {
|
|
libc::fchmodat(
|
|
root_dir.as_raw_fd(),
|
|
c_name.as_ptr(),
|
|
0o751,
|
|
libc::AT_SYMLINK_NOFOLLOW,
|
|
)
|
|
};
|
|
if rc != 0 {
|
|
return Err(std::io::Error::last_os_error())
|
|
.with_context(|| format!("chmod 0751 {}", path.display()));
|
|
}
|
|
tracing::info!(path = %path.display(), "created agent socket dir");
|
|
return Ok(());
|
|
}
|
|
let err = std::io::Error::last_os_error();
|
|
if err.kind() != std::io::ErrorKind::AlreadyExists {
|
|
return Err(err).with_context(|| format!("create {}", path.display()));
|
|
}
|
|
// SAFETY: `stat` is plain old data; the zeroed value is only read after
|
|
// `fstatat` has filled it in.
|
|
let mut st: libc::stat = unsafe { std::mem::zeroed() };
|
|
// SAFETY: as for `mkdirat`; `st` is a valid, writable `stat`.
|
|
let rc = unsafe {
|
|
libc::fstatat(
|
|
root_dir.as_raw_fd(),
|
|
c_name.as_ptr(),
|
|
&raw mut st,
|
|
libc::AT_SYMLINK_NOFOLLOW,
|
|
)
|
|
};
|
|
if rc != 0 {
|
|
return Err(std::io::Error::last_os_error())
|
|
.with_context(|| format!("stat (no-follow) {}", path.display()));
|
|
}
|
|
if st.st_mode & libc::S_IFMT != libc::S_IFDIR {
|
|
bail!(
|
|
"{} exists but is not a directory; refusing it as a bind source",
|
|
path.display()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::{
|
|
BindMount, BoundedRun, OwnedFd, PAUSED_MARKER_FILE, PrivRequest, StagedFile,
|
|
check_fd_agreement, clear_runner_credentials, contains_secret_shaped_run,
|
|
describe_forge_admin, ensure_plain_filename, ensure_socket_dir_in, git_overlay_flags,
|
|
limits_dropin_body, open_dir, open_export_dest, partial_name, publish_file,
|
|
redact_secret_line, remove_marker_in, run_bounded, single_output_path, toplevel_attr,
|
|
validate_credential_name, validate_snapshot_name, write_agent_dir_file,
|
|
write_bridge_dns_marker_in,
|
|
};
|
|
use std::path::PathBuf;
|
|
use std::sync::atomic::{AtomicU32, Ordering};
|
|
|
|
fn bind(container_path: &str) -> BindMount {
|
|
BindMount {
|
|
host_path: "/var/lib/hyperhive/whatever".to_owned(),
|
|
container_path: container_path.to_owned(),
|
|
read_only: true,
|
|
}
|
|
}
|
|
|
|
/// Pins the exact attr path we hand to `nix build` — it has to match
|
|
/// `hive-c0re`'s own `lifecycle::prebuild_toplevel` construction, since
|
|
/// that step's whole point is warming the store for this later build.
|
|
/// A drifted attr path defeats the cache-warming silently — no error,
|
|
/// just a slower `update`.
|
|
#[test]
|
|
fn toplevel_attr_matches_prebuild_toplevels_construction() {
|
|
assert_eq!(
|
|
toplevel_attr("atlas"),
|
|
"/var/lib/hyperhive/meta#nixosConfigurations.atlas.config.system.build.toplevel"
|
|
);
|
|
}
|
|
|
|
/// `--print-out-paths`' exact per-shape table (measured against real
|
|
/// `rustc` semantics, not just reasoned about) — a lone `"\n"` and a
|
|
/// trailing-space path are the two shapes a bare `.lines().collect()`
|
|
/// gets wrong, both catchable only by trimming + dropping empties
|
|
/// before counting rather than after.
|
|
#[test]
|
|
fn single_output_path_rejects_blank_and_trims_whitespace() {
|
|
assert_eq!(single_output_path(""), Err(0));
|
|
assert_eq!(single_output_path("\n"), Err(0));
|
|
assert_eq!(single_output_path("\n\n"), Err(0));
|
|
assert_eq!(single_output_path("/nix/store/abc\n"), Ok("/nix/store/abc"));
|
|
assert_eq!(
|
|
single_output_path("/nix/store/abc \n"),
|
|
Ok("/nix/store/abc")
|
|
);
|
|
assert_eq!(
|
|
single_output_path("/nix/store/abc\r\n"),
|
|
Ok("/nix/store/abc")
|
|
);
|
|
assert_eq!(
|
|
single_output_path("/nix/store/abc\n/nix/store/def\n"),
|
|
Err(2)
|
|
);
|
|
}
|
|
|
|
/// Every bound git repo gets its `.git` overlaid — the knowledge tree
|
|
/// and *each* config mount, an agent's own plus every child's.
|
|
///
|
|
/// The child case is the one worth pinning: that set grows at runtime
|
|
/// as agents gain children, so a rule written as a list of names would
|
|
/// silently stop covering new ones.
|
|
#[test]
|
|
fn every_bound_git_repo_gets_its_dot_git_hidden() {
|
|
let flags = git_overlay_flags(&[
|
|
bind("/knowledge"),
|
|
bind("/agents/atlas/config"),
|
|
bind("/agents/kiddo/config"),
|
|
]);
|
|
assert_eq!(
|
|
flags,
|
|
[
|
|
"--tmpfs=/knowledge/.git",
|
|
"--tmpfs=/agents/atlas/config/.git",
|
|
"--tmpfs=/agents/kiddo/config/.git",
|
|
]
|
|
);
|
|
}
|
|
|
|
/// ...and nothing else does. A blanket "overlay .git on every bind"
|
|
/// would mask a real `.git` under `state/`, where an agent legitimately
|
|
/// keeps working clones of its own.
|
|
#[test]
|
|
fn non_repo_mounts_are_left_alone() {
|
|
let flags = git_overlay_flags(&[
|
|
bind("/agents/atlas/state"),
|
|
bind("/shared"),
|
|
bind("/applied"),
|
|
bind("/agents/atlas/config-notes"),
|
|
]);
|
|
assert!(flags.is_empty(), "overlaid a non-repo mount: {flags:?}");
|
|
}
|
|
|
|
/// A request that streams into a caller-supplied descriptor.
|
|
fn fd_taking_request() -> PrivRequest {
|
|
PrivRequest::SendAgentSnapshotToFd {
|
|
agent_name: "atlas".to_owned(),
|
|
snapshot_name: "hive-migrate".to_owned(),
|
|
parent_snapshot_name: None,
|
|
}
|
|
}
|
|
|
|
/// A real descriptor — `/dev/null` rather than a fake, so the drop
|
|
/// that closes it on a rejection path is genuinely exercised.
|
|
fn some_fd() -> OwnedFd {
|
|
std::fs::File::open("/dev/null")
|
|
.expect("open /dev/null")
|
|
.into()
|
|
}
|
|
|
|
/// An op that streams into a passed descriptor cannot invent one:
|
|
/// falling back to anything (a temp file, the response socket) would
|
|
/// send an agent's state somewhere the caller never asked for.
|
|
#[test]
|
|
fn an_fd_taking_op_without_a_descriptor_is_rejected() {
|
|
let err = check_fd_agreement(&fd_taking_request(), None)
|
|
.expect_err("no descriptor arrived, so this must not proceed");
|
|
let msg = format!("{err:#}");
|
|
assert!(msg.contains("requires a passed file descriptor"), "{msg}");
|
|
}
|
|
|
|
/// The mirror case: a descriptor sent alongside an op that takes
|
|
/// none is a protocol error, not something to ignore. Returning the
|
|
/// error drops the `OwnedFd`, which closes it — the alternative
|
|
/// leaks one descriptor per stray request in a long-lived root
|
|
/// process.
|
|
#[test]
|
|
fn a_descriptor_sent_to_an_op_that_takes_none_is_rejected() {
|
|
let fd = some_fd();
|
|
let err = check_fd_agreement(&PrivRequest::DaemonReload, Some(&fd))
|
|
.expect_err("an unexpected descriptor must not be silently ignored");
|
|
let msg = format!("{err:#}");
|
|
assert!(
|
|
msg.contains("does not take a passed file descriptor"),
|
|
"{msg}"
|
|
);
|
|
}
|
|
|
|
/// Both agreeing combinations pass, so the check rejects mismatches
|
|
/// rather than descriptors in general.
|
|
#[test]
|
|
fn agreeing_combinations_are_accepted() {
|
|
let fd = some_fd();
|
|
check_fd_agreement(&fd_taking_request(), Some(&fd))
|
|
.expect("an fd-taking op with its descriptor is the normal case");
|
|
check_fd_agreement(&PrivRequest::DaemonReload, None)
|
|
.expect("every ordinary request arrives without a descriptor");
|
|
}
|
|
|
|
/// The whole rendered body, pinned literally — including the values,
|
|
/// so changing the restart policy is a visible test edit rather than a
|
|
/// silent one.
|
|
///
|
|
/// Placement is the part most worth pinning: `StartLimit*` are `[Unit]`
|
|
/// settings and systemd **silently ignores** them under `[Service]`, so
|
|
/// a bound that moved sections would look configured and do nothing.
|
|
/// An unset weight still emits no line at all, which is what keeps a
|
|
/// hive-c0re older than that field from changing what lands on disk.
|
|
#[test]
|
|
fn dropin_body_is_pinned_exactly() {
|
|
assert_eq!(
|
|
limits_dropin_body("/run/hyperhive/agents/iris", "4G", "200%", None, None),
|
|
"[Unit]\n\
|
|
ConditionPathIsDirectory=/run/hyperhive/agents/iris\n\
|
|
StartLimitIntervalSec=600\n\
|
|
StartLimitBurst=3\n\
|
|
\n\
|
|
[Service]\n\
|
|
RestartSec=5\n\
|
|
MemoryMax=4G\n\
|
|
CPUQuota=200%\n"
|
|
);
|
|
}
|
|
|
|
/// The bound is hive-wide **policy**, not a per-agent parameter: it is
|
|
/// rendered from constants and no caller-supplied value can omit or
|
|
/// alter it. This is the property that justifies keeping it out of the
|
|
/// wire protocol — if it ever varies by request, that argument is gone.
|
|
#[test]
|
|
fn the_start_limit_is_present_whatever_the_caller_passes() {
|
|
for (mem, cpu, cw, iw) in [
|
|
("4G", "200%", None, None),
|
|
("512M", "50%", Some(10), Some(10)),
|
|
("infinity", "infinity", Some(10_000), None),
|
|
] {
|
|
let body = limits_dropin_body("/rt/x", mem, cpu, cw, iw);
|
|
let unit = body
|
|
.split("[Service]")
|
|
.next()
|
|
.expect("the body always has a [Unit] section before [Service]");
|
|
assert!(
|
|
unit.contains("StartLimitIntervalSec=600") && unit.contains("StartLimitBurst=3"),
|
|
"start limit missing from [Unit] for ({mem}, {cpu}): {body}"
|
|
);
|
|
assert!(
|
|
body.contains("\nRestartSec=5\n"),
|
|
"restart backoff missing for ({mem}, {cpu}): {body}"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Weights are appended to the `[Service]` section, each omitted
|
|
/// independently when `None`.
|
|
#[test]
|
|
fn weights_are_emitted_only_when_set() {
|
|
let both = limits_dropin_body("/rt/x", "4G", "200%", Some(80), Some(80));
|
|
assert!(
|
|
both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"),
|
|
"{both}"
|
|
);
|
|
|
|
let cpu_only = limits_dropin_body("/rt/x", "4G", "200%", Some(80), None);
|
|
assert!(
|
|
cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"),
|
|
"{cpu_only}"
|
|
);
|
|
assert!(!cpu_only.contains("IOWeight"), "{cpu_only}");
|
|
|
|
let io_only = limits_dropin_body("/rt/x", "4G", "200%", None, Some(80));
|
|
assert!(
|
|
io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"),
|
|
"{io_only}"
|
|
);
|
|
assert!(!io_only.contains("CPUWeight"), "{io_only}");
|
|
}
|
|
|
|
#[test]
|
|
fn redacts_lines_mentioning_password_case_insensitively() {
|
|
assert_eq!(
|
|
redact_secret_line("New password: hunter2"),
|
|
"[redacted: line mentions a password]"
|
|
);
|
|
assert_eq!(
|
|
redact_secret_line("PASSWORD=hunter2"),
|
|
"[redacted: line mentions a password]"
|
|
);
|
|
assert_eq!(
|
|
redact_secret_line("User \"foo\" was successfully created."),
|
|
"User \"foo\" was successfully created."
|
|
);
|
|
}
|
|
|
|
/// `run_forge_admin`'s `bail!` names the invocation through this
|
|
/// function instead of `args.join(" ")` — the regression this test
|
|
/// guards. A `--password` value never survives to the error string.
|
|
#[test]
|
|
fn describe_forge_admin_does_not_reproduce_a_password_argument() {
|
|
let described = describe_forge_admin(&[
|
|
"user".to_owned(),
|
|
"create".to_owned(),
|
|
"--username".to_owned(),
|
|
"x".to_owned(),
|
|
"--password".to_owned(),
|
|
"S3cret".to_owned(),
|
|
]);
|
|
assert_eq!(described, "forgejo admin user create");
|
|
assert!(!described.contains("S3cret"), "{described}");
|
|
}
|
|
|
|
/// The `--password=value` single-argument form is also excluded: it
|
|
/// starts with `-`, so it never enters the kept verb path.
|
|
#[test]
|
|
fn describe_forge_admin_redacts_the_equals_sign_form_too() {
|
|
let described = describe_forge_admin(&[
|
|
"user".to_owned(),
|
|
"create".to_owned(),
|
|
"--password=S3cret".to_owned(),
|
|
]);
|
|
assert!(!described.contains("S3cret"), "{described}");
|
|
}
|
|
|
|
/// Control: a non-secret argument (the verb path) still appears, so
|
|
/// the message stays useful for diagnosing which call failed.
|
|
#[test]
|
|
fn describe_forge_admin_keeps_the_verb_path() {
|
|
assert_eq!(
|
|
describe_forge_admin(&["user".to_owned(), "change-password".to_owned()]),
|
|
"forgejo admin user change-password"
|
|
);
|
|
assert_eq!(
|
|
describe_forge_admin(&["--help".to_owned()]),
|
|
"forgejo admin"
|
|
);
|
|
assert_eq!(describe_forge_admin(&[]), "forgejo admin");
|
|
}
|
|
|
|
/// The regression this function exists for. The keyword rule passes
|
|
/// this line straight through — it says nothing about a password — so
|
|
/// only the shape rule catches it.
|
|
#[test]
|
|
fn redacts_access_token_line_which_mentions_no_password() {
|
|
let line =
|
|
"Access token was successfully created: 0123456789abcdef0123456789abcdef01234567";
|
|
assert!(
|
|
!line.to_ascii_lowercase().contains("password"),
|
|
"fixture must not contain the keyword, or it proves nothing"
|
|
);
|
|
assert_eq!(
|
|
redact_secret_line(line),
|
|
"[redacted: line contains a secret-shaped token]"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn secret_shape_boundaries() {
|
|
// Ordinary forgejo-admin output survives: no run is long enough.
|
|
assert_eq!(
|
|
redact_secret_line("Command 'user' 'create' finished with no errors."),
|
|
"Command 'user' 'create' finished with no errors."
|
|
);
|
|
// 31 chars is below the floor, 32 is at it.
|
|
assert!(!contains_secret_shaped_run(&"a".repeat(31)));
|
|
assert!(contains_secret_shaped_run(&"a".repeat(32)));
|
|
// Punctuation breaks the run — a sentence never trips it however long.
|
|
assert!(!contains_secret_shaped_run(
|
|
"this.is.a.very.long.dotted.identifier.but.not.a.secret"
|
|
));
|
|
// base64url and hex alphabets both count.
|
|
assert!(contains_secret_shaped_run(
|
|
"ZGVhZGJlZWZkZWFkYmVlZmRlYWRiZWVmZGVhZA=="
|
|
));
|
|
assert!(contains_secret_shaped_run(
|
|
"aG93-dy_there-aG93dy1theresomething"
|
|
));
|
|
}
|
|
|
|
/// argus on the review: a whitespace-delimited check is defeated by
|
|
/// punctuation glued to the secret — the punctuation joins the "word"
|
|
/// and fails the alphabet test for the whole run, while the credential
|
|
/// sits there in full. Scanning for a RUN rather than a WORD closes it.
|
|
/// These are the shapes that used to slip through.
|
|
#[test]
|
|
fn secret_is_caught_with_punctuation_glued_to_it() {
|
|
const TOK: &str = "0123456789abcdef0123456789abcdef01234567";
|
|
for line in [
|
|
format!("token: {TOK},"),
|
|
format!("token: {TOK}."),
|
|
format!("using [{TOK}] now"),
|
|
format!("value=\"{TOK}\""),
|
|
format!("(created {TOK})"),
|
|
// no whitespace anywhere -- one glued blob
|
|
format!("Bearer:{TOK};next"),
|
|
] {
|
|
assert_eq!(
|
|
redact_secret_line(&line),
|
|
"[redacted: line contains a secret-shaped token]",
|
|
"leaked through: {line}"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Unique scratch dir per test, no external tempfile dep.
|
|
fn scratch() -> PathBuf {
|
|
static CTR: AtomicU32 = AtomicU32::new(0);
|
|
let n = CTR.fetch_add(1, Ordering::Relaxed);
|
|
let dir = std::env::temp_dir().join(format!(
|
|
"hive-priv-nofollow-test-{}-{n}",
|
|
std::process::id()
|
|
));
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
dir
|
|
}
|
|
|
|
#[test]
|
|
fn rejects_non_plain_filenames() {
|
|
let dir = scratch();
|
|
let dir_fd = open_dir(&dir).unwrap();
|
|
for bad in ["", ".", "..", "a/b", "/etc/passwd", "../escape", "sub/tok"] {
|
|
assert!(
|
|
StagedFile::write(&dir_fd, &dir, bad, b"x", 0o600).is_err(),
|
|
"must reject filename {bad:?}"
|
|
);
|
|
// The wrapper's own check runs before its `create_dir_all`.
|
|
// Pointing it at a directory that does not exist yet is what makes
|
|
// this arm bite: the property worth pinning is that a bad leaf
|
|
// fails BEFORE any root-privileged filesystem work.
|
|
let absent = dir.join("never-created");
|
|
assert!(
|
|
write_agent_dir_file("a", &absent, bad, "x").is_err(),
|
|
"wrapper must reject filename {bad:?}"
|
|
);
|
|
assert!(
|
|
!absent.exists(),
|
|
"rejected write created a directory for {bad:?}"
|
|
);
|
|
}
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The temp name is the whole reason the rename is safe to watch: a
|
|
/// `systemd.path` unit globbing `matrix-token*` would fire on a temp that
|
|
/// merely suffixed the real name, on exactly the empty file the rename
|
|
/// exists to hide.
|
|
#[test]
|
|
fn a_temp_cannot_match_a_glob_on_the_name_it_publishes() {
|
|
for real in ["matrix-token", "matrix-token-alice", "forge-token"] {
|
|
let tmp = partial_name(real);
|
|
assert!(
|
|
!tmp.starts_with(real),
|
|
"{tmp:?} matches a `{real}*` path unit while half-written"
|
|
);
|
|
assert!(!tmp.contains('/'), "temp must stay in the same directory");
|
|
}
|
|
}
|
|
|
|
/// A published file arrives complete and alone. The residue arm is the
|
|
/// load-bearing one: a stranded temp holds the same secret, is owned by
|
|
/// root, and nothing else in the system would ever remove it.
|
|
#[test]
|
|
fn publishing_leaves_only_the_finished_file() {
|
|
use std::os::unix::fs::PermissionsExt as _;
|
|
let dir = scratch();
|
|
let names = |d: &PathBuf| {
|
|
let mut v: Vec<String> = std::fs::read_dir(d)
|
|
.unwrap()
|
|
.map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
|
|
.collect();
|
|
v.sort();
|
|
v
|
|
};
|
|
assert!(names(&dir).is_empty(), "control: scratch starts empty");
|
|
|
|
let path = dir.join("matrix-token-alice");
|
|
for content in ["SECRET", "ROTATED"] {
|
|
write_agent_dir_file("a", &dir, "matrix-token-alice", content).unwrap();
|
|
assert_eq!(names(&dir), ["matrix-token-alice"], "temp left behind");
|
|
assert_eq!(std::fs::read_to_string(&path).unwrap(), content);
|
|
}
|
|
let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
|
|
assert_eq!(mode, 0o600, "credential must be 0600 once published");
|
|
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// A failed publish must not strand the temp.
|
|
#[test]
|
|
fn a_failed_publish_removes_the_temp() {
|
|
let dir = scratch();
|
|
// A directory at the destination fails the `rename` (EISDIR) without
|
|
// needing to drop privileges; the write and chown ahead of it still run.
|
|
std::fs::create_dir(dir.join("forge-token")).unwrap();
|
|
|
|
let err = write_agent_dir_file("a", &dir, "forge-token", "SECRET")
|
|
.expect_err("publishing onto a directory must fail");
|
|
assert!(
|
|
format!("{err:#}").contains("publishing"),
|
|
"must fail AT the publish -- failing earlier leaves no temp to \
|
|
clean up and makes the next assertion vacuous. got: {err:#}"
|
|
);
|
|
assert_eq!(
|
|
entries(&dir),
|
|
["forge-token"],
|
|
"temp still holds the secret after a failed publish"
|
|
);
|
|
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// Names in `dir`, sorted — the residue check every publish test ends on.
|
|
fn entries(dir: &PathBuf) -> Vec<String> {
|
|
let mut v: Vec<String> = std::fs::read_dir(dir)
|
|
.unwrap()
|
|
.map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
|
|
.collect();
|
|
v.sort();
|
|
v
|
|
}
|
|
|
|
/// The agent plants a symlink where the token goes. The rename replaces
|
|
/// the link; the root-privileged write never reaches its target.
|
|
#[test]
|
|
fn a_symlink_at_the_name_is_replaced_not_followed() {
|
|
let dir = scratch();
|
|
let target = dir.join("target");
|
|
std::fs::write(&target, "original").unwrap();
|
|
std::os::unix::fs::symlink(&target, dir.join("forge-token")).unwrap();
|
|
|
|
write_agent_dir_file("a", &dir, "forge-token", "SECRET").unwrap();
|
|
assert_eq!(
|
|
std::fs::read_to_string(&target).unwrap(),
|
|
"original",
|
|
"symlink target must be untouched"
|
|
);
|
|
let meta = std::fs::symlink_metadata(dir.join("forge-token")).unwrap();
|
|
assert!(meta.file_type().is_file(), "the link must be replaced");
|
|
assert_eq!(
|
|
std::fs::read_to_string(dir.join("forge-token")).unwrap(),
|
|
"SECRET"
|
|
);
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// Any failure between staging and publishing drops the `StagedFile`.
|
|
/// Until the rename a reader sees only the old file, and afterwards the
|
|
/// old file is still whole and the temp is gone.
|
|
#[test]
|
|
fn a_failed_write_leaves_the_old_file_intact() {
|
|
let dir = scratch();
|
|
let dest = dir.join("hyperhive-limits.conf");
|
|
std::fs::write(&dest, "OLD").unwrap();
|
|
let dir_fd = open_dir(&dir).unwrap();
|
|
|
|
let staged =
|
|
StagedFile::write(&dir_fd, &dir, "hyperhive-limits.conf", b"NEW", 0o644).unwrap();
|
|
assert_eq!(std::fs::read_to_string(&dest).unwrap(), "OLD");
|
|
assert_eq!(
|
|
entries(&dir).len(),
|
|
2,
|
|
"control: the temp exists while staged"
|
|
);
|
|
drop(staged);
|
|
|
|
assert_eq!(std::fs::read_to_string(&dest).unwrap(), "OLD");
|
|
assert_eq!(entries(&dir), ["hyperhive-limits.conf"], "temp left behind");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// Two overlapping writes of one file each own a temp, so whichever
|
|
/// publishes last lands whole — never the other's bytes, never a mix.
|
|
#[test]
|
|
fn overlapping_writes_of_one_file_never_share_a_temp() {
|
|
let dir = scratch();
|
|
let dir_fd = open_dir(&dir).unwrap();
|
|
let name = "hyperhive-agents.conf";
|
|
|
|
let first = StagedFile::write(&dir_fd, &dir, name, b"roster A\n", 0o644).unwrap();
|
|
let second = StagedFile::write(&dir_fd, &dir, name, b"roster B, longer\n", 0o644).unwrap();
|
|
second.publish().unwrap();
|
|
first.publish().unwrap();
|
|
|
|
assert_eq!(
|
|
std::fs::read_to_string(dir.join(name)).unwrap(),
|
|
"roster A\n"
|
|
);
|
|
assert_eq!(entries(&dir), [name], "temp left behind");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The mode lands exactly as asked, whatever the umask would have made
|
|
/// of it (0666 under the usual 022 would otherwise come out 0644), and
|
|
/// the requested owner is on the file.
|
|
#[test]
|
|
fn publish_file_applies_mode_and_owner() {
|
|
use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _};
|
|
let dir = scratch();
|
|
let path = dir.join("c.conf");
|
|
let owner = std::fs::metadata(&dir).unwrap();
|
|
|
|
publish_file(&path, b"x\n", 0o666, Some((owner.uid(), owner.gid()))).unwrap();
|
|
let meta = std::fs::metadata(&path).unwrap();
|
|
assert_eq!(meta.permissions().mode() & 0o7777, 0o666);
|
|
assert_eq!((meta.uid(), meta.gid()), (owner.uid(), owner.gid()));
|
|
|
|
publish_file(&path, b"y\n", 0o600, None).unwrap();
|
|
let meta = std::fs::metadata(&path).unwrap();
|
|
assert_eq!(meta.permissions().mode() & 0o7777, 0o600);
|
|
assert_eq!(std::fs::read_to_string(&path).unwrap(), "y\n");
|
|
assert_eq!(entries(&dir), ["c.conf"], "temp left behind");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The container plants a symlink where the bridge-DNS marker goes; the
|
|
/// rename replaces it without writing through it.
|
|
#[test]
|
|
fn bridge_dns_marker_replaces_symlink_leaf() {
|
|
let dir = scratch();
|
|
let rootfs = dir.join("rootfs");
|
|
std::fs::create_dir_all(rootfs.join("etc")).unwrap();
|
|
let target = dir.join("target");
|
|
std::fs::write(&target, "original").unwrap();
|
|
let leaf = rootfs.join("etc/hyperhive-bridge-dns");
|
|
std::os::unix::fs::symlink(&target, &leaf).unwrap();
|
|
|
|
write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap();
|
|
assert_eq!(
|
|
std::fs::read_to_string(&target).unwrap(),
|
|
"original",
|
|
"symlink target must be untouched"
|
|
);
|
|
assert!(
|
|
std::fs::symlink_metadata(&leaf)
|
|
.unwrap()
|
|
.file_type()
|
|
.is_file()
|
|
);
|
|
assert_eq!(std::fs::read_to_string(&leaf).unwrap(), "10.0.0.1\n");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The container replaces its whole `/etc` with a symlink, so the leaf
|
|
/// name resolves inside a directory of its choosing.
|
|
#[test]
|
|
fn bridge_dns_marker_refuses_symlink_etc() {
|
|
let dir = scratch();
|
|
let rootfs = dir.join("rootfs");
|
|
std::fs::create_dir_all(&rootfs).unwrap();
|
|
let elsewhere = dir.join("elsewhere");
|
|
std::fs::create_dir_all(&elsewhere).unwrap();
|
|
let target = elsewhere.join("hyperhive-bridge-dns");
|
|
std::fs::write(&target, "original").unwrap();
|
|
std::os::unix::fs::symlink(&elsewhere, rootfs.join("etc")).unwrap();
|
|
|
|
let res = write_bridge_dns_marker_in(&rootfs, "10.0.0.1");
|
|
assert!(res.is_err(), "O_NOFOLLOW must refuse a symlinked etc");
|
|
assert_eq!(
|
|
std::fs::read_to_string(&target).unwrap(),
|
|
"original",
|
|
"file behind the symlinked etc must be untouched"
|
|
);
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// A FIFO at the leaf is replaced, never opened, so it cannot block the
|
|
/// helper.
|
|
#[test]
|
|
fn bridge_dns_marker_replaces_fifo_leaf() {
|
|
use std::os::unix::ffi::OsStrExt as _;
|
|
let dir = scratch();
|
|
let rootfs = dir.join("rootfs");
|
|
std::fs::create_dir_all(rootfs.join("etc")).unwrap();
|
|
let fifo = rootfs.join("etc/hyperhive-bridge-dns");
|
|
let c_fifo = std::ffi::CString::new(fifo.as_os_str().as_bytes()).unwrap();
|
|
// SAFETY: `c_fifo` is a valid NUL-terminated path.
|
|
assert_eq!(unsafe { libc::mkfifo(c_fifo.as_ptr(), 0o644) }, 0);
|
|
|
|
write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap();
|
|
assert!(
|
|
std::fs::symlink_metadata(&fifo)
|
|
.unwrap()
|
|
.file_type()
|
|
.is_file()
|
|
);
|
|
assert_eq!(std::fs::read_to_string(&fifo).unwrap(), "10.0.0.1\n");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The control: a plain rootfs with no `etc` yet (a fresh install) gets
|
|
/// the dir and a 0644 marker, and a second write replaces it.
|
|
#[test]
|
|
fn bridge_dns_marker_written_into_plain_etc() {
|
|
use std::os::unix::fs::PermissionsExt as _;
|
|
let dir = scratch();
|
|
let rootfs = dir.join("rootfs");
|
|
std::fs::create_dir_all(&rootfs).unwrap();
|
|
|
|
write_bridge_dns_marker_in(&rootfs, "10.0.0.1").unwrap();
|
|
let path = rootfs.join("etc/hyperhive-bridge-dns");
|
|
assert_eq!(std::fs::read_to_string(&path).unwrap(), "10.0.0.1\n");
|
|
let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
|
|
assert_eq!(mode, 0o644, "marker must stay readable in the container");
|
|
|
|
write_bridge_dns_marker_in(&rootfs, "10.9.8.7").unwrap();
|
|
assert_eq!(std::fs::read_to_string(&path).unwrap(), "10.9.8.7\n");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// Whether `pid` is gone, or only a zombie awaiting its reaper: either
|
|
/// way it is no longer running.
|
|
fn is_dead(pid: &str) -> bool {
|
|
match std::fs::read_to_string(format!("/proc/{pid}/stat")) {
|
|
Err(_) => true,
|
|
// The state letter follows the parenthesised command name.
|
|
Ok(stat) => stat
|
|
.rsplit_once(") ")
|
|
.is_some_and(|(_, rest)| rest.starts_with('Z')),
|
|
}
|
|
}
|
|
|
|
/// A child still running at the limit comes back `TimedOut`, and the kill
|
|
/// reaches its whole process group: the backgrounded grandchild dies too,
|
|
/// not just the shell that started it.
|
|
#[tokio::test]
|
|
async fn a_build_past_its_limit_is_killed_with_its_group() {
|
|
use std::time::{Duration, Instant};
|
|
let dir = scratch();
|
|
let pidfile = dir.join("grandchild");
|
|
let mut cmd = tokio::process::Command::new("sh");
|
|
cmd.arg("-c")
|
|
.arg(format!("sleep 600 & echo $! > {}; wait", pidfile.display()));
|
|
|
|
let started = Instant::now();
|
|
let run = run_bounded(cmd, Duration::from_secs(2), None)
|
|
.await
|
|
.unwrap();
|
|
assert!(matches!(run, BoundedRun::TimedOut), "must time out");
|
|
assert!(
|
|
started.elapsed() < Duration::from_mins(1),
|
|
"the limit did not bound the run"
|
|
);
|
|
|
|
let pid = std::fs::read_to_string(&pidfile).unwrap();
|
|
let pid = pid.trim();
|
|
let deadline = Instant::now() + Duration::from_secs(5);
|
|
while !is_dead(pid) && Instant::now() < deadline {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
}
|
|
assert!(is_dead(pid), "grandchild {pid} outlived the timeout");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The control: a child that exits inside its limit is `Exited`, with its
|
|
/// status and both streams kept apart.
|
|
#[tokio::test]
|
|
async fn a_build_inside_its_limit_returns_its_output() {
|
|
let mut cmd = tokio::process::Command::new("sh");
|
|
cmd.arg("-c")
|
|
.arg("echo progress >&2; echo /nix/store/abc-toplevel");
|
|
match run_bounded(cmd, std::time::Duration::from_mins(1), None)
|
|
.await
|
|
.unwrap()
|
|
{
|
|
BoundedRun::Exited {
|
|
status,
|
|
stdout,
|
|
stderr,
|
|
} => {
|
|
assert!(status.success());
|
|
assert_eq!(stdout, "/nix/store/abc-toplevel\n");
|
|
assert_eq!(stderr, "progress");
|
|
}
|
|
BoundedRun::TimedOut => panic!("a fast child must not time out"),
|
|
}
|
|
}
|
|
|
|
/// A missing socket dir is created `0751`, owned by the caller (root in
|
|
/// production), which is the mode the container's activation expects to
|
|
/// hand over and the one dialers traverse.
|
|
#[test]
|
|
fn socket_dir_created_0751_when_absent() {
|
|
use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _};
|
|
let dir = scratch();
|
|
ensure_socket_dir_in(&dir, "probe").unwrap();
|
|
let meta = std::fs::symlink_metadata(dir.join("probe")).unwrap();
|
|
assert!(meta.is_dir());
|
|
assert_eq!(meta.permissions().mode() & 0o7777, 0o751);
|
|
// SAFETY: `geteuid` has no preconditions.
|
|
assert_eq!(meta.uid(), unsafe { libc::geteuid() });
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// An existing directory keeps its mode and contents: by the second start
|
|
/// it belongs to the agent user, and a live socket may sit inside it.
|
|
#[test]
|
|
fn socket_dir_existing_directory_left_alone() {
|
|
use std::os::unix::fs::PermissionsExt as _;
|
|
let dir = scratch();
|
|
let sub = dir.join("probe");
|
|
std::fs::create_dir(&sub).unwrap();
|
|
std::fs::set_permissions(&sub, std::fs::Permissions::from_mode(0o700)).unwrap();
|
|
std::fs::write(sub.join("web.sock"), "").unwrap();
|
|
|
|
ensure_socket_dir_in(&dir, "probe").unwrap();
|
|
let mode = std::fs::metadata(&sub).unwrap().permissions().mode() & 0o7777;
|
|
assert_eq!(mode, 0o700, "an existing dir's mode must not be reset");
|
|
assert!(sub.join("web.sock").exists());
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// A symlink at the leaf would make nspawn bind wherever it points; it is
|
|
/// refused, whether it points at a directory or nowhere.
|
|
#[test]
|
|
fn socket_dir_refuses_symlink_leaf() {
|
|
use std::os::unix::fs::PermissionsExt as _;
|
|
let dir = scratch();
|
|
let elsewhere = dir.join("elsewhere");
|
|
std::fs::create_dir(&elsewhere).unwrap();
|
|
std::fs::set_permissions(&elsewhere, std::fs::Permissions::from_mode(0o700)).unwrap();
|
|
std::os::unix::fs::symlink(&elsewhere, dir.join("probe")).unwrap();
|
|
std::os::unix::fs::symlink(dir.join("missing"), dir.join("dangling")).unwrap();
|
|
|
|
assert!(ensure_socket_dir_in(&dir, "probe").is_err());
|
|
assert!(ensure_socket_dir_in(&dir, "dangling").is_err());
|
|
let mode = std::fs::metadata(&elsewhere).unwrap().permissions().mode() & 0o7777;
|
|
assert_eq!(mode, 0o700, "the symlink target must be untouched");
|
|
assert!(!dir.join("missing").exists());
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// A regular file where the dir should be is refused, not bound.
|
|
#[test]
|
|
fn socket_dir_refuses_non_directory_leaf() {
|
|
let dir = scratch();
|
|
std::fs::write(dir.join("probe"), "x").unwrap();
|
|
let err = ensure_socket_dir_in(&dir, "probe").unwrap_err();
|
|
assert!(format!("{err:#}").contains("not a directory"), "{err:#}");
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// A symlinked root would place the new dir in a tree of the planter's
|
|
/// choosing; the `O_NOFOLLOW` open refuses it before anything is created.
|
|
#[test]
|
|
fn socket_dir_refuses_symlinked_root() {
|
|
let dir = scratch();
|
|
let elsewhere = dir.join("elsewhere");
|
|
std::fs::create_dir(&elsewhere).unwrap();
|
|
let root = dir.join("root");
|
|
std::os::unix::fs::symlink(&elsewhere, &root).unwrap();
|
|
|
|
assert!(ensure_socket_dir_in(&root, "probe").is_err());
|
|
assert!(!elsewhere.join("probe").exists());
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// Names that are not a plain agent ident are refused by validation,
|
|
/// before anything is created. Runs against a scratch parent, so the
|
|
/// refusal cannot come from a missing `/run/hive-agent`; `.`/`..` exist
|
|
/// as directories and `Atlas` would be created, so each bad name gets
|
|
/// through only if validation is gone. Control: a plain name succeeds
|
|
/// in the same parent.
|
|
#[test]
|
|
fn socket_dir_refuses_bogus_names() {
|
|
let dir = scratch();
|
|
for bad in ["", ".", "..", "../etc", "a/b", "Atlas", "a b", "a\0b"] {
|
|
let err = ensure_socket_dir_in(&dir, bad).unwrap_err();
|
|
assert!(
|
|
format!("{err:#}").contains("invalid name"),
|
|
"agent name {bad:?} must be refused by validation, got: {err:#}"
|
|
);
|
|
}
|
|
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0);
|
|
ensure_socket_dir_in(&dir, "atlas").unwrap();
|
|
assert!(dir.join("atlas").is_dir());
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// An older hive-c0re's `SyncAgentTmpfiles` still decodes, payload and
|
|
/// all, so it gets the legacy cleanup rather than a parse error. Control:
|
|
/// an unknown op does not decode.
|
|
#[test]
|
|
fn legacy_sync_agent_tmpfiles_request_still_decodes() {
|
|
let legacy =
|
|
r#"{"op":"sync_agent_tmpfiles","agents":[{"name":"atlas","uid":1000,"gid":994}]}"#;
|
|
assert!(matches!(
|
|
serde_json::from_str::<PrivRequest>(legacy),
|
|
Ok(PrivRequest::SyncAgentTmpfiles)
|
|
));
|
|
assert!(serde_json::from_str::<PrivRequest>(r#"{"op":"sync_agent_tmpfilez"}"#).is_err());
|
|
}
|
|
|
|
/// The runner-credential clear, in all three states that matter. The
|
|
/// PRESENCE arm is the load-bearing one: an implementation that did nothing
|
|
/// at all would pass the "absent is fine" arm perfectly, and the whole point
|
|
/// of the call is that the file is *gone* afterwards — upstream re-registers
|
|
/// on absence and on nothing else we control.
|
|
#[test]
|
|
fn clearing_runner_credentials_removes_it_and_tolerates_absence() {
|
|
let dir = scratch();
|
|
let path = dir.join(".runner");
|
|
let path_str = path.to_str().unwrap();
|
|
|
|
// Absent → success (this is the state we are aiming for).
|
|
assert!(!path.exists());
|
|
clear_runner_credentials(path_str).unwrap();
|
|
|
|
// Present → success AND actually gone. Without this arm a no-op passes.
|
|
std::fs::write(&path, r#"{"id":7,"address":"http://old.invalid"}"#).unwrap();
|
|
assert!(
|
|
path.exists(),
|
|
"control: the file must exist before the clear"
|
|
);
|
|
clear_runner_credentials(path_str).unwrap();
|
|
assert!(
|
|
!path.exists(),
|
|
"stale credentials must be GONE, or the restart takes upstream's \
|
|
already-registered branch and registration silently never happens"
|
|
);
|
|
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// A failure that is not `NotFound` must propagate, never read as success.
|
|
/// A directory in the file's place makes `remove_file` fail with a non-
|
|
/// `NotFound` error without needing to drop privileges in a test.
|
|
#[test]
|
|
fn clearing_runner_credentials_propagates_a_real_failure() {
|
|
let dir = scratch();
|
|
let path = dir.join(".runner");
|
|
std::fs::create_dir(&path).unwrap();
|
|
|
|
let err = clear_runner_credentials(path.to_str().unwrap())
|
|
.expect_err("a non-NotFound failure must NOT be reported as success");
|
|
assert!(
|
|
format!("{err:#}").contains("remove stale runner credentials"),
|
|
"error must name what it failed to do, got: {err:#}"
|
|
);
|
|
assert!(
|
|
path.exists(),
|
|
"nothing was removed, and the caller must know"
|
|
);
|
|
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The pause marker round-trips through the same root-only path the
|
|
/// credential writes use, and BOTH directions are idempotent — the
|
|
/// dashboard toggle and `hivectl pause|resume` fire blind, without
|
|
/// reading the current state first.
|
|
#[test]
|
|
fn pause_marker_create_and_remove_are_idempotent() {
|
|
let dir = scratch();
|
|
let path = dir.join(PAUSED_MARKER_FILE);
|
|
|
|
for _ in 0..2 {
|
|
write_agent_dir_file("a", &dir, PAUSED_MARKER_FILE, "").unwrap();
|
|
assert!(path.exists(), "marker must exist after pause");
|
|
assert_eq!(std::fs::read_to_string(&path).unwrap(), "");
|
|
}
|
|
for _ in 0..2 {
|
|
remove_marker_in(&dir, PAUSED_MARKER_FILE).unwrap();
|
|
assert!(!path.exists(), "marker must be gone after resume");
|
|
}
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// A resume must never follow an agent-planted symlink at the marker
|
|
/// path: this unlink runs as root, so following it would let an agent
|
|
/// delete an arbitrary file on the host.
|
|
#[test]
|
|
fn resume_unlinks_the_symlink_not_its_target() {
|
|
let dir = scratch();
|
|
let target = dir.join("target");
|
|
std::fs::write(&target, "original").unwrap();
|
|
let link = dir.join(PAUSED_MARKER_FILE);
|
|
std::os::unix::fs::symlink(&target, &link).unwrap();
|
|
|
|
remove_marker_in(&dir, PAUSED_MARKER_FILE).unwrap();
|
|
// `exists()` follows the link, so it can't tell "link removed" from
|
|
// "target removed, dangling link left" — stat the link itself.
|
|
assert!(
|
|
std::fs::symlink_metadata(&link).is_err(),
|
|
"the link itself must be unlinked"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(&target).unwrap(),
|
|
"original",
|
|
"symlink target must survive the root unlink"
|
|
);
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// The only gate on the root-privileged socket for a `--load-credential`
|
|
/// name. The doc comment is explicit that `.` is excluded on purpose
|
|
/// ("this name gets interpolated into filesystem paths") — a future
|
|
/// "let's allow dots for version numbers" loosening must fail this,
|
|
/// not just the `/` case below.
|
|
#[test]
|
|
fn validate_credential_name_rejects_dot_slash_and_empty_but_allows_the_charset() {
|
|
assert!(
|
|
validate_credential_name("").is_err(),
|
|
"empty must be rejected"
|
|
);
|
|
assert!(validate_credential_name("valid-name_123").is_ok());
|
|
assert!(
|
|
validate_credential_name("bad.name").is_err(),
|
|
"dot must be rejected — see the fn's doc comment"
|
|
);
|
|
assert!(validate_credential_name("bad/name").is_err());
|
|
}
|
|
|
|
/// A snapshot label must additionally carry the `hive-` prefix (it
|
|
/// doubles as the allow-list gating the btrfs snapshot/delete
|
|
/// shellouts) on top of [`validate_credential_name`]'s charset rule.
|
|
#[test]
|
|
fn validate_snapshot_name_requires_hive_prefix() {
|
|
assert!(validate_snapshot_name("not-hive-prefixed").is_err());
|
|
assert!(validate_snapshot_name("hive-valid").is_ok());
|
|
}
|
|
|
|
/// `ensure_plain_filename` is the gate between an agent-writable
|
|
/// directory and a root-privileged file write. `.`/`..` would resolve
|
|
/// to the directory itself or its parent; any `/` climbs into a path
|
|
/// component the caller never named.
|
|
#[test]
|
|
fn ensure_plain_filename_rejects_dot_dotdot_and_any_slash() {
|
|
assert!(ensure_plain_filename("test", ".").is_err());
|
|
assert!(ensure_plain_filename("test", "..").is_err());
|
|
assert!(ensure_plain_filename("test", "sub/dir").is_err());
|
|
assert!(ensure_plain_filename("test", "matrix-token").is_ok());
|
|
}
|
|
|
|
/// Unique scratch dir per test, no external tempfile dep. Same pattern
|
|
/// as `scratch()` above, kept separate so a rename of one doesn't
|
|
/// collide with the other's directory names under parallel test runs.
|
|
fn snapshot_export_scratch() -> PathBuf {
|
|
static CTR: AtomicU32 = AtomicU32::new(0);
|
|
let n = CTR.fetch_add(1, Ordering::Relaxed);
|
|
let dir = std::env::temp_dir().join(format!(
|
|
"hive-priv-snapshot-export-test-{}-{n}",
|
|
std::process::id()
|
|
));
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
dir
|
|
}
|
|
|
|
/// The fix: a missing parent must be caught before `dest` is created,
|
|
/// not after. Fails against the pre-fix ordering (`dest` created via
|
|
/// `create_new` first, parent checked second) — see the issue for the
|
|
/// inverted run.
|
|
#[test]
|
|
fn missing_parent_leaves_no_file_at_dest() {
|
|
let dir = snapshot_export_scratch();
|
|
let snap = dir.join("snap");
|
|
std::fs::write(&snap, b"").unwrap();
|
|
let parent = dir.join("parent-that-does-not-exist");
|
|
let dest = dir.join("dest");
|
|
|
|
let err = open_export_dest(&snap, Some(&parent), &dest).unwrap_err();
|
|
assert!(
|
|
err.to_string().contains("does not exist"),
|
|
"wrong error: {err}"
|
|
);
|
|
assert!(
|
|
!dest.exists(),
|
|
"a failed export must not leave a file at dest"
|
|
);
|
|
}
|
|
|
|
/// A retry against the same `dest` after a precondition failure must
|
|
/// not trip the no-overwrite guard — there is nothing there to
|
|
/// overwrite, since the failed attempt never created `dest` (the test
|
|
/// above). This is the behavior a "stale parent falls back to a full
|
|
/// send, retried without -p" caller needs against a fixed `--dest`.
|
|
#[test]
|
|
fn retry_after_precondition_failure_succeeds() {
|
|
let dir = snapshot_export_scratch();
|
|
let snap = dir.join("snap");
|
|
std::fs::write(&snap, b"").unwrap();
|
|
let parent = dir.join("parent-that-does-not-exist");
|
|
let dest = dir.join("dest");
|
|
|
|
assert!(open_export_dest(&snap, Some(&parent), &dest).is_err());
|
|
assert!(!dest.exists());
|
|
|
|
// Retry without the missing parent, same dest.
|
|
let file = open_export_dest(&snap, None, &dest);
|
|
assert!(
|
|
file.is_ok(),
|
|
"retry against a dest no previous attempt touched must succeed: {:?}",
|
|
file.err()
|
|
);
|
|
assert!(dest.exists());
|
|
}
|
|
|
|
/// Control for the two tests above: an export that actually landed at
|
|
/// `dest` must still refuse a second one. Without this, a bug that
|
|
/// simply stopped creating `dest` at all (rather than fixing the
|
|
/// ordering) would also make the retry test above pass.
|
|
#[test]
|
|
fn existing_dest_still_refuses_overwrite() {
|
|
let dir = snapshot_export_scratch();
|
|
let snap = dir.join("snap");
|
|
std::fs::write(&snap, b"").unwrap();
|
|
let dest = dir.join("dest");
|
|
std::fs::write(&dest, b"already here").unwrap();
|
|
|
|
let err = open_export_dest(&snap, None, &dest).unwrap_err();
|
|
assert!(
|
|
err.to_string().contains("already exists"),
|
|
"wrong error: {err}"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read(&dest).unwrap(),
|
|
b"already here",
|
|
"must not have touched the existing file"
|
|
);
|
|
}
|
|
}
|