//! Minimal privileged helper for hive-c0re. //! //! Runs as root. Exposes a narrow unix socket at `/run/hive/priv.sock` //! that accepts `PrivRequest` JSON lines and executes only the //! operations that genuinely require root. All coordination logic, //! broker, HTTP, and scheduling stay in the unprivileged hive-c0re //! process. //! //! **Security model**: every request is validated against a strict //! container-name allowlist before any filesystem or process operation. //! Only containers whose names match the hive convention (`h-*`, //! the manager container, or known sibling service containers) are //! accepted. Every variant maps to a single known operation — no //! arbitrary command pass-through. //! //! **Socket activation**: when systemd passes the listener socket via //! `LISTEN_FDS=1` + `LISTEN_PID=`, the inherited fd 3 is used //! instead of binding a fresh socket. use std::path::{Path, PathBuf}; use anyhow::{Context as _, Result, bail}; use hive_sh4re::priv_proto::{ AGENT_PREFIX, AGENT_STATE_ROOT, BindMount, CredentialMount, InfraAction, InfraContainer, JournalQuery, META_DIR, NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream, PrivStreamLine, SIBLING_CONTAINERS, }; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::net::unix::OwnedWriteHalf; use tokio::net::{UnixListener, UnixStream}; use tokio::process::Command; /// Root of the per-agent unix-socket dirs on the host. const SOCKET_DIR_ROOT: &str = "/run/hive-agent"; #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()), ) .init(); let listener = socket_listener()?; tracing::info!("hive-priv listening"); loop { match listener.accept().await { Ok((stream, _)) => { tokio::spawn(handle(stream)); } Err(e) => { tracing::error!(error = %e, "accept failed"); } } } } fn socket_listener() -> Result { // hive-priv is ALWAYS socket-activated by the `hive-priv.socket` unit // (fd 3 via LISTEN_FDS). There is intentionally no self-bind fallback, // so dev and prod take the same path; see docs/boundary.md. let listen_fds: Option = std::env::var("LISTEN_FDS") .ok() .and_then(|s| s.parse().ok()); let listen_pid: Option = std::env::var("LISTEN_PID") .ok() .and_then(|s| s.parse().ok()); let activated = matches!(listen_fds, Some(n) if n >= 1) && listen_pid == Some(std::process::id()); if !activated { bail!( "hive-priv requires systemd socket activation (expected LISTEN_FDS>=1 + \ LISTEN_PID= for {PRIV_SOCK}); run it via the hive-priv.socket unit, \ not directly" ); } // SAFETY: systemd has passed us a ready UnixListener on fd 3. let std_listener = unsafe { use std::os::unix::io::FromRawFd; std::os::unix::net::UnixListener::from_raw_fd(3) }; std_listener .set_nonblocking(true) .context("set socket non-blocking")?; let listener = tokio::net::UnixListener::from_std(std_listener).context("wrap systemd socket")?; tracing::info!("using systemd-activated socket"); Ok(listener) } async fn handle(stream: UnixStream) { let (reader, mut writer) = stream.into_split(); let mut lines = BufReader::new(reader).lines(); while let Ok(Some(line)) = lines.next_line().await { let resp = dispatch(&line, &mut writer).await; // Write the terminal PrivResponse as a PrivEvent::Done. Wire-identical // to a bare PrivResponse (untagged), so old hive-c0re callers that // deserialise directly to PrivResponse continue to work. let event = PrivEvent::Done(resp); let mut json = serde_json::to_string(&event).unwrap_or_else(|e| { format!("{{\"ok\":false,\"stdout\":\"\",\"stderr\":\"\",\"error\":\"serialise failed: {e}\"}}") }); json.push('\n'); if let Err(e) = writer.write_all(json.as_bytes()).await { tracing::warn!(error = %e, "write response failed"); break; } } } async fn dispatch(line: &str, writer: &mut OwnedWriteHalf) -> PrivResponse { match serde_json::from_str::(line) { Ok(req) => match exec(req, writer).await { Ok((stdout, stderr)) => PrivResponse { ok: true, stdout, stderr, error: None, }, Err(e) => PrivResponse { ok: false, stdout: String::new(), stderr: String::new(), error: Some(format!("{e:#}")), }, }, Err(e) => PrivResponse { ok: false, stdout: String::new(), stderr: String::new(), error: Some(format!("parse request: {e}")), }, } } /// Write one `PrivEvent::Line` to the client. Best-effort: a write /// failure is logged but doesn't abort the running subprocess. async fn write_line_event(writer: &mut OwnedWriteHalf, stream: PrivStream, data: &str) { let event = PrivEvent::Line(PrivStreamLine { stream, data: data.to_owned(), }); if let Ok(mut json) = serde_json::to_string(&event) { json.push('\n'); if let Err(e) = writer.write_all(json.as_bytes()).await { tracing::warn!(error = %e, "write_line_event: write failed"); } } } /// Execute a validated `PrivRequest`. Returns `(stdout, stderr)` on success. /// For streaming ops (`CreateContainer`/`UpdateContainer` with `stream: true`) /// output lines are forwarded to `writer` as `PrivEvent::Line` messages and /// the returned strings are empty. // One match arm per priv op — a flat 1:1 dispatch table. The length tracks // the op count, not complexity; splitting it would just scatter the mapping. #[allow(clippy::too_many_lines)] async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String, String)> { match req { PrivRequest::StartContainer { ref name } => { validate_container_name(name)?; container_run(&["start", &container_system_name(name)]).await } PrivRequest::StopContainer { ref name } => { validate_container_name(name)?; container_run(&["stop", &container_system_name(name)]).await } PrivRequest::KillContainer { ref name } => { validate_container_name(name)?; container_run(&["kill", &container_system_name(name)]).await } PrivRequest::UpdateContainer { ref name, stream } => { container_flake_action("update", name, stream, writer).await } PrivRequest::CreateContainer { ref name, stream } => { container_flake_action("create", name, stream, writer).await } PrivRequest::DestroyContainer { ref name } => { validate_container_name(name)?; container_run(&["destroy", &container_system_name(name)]).await } PrivRequest::ListContainers => container_run(&["list"]).await, PrivRequest::ReadContainerJournal { ref container, ref query, } => { validate_container_system_name(container)?; read_container_journal(container, query).await } PrivRequest::WriteNspawnFlags { ref container, ref binds, ref isolation, ref load_credentials, } => handle_write_nspawn_flags(container, binds, isolation.as_ref(), load_credentials), PrivRequest::WriteResourceLimits { ref container, ref memory_max, ref cpu_quota, } => write_resource_limits(container, memory_max, cpu_quota), PrivRequest::RemoveServiceDropin { ref container } => remove_service_dropin(container), PrivRequest::DaemonReload => daemon_reload().await, PrivRequest::ReloadGatewayNginx => sync_gateway_nginx().await, PrivRequest::ChownSocketDir { ref agent_name, uid, gid, } => chown_socket_dir(agent_name, uid, gid), PrivRequest::ChmodSocketDir { ref agent_name, mode, } => chmod_socket_dir(agent_name, mode), PrivRequest::RunForgeAdmin { ref args } => { for arg in args { validate_forge_admin_arg(arg)?; } run_forge_admin(args).await } PrivRequest::WriteAgentForgeToken { ref agent_name, ref token, } => { validate_agent_name(agent_name)?; write_agent_state_file(agent_name, "forge-token", &format!("{token}\n")) } PrivRequest::WriteAgentMatrixToken { ref agent_name, ref token, ref account, } => { validate_agent_name(agent_name)?; // Build the token filename. `None` → the hive account's // `matrix-token`; `Some(a)` → `matrix-token-`. The account // suffix MUST be validated as a plain identifier (no `/`, `.`, // `..`) before it goes into the filename, or a crafted account // could traverse out of the state dir — `write_agent_state_file` // trusts its `filename` argument. let filename = match account { None => "matrix-token".to_owned(), Some(a) => { validate_name_chars(a)?; format!("matrix-token-{a}") } }; write_agent_state_file(agent_name, &filename, &format!("{token}\n")) } PrivRequest::RestartMatrixDaemon { ref agent_name } => { restart_matrix_daemon(agent_name).await } PrivRequest::ControlInfraContainer { container, action } => { control_infra_container(container, action).await } PrivRequest::EnsureAgentSubvolume { ref agent_name } => { validate_agent_name(agent_name)?; ensure_agent_subvolume(agent_name).await } PrivRequest::DeleteAgentSubvolume { ref agent_name } => { validate_agent_name(agent_name)?; delete_agent_subvolume(agent_name).await } PrivRequest::EnsureBtrfsQuota => ensure_btrfs_quota().await, PrivRequest::ReadSubvolumeUsage { ref agent_name } => { validate_agent_name(agent_name)?; read_subvolume_usage(agent_name).await } PrivRequest::SetSubvolumeQuota { ref agent_name, limit_bytes, } => { validate_agent_name(agent_name)?; set_subvolume_quota(agent_name, limit_bytes).await } PrivRequest::UpgradeAgentSubvolume { ref agent_name } => { validate_agent_name(agent_name)?; upgrade_agent_subvolume(agent_name).await } } } /// Shared body for `CreateContainer` / `UpdateContainer`: validate the /// name, build the `nixos-container … --flake ` argv, and /// run it (streaming line events to `writer` when `stream` is set). async fn container_flake_action( verb: &str, name: &str, stream: bool, writer: &mut OwnedWriteHalf, ) -> Result<(String, String)> { validate_container_name(name)?; let flake_ref = agent_flake_ref(name); let args = [verb, &container_system_name(name), "--flake", &flake_ref]; if stream { container_run_streaming(&args, writer).await } else { container_run(&args).await } } /// `WriteNspawnFlags` — validate the container + every bind path + every /// credential entry, then write the container's nspawn flag overrides. fn handle_write_nspawn_flags( container: &str, binds: &[BindMount], isolation: Option<&NetworkIsolation>, load_credentials: &[CredentialMount], ) -> Result<(String, String)> { validate_container_system_name(container)?; for bind in binds { validate_bind_path(&bind.host_path)?; validate_bind_path(&bind.container_path)?; } for cred in load_credentials { validate_credential_name(&cred.name)?; // Same path rules as binds (absolute, no colon/newline/quote/null): // the colon ban is essential since `--load-credential=name:path` // uses `:` as the name/path separator. validate_bind_path(&cred.host_path)?; } write_nspawn_flags(container, binds, isolation, load_credentials)?; Ok((String::new(), String::new())) } /// A systemd credential id must be a short token — restrict to /// `[A-Za-z0-9_.-]` so it can't inject extra `--load-credential` argv or /// break the `name:path` shape. fn validate_credential_name(name: &str) -> Result<()> { if name.is_empty() || !name .bytes() .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'-')) { bail!("invalid credential name {name:?}: must be non-empty [A-Za-z0-9_.-]"); } Ok(()) } /// `RemoveServiceDropin` — remove the container service's drop-in dir /// if present (idempotent). fn remove_service_dropin(container: &str) -> Result<(String, String)> { validate_container_system_name(container)?; let dir = format!("/run/systemd/system/container@{container}.service.d"); if Path::new(&dir).exists() { std::fs::remove_dir_all(&dir).with_context(|| format!("remove {dir}"))?; } Ok((String::new(), String::new())) } /// `ChownSocketDir` — chown the agent's host socket dir to its /// container uid/gid. fn chown_socket_dir(agent_name: &str, uid: u32, gid: u32) -> Result<(String, String)> { validate_agent_name(agent_name)?; let path = socket_dir_path(agent_name); std::os::unix::fs::chown(&path, Some(uid), Some(gid)) .with_context(|| format!("chown {} to {uid}:{gid}", path.display()))?; Ok((String::new(), String::new())) } /// `ChmodSocketDir` — set the mode on the agent's host socket dir. fn chmod_socket_dir(agent_name: &str, mode: u32) -> Result<(String, String)> { use std::os::unix::fs::PermissionsExt as _; validate_agent_name(agent_name)?; let path = socket_dir_path(agent_name); std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode)) .with_context(|| format!("chmod {:o} {}", mode, path.display()))?; Ok((String::new(), String::new())) } /// `WriteResourceLimits` — drop a systemd `MemoryMax`/`CPUQuota` /// override into the container service's drop-in dir. fn write_resource_limits( container: &str, memory_max: &str, cpu_quota: &str, ) -> Result<(String, String)> { validate_container_system_name(container)?; let dir = format!("/run/systemd/system/container@{container}.service.d"); std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?; let path = format!("{dir}/hyperhive-limits.conf"); let content = format!("[Service]\nMemoryMax={memory_max}\nCPUQuota={cpu_quota}\n"); std::fs::write(&path, content).with_context(|| format!("write {path}"))?; Ok((String::new(), String::new())) } /// `DaemonReload` — `systemctl daemon-reload` on the host. async fn daemon_reload() -> Result<(String, String)> { let out = Command::new("systemctl") .arg("daemon-reload") .output() .await .context("invoke systemctl daemon-reload")?; if !out.status.success() { bail!( "systemctl daemon-reload failed ({}): {}", out.status, String::from_utf8_lossy(&out.stderr).trim() ); } Ok((String::new(), String::new())) } /// `RestartMatrixDaemon` — restart the matrix daemon unit inside the /// agent's container. async fn restart_matrix_daemon(agent_name: &str) -> Result<(String, String)> { validate_agent_name(agent_name)?; let machine = format!("--machine=h-{agent_name}"); let unit = "hive-matrix-daemon.service"; let out = Command::new("systemctl") .args([&machine, "restart", unit]) .output() .await .with_context(|| format!("systemctl restart {unit} in container h-{agent_name}"))?; if !out.status.success() { bail!( "systemctl restart {unit} in h-{agent_name} exited {}: {}", out.status, String::from_utf8_lossy(&out.stderr).trim() ); } Ok(( String::from_utf8_lossy(&out.stdout).into_owned(), String::from_utf8_lossy(&out.stderr).into_owned(), )) } /// `ControlInfraContainer` — start/stop/restart a hive infrastructure /// container via `systemctl container@.service`. The /// [`InfraContainer`] enum is the allowlist: serde already rejected any /// unknown / unsafe name (hive-c0re has no variant, so a stop can't sever /// the daemon socket) at deserialisation, so no root-side `.contains()` /// check is needed here. Serves both the hive-wide `hivectl stop`/`start` /// flow and an `infra_admin` agent's `restart` (action = Restart). async fn control_infra_container( container: InfraContainer, action: InfraAction, ) -> Result<(String, String)> { let verb = action.systemctl_verb(); let unit = format!("container@{}.service", container.unit_name()); let out = Command::new("systemctl") .args([verb, &unit]) .output() .await .with_context(|| format!("systemctl {verb} {unit}"))?; if !out.status.success() { bail!( "systemctl {verb} {unit} exited {}: {}", out.status, String::from_utf8_lossy(&out.stderr).trim() ); } tracing::info!(target: "infra-control", "{verb} {unit}"); Ok(( String::from_utf8_lossy(&out.stdout).into_owned(), String::from_utf8_lossy(&out.stderr).into_owned(), )) } /// Shared helper for `WriteAgentForgeToken` and `WriteAgentMatrixToken`. /// Writes `content` to `AGENT_STATE_ROOT//state/`, /// chowns to the agent user (derived from the state dir's existing owner), /// and chmods 0600. Running as root (hive-priv), so this succeeds /// regardless of the file's prior owner/permissions. fn write_agent_state_file( agent_name: &str, filename: &str, content: &str, ) -> Result<(String, String)> { use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; let state_dir = PathBuf::from(AGENT_STATE_ROOT) .join(agent_name) .join("state"); // NOTE: `create_dir_all` is normally a no-op — lifecycle creates and chowns // the state dir during spawn. On the rare edge where the dir doesn't exist // yet (container being provisioned for the first time), the newly created dir // is root:root. The `stat state_dir` chown below will then see uid=0 and // leave the file root-owned (0600). The agent won't be able to read it until // its lifecycle completes. If that happens, a `systemctl restart hive-c0re` // after provisioning will re-mint and re-write the token correctly. std::fs::create_dir_all(&state_dir) .with_context(|| format!("create state dir {}", state_dir.display()))?; let path = state_dir.join(filename); std::fs::write(&path, content).with_context(|| format!("write {}", path.display()))?; std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) .with_context(|| format!("chmod 600 {}", path.display()))?; // Chown to the state dir's owner so the agent process can read the file. // If stat fails (e.g. dir just created, owner is root), the file stays // root-owned and 0600 — still unreadable by others, just not agent-readable. // Log a warning so operators can diagnose. match std::fs::metadata(&state_dir) { Ok(meta) => { if let Err(e) = std::os::unix::fs::chown(&path, Some(meta.uid()), Some(meta.gid())) { tracing::warn!( agent = %agent_name, path = %path.display(), error = %e, "write_agent_state_file: chown failed" ); } } Err(e) => { tracing::warn!( agent = %agent_name, error = %e, "write_agent_state_file: stat state_dir failed, leaving file root-owned" ); } } tracing::info!(agent = %agent_name, file = %filename, "wrote agent state file"); Ok((String::new(), String::new())) } /// btrfs superblock magic, as reported by `statfs(2)`'s `f_type`. const BTRFS_SUPER_MAGIC: i64 = 0x9123_683E; /// Inode number of a btrfs subvolume root (`BTRFS_FIRST_FREE_OBJECTID`). /// Every subvolume's top directory has this inode; plain directories do /// not, so `statfs == btrfs && st_ino == 256` reliably identifies a /// subvolume root. const BTRFS_SUBVOL_ROOT_INO: u64 = 256; /// Whether `path` lives on a btrfs filesystem (via `statfs(2)`). fn is_on_btrfs(path: &Path) -> Result { use std::os::unix::ffi::OsStrExt as _; let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()) .with_context(|| format!("path {} has an interior null byte", path.display()))?; // SAFETY: `c_path` is a valid NUL-terminated C string that outlives the // call; `statfs` only writes into the zero-initialised `buf`. let mut buf: libc::statfs = unsafe { std::mem::zeroed() }; let rc = unsafe { libc::statfs(c_path.as_ptr(), &raw mut buf) }; if rc != 0 { return Err(std::io::Error::last_os_error()) .with_context(|| format!("statfs {}", path.display())); } Ok(buf.f_type == BTRFS_SUPER_MAGIC) } /// Whether `path` is the root of a btrfs subvolume (on btrfs and inode 256). fn is_btrfs_subvolume(path: &Path) -> bool { use std::os::unix::fs::MetadataExt as _; let on_btrfs = is_on_btrfs(path).unwrap_or(false); let ino_match = std::fs::metadata(path).is_ok_and(|m| m.ino() == BTRFS_SUBVOL_ROOT_INO); on_btrfs && ino_match } /// `EnsureAgentSubvolume` — make the agent's state root a btrfs subvolume /// when the FS supports it. Idempotent + progressive: no-op when the root /// already exists or the FS isn't btrfs. See the wire doc on the variant. async fn ensure_agent_subvolume(agent_name: &str) -> Result<(String, String)> { use std::os::unix::fs::MetadataExt as _; let root = PathBuf::from(AGENT_STATE_ROOT); let agent_root = root.join(agent_name); // Progressive: existing agents (plain dir OR already a subvol) are left // untouched — never auto-migrated. if agent_root.exists() { return Ok((String::new(), String::new())); } // Only btrfs supports subvolumes; on anything else hive-c0re's normal // `create_dir_all` makes a plain dir (the pre-subvolume behaviour). The // parent must exist for both statfs and `btrfs subvolume create`. std::fs::create_dir_all(&root) .with_context(|| format!("create agents root {}", root.display()))?; if !is_on_btrfs(&root)? { return Ok((String::new(), String::new())); } let out = Command::new("btrfs") .args(["subvolume", "create"]) .arg(&agent_root) .output() .await .with_context(|| format!("spawn btrfs subvolume create {}", agent_root.display()))?; if !out.status.success() { bail!( "btrfs subvolume create {} failed: {}", agent_root.display(), String::from_utf8_lossy(&out.stderr).trim() ); } // The subvol root is created root-owned, but hive-c0re (the `hive-core` // user) must be able to mkdir state/ claude/ harness/ inside it — exactly // as it would in a plain dir. Chown it to AGENT_STATE_ROOT's owner // (hive-core). This MUST succeed: a root-owned subvol would make the // downstream dir creation fail with a confusing permission error, and the // c0re-side exists-check would then skip re-running this op on retry, // wedging the agent. So on any failure roll the subvol back and bail — the // create path surfaces a clear error and a retry starts clean. let chown_result = std::fs::metadata(&root) .with_context(|| format!("stat agents root {} for ownership", root.display())) .and_then(|meta| { std::os::unix::fs::chown(&agent_root, Some(meta.uid()), Some(meta.gid())).with_context( || format!("chown subvol {} to agents-root owner", agent_root.display()), ) }); if let Err(e) = chown_result { // Best-effort rollback so we never leave a root-owned subvol behind. let _ = Command::new("btrfs") .args(["subvolume", "delete"]) .arg(&agent_root) .output() .await; return Err(e.context(format!( "rolled back subvolume {} after chown failed", agent_root.display() ))); } tracing::info!(agent = %agent_name, path = %agent_root.display(), "created agent state subvolume"); Ok((String::new(), String::new())) } /// `DeleteAgentSubvolume` — delete the agent's state root iff it's a btrfs /// subvolume (purge path only). No-op for plain dirs / missing paths; /// hive-c0re's own `remove_dir_all` covers those. See the wire doc. async fn delete_agent_subvolume(agent_name: &str) -> Result<(String, String)> { let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); if !is_btrfs_subvolume(&agent_root) { return Ok((String::new(), String::new())); } let out = Command::new("btrfs") .args(["subvolume", "delete"]) .arg(&agent_root) .output() .await .with_context(|| format!("spawn btrfs subvolume delete {}", agent_root.display()))?; if !out.status.success() { bail!( "btrfs subvolume delete {} failed: {}", agent_root.display(), String::from_utf8_lossy(&out.stderr).trim() ); } tracing::info!(agent = %agent_name, path = %agent_root.display(), "deleted agent state subvolume"); Ok((String::new(), String::new())) } /// `EnsureBtrfsQuota` — enable btrfs qgroup accounting on the filesystem /// holding `AGENT_STATE_ROOT`. Idempotent + statfs-gated (no-op off btrfs). /// Operator opt-in only; see the wire doc. async fn ensure_btrfs_quota() -> Result<(String, String)> { let root = PathBuf::from(AGENT_STATE_ROOT); std::fs::create_dir_all(&root) .with_context(|| format!("create agents root {}", root.display()))?; if !is_on_btrfs(&root)? { // Non-btrfs host: quota/qgroups don't apply. No-op success so the // operator-facing verb degrades cleanly. return Ok(( String::new(), "filesystem is not btrfs — quota not applicable".to_owned(), )); } let out = Command::new("btrfs") .args(["quota", "enable"]) .arg(&root) .output() .await .with_context(|| format!("spawn btrfs quota enable {}", root.display()))?; if !out.status.success() { bail!( "btrfs quota enable {} failed: {}", root.display(), String::from_utf8_lossy(&out.stderr).trim() ); } tracing::info!(path = %root.display(), "enabled btrfs qgroup accounting"); Ok((String::new(), String::new())) } /// `ReadSubvolumeUsage` — return an agent subvolume's qgroup rows /// (`btrfs qgroup show -f --raw <…/agent_name>`) verbatim in stdout for /// hive-c0re to parse. `-f` lists the qgroups impacting the given path, /// excluding ancestral qgroups (per btrfs-qgroup-show(8)) — so it scopes /// to this subvolume and never mixes in other agents'. hive-c0re then /// selects the level-0 (`0/`) leaf row. See the wire doc. async fn read_subvolume_usage(agent_name: &str) -> Result<(String, String)> { let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); let out = Command::new("btrfs") .args(["qgroup", "show", "-f", "--raw"]) .arg(&agent_root) .output() .await .with_context(|| format!("spawn btrfs qgroup show {}", agent_root.display()))?; if !out.status.success() { // The common failure is "quota not enabled" — pass the stderr // through so hive-c0re can surface it gracefully. bail!( "btrfs qgroup show {} failed: {}", agent_root.display(), String::from_utf8_lossy(&out.stderr).trim() ); } Ok(( String::from_utf8_lossy(&out.stdout).into_owned(), String::new(), )) } /// Best-effort removal of a leftover migration path from a prior aborted /// upgrade: try `btrfs subvolume delete` (in case it's a half-created /// subvolume) then a plain recursive remove. Both failures are ignored — /// the path may simply not exist. async fn cleanup_stale_path(path: &Path) { if path.exists() { let _ = Command::new("btrfs") .args(["subvolume", "delete"]) .arg(path) .output() .await; let _ = std::fs::remove_dir_all(path); } } /// Stage a populated subvolume at `tmp` mirroring `agent_root`: create the /// subvolume, copy `agent_root`'s contents into it preserving /// ownership/permissions/xattrs, then match the subvolume root's owner + mode /// to the original. On any failure the partially-staged `tmp` is cleaned up /// (so the caller can bail with the original dir still untouched). async fn stage_upgrade_subvolume(agent_root: &Path, tmp: &Path) -> Result<()> { use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; // Fresh subvolume to receive the copy. let out = Command::new("btrfs") .args(["subvolume", "create"]) .arg(tmp) .output() .await .with_context(|| format!("spawn btrfs subvolume create {}", tmp.display()))?; if !out.status.success() { bail!( "btrfs subvolume create {} failed: {}", tmp.display(), String::from_utf8_lossy(&out.stderr).trim() ); } // Copy contents preserving everything (`-a` = --preserve=all → mode, // ownership, timestamps, links, xattrs); reflink for fast CoW clones on the // same btrfs. `/.` copies the directory's contents (incl. dotfiles) // into the subvolume rather than nesting it. let copy = Command::new("cp") .arg("-a") .arg("--reflink=auto") .arg(format!("{}/.", agent_root.display())) .arg(tmp) .output() .await .with_context(|| format!("spawn cp into {}", tmp.display()))?; if !copy.status.success() { cleanup_stale_path(tmp).await; bail!( "copy {} -> {} failed (original left untouched): {}", agent_root.display(), tmp.display(), String::from_utf8_lossy(©.stderr).trim() ); } // Match the new subvolume root's ownership + mode to the original dir. // `cp -a /.` copies the *contents* but the subvolume root keeps its // create-time root ownership, so set it explicitly — the swapped-in // subvolume must be indistinguishable from the original to hive-c0re. let apply = std::fs::metadata(agent_root) .with_context(|| format!("stat {} for ownership", agent_root.display())) .and_then(|m| { std::os::unix::fs::chown(tmp, Some(m.uid()), Some(m.gid())) .with_context(|| format!("chown {} to match original", tmp.display()))?; std::fs::set_permissions(tmp, std::fs::Permissions::from_mode(m.mode())) .with_context(|| format!("chmod {} to match original", tmp.display()))?; Ok(()) }); if let Err(e) = apply { cleanup_stale_path(tmp).await; return Err(e.context("upgrade aborted before swap; original left untouched")); } Ok(()) } /// `UpgradeAgentSubvolume` — convert an existing plain-dir agent state root /// into a btrfs subvolume in place. Operator opt-in; the caller (hivectl) /// stops the agent first and restarts it after. See the wire doc. /// /// Migration: stage a sibling subvolume mirroring the dir /// ([`stage_upgrade_subvolume`]), then rename the original aside and the /// subvolume into place, then remove the original. Any failure before the /// rename-swap leaves the original dir untouched. async fn upgrade_agent_subvolume(agent_name: &str) -> Result<(String, String)> { let root = PathBuf::from(AGENT_STATE_ROOT); let agent_root = root.join(agent_name); if !agent_root.exists() { // A crash between the two swap renames (original → `..old` // succeeded, `..migrating` → agent_root did not) leaves the // agent root missing but the original data intact under `..old`. // Point at the recovery rather than a bare "nothing to do" so the // operator isn't left guessing where the data went. let old = root.join(format!(".{agent_name}.old")); if old.exists() { bail!( "no state dir at {agent} — but {old} holds the original data from an \ interrupted upgrade (host crashed mid-swap). Restore it with \ `mv {old} {agent}`, then re-run the upgrade.", agent = agent_root.display(), old = old.display(), ); } bail!( "no state dir to upgrade at {} — nothing to do", agent_root.display() ); } // Idempotent: already a subvolume → nothing to do. if is_btrfs_subvolume(&agent_root) { return Ok((String::new(), String::new())); } if !is_on_btrfs(&root)? { bail!( "{} is not on btrfs — subvolumes are unsupported, cannot upgrade", root.display() ); } // Sibling temp paths on the same filesystem (so the copy can reflink and // the swap renames are atomic). Leading dots keep them out of the agent // namespace (`validate_agent_name` rejects dot-prefixed names). let tmp = root.join(format!(".{agent_name}.migrating")); let old = root.join(format!(".{agent_name}.old")); // Clear any debris from a previously interrupted run before starting. cleanup_stale_path(&tmp).await; cleanup_stale_path(&old).await; stage_upgrade_subvolume(&agent_root, &tmp).await?; // Swap. `rename` is atomic within a filesystem. The window between the // two renames is the only unsafe point: a crash there leaves the agent // root missing but both `.old` (original) and the new subvolume present // — recoverable by hand, hence the loud logging. if let Err(e) = std::fs::rename(&agent_root, &old) { cleanup_stale_path(&tmp).await; return Err(anyhow::Error::new(e).context(format!( "rename {} -> {} failed; original left untouched", agent_root.display(), old.display() ))); } if let Err(e) = std::fs::rename(&tmp, &agent_root) { // Restore the original from its renamed-aside copy. let restored = std::fs::rename(&old, &agent_root).is_ok(); cleanup_stale_path(&tmp).await; return Err(anyhow::Error::new(e).context(format!( "rename {} -> {} failed; original {}", tmp.display(), agent_root.display(), if restored { "restored" } else { "COULD NOT BE RESTORED — manual recovery needed" } ))); } // 5. Success: drop the original (a plain dir) and report. if let Err(e) = std::fs::remove_dir_all(&old) { // The migration succeeded; a leftover `.old` is cosmetic. Warn only. tracing::warn!( agent = %agent_name, path = %old.display(), "upgraded subvolume but failed to remove old dir: {e}" ); } tracing::info!( agent = %agent_name, path = %agent_root.display(), "upgraded agent state dir to btrfs subvolume" ); Ok(( format!("upgraded {} to a btrfs subvolume", agent_root.display()), String::new(), )) } /// `SetSubvolumeQuota` — set or clear a qgroup size limit on an agent /// subvolume (`btrfs qgroup limit <…/agent_name>`). See the /// wire doc. async fn set_subvolume_quota( agent_name: &str, limit_bytes: Option, ) -> Result<(String, String)> { let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); let limit = limit_bytes.map_or_else(|| "none".to_owned(), |n| n.to_string()); let out = Command::new("btrfs") .args(["qgroup", "limit", &limit]) .arg(&agent_root) .output() .await .with_context(|| format!("spawn btrfs qgroup limit {}", agent_root.display()))?; if !out.status.success() { bail!( "btrfs qgroup limit {limit} {} failed: {}", agent_root.display(), String::from_utf8_lossy(&out.stderr).trim() ); } tracing::info!(agent = %agent_name, %limit, "set agent subvolume quota"); Ok((String::new(), String::new())) } /// Validate a single argument destined for `forgejo admin`. Rejects /// null bytes and newlines (which could corrupt the subprocess args list /// or log output). Shell metacharacters are harmless since the command /// is spawned directly (no shell), but we reject them defensively. fn validate_forge_admin_arg(arg: &str) -> Result<()> { if arg.bytes().any(|b| b == 0 || b == b'\n' || b == b'\r') { bail!("forge admin arg {arg:?} contains null byte or newline"); } Ok(()) } /// Run `forgejo admin ` inside the `hive-forge` container as the /// `forgejo` unix user. Requires root (for nsenter into the container's /// namespaces). Returns `(stdout, stderr)`. async fn run_forge_admin(args: &[String]) -> Result<(String, String)> { let mut cmd_args: Vec<&str> = vec![ "run", "hive-forge", "--", "runuser", "-u", "forgejo", "--", "forgejo", "--work-path", "/var/lib/forgejo", "admin", ]; for a in args { cmd_args.push(a.as_str()); } let out = Command::new("nixos-container") .args(&cmd_args) .output() .await .context("invoke nixos-container run hive-forge -- forgejo admin")?; let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); for line in stdout.lines() { tracing::info!(target: "forgejo-admin", "{line}"); } for line in stderr.lines() { tracing::warn!(target: "forgejo-admin", "{line}"); } if !out.status.success() { bail!( "forgejo admin {} failed ({}): {}", args.join(" "), out.status, stderr.trim() ); } Ok((stdout, stderr)) } /// Invoke `nixos-container` with the given args, log output to journald. async fn container_run(args: &[&str]) -> Result<(String, String)> { let out = Command::new("nixos-container") .args(args) .output() .await .context("invoke nixos-container")?; let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); for line in stdout.lines() { tracing::info!(target: "nixos-container", "{line}"); } for line in stderr.lines() { tracing::warn!(target: "nixos-container", "{line}"); } if !out.status.success() { bail!( "nixos-container {} failed ({}): {}", args.join(" "), out.status, stderr.trim() ); } Ok((stdout, stderr)) } /// Invoke `nixos-container` with the given args and forward output lines /// to the caller as `PrivEvent::Line` messages in real time, logging each /// line to journald as it arrives. Returns `(String::new(), String::new())` /// on success (all output was streamed); the error string includes stderr /// tail on failure. async fn container_run_streaming( args: &[&str], writer: &mut OwnedWriteHalf, ) -> Result<(String, String)> { use tokio::io::AsyncBufReadExt as _; use tokio::process::Command; let mut child = Command::new("nixos-container") .args(args) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn() .context("invoke nixos-container (streaming)")?; let stdout = child.stdout.take().expect("stdout piped"); let stderr = child.stderr.take().expect("stderr piped"); let mut stdout_lines = BufReader::new(stdout).lines(); let mut stderr_lines = BufReader::new(stderr).lines(); // Collect stderr for the error message; stream both to the client. let mut stderr_buf = String::new(); // Drive stdout and stderr concurrently. `tokio::select!` interleaves // them without bias — both streams drain at roughly the same rate // as the subprocess produces output. loop { tokio::select! { line = stdout_lines.next_line() => { match line { Ok(Some(l)) => { tracing::info!(target: "nixos-container", "{l}"); write_line_event(writer, PrivStream::Stdout, &l).await; } Ok(None) => break, Err(e) => { tracing::warn!(error = %e, "nixos-container stdout read error"); break; } } } line = stderr_lines.next_line() => { match line { Ok(Some(l)) => { tracing::warn!(target: "nixos-container", "{l}"); write_line_event(writer, PrivStream::Stderr, &l).await; if !stderr_buf.is_empty() { stderr_buf.push('\n'); } stderr_buf.push_str(&l); } Ok(None) => {} Err(e) => { tracing::warn!(error = %e, "nixos-container stderr read error"); } } } } } // Drain any remaining stderr after stdout closed. while let Ok(Some(l)) = stderr_lines.next_line().await { tracing::warn!(target: "nixos-container", "{l}"); write_line_event(writer, PrivStream::Stderr, &l).await; if !stderr_buf.is_empty() { stderr_buf.push('\n'); } stderr_buf.push_str(&l); } let status = child.wait().await.context("wait nixos-container")?; if !status.success() { // Only the last stderr line is embedded — the full stderr was // already forwarded line-by-line as PrivEvent::Line messages and // is captured in build_logs.sqlite by the caller. Keeping the // error message short avoids bloating the anyhow chain. bail!( "nixos-container {} failed ({}): {}", args.join(" "), status, stderr_buf.lines().last().unwrap_or("").trim() ); } Ok((String::new(), String::new())) } /// Read a container's journal as root via `journalctl -M`. Returns /// `(stdout, stderr)`. Unlike `container_run` a non-zero exit is *not* a /// hard error — journalctl's own diagnostic (folded into `stderr` with /// the exit status) is what the caller surfaces to the operator, so the /// helper never bails. async fn read_container_journal(container: &str, query: &JournalQuery) -> Result<(String, String)> { let mut args: Vec = vec![ "-M".to_owned(), container.to_owned(), "--no-pager".to_owned(), format!("--output={}", query.output.as_journalctl()), "-n".to_owned(), query.lines.to_string(), ]; if query.boot { args.push("-b".to_owned()); } if let Some(u) = &query.unit { args.push("-u".to_owned()); args.push(u.clone()); } if let Some(p) = &query.priority { args.push("-p".to_owned()); args.push(p.clone()); } // `--grep=`/`--since=`/`--until=` use the `=`-joined form so a value // can never be parsed as a separate journalctl flag. if let Some(g) = &query.grep { args.push(format!("--grep={g}")); } if let Some(s) = &query.since { args.push(format!("--since={s}")); } if let Some(u) = &query.until { args.push(format!("--until={u}")); } let out = Command::new("journalctl") .args(&args) .output() .await .context("invoke journalctl -M")?; let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); let stderr = if out.status.success() { String::from_utf8_lossy(&out.stderr).into_owned() } else { format!( "journalctl -M {container} exited {}: {}", out.status, String::from_utf8_lossy(&out.stderr).trim() ) }; Ok((stdout, stderr)) } /// Synchronise the nginx unit inside the `hive-gateway` container. /// /// Queries `ActiveState` via `systemctl --machine=hive-gateway` (requires /// root — machine-bus transport enters the container namespace), then /// dispatches: /// - `active` → `systemctl reload nginx` (SIGHUP, zero-downtime) /// - `failed` → `systemctl reset-failed nginx` + `systemctl start nginx` /// - otherwise → `systemctl start nginx` /// /// Returns `(String::new(), String::new())` on success so it fits the /// `exec` return type directly. async fn sync_gateway_nginx() -> Result<(String, String)> { let state_out = Command::new("systemctl") .args([ "--machine=hive-gateway", "show", "--property=ActiveState", "--value", "nginx", ]) .output() .await .context("query nginx ActiveState in hive-gateway")?; if !state_out.status.success() { tracing::warn!( exit_code = ?state_out.status.code(), stderr = %String::from_utf8_lossy(&state_out.stderr).trim(), "systemctl show ActiveState exited non-zero — gateway container may be down" ); } let state = String::from_utf8_lossy(&state_out.stdout).trim().to_owned(); // State-aware dispatch: reload when running; reset+start after // start-limit failure; plain start when inactive or unknown. match state.as_str() { "active" => { let out = Command::new("systemctl") .args(["--machine=hive-gateway", "reload", "nginx"]) .output() .await .context("reload nginx in hive-gateway")?; if !out.status.success() { bail!( "gateway nginx reload failed ({}): {}", out.status, String::from_utf8_lossy(&out.stderr).trim() ); } } "failed" => { // Clear start-limit so the next start can proceed. let _ = Command::new("systemctl") .args(["--machine=hive-gateway", "reset-failed", "nginx"]) .status() .await; let out = Command::new("systemctl") .args(["--machine=hive-gateway", "start", "nginx"]) .output() .await .context("start nginx after reset-failed in hive-gateway")?; if !out.status.success() { bail!( "gateway nginx start (after reset-failed) failed ({}): {}", out.status, String::from_utf8_lossy(&out.stderr).trim() ); } } _ => { // inactive, activating, deactivating, unknown — just start. let out = Command::new("systemctl") .args(["--machine=hive-gateway", "start", "nginx"]) .output() .await .context("start nginx in hive-gateway")?; if !out.status.success() { bail!( "gateway nginx start failed (state={state:?}) ({}): {}", out.status, String::from_utf8_lossy(&out.stderr).trim() ); } } } Ok((String::new(), String::new())) } /// Return the system container name for a logical agent name. /// All agents (including the manager) use the `h-` prefix. fn container_system_name(name: &str) -> String { format!("{AGENT_PREFIX}{name}") } /// Path of the per-agent unix-socket dir on the host. fn socket_dir_path(agent_name: &str) -> PathBuf { PathBuf::from(format!("{SOCKET_DIR_ROOT}/{agent_name}")) } /// Validate a logical agent name (the name hive-c0re uses internally, /// before the `h-` container prefix is applied). fn validate_agent_name(name: &str) -> Result<()> { validate_name_chars(name)?; Ok(()) } /// Validate a logical agent name and check it maps to a hive-managed container. fn validate_container_name(name: &str) -> Result<()> { if SIBLING_CONTAINERS.contains(&name) { return Ok(()); } validate_name_chars(name)?; Ok(()) } /// Validate a system-level container name (already has `h-` prefix for /// all agents including the manager, or is a sibling service name). fn validate_container_system_name(name: &str) -> Result<()> { if SIBLING_CONTAINERS.contains(&name) { return Ok(()); } if let Some(suffix) = name.strip_prefix(AGENT_PREFIX) { validate_name_chars(suffix)?; return Ok(()); } bail!("container name {name:?} is not managed by hive"); } fn validate_name_chars(name: &str) -> Result<()> { if name.is_empty() || !name .chars() .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') { bail!("invalid name {name:?}: must be non-empty lowercase ascii + digits + hyphens"); } Ok(()) } /// Derive the meta-flake ref for an agent by name. fn agent_flake_ref(name: &str) -> String { format!("{META_DIR}#{name}") } /// Validate a bind-mount path: must be absolute, non-empty, and contain /// no newlines, null bytes, or double-quotes (which would break the /// `EXTRA_NSPAWN_FLAGS="..."` conf line format). fn validate_bind_path(path: &str) -> Result<()> { if path.is_empty() || !path.starts_with('/') || path .bytes() .any(|b| b == 0 || b == b'\n' || b == b'"' || b == b':') { bail!( "invalid bind path {path:?}: must be an absolute path with no colons, newlines, null bytes, or double-quotes" ); } Ok(()) } /// Update `/etc/nixos-containers/.conf`: strips network-isolation /// vars (`PRIVATE_NETWORK`, `HOST_ADDRESS*`, `LOCAL_ADDRESS*`, `HOST_BRIDGE`, /// Update `/etc/nixos-containers/.conf`: strip old network vars, /// write network isolation settings, then append `EXTRA_NSPAWN_FLAGS`. /// When `isolation` is `Some`, writes `PRIVATE_NETWORK=1` + veth wiring; /// when `None`, writes `PRIVATE_NETWORK=0`. fn write_nspawn_flags( container: &str, binds: &[BindMount], isolation: Option<&NetworkIsolation>, load_credentials: &[CredentialMount], ) -> Result<()> { use std::fmt::Write as _; let path = format!("/etc/nixos-containers/{container}.conf"); let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?; let lines: Vec<&str> = original .lines() .filter(|line| { let t = line.trim_start(); !t.starts_with("EXTRA_NSPAWN_FLAGS=") && !t.starts_with("PRIVATE_NETWORK=") && !t.starts_with("HOST_ADDRESS=") && !t.starts_with("LOCAL_ADDRESS=") && !t.starts_with("HOST_ADDRESS6=") && !t.starts_with("LOCAL_ADDRESS6=") && !t.starts_with("HOST_BRIDGE=") }) .collect(); let mut out = lines.join("\n"); if !out.is_empty() { out.push('\n'); } if let Some(iso) = isolation { out.push_str("PRIVATE_NETWORK=1\n"); // HOST_ADDRESS = the bridge gateway IP. nixos-container's // container-side setup only installs a default route // (`ip route add default via $HOST_ADDRESS`) when HOST_ADDRESS is // non-empty; leaving it blank gave the container an address but no // route off the bridge subnet (no internet, no api.anthropic.com). // In bridge mode (HOST_BRIDGE set) the host-side address/route // setup is skipped, so this only affects the container's route — // exactly what we want. let _ = writeln!(out, "HOST_ADDRESS={}", iso.gateway_ip); let _ = writeln!(out, "LOCAL_ADDRESS={}", iso.agent_ip); out.push_str("HOST_ADDRESS6=\n"); out.push_str("LOCAL_ADDRESS6=\n"); let _ = writeln!(out, "HOST_BRIDGE={}", iso.bridge); } else { out.push_str("PRIVATE_NETWORK=0\n"); out.push_str("HOST_ADDRESS=\n"); out.push_str("LOCAL_ADDRESS=\n"); out.push_str("HOST_ADDRESS6=\n"); out.push_str("LOCAL_ADDRESS6=\n"); out.push_str("HOST_BRIDGE=\n"); } let mut flags: Vec = binds .iter() .map(|b| { let flag = if b.read_only { "--bind-ro" } else { "--bind" }; format!("{flag}={}:{}", b.host_path, b.container_path) }) .collect(); // Credential forwarding: nspawn loads each host secret into the // container's credential store under ``; inner units inherit it // via `LoadCredential=`. Validated (name charset + bind-path // rules) in handle_write_nspawn_flags above. for cred in load_credentials { flags.push(format!( "--load-credential={}:{}", cred.name, cred.host_path )); } let flags_joined = flags.join(" "); let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\""); std::fs::write(&path, out).with_context(|| format!("write {path}"))?; // DNS marker for the in-container resolver oneshot. nixos-container // copies the *host's* /etc/resolv.conf into the container at every // start (its host resolver — e.g. 127.0.0.53 — is unreachable from a // private netns, and isn't authoritative for the hive's own zones // anyway). The `hyperhive-isolated-dns` oneshot in harness-base.nix // rewrites resolv.conf to point at the bridge resolver, but only when // this marker exists; it carries the gateway IP so the container // doesn't have to re-derive it. Written on isolate, removed otherwise, // so the same shared container toplevel behaves correctly in both modes. write_bridge_dns_marker(container, isolation)?; Ok(()) } /// Path to the in-container DNS marker (the container's own `/etc`). fn bridge_dns_marker_path(container: &str) -> String { format!("/var/lib/nixos-containers/{container}/etc/hyperhive-bridge-dns") } /// Write (isolated) or remove (host-netns) the bridge-DNS marker the /// `hyperhive-isolated-dns` oneshot keys off. The marker file contains /// just the gateway IP. Best-effort on removal (absence is the goal). fn write_bridge_dns_marker(container: &str, isolation: Option<&NetworkIsolation>) -> Result<()> { let path = bridge_dns_marker_path(container); match isolation { Some(iso) => { std::fs::write(&path, format!("{}\n", iso.gateway_ip)) .with_context(|| format!("write bridge-DNS marker {path}"))?; } None => match std::fs::remove_file(&path) { Ok(()) => {} Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(e) => return Err(e).with_context(|| format!("remove bridge-DNS marker {path}")), }, } Ok(()) }