//! `nixos-container` lifecycle + per-agent config flake generation. mod agent_identity; mod git; mod host_config; mod setup; #[cfg(test)] mod tests; pub use git::{git_command, git_command_authed}; pub use host_config::write_dropins; pub use setup::{ensure_agent_state_subvolume, ensure_claude_dir, ensure_state_dir}; use anyhow::{Context, Result, bail}; use tokio::process::Command; use crate::coordinator::{AgentPaths, HiveEnv}; // `AGENT_PREFIX` (`h-`) + `container_name` are shared with the host-side // `hivectl` CLI, so they live in `hive-host-sock`; re-exported here so this // module stays the daemon's facade for its own callsites. pub use hive_host_sock::{AGENT_PREFIX, container_name}; /// Max agent-name length. `nixos-container` caps the total container name at /// 11 chars (it gets encoded into network interface names) and `AGENT_PREFIX` /// (`h-`) takes 2, so the agent name itself is at most `MAX_AGENT_NAME` chars. pub const MAX_AGENT_NAME: usize = 9; /// Mount point of the per-agent runtime directory inside the container. pub const CONTAINER_RUNTIME_MOUNT: &str = "/run/hive"; /// Where the per-agent Claude credentials dir mounts inside the /// container. The harness service runs as a non-root unix user /// whose home is `/home//`, so the mount path varies per /// agent — `container_claude_mount(name)` returns /// `/home//.claude` for every agent including the manager. /// `claude` inside the container reads /// `$HOME/.claude` and the service environment sets `HOME` to the /// same path, so the OAuth session survives container restarts. #[must_use] pub fn container_claude_mount(name: &str) -> String { format!("/home/{name}/.claude") } pub use hive_meta_flake::agent_web_port; /// Extract the bridge gateway IP from `HIVE_NETWORK_SUBNET`. /// /// `HIVE_NETWORK_SUBNET` carries the host-side bridge address verbatim /// (e.g. `10.42.0.1/24`), **not** the canonical network address — see /// the note in `set_nspawn_flags` + `docs/4-implementation/networking/network.md`. The IP part is /// therefore the bridge IP itself: the host end of the bridge, the /// default-route target for isolated containers, and the address the /// hive dnsmasq resolver binds. Returns the dotted-decimal IP with the /// `/` stripped, or `None` if the input isn't a valid /// `/` pair. /// /// Deliberately returns the operator-configured address verbatim rather /// than deriving `network + 1`: an operator who sets `bridgeIp` to a /// non-`.1` host address (e.g. `10.42.0.254`) runs the bridge + resolver /// there, so that — not `.1` — is the real gateway. #[must_use] pub fn bridge_gateway_ip(subnet_cidr: &str) -> Option { let (ip_str, prefix_str) = subnet_cidr.split_once('/')?; // Validate the prefix is a sane IPv4 CIDR length and the address is // dotted-decimal IPv4 so a malformed `HIVE_NETWORK_SUBNET` can't // smuggle a bogus HOST_ADDRESS into the nspawn conf. let prefix_len: u32 = prefix_str.parse().ok()?; if prefix_len > 32 { return None; } let octets: Vec = ip_str .split('.') .map(|o| o.parse::().ok()) .collect::>>()?; if octets.len() != 4 { return None; } Some(ip_str.to_owned()) } /// Build the network-isolation settings every container is configured /// with, from the variables the hyperhive NixOS module derives from /// `services.hyperhive.network.*` onto the `hive-c0re` unit. /// /// Isolation is the only supported mode: the on/off toggle is gone, so /// there is no non-isolated branch to fall back to and a missing or /// malformed value means the daemon is misconfigured — not that a /// container should quietly come up sharing the host's netns. Silently /// degrading here dropped a security boundary with nothing in the log /// to say so. /// /// Split from [`network_isolation_from_env`] so the parsing is testable /// without touching process environment. pub fn network_isolation_from_vars( bridge: Option<&str>, subnet: Option<&str>, ) -> Result { let bridge = bridge.filter(|s| !s.is_empty()).context( "HIVE_NETWORK_BRIDGE is unset or empty — the hyperhive NixOS module sets it \ from services.hyperhive.network.bridgeName on the hive-c0re unit, so this \ means the daemon is running outside its unit or with a broken module \ evaluation", )?; let subnet = subnet.filter(|s| !s.is_empty()).context( "HIVE_NETWORK_SUBNET is unset or empty — the hyperhive NixOS module sets it \ from services.hyperhive.network.bridgeIp and .bridgePrefixLength on the \ hive-c0re unit, so this means the daemon is running outside its unit or \ with a broken module evaluation", )?; let gateway_ip = bridge_gateway_ip(subnet).with_context(|| { format!( "HIVE_NETWORK_SUBNET={subnet} is not a valid / pair; \ it comes from services.hyperhive.network.bridgeIp + bridgePrefixLength" ) })?; Ok(hive_priv_sock::NetworkIsolation { bridge: bridge.to_owned(), gateway_ip, }) } /// [`network_isolation_from_vars`] over the real process environment. /// /// Called once at daemon startup so a bad value fails the unit loudly, /// and again per container — the variables are process-global, so the /// second call cannot start failing once the first has passed. pub fn network_isolation_from_env() -> Result { let bridge = std::env::var("HIVE_NETWORK_BRIDGE").ok(); let subnet = std::env::var("HIVE_NETWORK_SUBNET").ok(); network_isolation_from_vars(bridge.as_deref(), subnet.as_deref()) } /// The hive's facts every container is handed at runtime, so that no agent /// build carries a value that differs between hosts. The name, domain, egress /// proxy and its direct hosts, OTEL resource labels and queue address come from hive-c0re's own unit environment /// (`nix/host-modules/hive-c0re/environment.nix`), where an unset or empty /// variable is `None`; the dashboard port and pinned claude from `hive`. fn hive_facts(hive: &HiveEnv) -> hive_priv_sock::HiveFacts { hive_facts_from(hive, |name| std::env::var(name).ok()) } /// [`hive_facts`] over a given environment lookup, for tests. fn hive_facts_from( hive: &HiveEnv, var: impl Fn(&str) -> Option, ) -> hive_priv_sock::HiveFacts { let read = |name: &str| var(name).filter(|v| !v.is_empty()); hive_priv_sock::HiveFacts { name: read("HYPERHIVE_HIVE_NAME"), domain: read("HYPERHIVE_HIVE_DOMAIN"), dashboard_port: Some(hive.dashboard_port), claude_code_path: hive.claude_code_path.clone(), otel_extra_resource_attributes: read("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES"), agent_nats_url: read("HIVE_AGENT_NATS_URL"), squid_proxy: read("HIVE_SQUID_PROXY"), egress_no_proxy: read("HIVE_EGRESS_NO_PROXY"), } } fn validate(name: &str) -> Result<()> { if name.is_empty() { bail!("agent name must not be empty"); } if name.len() > MAX_AGENT_NAME { bail!( "agent name '{name}' is too long ({} chars); max {MAX_AGENT_NAME}", name.len() ); } Ok(()) } /// First name (≠ `self_name`) currently running whose hashed port /// matches this agent's. The harness inside the colliding container /// would otherwise loop on `AddrInUse` forever; we surface the /// conflict here so spawn / rebuild fails loudly with an actionable /// message instead. async fn port_collision(self_name: &str) -> Result> { let port = agent_web_port(self_name); let raw = list() .await .context("listing containers for the port-collision check")?; for c in raw { let Some(other) = c.strip_prefix(AGENT_PREFIX) else { continue; }; if other == self_name { continue; } if agent_web_port(other) == port && is_running(other).await { return Ok(Some(other.to_owned())); } } Ok(None) } /// First-spawn pre-create provisioning — the job queue's `Provision` /// node. Fail fast on a port collision, set up the state /// subvolume + claude/notes dirs, then register the new /// agent in the host-side files that follow the agent set /// ([`crate::agent_set::refresh`]). Does NOT /// create the container — that's `create_only` / the `Create` node. pub async fn provision_container(name: &str, paths: &AgentPaths) -> Result<()> { validate(name)?; if let Some(other) = port_collision(name).await? { bail!( "port {} is already taken by '{other}' — rename one of them and retry", agent_web_port(name) ); } // Read before the build: an archetype agent gets no config repo, and // the build takes it from the archetype. crate::archetypes::refresh(name).await?; ensure_agent_state_subvolume(name).await?; ensure_claude_dir(&paths.claude)?; ensure_state_dir(&paths.notes)?; let agents = agents_after_spawn(name).await?; crate::agent_set::refresh(&agents).await } /// The `nixos-container create` proper — the job queue's `Create` node. /// Assumes `provision_container` already registered the agent. /// `toplevel` is the store path of the system closure to create it from. pub async fn create_only(name: &str, toplevel: &str) -> Result<()> { priv_run_inner("create", name, Some(toplevel)).await } /// Rebuild-path preamble, run by the job queue's `Prebuild` node: fail fast on /// a port collision, then make sure the state dirs exist. Container untouched. pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> { validate(name)?; if let Some(other) = port_collision(name).await? { bail!( "port {} is already taken by '{other}' — rename one of them and retry", agent_web_port(name) ); } ensure_agent_state_subvolume(name).await?; ensure_claude_dir(&paths.claude)?; ensure_state_dir(&paths.notes)?; Ok(()) } /// Profile-swap for an existing, stopped container: re-apply the /// drop-ins, then point the container at the closure `toplevel`. The job /// queue's `Swap` node. Requires the container stopped (the queue's /// `StopForUpdate` upstream); does NOT start it — the DAG's tail `Reconcile` owns /// bringing the agent back to its wanted power state. `toplevel` is as for /// [`create_only`]. pub async fn swap_update( name: &str, hive: &HiveEnv, paths: &AgentPaths, toplevel: &str, ) -> Result<()> { write_dropins(name, hive, paths).await?; priv_run_inner("update", name, Some(toplevel)).await } /// Build the `AgentSpec` list for [`crate::agent_set::refresh`] from /// `nixos-container list` + a hypothetical extra name not yet in the list (for spawn /// where the new agent's container doesn't exist yet). Pass empty /// `name_to_add` from rebuild paths where the agent is already in the /// container list. /// /// Propagates errors from `list()` rather than swallowing them. /// Using `.unwrap_or_default()` here would silently produce an empty /// agent list when `nixos-container list` fails (priv helper down, race), /// which would drop every agent from the topology and the gateway. Callers /// that can tolerate failures (e.g. migration) /// handle the `Err` themselves with `.unwrap_or_default()`. async fn agents_for_meta(name_to_add: Option<&str>) -> Result> { let containers = list().await?; let archetypes = crate::archetypes::read_at(&crate::archetypes::path())?; let mut out: Vec = containers .into_iter() .filter_map(|c| { let name = c.strip_prefix(AGENT_PREFIX)?.to_owned(); Some(hive_meta_flake::AgentSpec { port: agent_web_port(&name), archetype: archetypes.get(&name).cloned(), pin: None, name, }) }) .collect(); if let Some(extra) = name_to_add && !out.iter().any(|a| a.name == extra) { out.push(hive_meta_flake::AgentSpec { port: agent_web_port(extra), archetype: archetypes.get(extra).cloned(), pin: None, name: extra.to_owned(), }); } out.sort_by(|a, b| a.name.cmp(&b.name)); Ok(out) } async fn agents_after_spawn(name: &str) -> Result> { agents_for_meta(Some(name)).await } /// Public enumeration of currently-existing agents (whatever /// `nixos-container list` says), sorted, no extras. For callers /// outside this module that need to reseed meta after lifecycle /// changes — destroy, startup reconciliation, etc. pub async fn agents_for_meta_listing() -> Result> { agents_for_meta(None).await } /// Per-agent "stop failed" warning guards, keyed by agent name. Cleared the /// next time that agent's container stops cleanly — a `Mutex>` /// rather than a bare `set_boot_warning` because this condition (unlike a /// one-shot boot step) genuinely resolves later, once an operator has /// intervened, and the warning should clear with it instead of surviving /// until the next hive-c0re restart. fn stop_failed_guards() -> &'static std::sync::Mutex> { static REG: std::sync::OnceLock< std::sync::Mutex>, > = std::sync::OnceLock::new(); REG.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new())) } /// The per-agent warning `kind`, leaked on the first call for a name and /// returned unchanged afterwards, whether or not the warning was cleared in /// between. The leak is one short string per distinct agent name. fn stop_kind(name: &str) -> &'static str { static KINDS: std::sync::OnceLock< std::sync::Mutex>, > = std::sync::OnceLock::new(); let mut kinds = KINDS .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new())) .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); kinds .entry(name.to_owned()) .or_insert_with(|| Box::leak(format!("agent_stop_failed_{name}").into_boxed_str())) } /// Stop `name`'s container. See `hive-priv`'s `stop_and_release` for why /// a failure here means the container is likely wedged rather than just /// slow — SIGKILL doesn't recover that case in practice, so this /// surfaces it on the dashboard instead of silently retrying. pub async fn kill(name: &str) -> Result<()> { validate(name)?; let result = priv_run("stop", name).await; // Lock only for the synchronous map update — never held across the // `.await` above. let mut guards = stop_failed_guards() .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); match result { Ok(()) => { // A clean stop clears any earlier "this looked wedged" warning // for the same agent. guards.remove(name); Ok(()) } Err(e) => { tracing::error!(%name, error = %e, "container stop failed"); guards.insert( name.to_owned(), crate::warnings::set_warning( stop_kind(name), "crit", format!("{name}: stop failed — {e}"), ), ); Err(e) } } } /// Start a container. Success is defined as the container's unit reaching /// `active`, **not** the `nixos-container start` exit code — a start-job /// timeout on a slow boot (e.g. DHCP taking ~10s) exits non-zero while /// `container@.service` keeps retrying and the container comes up seconds /// later. So a zero exit returns immediately (already active), and a non-zero /// exit polls the unit state for [`START_SETTLE_TIMEOUT`] before concluding /// the start actually failed. Every caller (dashboard restart, reconcile /// start, the cold-start fallback) gets this truth for free. async fn start(name: &str) -> Result<()> { validate(name)?; if priv_run("start", name).await.is_ok() { return Ok(()); } let container = container_name(name); if wait_until_running(name, START_SETTLE_TIMEOUT).await { tracing::info!( container = %container, "start exited non-zero but the container reached active on systemd's retry; treating as success" ); return Ok(()); } Err(anyhow::anyhow!( "container {container} did not reach active within {}s of start", START_SETTLE_TIMEOUT.as_secs() )) } /// Opaque token produced by [`converge_start_preamble`]. /// [`start_with_fallback`] requires this as proof that the pre-start /// preamble (runtime dir + drop-ins) ran. Dropping the token without /// calling `start_with_fallback` is a no-op. #[must_use = "call lifecycle::start_with_fallback(token) to start the container"] pub struct StartableAgent { name: String, } /// Run the per-agent start preamble: ensure the runtime dir exists and write /// the nspawn / resource-limits drop-ins. Returns a [`StartableAgent`] token /// as typed proof that the preamble ran; pass it to [`start_with_fallback`]. /// Callers that omit this step cannot call `start_with_fallback` — the type /// system makes forgetting the preamble a compile error. /// /// # Errors /// /// Returns an error if `ensure_agent_runtime_dir` or `write_dropins` fails. pub async fn converge_start_preamble( name: &str, hive: &HiveEnv, paths: &AgentPaths, ) -> Result { ensure_agent_runtime_dir(name)?; write_dropins(name, hive, paths).await?; Ok(StartableAgent { name: name.to_owned(), }) } /// Start with the cold-start fallback: when a plain start fails (the /// activation-error shape), retry once via stop + kill + start before /// giving up. Used by the queue's fast-lane `Start` handler and the /// inline start-after-rebuild path. /// See `docs/4-implementation/scheduler/coordinator.md::Cold-start fallback`. /// /// Requires a [`StartableAgent`] token from [`converge_start_preamble`] /// to prove the preamble ran. For internal use within this module (where /// the preamble is already enforced structurally) call /// `start_with_fallback_inner` directly. /// /// # Errors /// /// Propagates the retry's start error (annotated with the original /// failure) when the fallback also fails. pub async fn start_with_fallback(token: StartableAgent) -> Result<()> { start_with_fallback_inner(&token.name).await } /// How long to wait for a container's unit to reach `active` after a /// `nixos-container start` that returned a non-zero exit. A start-job timeout /// on a slow boot (e.g. DHCP taking ~10s) returns an error while /// `container@.service` keeps retrying and the container comes up seconds /// later — so the exit code is NOT authoritative, the unit state is. 60s /// comfortably covers an observed slow boot (the reported incident settled /// ~16s after the "failure") without pinning a queue node on a genuine /// never-boots failure for too long. const START_SETTLE_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(1); /// Poll cadence while waiting for a container to reach `active`. const START_SETTLE_POLL: std::time::Duration = std::time::Duration::from_secs(2); /// Poll [`is_running`] until the container's unit is active or `timeout` /// elapses; returns true as soon as it's active. async fn wait_until_running(name: &str, timeout: std::time::Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; loop { if is_running(name).await { return true; } if tokio::time::Instant::now() >= deadline { return false; } tokio::time::sleep(START_SETTLE_POLL).await; } } /// Internal implementation of the cold-start fallback, behind /// [`start_with_fallback`] (public, token-gated) — the inner form exists for /// callers that have already run the drop-in preamble themselves. /// /// [`start`] already treats unit-active (not the exit code) as success and /// waits out a slow boot, so this only layers the activation-error recovery on /// top: if the container is still down after `start`, tear it down and try /// `start` once more. The teardown prefers a graceful `stop` and only /// escalates to SIGKILL when that `stop` itself fails. async fn start_with_fallback_inner(name: &str) -> Result<()> { validate(name)?; if start(name).await.is_ok() { return Ok(()); } tracing::warn!( %name, "start did not bring the container up; hard-resetting (stop, kill only if that fails) and retrying once" ); // Graceful `stop` first; only escalate to SIGKILL if the stop itself // fails (e.g. a wedged container whose `machinectl poweroff` never // completes). A clean stop is enough on its own — SIGKILL is the // last-resort teardown for when graceful shutdown can't finish. if let Err(stop_err) = priv_run("stop", name).await { tracing::warn!( %name, error = %stop_err, "graceful stop failed; escalating to SIGKILL" ); priv_run("kill", name).await.unwrap_or_else(|e| { tracing::warn!( %name, error = %e, "kill after failed stop also failed (ignored)" ); }); } start(name) .await .with_context(|| format!("cold-start fallback also failed for {name}")) } /// The part of a container unit's systemd state we act on. /// /// Deliberately not the full set: everything outside `Active`/`Failed` is /// one bucket, because the only distinction anything downstream makes is /// *gave up* versus *anything else*. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum UnitState { /// Running. Active, /// systemd gave up on it — most often by exhausting the bounded start /// limit. Its own variant precisely because it is the difference /// between **gave up** and **stopped on purpose**, which every other /// non-running state collapses into. Failed, /// `inactive` (the deliberate stop), `activating`, `deactivating`, an /// unknown unit, or a `systemctl` we could not run at all. Other, } impl UnitState { /// Parse `systemctl is-active`'s stdout. /// /// Split out from the shell-out so the mapping is testable without a /// systemd to ask — the states we care about are the two we name, and /// everything else must land in `Other` rather than being guessed at. fn from_is_active(stdout: &str) -> Self { match stdout.trim() { "active" => Self::Active, "failed" => Self::Failed, _ => Self::Other, } } } /// The container unit's state, in one `systemctl` call. /// /// `is-active` **prints** the state and exits 0 only for `active`, so /// dropping `--quiet` yields both facts from the call [`is_running`] /// already makes. That matters: the status path runs this per agent (see /// the spawn-count note in `socket_server::lifecycle_handlers`), so a /// caller wanting *running* and *failed* must not pay for two subprocesses. pub async fn unit_state(name: &str) -> UnitState { let container = container_name(name); let unit = format!("container@{container}.service"); // Not "the unit is fine" and not "the unit failed" — we simply do not // know, and saying `Failed` here would report a gave-up agent every time // `systemctl` itself was unavailable or hung. systemctl_is_active(&[&unit]) .await .map_or(UnitState::Other, |out| { UnitState::from_is_active(&String::from_utf8_lossy(&out.stdout)) }) } /// True when the container's systemd unit is active. Used by the dashboard /// to gate stop/restart buttons. /// /// Kept boolean on purpose: nearly every caller is reconcile/power logic /// asking "is it up? if not, start it", and that question has two answers. /// A caller that needs to tell *failed* from *stopped* wants /// [`unit_state`] instead. /// /// Unbounded: its callers record or act on the answer, so a hung `systemctl` /// must not read as "not running". Periodic workers use [`running_state`]. pub async fn is_running(name: &str) -> bool { let container = container_name(name); let unit = format!("container@{container}.service"); Command::new("systemctl") .args(["is-active", "--quiet", &unit]) .status() .await .is_ok_and(|s| s.success()) } /// Whether the container's systemd unit is active, for periodic workers that /// must not stall on a hung `systemctl`; `None` when it could not be run or /// did not answer within [`IS_ACTIVE_TIMEOUT`]. pub async fn running_state(name: &str) -> Option { let container = container_name(name); let unit = format!("container@{container}.service"); systemctl_is_active(&["--quiet", &unit]) .await .map(|out| out.status.success()) } /// Whether an agent's harness unit (`hive-agent.service`) is up inside its /// running container. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HarnessState { /// Active, or on its way: activating, deactivating or reloading. Present, /// Loaded and `inactive` or `failed`: nothing is running it and nothing /// is about to. Stopped, /// Not readable: hive-priv is older than the op, the container's systemd /// did not answer, or the unit is not loaded. Unknown, } /// Read the harness unit's state out of `AgentHarness` probe output /// (`LoadState=` and `ActiveState=` lines). fn parse_harness_state(out: &str) -> HarnessState { let field = |key: &str| { out.lines() .find_map(|l| l.strip_prefix(key).and_then(|v| v.strip_prefix('='))) .map(str::trim) }; match (field("LoadState"), field("ActiveState")) { (Some("loaded"), Some("inactive" | "failed")) => HarnessState::Stopped, (Some("loaded"), Some("active" | "activating" | "deactivating" | "reloading")) => { HarnessState::Present } _ => HarnessState::Unknown, } } /// The state of `name`'s harness unit, asked of hive-priv. Bounded by /// [`IS_ACTIVE_TIMEOUT`], because the tick calls it per agent inline. /// /// Any failure reads as [`HarnessState::Unknown`], so reconcile keeps its /// container-only behaviour: a hive-priv that predates the op answers an /// error, and that is logged at `warn` the first time and `debug` after. pub async fn harness_state(name: &str) -> HarnessState { let probe = crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Probe); if let Ok(answer) = tokio::time::timeout(IS_ACTIVE_TIMEOUT, probe).await { harness_of_answer(name, answer) } else { tracing::debug!(%name, "reading the agent's harness state timed out"); HarnessState::Unknown } } /// The state a probe answer means; an error is [`HarnessState::Unknown`]. fn harness_of_answer(name: &str, answer: Result) -> HarnessState { use std::sync::atomic::{AtomicBool, Ordering}; static WARNED: AtomicBool = AtomicBool::new(false); match answer { Ok(out) => parse_harness_state(&out), Err(e) if !WARNED.swap(true, Ordering::Relaxed) => { tracing::warn!(%name, error = ?e, "cannot read the agent's harness state; reconcile only watches the container (logged once)"); HarnessState::Unknown } Err(e) => { tracing::debug!(%name, error = ?e, "cannot read the agent's harness state"); HarnessState::Unknown } } } /// Queue a start of `name`'s harness unit inside its running container. /// /// # Errors /// hive-priv is unreachable, refuses, or `systemctl` fails. pub async fn start_harness(name: &str) -> Result<()> { crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Start) .await .map(drop) } /// How long one `systemctl is-active` may run before the unit's state counts /// as unreadable. Periodic workers make this call per agent inline. const IS_ACTIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); /// Run `systemctl is-active `, killed after [`IS_ACTIVE_TIMEOUT`]. async fn systemctl_is_active(args: &[&str]) -> Option { let output = Command::new("systemctl") .arg("is-active") .args(args) .kill_on_drop(true) .output(); output_within(IS_ACTIVE_TIMEOUT, output, args).await } /// Await a child's output for at most `limit`. `None` when it failed to run /// or timed out; the timeout drops `fut`, which kills a child spawned with /// `kill_on_drop(true)`. async fn output_within( limit: std::time::Duration, fut: impl std::future::Future>, args: &[&str], ) -> Option { if let Ok(result) = tokio::time::timeout(limit, fut).await { result.ok() } else { tracing::warn!( ?args, timeout_s = limit.as_secs(), "systemctl is-active timed out" ); None } } /// Fully tear down a sub-agent's container: stop + remove via `nixos-container /// destroy`, then clean our own systemd drop-in. Leaves it to the caller to /// wipe `/var/lib/hyperhive/...` state and the per-agent runtime dir. /// /// Fails when the container survives the destroy. The destroy DAG purges the /// agent's state only after this returns `Ok`, so a swallowed failure here /// deletes the state of a container that still exists. pub async fn destroy(name: &str) -> Result<()> { validate(name)?; let container = container_name(name); // nixos-container destroy handles stop + removal of /var/lib/nixos-containers/ // and /etc/nixos-containers/.conf. When it errors, the container list // decides: a container that is already gone is still a successful destroy. if let Err(e) = priv_run("destroy", name).await { confirm_gone_after_failed_destroy(e, &container, list().await)?; } // Remove the systemd resource-limits drop-in via hive-priv. if let Err(e) = crate::priv_client::remove_service_dropin(&container).await { tracing::warn!(error = ?e, "remove service drop-in failed (non-fatal)"); } Ok(()) } /// Settle a failed `nixos-container destroy` against the container list taken /// after it. `Ok` only when the list is readable and no longer names the /// container; a list that cannot be read proves nothing, so it fails too. fn confirm_gone_after_failed_destroy( destroy_err: anyhow::Error, container: &str, listed: Result>, ) -> Result<()> { match listed { Ok(names) if !names.iter().any(|n| n == container) => { tracing::warn!( error = ?destroy_err, %container, "nixos-container destroy returned an error, but the container is gone" ); Ok(()) } Ok(_) => Err(destroy_err.context(format!("container {container} still exists"))), Err(list_err) => Err(destroy_err.context(format!( "cannot confirm whether container {container} still exists: {list_err:#}" ))), } } pub async fn list() -> Result> { let stdout = crate::priv_client::list_containers().await?; Ok(stdout .lines() .map(str::trim) .filter(|line| line.starts_with(AGENT_PREFIX)) .map(str::to_owned) .collect()) } /// Ensure the per-agent runtime directory `/run/hyperhive/agents/` /// exists. It is a bind source and `/run` is a tmpfs, so every start path /// creates it before `nixos-container start`. /// /// Pure filesystem op — no `Coordinator` dependency — so callers that only /// need the dir do not have to hold an `Arc`. /// /// # Errors /// Returns an error if `create_dir_all` fails. pub fn ensure_agent_runtime_dir(name: &str) -> Result<()> { let dir = crate::paths::agent_runtime_dir(name); std::fs::create_dir_all(&dir) .with_context(|| format!("create agent runtime dir {}", dir.display())) } /// Whether a `nix`/`nixos-container` stderr line is worth WARN (nix's own /// warnings and errors) rather than routine progress chatter (`copying N /// paths…`, `fetching git input…`, `structuredAttrs is enabled`, …) — nix /// writes the latter to stderr too, and it dwarfs the former in volume. /// Matches nix's own prefixes (`warning:`, `evaluation warning:`, /// `error:`) rather than only `warning:`, so a real build failure /// (`error: builder for '...' failed`, `error: 1 dependencies ... failed /// to build`) never gets buried at INFO alongside the chatter. fn is_actionable_nix_line(line: &str) -> bool { let trimmed = line.trim_start(); trimmed.starts_with("warning:") || trimmed.starts_with("evaluation warning:") || trimmed.starts_with("error:") } /// What every log line of one container operation carries, so the swarm log /// store can group a build's lines and the builds board can list the builds: /// `agent` and `kind`. A `start` line opens the build and a `finish` line /// closes it with its `status` and `runtime_secs`; the output lines between /// them have no `event`. #[derive(Clone)] struct BuildTag { agent: String, kind: String, cmdline: String, } impl BuildTag { fn new(agent: &str, kind: &str, cmdline: &str) -> Self { Self { agent: agent.to_owned(), kind: kind.to_owned(), cmdline: cmdline.to_owned(), } } fn started(&self) { tracing::info!( target: "nixos-container", event = "start", agent = %self.agent, kind = %self.kind, cmdline = %self.cmdline, "{} started", self.cmdline ); } /// One output line; a nix warning or error is logged at warn, the rest at /// info. fn line(&self, stream: hive_priv_sock::PrivStream, line: &str) { let actionable = matches!(stream, hive_priv_sock::PrivStream::Stderr) && is_actionable_nix_line(line); if actionable { tracing::warn!( target: "nixos-container", agent = %self.agent, kind = %self.kind, cmdline = %self.cmdline, "{line}" ); } else { tracing::info!( target: "nixos-container", agent = %self.agent, kind = %self.kind, cmdline = %self.cmdline, "{line}" ); } } fn finished(&self, succeeded: bool, runtime: std::time::Duration) { let status = if succeeded { "ok" } else { "fail" }; if succeeded { tracing::info!( target: "nixos-container", event = "finish", agent = %self.agent, kind = %self.kind, status, runtime_secs = runtime.as_secs(), cmdline = %self.cmdline, "{} finished: {status}", self.cmdline ); } else { tracing::warn!( target: "nixos-container", event = "finish", agent = %self.agent, kind = %self.kind, status, runtime_secs = runtime.as_secs(), cmdline = %self.cmdline, "{} finished: {status}", self.cmdline ); } } } /// Build the per-line callback for `create_container_streaming` / /// `update_container_streaming`: log each line through its `BuildTag`. fn make_log_callback(tag: BuildTag) -> impl FnMut(hive_priv_sock::PrivStream, &str) { move |stream, line| tag.line(stream, line) } /// Log a failed `start`'s error and the container unit's own journal (from /// hive-priv) line by line. An `Err` journal (an older hive-priv does not know /// the op) leaves just the error. fn record_failure(tag: &BuildTag, err: &anyhow::Error, journal: Result) { use hive_priv_sock::PrivStream; for line in format!("{err:#}").lines() { tag.line(PrivStream::Stderr, line); } match journal { Ok(text) => text.lines().for_each(|l| tag.line(PrivStream::Stderr, l)), Err(e) => tracing::debug!(error = ?e, "start journal unavailable"), } } /// A `create` or `update` asked for without a closure. The /// public entry points take the closure, so this guards the string dispatch /// below only. fn missing_toplevel(kind: &str, name: &str) -> anyhow::Error { anyhow::anyhow!("{name}: {kind} needs a closure to apply") } /// Execute a container operation via hive-priv. hive-priv runs as root and /// forwards output lines to hive-c0re in real time via the streaming priv /// protocol; each line is logged through the op's `BuildTag`. async fn priv_run(kind: &str, name: &str) -> Result<()> { priv_run_inner(kind, name, None).await } /// [`priv_run`] with the closure a `create` or `update` applies. async fn priv_run_inner(kind: &str, name: &str, toplevel: Option<&str>) -> Result<()> { let container = container_name(name); let cmdline = format!("nixos-container {kind} {container}"); // Long-running ops use the streaming protocol so lines are logged in real // time rather than as a batch at completion. let tag = BuildTag::new(name, kind, &cmdline); tag.started(); let began = std::time::Instant::now(); let result: Result<()> = match kind { "create" => match toplevel { Some(toplevel) => { crate::priv_client::create_container_streaming( name, toplevel, make_log_callback(tag.clone()), ) .await } None => Err(missing_toplevel(kind, name)), }, "update" => match toplevel { Some(toplevel) => { crate::priv_client::update_container_streaming( name, toplevel, make_log_callback(tag.clone()), ) .await } None => Err(missing_toplevel(kind, name)), }, "start" => { let attempt = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); let started = crate::priv_client::start_container(name).await; if let Err(e) = &started { let journal = crate::priv_client::start_failure_log(name, attempt).await; record_failure(&tag, e, journal); } started } "stop" => crate::priv_client::stop_container(name).await, "kill" => crate::priv_client::kill_container(name).await, "destroy" => crate::priv_client::destroy_container(name).await, other => Err(anyhow::anyhow!("unknown container op: {other}")), }; let succeeded = result.is_ok(); tag.finished(succeeded, began.elapsed()); result.map_err(|e| anyhow::anyhow!("{e:#}")) }