hive-priv gains AgentHarness { agent_name, action: probe | start }, which
runs systemctl -M h-<name> against the fixed unit hive-agent.service.
run_reconcile and the 1-min tick read the unit for a running wanted-up
agent and start it when it is inactive or failed. A hive-priv without the
op reads as unknown state and leaves reconcile container-only.
977 lines
39 KiB
Rust
977 lines
39 KiB
Rust
//! `nixos-container` lifecycle + per-agent config flake generation.
|
|
|
|
mod agent_identity;
|
|
mod git;
|
|
mod host_config;
|
|
mod setup;
|
|
#[cfg(test)]
|
|
mod tests;
|
|
|
|
pub use git::{git_command, git_command_authed};
|
|
pub use host_config::write_dropins;
|
|
pub use setup::{ensure_agent_state_subvolume, ensure_claude_dir, ensure_state_dir};
|
|
|
|
use anyhow::{Context, Result, bail};
|
|
use tokio::process::Command;
|
|
|
|
use crate::coordinator::{AgentPaths, HiveEnv};
|
|
|
|
// `AGENT_PREFIX` (`h-`) + `container_name` are shared with the host-side
|
|
// `hivectl` CLI, so they live in `hive-host-sock`; re-exported here so this
|
|
// module stays the daemon's facade for its own callsites.
|
|
pub use hive_host_sock::{AGENT_PREFIX, container_name};
|
|
|
|
/// Max agent-name length. `nixos-container` caps the total container name at
|
|
/// 11 chars (it gets encoded into network interface names) and `AGENT_PREFIX`
|
|
/// (`h-`) takes 2, so the agent name itself is at most `MAX_AGENT_NAME` chars.
|
|
pub const MAX_AGENT_NAME: usize = 9;
|
|
/// Mount point of the per-agent runtime directory inside the container.
|
|
pub const CONTAINER_RUNTIME_MOUNT: &str = "/run/hive";
|
|
|
|
/// Where the per-agent Claude credentials dir mounts inside the
|
|
/// container. The harness service runs as a non-root unix user
|
|
/// whose home is `/home/<agent>/`, so the mount path varies per
|
|
/// agent — `container_claude_mount(name)` returns
|
|
/// `/home/<name>/.claude` for every agent including the manager.
|
|
/// `claude` inside the container reads
|
|
/// `$HOME/.claude` and the service environment sets `HOME` to the
|
|
/// same path, so the OAuth session survives container restarts.
|
|
#[must_use]
|
|
pub fn container_claude_mount(name: &str) -> String {
|
|
format!("/home/{name}/.claude")
|
|
}
|
|
|
|
pub use hive_meta_flake::agent_web_port;
|
|
|
|
/// Extract the bridge gateway IP from `HIVE_NETWORK_SUBNET`.
|
|
///
|
|
/// `HIVE_NETWORK_SUBNET` carries the host-side bridge address verbatim
|
|
/// (e.g. `10.42.0.1/24`), **not** the canonical network address — see
|
|
/// the note in `set_nspawn_flags` + `docs/4-implementation/networking/network.md`. The IP part is
|
|
/// therefore the bridge IP itself: the host end of the bridge, the
|
|
/// default-route target for isolated containers, and the address the
|
|
/// hive dnsmasq resolver binds. Returns the dotted-decimal IP with the
|
|
/// `/<prefix>` stripped, or `None` if the input isn't a valid
|
|
/// `<ipv4>/<prefix>` pair.
|
|
///
|
|
/// Deliberately returns the operator-configured address verbatim rather
|
|
/// than deriving `network + 1`: an operator who sets `bridgeIp` to a
|
|
/// non-`.1` host address (e.g. `10.42.0.254`) runs the bridge + resolver
|
|
/// there, so that — not `.1` — is the real gateway.
|
|
#[must_use]
|
|
pub fn bridge_gateway_ip(subnet_cidr: &str) -> Option<String> {
|
|
let (ip_str, prefix_str) = subnet_cidr.split_once('/')?;
|
|
// Validate the prefix is a sane IPv4 CIDR length and the address is
|
|
// dotted-decimal IPv4 so a malformed `HIVE_NETWORK_SUBNET` can't
|
|
// smuggle a bogus HOST_ADDRESS into the nspawn conf.
|
|
let prefix_len: u32 = prefix_str.parse().ok()?;
|
|
if prefix_len > 32 {
|
|
return None;
|
|
}
|
|
let octets: Vec<u8> = ip_str
|
|
.split('.')
|
|
.map(|o| o.parse::<u8>().ok())
|
|
.collect::<Option<Vec<_>>>()?;
|
|
if octets.len() != 4 {
|
|
return None;
|
|
}
|
|
Some(ip_str.to_owned())
|
|
}
|
|
|
|
/// Build the network-isolation settings every container is configured
|
|
/// with, from the variables the hyperhive NixOS module derives from
|
|
/// `services.hyperhive.network.*` onto the `hive-c0re` unit.
|
|
///
|
|
/// Isolation is the only supported mode: the on/off toggle is gone, so
|
|
/// there is no non-isolated branch to fall back to and a missing or
|
|
/// malformed value means the daemon is misconfigured — not that a
|
|
/// container should quietly come up sharing the host's netns. Silently
|
|
/// degrading here dropped a security boundary with nothing in the log
|
|
/// to say so.
|
|
///
|
|
/// Split from [`network_isolation_from_env`] so the parsing is testable
|
|
/// without touching process environment.
|
|
pub fn network_isolation_from_vars(
|
|
bridge: Option<&str>,
|
|
subnet: Option<&str>,
|
|
) -> Result<hive_priv_sock::NetworkIsolation> {
|
|
let bridge = bridge.filter(|s| !s.is_empty()).context(
|
|
"HIVE_NETWORK_BRIDGE is unset or empty — the hyperhive NixOS module sets it \
|
|
from services.hyperhive.network.bridgeName on the hive-c0re unit, so this \
|
|
means the daemon is running outside its unit or with a broken module \
|
|
evaluation",
|
|
)?;
|
|
let subnet = subnet.filter(|s| !s.is_empty()).context(
|
|
"HIVE_NETWORK_SUBNET is unset or empty — the hyperhive NixOS module sets it \
|
|
from services.hyperhive.network.bridgeIp and .bridgePrefixLength on the \
|
|
hive-c0re unit, so this means the daemon is running outside its unit or \
|
|
with a broken module evaluation",
|
|
)?;
|
|
let gateway_ip = bridge_gateway_ip(subnet).with_context(|| {
|
|
format!(
|
|
"HIVE_NETWORK_SUBNET={subnet} is not a valid <ipv4>/<prefix> pair; \
|
|
it comes from services.hyperhive.network.bridgeIp + bridgePrefixLength"
|
|
)
|
|
})?;
|
|
Ok(hive_priv_sock::NetworkIsolation {
|
|
bridge: bridge.to_owned(),
|
|
gateway_ip,
|
|
})
|
|
}
|
|
|
|
/// [`network_isolation_from_vars`] over the real process environment.
|
|
///
|
|
/// Called once at daemon startup so a bad value fails the unit loudly,
|
|
/// and again per container — the variables are process-global, so the
|
|
/// second call cannot start failing once the first has passed.
|
|
pub fn network_isolation_from_env() -> Result<hive_priv_sock::NetworkIsolation> {
|
|
let bridge = std::env::var("HIVE_NETWORK_BRIDGE").ok();
|
|
let subnet = std::env::var("HIVE_NETWORK_SUBNET").ok();
|
|
network_isolation_from_vars(bridge.as_deref(), subnet.as_deref())
|
|
}
|
|
|
|
/// The hive's facts every container is handed at runtime, so that no agent
|
|
/// build carries a value that differs between hosts. The name, domain, egress
|
|
/// proxy and its direct hosts, OTEL resource labels and queue address come from hive-c0re's own unit environment
|
|
/// (`nix/host-modules/hive-c0re/environment.nix`), where an unset or empty
|
|
/// variable is `None`; the dashboard port and pinned claude from `hive`.
|
|
fn hive_facts(hive: &HiveEnv) -> hive_priv_sock::HiveFacts {
|
|
hive_facts_from(hive, |name| std::env::var(name).ok())
|
|
}
|
|
|
|
/// [`hive_facts`] over a given environment lookup, for tests.
|
|
fn hive_facts_from(
|
|
hive: &HiveEnv,
|
|
var: impl Fn(&str) -> Option<String>,
|
|
) -> hive_priv_sock::HiveFacts {
|
|
let read = |name: &str| var(name).filter(|v| !v.is_empty());
|
|
hive_priv_sock::HiveFacts {
|
|
name: read("HYPERHIVE_HIVE_NAME"),
|
|
domain: read("HYPERHIVE_HIVE_DOMAIN"),
|
|
dashboard_port: Some(hive.dashboard_port),
|
|
claude_code_path: hive.claude_code_path.clone(),
|
|
otel_extra_resource_attributes: read("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES"),
|
|
agent_nats_url: read("HIVE_AGENT_NATS_URL"),
|
|
squid_proxy: read("HIVE_SQUID_PROXY"),
|
|
egress_no_proxy: read("HIVE_EGRESS_NO_PROXY"),
|
|
}
|
|
}
|
|
|
|
fn validate(name: &str) -> Result<()> {
|
|
if name.is_empty() {
|
|
bail!("agent name must not be empty");
|
|
}
|
|
if name.len() > MAX_AGENT_NAME {
|
|
bail!(
|
|
"agent name '{name}' is too long ({} chars); max {MAX_AGENT_NAME}",
|
|
name.len()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// First name (≠ `self_name`) currently running whose hashed port
|
|
/// matches this agent's. The harness inside the colliding container
|
|
/// would otherwise loop on `AddrInUse` forever; we surface the
|
|
/// conflict here so spawn / rebuild fails loudly with an actionable
|
|
/// message instead.
|
|
async fn port_collision(self_name: &str) -> Result<Option<String>> {
|
|
let port = agent_web_port(self_name);
|
|
let raw = list()
|
|
.await
|
|
.context("listing containers for the port-collision check")?;
|
|
for c in raw {
|
|
let Some(other) = c.strip_prefix(AGENT_PREFIX) else {
|
|
continue;
|
|
};
|
|
if other == self_name {
|
|
continue;
|
|
}
|
|
if agent_web_port(other) == port && is_running(other).await {
|
|
return Ok(Some(other.to_owned()));
|
|
}
|
|
}
|
|
Ok(None)
|
|
}
|
|
|
|
/// First-spawn pre-create provisioning — the job queue's `Provision`
|
|
/// node. Fail fast on a port collision, set up the state
|
|
/// subvolume + claude/notes dirs, then register the new
|
|
/// agent in the host-side files that follow the agent set
|
|
/// ([`crate::agent_set::refresh`]). Does NOT
|
|
/// create the container — that's `create_only` / the `Create` node.
|
|
pub async fn provision_container(name: &str, paths: &AgentPaths) -> Result<()> {
|
|
validate(name)?;
|
|
if let Some(other) = port_collision(name).await? {
|
|
bail!(
|
|
"port {} is already taken by '{other}' — rename one of them and retry",
|
|
agent_web_port(name)
|
|
);
|
|
}
|
|
// Read before the build: an archetype agent gets no config repo, and
|
|
// the build takes it from the archetype.
|
|
crate::archetypes::refresh(name).await?;
|
|
ensure_agent_state_subvolume(name).await?;
|
|
ensure_claude_dir(&paths.claude)?;
|
|
ensure_state_dir(&paths.notes)?;
|
|
let agents = agents_after_spawn(name).await?;
|
|
crate::agent_set::refresh(&agents).await
|
|
}
|
|
|
|
/// The `nixos-container create` proper — the job queue's `Create` node.
|
|
/// Assumes `provision_container` already registered the agent.
|
|
/// `toplevel` is the store path of the system closure to create it from.
|
|
pub async fn create_only(name: &str, toplevel: &str) -> Result<()> {
|
|
priv_run_inner("create", name, Some(toplevel)).await
|
|
}
|
|
|
|
/// Rebuild-path preamble, run by the job queue's `Prebuild` node: fail fast on
|
|
/// a port collision, then make sure the state dirs exist. Container untouched.
|
|
pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> {
|
|
validate(name)?;
|
|
if let Some(other) = port_collision(name).await? {
|
|
bail!(
|
|
"port {} is already taken by '{other}' — rename one of them and retry",
|
|
agent_web_port(name)
|
|
);
|
|
}
|
|
ensure_agent_state_subvolume(name).await?;
|
|
ensure_claude_dir(&paths.claude)?;
|
|
ensure_state_dir(&paths.notes)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Profile-swap for an existing, stopped container: re-apply the
|
|
/// drop-ins, then point the container at the closure `toplevel`. The job
|
|
/// queue's `Swap` node. Requires the container stopped (the queue's
|
|
/// `StopForUpdate` upstream); does NOT start it — the DAG's tail `Reconcile` owns
|
|
/// bringing the agent back to its wanted power state. `toplevel` is as for
|
|
/// [`create_only`].
|
|
pub async fn swap_update(
|
|
name: &str,
|
|
hive: &HiveEnv,
|
|
paths: &AgentPaths,
|
|
toplevel: &str,
|
|
) -> Result<()> {
|
|
write_dropins(name, hive, paths).await?;
|
|
priv_run_inner("update", name, Some(toplevel)).await
|
|
}
|
|
|
|
/// Build the `AgentSpec` list for [`crate::agent_set::refresh`] from
|
|
/// `nixos-container list` + a hypothetical extra name not yet in the list (for spawn
|
|
/// where the new agent's container doesn't exist yet). Pass empty
|
|
/// `name_to_add` from rebuild paths where the agent is already in the
|
|
/// container list.
|
|
///
|
|
/// Propagates errors from `list()` rather than swallowing them.
|
|
/// Using `.unwrap_or_default()` here would silently produce an empty
|
|
/// agent list when `nixos-container list` fails (priv helper down, race),
|
|
/// which would drop every agent from the topology and the gateway. Callers
|
|
/// that can tolerate failures (e.g. migration)
|
|
/// handle the `Err` themselves with `.unwrap_or_default()`.
|
|
async fn agents_for_meta(name_to_add: Option<&str>) -> Result<Vec<hive_meta_flake::AgentSpec>> {
|
|
let containers = list().await?;
|
|
let archetypes = crate::archetypes::read_at(&crate::archetypes::path())?;
|
|
let mut out: Vec<hive_meta_flake::AgentSpec> = containers
|
|
.into_iter()
|
|
.filter_map(|c| {
|
|
let name = c.strip_prefix(AGENT_PREFIX)?.to_owned();
|
|
Some(hive_meta_flake::AgentSpec {
|
|
port: agent_web_port(&name),
|
|
archetype: archetypes.get(&name).cloned(),
|
|
pin: None,
|
|
name,
|
|
})
|
|
})
|
|
.collect();
|
|
if let Some(extra) = name_to_add
|
|
&& !out.iter().any(|a| a.name == extra)
|
|
{
|
|
out.push(hive_meta_flake::AgentSpec {
|
|
port: agent_web_port(extra),
|
|
archetype: archetypes.get(extra).cloned(),
|
|
pin: None,
|
|
name: extra.to_owned(),
|
|
});
|
|
}
|
|
out.sort_by(|a, b| a.name.cmp(&b.name));
|
|
Ok(out)
|
|
}
|
|
|
|
async fn agents_after_spawn(name: &str) -> Result<Vec<hive_meta_flake::AgentSpec>> {
|
|
agents_for_meta(Some(name)).await
|
|
}
|
|
|
|
/// Public enumeration of currently-existing agents (whatever
|
|
/// `nixos-container list` says), sorted, no extras. For callers
|
|
/// outside this module that need to reseed meta after lifecycle
|
|
/// changes — destroy, startup reconciliation, etc.
|
|
pub async fn agents_for_meta_listing() -> Result<Vec<hive_meta_flake::AgentSpec>> {
|
|
agents_for_meta(None).await
|
|
}
|
|
|
|
/// Per-agent "stop failed" warning guards, keyed by agent name. Cleared the
|
|
/// next time that agent's container stops cleanly — a `Mutex<HashMap<...>>`
|
|
/// rather than a bare `set_boot_warning` because this condition (unlike a
|
|
/// one-shot boot step) genuinely resolves later, once an operator has
|
|
/// intervened, and the warning should clear with it instead of surviving
|
|
/// until the next hive-c0re restart.
|
|
fn stop_failed_guards()
|
|
-> &'static std::sync::Mutex<std::collections::HashMap<String, crate::warnings::WarningGuard>> {
|
|
static REG: std::sync::OnceLock<
|
|
std::sync::Mutex<std::collections::HashMap<String, crate::warnings::WarningGuard>>,
|
|
> = std::sync::OnceLock::new();
|
|
REG.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
|
|
}
|
|
|
|
/// The per-agent warning `kind`, leaked on the first call for a name and
|
|
/// returned unchanged afterwards, whether or not the warning was cleared in
|
|
/// between. The leak is one short string per distinct agent name.
|
|
fn stop_kind(name: &str) -> &'static str {
|
|
static KINDS: std::sync::OnceLock<
|
|
std::sync::Mutex<std::collections::HashMap<String, &'static str>>,
|
|
> = std::sync::OnceLock::new();
|
|
let mut kinds = KINDS
|
|
.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
kinds
|
|
.entry(name.to_owned())
|
|
.or_insert_with(|| Box::leak(format!("agent_stop_failed_{name}").into_boxed_str()))
|
|
}
|
|
|
|
/// Stop `name`'s container. See `hive-priv`'s `stop_and_release` for why
|
|
/// a failure here means the container is likely wedged rather than just
|
|
/// slow — SIGKILL doesn't recover that case in practice, so this
|
|
/// surfaces it on the dashboard instead of silently retrying.
|
|
pub async fn kill(name: &str) -> Result<()> {
|
|
validate(name)?;
|
|
let result = priv_run("stop", name).await;
|
|
// Lock only for the synchronous map update — never held across the
|
|
// `.await` above.
|
|
let mut guards = stop_failed_guards()
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
match result {
|
|
Ok(()) => {
|
|
// A clean stop clears any earlier "this looked wedged" warning
|
|
// for the same agent.
|
|
guards.remove(name);
|
|
Ok(())
|
|
}
|
|
Err(e) => {
|
|
tracing::error!(%name, error = %e, "container stop failed");
|
|
guards.insert(
|
|
name.to_owned(),
|
|
crate::warnings::set_warning(
|
|
stop_kind(name),
|
|
"crit",
|
|
format!("{name}: stop failed — {e}"),
|
|
),
|
|
);
|
|
Err(e)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Start a container. Success is defined as the container's unit reaching
|
|
/// `active`, **not** the `nixos-container start` exit code — a start-job
|
|
/// timeout on a slow boot (e.g. DHCP taking ~10s) exits non-zero while
|
|
/// `container@<c>.service` keeps retrying and the container comes up seconds
|
|
/// later. So a zero exit returns immediately (already active), and a non-zero
|
|
/// exit polls the unit state for [`START_SETTLE_TIMEOUT`] before concluding
|
|
/// the start actually failed. Every caller (dashboard restart, reconcile
|
|
/// start, the cold-start fallback) gets this truth for free.
|
|
async fn start(name: &str) -> Result<()> {
|
|
validate(name)?;
|
|
if priv_run("start", name).await.is_ok() {
|
|
return Ok(());
|
|
}
|
|
let container = container_name(name);
|
|
if wait_until_running(name, START_SETTLE_TIMEOUT).await {
|
|
tracing::info!(
|
|
container = %container,
|
|
"start exited non-zero but the container reached active on systemd's retry; treating as success"
|
|
);
|
|
return Ok(());
|
|
}
|
|
Err(anyhow::anyhow!(
|
|
"container {container} did not reach active within {}s of start",
|
|
START_SETTLE_TIMEOUT.as_secs()
|
|
))
|
|
}
|
|
|
|
/// Opaque token produced by [`converge_start_preamble`].
|
|
/// [`start_with_fallback`] requires this as proof that the pre-start
|
|
/// preamble (runtime dir + drop-ins) ran. Dropping the token without
|
|
/// calling `start_with_fallback` is a no-op.
|
|
#[must_use = "call lifecycle::start_with_fallback(token) to start the container"]
|
|
pub struct StartableAgent {
|
|
name: String,
|
|
}
|
|
|
|
/// Run the per-agent start preamble: ensure the runtime dir exists and write
|
|
/// the nspawn / resource-limits drop-ins. Returns a [`StartableAgent`] token
|
|
/// as typed proof that the preamble ran; pass it to [`start_with_fallback`].
|
|
/// Callers that omit this step cannot call `start_with_fallback` — the type
|
|
/// system makes forgetting the preamble a compile error.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns an error if `ensure_agent_runtime_dir` or `write_dropins` fails.
|
|
pub async fn converge_start_preamble(
|
|
name: &str,
|
|
hive: &HiveEnv,
|
|
paths: &AgentPaths,
|
|
) -> Result<StartableAgent> {
|
|
ensure_agent_runtime_dir(name)?;
|
|
write_dropins(name, hive, paths).await?;
|
|
Ok(StartableAgent {
|
|
name: name.to_owned(),
|
|
})
|
|
}
|
|
|
|
/// Start with the cold-start fallback: when a plain start fails (the
|
|
/// activation-error shape), retry once via stop + kill + start before
|
|
/// giving up. Used by the queue's fast-lane `Start` handler and the
|
|
/// inline start-after-rebuild path.
|
|
/// See `docs/4-implementation/scheduler/coordinator.md::Cold-start fallback`.
|
|
///
|
|
/// Requires a [`StartableAgent`] token from [`converge_start_preamble`]
|
|
/// to prove the preamble ran. For internal use within this module (where
|
|
/// the preamble is already enforced structurally) call
|
|
/// `start_with_fallback_inner` directly.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Propagates the retry's start error (annotated with the original
|
|
/// failure) when the fallback also fails.
|
|
pub async fn start_with_fallback(token: StartableAgent) -> Result<()> {
|
|
start_with_fallback_inner(&token.name).await
|
|
}
|
|
|
|
/// How long to wait for a container's unit to reach `active` after a
|
|
/// `nixos-container start` that returned a non-zero exit. A start-job timeout
|
|
/// on a slow boot (e.g. DHCP taking ~10s) returns an error while
|
|
/// `container@<c>.service` keeps retrying and the container comes up seconds
|
|
/// later — so the exit code is NOT authoritative, the unit state is. 60s
|
|
/// comfortably covers an observed slow boot (the reported incident settled
|
|
/// ~16s after the "failure") without pinning a queue node on a genuine
|
|
/// never-boots failure for too long.
|
|
const START_SETTLE_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(1);
|
|
|
|
/// Poll cadence while waiting for a container to reach `active`.
|
|
const START_SETTLE_POLL: std::time::Duration = std::time::Duration::from_secs(2);
|
|
|
|
/// Poll [`is_running`] until the container's unit is active or `timeout`
|
|
/// elapses; returns true as soon as it's active.
|
|
async fn wait_until_running(name: &str, timeout: std::time::Duration) -> bool {
|
|
let deadline = tokio::time::Instant::now() + timeout;
|
|
loop {
|
|
if is_running(name).await {
|
|
return true;
|
|
}
|
|
if tokio::time::Instant::now() >= deadline {
|
|
return false;
|
|
}
|
|
tokio::time::sleep(START_SETTLE_POLL).await;
|
|
}
|
|
}
|
|
|
|
/// Internal implementation of the cold-start fallback, behind
|
|
/// [`start_with_fallback`] (public, token-gated) — the inner form exists for
|
|
/// callers that have already run the drop-in preamble themselves.
|
|
///
|
|
/// [`start`] already treats unit-active (not the exit code) as success and
|
|
/// waits out a slow boot, so this only layers the activation-error recovery on
|
|
/// top: if the container is still down after `start`, tear it down and try
|
|
/// `start` once more. The teardown prefers a graceful `stop` and only
|
|
/// escalates to SIGKILL when that `stop` itself fails.
|
|
async fn start_with_fallback_inner(name: &str) -> Result<()> {
|
|
validate(name)?;
|
|
if start(name).await.is_ok() {
|
|
return Ok(());
|
|
}
|
|
tracing::warn!(
|
|
%name,
|
|
"start did not bring the container up; hard-resetting (stop, kill only if that fails) and retrying once"
|
|
);
|
|
// Graceful `stop` first; only escalate to SIGKILL if the stop itself
|
|
// fails (e.g. a wedged container whose `machinectl poweroff` never
|
|
// completes). A clean stop is enough on its own — SIGKILL is the
|
|
// last-resort teardown for when graceful shutdown can't finish.
|
|
if let Err(stop_err) = priv_run("stop", name).await {
|
|
tracing::warn!(
|
|
%name,
|
|
error = %stop_err,
|
|
"graceful stop failed; escalating to SIGKILL"
|
|
);
|
|
priv_run("kill", name).await.unwrap_or_else(|e| {
|
|
tracing::warn!(
|
|
%name,
|
|
error = %e,
|
|
"kill after failed stop also failed (ignored)"
|
|
);
|
|
});
|
|
}
|
|
start(name)
|
|
.await
|
|
.with_context(|| format!("cold-start fallback also failed for {name}"))
|
|
}
|
|
|
|
/// The part of a container unit's systemd state we act on.
|
|
///
|
|
/// Deliberately not the full set: everything outside `Active`/`Failed` is
|
|
/// one bucket, because the only distinction anything downstream makes is
|
|
/// *gave up* versus *anything else*.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum UnitState {
|
|
/// Running.
|
|
Active,
|
|
/// systemd gave up on it — most often by exhausting the bounded start
|
|
/// limit. Its own variant precisely because it is the difference
|
|
/// between **gave up** and **stopped on purpose**, which every other
|
|
/// non-running state collapses into.
|
|
Failed,
|
|
/// `inactive` (the deliberate stop), `activating`, `deactivating`, an
|
|
/// unknown unit, or a `systemctl` we could not run at all.
|
|
Other,
|
|
}
|
|
|
|
impl UnitState {
|
|
/// Parse `systemctl is-active`'s stdout.
|
|
///
|
|
/// Split out from the shell-out so the mapping is testable without a
|
|
/// systemd to ask — the states we care about are the two we name, and
|
|
/// everything else must land in `Other` rather than being guessed at.
|
|
fn from_is_active(stdout: &str) -> Self {
|
|
match stdout.trim() {
|
|
"active" => Self::Active,
|
|
"failed" => Self::Failed,
|
|
_ => Self::Other,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The container unit's state, in one `systemctl` call.
|
|
///
|
|
/// `is-active` **prints** the state and exits 0 only for `active`, so
|
|
/// dropping `--quiet` yields both facts from the call [`is_running`]
|
|
/// already makes. That matters: the status path runs this per agent (see
|
|
/// the spawn-count note in `socket_server::lifecycle_handlers`), so a
|
|
/// caller wanting *running* and *failed* must not pay for two subprocesses.
|
|
pub async fn unit_state(name: &str) -> UnitState {
|
|
let container = container_name(name);
|
|
let unit = format!("container@{container}.service");
|
|
// Not "the unit is fine" and not "the unit failed" — we simply do not
|
|
// know, and saying `Failed` here would report a gave-up agent every time
|
|
// `systemctl` itself was unavailable or hung.
|
|
systemctl_is_active(&[&unit])
|
|
.await
|
|
.map_or(UnitState::Other, |out| {
|
|
UnitState::from_is_active(&String::from_utf8_lossy(&out.stdout))
|
|
})
|
|
}
|
|
|
|
/// True when the container's systemd unit is active. Used by the dashboard
|
|
/// to gate stop/restart buttons.
|
|
///
|
|
/// Kept boolean on purpose: nearly every caller is reconcile/power logic
|
|
/// asking "is it up? if not, start it", and that question has two answers.
|
|
/// A caller that needs to tell *failed* from *stopped* wants
|
|
/// [`unit_state`] instead.
|
|
///
|
|
/// Unbounded: its callers record or act on the answer, so a hung `systemctl`
|
|
/// must not read as "not running". Periodic workers use [`running_state`].
|
|
pub async fn is_running(name: &str) -> bool {
|
|
let container = container_name(name);
|
|
let unit = format!("container@{container}.service");
|
|
Command::new("systemctl")
|
|
.args(["is-active", "--quiet", &unit])
|
|
.status()
|
|
.await
|
|
.is_ok_and(|s| s.success())
|
|
}
|
|
|
|
/// Whether the container's systemd unit is active, for periodic workers that
|
|
/// must not stall on a hung `systemctl`; `None` when it could not be run or
|
|
/// did not answer within [`IS_ACTIVE_TIMEOUT`].
|
|
pub async fn running_state(name: &str) -> Option<bool> {
|
|
let container = container_name(name);
|
|
let unit = format!("container@{container}.service");
|
|
systemctl_is_active(&["--quiet", &unit])
|
|
.await
|
|
.map(|out| out.status.success())
|
|
}
|
|
|
|
/// Whether an agent's harness unit (`hive-agent.service`) is up inside its
|
|
/// running container.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum HarnessState {
|
|
/// Active, or on its way: activating, deactivating or reloading.
|
|
Present,
|
|
/// Loaded and `inactive` or `failed`: nothing is running it and nothing
|
|
/// is about to.
|
|
Stopped,
|
|
/// Not readable: hive-priv is older than the op, the container's systemd
|
|
/// did not answer, or the unit is not loaded.
|
|
Unknown,
|
|
}
|
|
|
|
/// Read the harness unit's state out of `AgentHarness` probe output
|
|
/// (`LoadState=` and `ActiveState=` lines).
|
|
fn parse_harness_state(out: &str) -> HarnessState {
|
|
let field = |key: &str| {
|
|
out.lines()
|
|
.find_map(|l| l.strip_prefix(key).and_then(|v| v.strip_prefix('=')))
|
|
.map(str::trim)
|
|
};
|
|
match (field("LoadState"), field("ActiveState")) {
|
|
(Some("loaded"), Some("inactive" | "failed")) => HarnessState::Stopped,
|
|
(Some("loaded"), Some("active" | "activating" | "deactivating" | "reloading")) => {
|
|
HarnessState::Present
|
|
}
|
|
_ => HarnessState::Unknown,
|
|
}
|
|
}
|
|
|
|
/// The state of `name`'s harness unit, asked of hive-priv. Bounded by
|
|
/// [`IS_ACTIVE_TIMEOUT`], because the tick calls it per agent inline.
|
|
///
|
|
/// Any failure reads as [`HarnessState::Unknown`], so reconcile keeps its
|
|
/// container-only behaviour: a hive-priv that predates the op answers an
|
|
/// error, and that is logged at `warn` the first time and `debug` after.
|
|
pub async fn harness_state(name: &str) -> HarnessState {
|
|
let probe = crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Probe);
|
|
if let Ok(answer) = tokio::time::timeout(IS_ACTIVE_TIMEOUT, probe).await {
|
|
harness_of_answer(name, answer)
|
|
} else {
|
|
tracing::debug!(%name, "reading the agent's harness state timed out");
|
|
HarnessState::Unknown
|
|
}
|
|
}
|
|
|
|
/// The state a probe answer means; an error is [`HarnessState::Unknown`].
|
|
fn harness_of_answer(name: &str, answer: Result<String>) -> HarnessState {
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
static WARNED: AtomicBool = AtomicBool::new(false);
|
|
match answer {
|
|
Ok(out) => parse_harness_state(&out),
|
|
Err(e) if !WARNED.swap(true, Ordering::Relaxed) => {
|
|
tracing::warn!(%name, error = ?e, "cannot read the agent's harness state; reconcile only watches the container (logged once)");
|
|
HarnessState::Unknown
|
|
}
|
|
Err(e) => {
|
|
tracing::debug!(%name, error = ?e, "cannot read the agent's harness state");
|
|
HarnessState::Unknown
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Queue a start of `name`'s harness unit inside its running container.
|
|
///
|
|
/// # Errors
|
|
/// hive-priv is unreachable, refuses, or `systemctl` fails.
|
|
pub async fn start_harness(name: &str) -> Result<()> {
|
|
crate::priv_client::agent_harness(name, hive_priv_sock::HarnessAction::Start)
|
|
.await
|
|
.map(drop)
|
|
}
|
|
|
|
/// How long one `systemctl is-active` may run before the unit's state counts
|
|
/// as unreadable. Periodic workers make this call per agent inline.
|
|
const IS_ACTIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
|
|
|
/// Run `systemctl is-active <args>`, killed after [`IS_ACTIVE_TIMEOUT`].
|
|
async fn systemctl_is_active(args: &[&str]) -> Option<std::process::Output> {
|
|
let output = Command::new("systemctl")
|
|
.arg("is-active")
|
|
.args(args)
|
|
.kill_on_drop(true)
|
|
.output();
|
|
output_within(IS_ACTIVE_TIMEOUT, output, args).await
|
|
}
|
|
|
|
/// Await a child's output for at most `limit`. `None` when it failed to run
|
|
/// or timed out; the timeout drops `fut`, which kills a child spawned with
|
|
/// `kill_on_drop(true)`.
|
|
async fn output_within<T>(
|
|
limit: std::time::Duration,
|
|
fut: impl std::future::Future<Output = std::io::Result<T>>,
|
|
args: &[&str],
|
|
) -> Option<T> {
|
|
if let Ok(result) = tokio::time::timeout(limit, fut).await {
|
|
result.ok()
|
|
} else {
|
|
tracing::warn!(
|
|
?args,
|
|
timeout_s = limit.as_secs(),
|
|
"systemctl is-active timed out"
|
|
);
|
|
None
|
|
}
|
|
}
|
|
|
|
/// Fully tear down a sub-agent's container: stop + remove via `nixos-container
|
|
/// destroy`, then clean our own systemd drop-in. Leaves it to the caller to
|
|
/// wipe `/var/lib/hyperhive/...` state and the per-agent runtime dir.
|
|
///
|
|
/// Fails when the container survives the destroy. The destroy DAG purges the
|
|
/// agent's state only after this returns `Ok`, so a swallowed failure here
|
|
/// deletes the state of a container that still exists.
|
|
pub async fn destroy(name: &str) -> Result<()> {
|
|
validate(name)?;
|
|
let container = container_name(name);
|
|
// nixos-container destroy handles stop + removal of /var/lib/nixos-containers/<C>
|
|
// and /etc/nixos-containers/<C>.conf. When it errors, the container list
|
|
// decides: a container that is already gone is still a successful destroy.
|
|
if let Err(e) = priv_run("destroy", name).await {
|
|
confirm_gone_after_failed_destroy(e, &container, list().await)?;
|
|
}
|
|
// Remove the systemd resource-limits drop-in via hive-priv.
|
|
if let Err(e) = crate::priv_client::remove_service_dropin(&container).await {
|
|
tracing::warn!(error = ?e, "remove service drop-in failed (non-fatal)");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Settle a failed `nixos-container destroy` against the container list taken
|
|
/// after it. `Ok` only when the list is readable and no longer names the
|
|
/// container; a list that cannot be read proves nothing, so it fails too.
|
|
fn confirm_gone_after_failed_destroy(
|
|
destroy_err: anyhow::Error,
|
|
container: &str,
|
|
listed: Result<Vec<String>>,
|
|
) -> Result<()> {
|
|
match listed {
|
|
Ok(names) if !names.iter().any(|n| n == container) => {
|
|
tracing::warn!(
|
|
error = ?destroy_err,
|
|
%container,
|
|
"nixos-container destroy returned an error, but the container is gone"
|
|
);
|
|
Ok(())
|
|
}
|
|
Ok(_) => Err(destroy_err.context(format!("container {container} still exists"))),
|
|
Err(list_err) => Err(destroy_err.context(format!(
|
|
"cannot confirm whether container {container} still exists: {list_err:#}"
|
|
))),
|
|
}
|
|
}
|
|
|
|
pub async fn list() -> Result<Vec<String>> {
|
|
let stdout = crate::priv_client::list_containers().await?;
|
|
Ok(stdout
|
|
.lines()
|
|
.map(str::trim)
|
|
.filter(|line| line.starts_with(AGENT_PREFIX))
|
|
.map(str::to_owned)
|
|
.collect())
|
|
}
|
|
|
|
/// Ensure the per-agent runtime directory `/run/hyperhive/agents/<name>`
|
|
/// exists. It is a bind source and `/run` is a tmpfs, so every start path
|
|
/// creates it before `nixos-container start`.
|
|
///
|
|
/// Pure filesystem op — no `Coordinator` dependency — so callers that only
|
|
/// need the dir do not have to hold an `Arc<Coordinator>`.
|
|
///
|
|
/// # Errors
|
|
/// Returns an error if `create_dir_all` fails.
|
|
pub fn ensure_agent_runtime_dir(name: &str) -> Result<()> {
|
|
let dir = crate::paths::agent_runtime_dir(name);
|
|
std::fs::create_dir_all(&dir)
|
|
.with_context(|| format!("create agent runtime dir {}", dir.display()))
|
|
}
|
|
|
|
/// Whether a `nix`/`nixos-container` stderr line is worth WARN (nix's own
|
|
/// warnings and errors) rather than routine progress chatter (`copying N
|
|
/// paths…`, `fetching git input…`, `structuredAttrs is enabled`, …) — nix
|
|
/// writes the latter to stderr too, and it dwarfs the former in volume.
|
|
/// Matches nix's own prefixes (`warning:`, `evaluation warning:`,
|
|
/// `error:`) rather than only `warning:`, so a real build failure
|
|
/// (`error: builder for '...' failed`, `error: 1 dependencies ... failed
|
|
/// to build`) never gets buried at INFO alongside the chatter.
|
|
fn is_actionable_nix_line(line: &str) -> bool {
|
|
let trimmed = line.trim_start();
|
|
trimmed.starts_with("warning:")
|
|
|| trimmed.starts_with("evaluation warning:")
|
|
|| trimmed.starts_with("error:")
|
|
}
|
|
|
|
/// What every log line of one container operation carries, so the swarm log
|
|
/// store can group a build's lines and the builds board can list the builds:
|
|
/// `agent` and `kind`. A `start` line opens the build and a `finish` line
|
|
/// closes it with its `status` and `runtime_secs`; the output lines between
|
|
/// them have no `event`.
|
|
#[derive(Clone)]
|
|
struct BuildTag {
|
|
agent: String,
|
|
kind: String,
|
|
cmdline: String,
|
|
}
|
|
|
|
impl BuildTag {
|
|
fn new(agent: &str, kind: &str, cmdline: &str) -> Self {
|
|
Self {
|
|
agent: agent.to_owned(),
|
|
kind: kind.to_owned(),
|
|
cmdline: cmdline.to_owned(),
|
|
}
|
|
}
|
|
|
|
fn started(&self) {
|
|
tracing::info!(
|
|
target: "nixos-container",
|
|
event = "start",
|
|
agent = %self.agent,
|
|
kind = %self.kind,
|
|
cmdline = %self.cmdline,
|
|
"{} started",
|
|
self.cmdline
|
|
);
|
|
}
|
|
|
|
/// One output line; a nix warning or error is logged at warn, the rest at
|
|
/// info.
|
|
fn line(&self, stream: hive_priv_sock::PrivStream, line: &str) {
|
|
let actionable =
|
|
matches!(stream, hive_priv_sock::PrivStream::Stderr) && is_actionable_nix_line(line);
|
|
if actionable {
|
|
tracing::warn!(
|
|
target: "nixos-container",
|
|
agent = %self.agent,
|
|
kind = %self.kind,
|
|
cmdline = %self.cmdline,
|
|
"{line}"
|
|
);
|
|
} else {
|
|
tracing::info!(
|
|
target: "nixos-container",
|
|
agent = %self.agent,
|
|
kind = %self.kind,
|
|
cmdline = %self.cmdline,
|
|
"{line}"
|
|
);
|
|
}
|
|
}
|
|
|
|
fn finished(&self, succeeded: bool, runtime: std::time::Duration) {
|
|
let status = if succeeded { "ok" } else { "fail" };
|
|
if succeeded {
|
|
tracing::info!(
|
|
target: "nixos-container",
|
|
event = "finish",
|
|
agent = %self.agent,
|
|
kind = %self.kind,
|
|
status,
|
|
runtime_secs = runtime.as_secs(),
|
|
cmdline = %self.cmdline,
|
|
"{} finished: {status}",
|
|
self.cmdline
|
|
);
|
|
} else {
|
|
tracing::warn!(
|
|
target: "nixos-container",
|
|
event = "finish",
|
|
agent = %self.agent,
|
|
kind = %self.kind,
|
|
status,
|
|
runtime_secs = runtime.as_secs(),
|
|
cmdline = %self.cmdline,
|
|
"{} finished: {status}",
|
|
self.cmdline
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Build the per-line callback for `create_container_streaming` /
|
|
/// `update_container_streaming`: log each line through its `BuildTag`.
|
|
fn make_log_callback(tag: BuildTag) -> impl FnMut(hive_priv_sock::PrivStream, &str) {
|
|
move |stream, line| tag.line(stream, line)
|
|
}
|
|
|
|
/// Log a failed `start`'s error and the container unit's own journal (from
|
|
/// hive-priv) line by line. An `Err` journal (an older hive-priv does not know
|
|
/// the op) leaves just the error.
|
|
fn record_failure(tag: &BuildTag, err: &anyhow::Error, journal: Result<String>) {
|
|
use hive_priv_sock::PrivStream;
|
|
for line in format!("{err:#}").lines() {
|
|
tag.line(PrivStream::Stderr, line);
|
|
}
|
|
match journal {
|
|
Ok(text) => text.lines().for_each(|l| tag.line(PrivStream::Stderr, l)),
|
|
Err(e) => tracing::debug!(error = ?e, "start journal unavailable"),
|
|
}
|
|
}
|
|
|
|
/// A `create` or `update` asked for without a closure. The
|
|
/// public entry points take the closure, so this guards the string dispatch
|
|
/// below only.
|
|
fn missing_toplevel(kind: &str, name: &str) -> anyhow::Error {
|
|
anyhow::anyhow!("{name}: {kind} needs a closure to apply")
|
|
}
|
|
|
|
/// Execute a container operation via hive-priv. hive-priv runs as root and
|
|
/// forwards output lines to hive-c0re in real time via the streaming priv
|
|
/// protocol; each line is logged through the op's `BuildTag`.
|
|
async fn priv_run(kind: &str, name: &str) -> Result<()> {
|
|
priv_run_inner(kind, name, None).await
|
|
}
|
|
|
|
/// [`priv_run`] with the closure a `create` or `update` applies.
|
|
async fn priv_run_inner(kind: &str, name: &str, toplevel: Option<&str>) -> Result<()> {
|
|
let container = container_name(name);
|
|
let cmdline = format!("nixos-container {kind} {container}");
|
|
|
|
// Long-running ops use the streaming protocol so lines are logged in real
|
|
// time rather than as a batch at completion.
|
|
let tag = BuildTag::new(name, kind, &cmdline);
|
|
tag.started();
|
|
let began = std::time::Instant::now();
|
|
let result: Result<()> = match kind {
|
|
"create" => match toplevel {
|
|
Some(toplevel) => {
|
|
crate::priv_client::create_container_streaming(
|
|
name,
|
|
toplevel,
|
|
make_log_callback(tag.clone()),
|
|
)
|
|
.await
|
|
}
|
|
None => Err(missing_toplevel(kind, name)),
|
|
},
|
|
"update" => match toplevel {
|
|
Some(toplevel) => {
|
|
crate::priv_client::update_container_streaming(
|
|
name,
|
|
toplevel,
|
|
make_log_callback(tag.clone()),
|
|
)
|
|
.await
|
|
}
|
|
None => Err(missing_toplevel(kind, name)),
|
|
},
|
|
"start" => {
|
|
let attempt = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map_or(0, |d| d.as_secs());
|
|
let started = crate::priv_client::start_container(name).await;
|
|
if let Err(e) = &started {
|
|
let journal = crate::priv_client::start_failure_log(name, attempt).await;
|
|
record_failure(&tag, e, journal);
|
|
}
|
|
started
|
|
}
|
|
"stop" => crate::priv_client::stop_container(name).await,
|
|
"kill" => crate::priv_client::kill_container(name).await,
|
|
"destroy" => crate::priv_client::destroy_container(name).await,
|
|
other => Err(anyhow::anyhow!("unknown container op: {other}")),
|
|
};
|
|
|
|
let succeeded = result.is_ok();
|
|
tag.finished(succeeded, began.elapsed());
|
|
|
|
result.map_err(|e| anyhow::anyhow!("{e:#}"))
|
|
}
|