hyperhive/hive-c0re/src/socket_server/lifecycle_handlers.rs
atlas 860484a193 refactor(#2439): build hive-wide stop/start/restart DAGs dynamically
Hive-wide `stop` / `start` / `restart` emit ONE DAG with a per-agent
subgraph each (concurrent on their own leases) instead of N DAGs — and each
subgraph is now built dynamically from the agent's live running state rather
than a fixed template shape:

- online agent: the full stop→reconcile (restart: stop-for-update→reconcile)
  chain; `graceful` prepends signal→drain.
- offline agent: just `SetWanted → Reconcile` (nothing to quiesce/stop; a
  restart of a down agent is really a start).

The head `SetWanted` (intent) and tail `Reconcile` (convergence guarantee)
are always present; only the mechanical `Signal`/`Drain`/`StopForUpdate`
nodes are state-conditional. Keeping `Reconcile` in every shape closes the
TOCTOU window — a race-up between the `is_running` read and node exec is
still converged in-DAG (with `StopForUpdate`-noop as the backstop) — with no
reliance on an external reconcile sweep.

The state-aware assembly needs an async `is_running` read, so it moves out
of the pure/sync `templates.rs` into `submit.rs`, layered as pure
`*_chain(running)` → pure `*_spec(targets)` (the unit-test seam) → async
`*_many` (reads live state + submits). `templates.rs` keeps only the shared
pure primitives (`node`/`after_ok`/`rebuild_nodes`).

Callers await the now-async submit fns (server, dashboard, socket_server).
Tests exercise both the online and offline shapes via the pure `*_spec`
seam. docs/coordinator.md shapes updated.
2026-07-15 00:29:54 +02:00

207 lines
7.6 KiB
Rust

//! Container-lifecycle request handlers (`Start` / `Restart` / `Kill` /
//! `Update` / `ListDescendants`), including the capability-gated
//! infra-container restart path. All are topology-guarded via
//! `super::require_descendant`.
use std::sync::Arc;
use hive_sh4re::AgentResponse;
use super::require_descendant;
use crate::coordinator::Coordinator;
/// `Start` — start a container, kicking its next turn. The caller must be an
/// ancestor of `name` in the topology (the root covers every agent).
pub(super) async fn handle_start(
coord: &Arc<Coordinator>,
agent: &str,
name: &str,
) -> AgentResponse {
if let Some(err) = require_descendant(agent, name, "start") {
return err;
}
tracing::info!(%agent, %name, "start container");
// Persist `wanted = Up` and submit the Start DAG; the submit layer
// upgrades a stale-rev start to a full rebuild so the container
// runs current nix derivations before it starts.
crate::job_queue::submit::start(
coord,
name,
crate::job_queue::Source::Manual,
format!("agent `{agent}` start tool"),
)
.await;
AgentResponse::Ok
}
/// `Restart` — enqueue a restart for a container. The caller must be an
/// ancestor of `name` in the topology. The infra-container branch is
/// orthogonal: it is gated on the `infra_admin` capability and audited, so it
/// stays ahead of the topology guard.
pub(super) async fn handle_restart(
coord: &Arc<Coordinator>,
agent: &str,
name: &str,
) -> AgentResponse {
// Infra-container restart: an agent holding the `infra_admin`
// capability can restart a hive infrastructure container (hive-ci /
// hive-gateway / hive-forge / hive-matrix) by passing its name to the
// same restart tool. The `InfraContainer` enum parse both recognises
// these (never agent children, so disjoint from the child path below)
// and yields the typed value the restart path needs.
if let Ok(container) = name.parse::<hive_priv_sock::InfraContainer>() {
return handle_restart_infra(coord, agent, container).await;
}
if let Some(err) = require_descendant(agent, name, "restart") {
return err;
}
tracing::info!(%agent, %name, "submit restart");
crate::job_queue::submit::restart(
coord,
name,
crate::job_queue::Source::Manual,
format!("agent `{agent}` restart tool"),
)
.await;
AgentResponse::Ok
}
/// Restart a hive infrastructure container on behalf of an agent that
/// holds the `infra_admin` capability. The `container` is already a valid
/// [`InfraContainer`] (the caller parsed it); this gates on the capability
/// and routes the systemctl restart through hive-priv. Direct, not
/// approval-gated.
async fn handle_restart_infra(
coord: &Arc<Coordinator>,
agent: &str,
container: hive_priv_sock::InfraContainer,
) -> AgentResponse {
let name = container.unit_name();
// Record the attempt in the operator-visible privileged-action audit
// trail, then emit a live `AuditEntryAdded` so the dashboard audit view
// appends it off `/dashboard/stream`. Best-effort: `record` returns the
// canonical row (or `None` on a sqlite blip), and we stream exactly that
// row so the stored + streamed views can't drift. `action` is stable so
// the dashboard can group/filter.
let audit = |outcome: crate::audit_log::AuditOutcome, detail: Option<&str>| {
if let Some(entry) = coord
.audit_log
.record(agent, "restart_infra", name, outcome, detail)
{
coord.emit_audit_entry(entry);
}
};
if !crate::capabilities::has_cap(agent, hive_sh4re::Capability::InfraAdmin) {
tracing::warn!(%agent, %name, "agent: infra restart denied (no infra_admin capability)");
audit(
crate::audit_log::AuditOutcome::Err,
Some("denied: missing infra_admin capability"),
);
return AgentResponse::Err {
message: format!(
"restarting infra container `{name}` requires the `infra_admin` capability"
),
};
}
tracing::info!(%agent, %name, "agent: restart infra container");
match crate::priv_client::restart_infra_container(container).await {
Ok(()) => {
audit(crate::audit_log::AuditOutcome::Ok, None);
AgentResponse::Ok
}
Err(e) => {
let msg = format!("{e:#}");
audit(crate::audit_log::AuditOutcome::Err, Some(&msg));
AgentResponse::Err { message: msg }
}
}
}
/// `Kill` — kill a container, unregister it, notify the manager. The caller
/// must be an ancestor of `name` in the topology.
pub(super) async fn handle_kill(
coord: &Arc<Coordinator>,
agent: &str,
name: &str,
) -> AgentResponse {
if let Some(err) = require_descendant(agent, name, "kill") {
return err;
}
tracing::info!(%agent, %name, "kill container");
// Persist the intent even if the kill fails — otherwise the next
// reconcile would restart the container.
if let Err(e) = coord.power.set(name, crate::power::Wanted::Offline) {
tracing::warn!(%name, error = ?e, "agent_power: set wanted=offline failed");
}
let result: anyhow::Result<()> = async {
crate::lifecycle::kill(name).await?;
coord.unregister_agent(name);
Ok(())
}
.await;
match result {
Ok(()) => {
coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
agent: name.to_owned(),
});
AgentResponse::Ok
}
Err(e) => AgentResponse::Err {
message: format!("{e:#}"),
},
}
}
/// `Update` — enqueue a rebuild for a container. The caller must be an
/// ancestor of `name` in the topology.
pub(super) fn handle_update(coord: &Arc<Coordinator>, agent: &str, name: &str) -> AgentResponse {
if let Some(err) = require_descendant(agent, name, "rebuild") {
return err;
}
tracing::info!(%agent, %name, "submit rebuild");
crate::job_queue::submit::rebuild(
coord,
name,
crate::job_queue::Source::Manual,
format!("agent `{agent}` update tool"),
);
AgentResponse::Ok
}
/// `ListDescendants` — every topological descendant of `agent` with
/// its running/stopped state, parents before children.
pub(super) async fn handle_list_descendants(agent: &str) -> AgentResponse {
tracing::debug!(%agent, "agent: list descendants");
// All containers known to nixos-container (running only).
let running_set: std::collections::HashSet<String> = match crate::lifecycle::list().await {
Ok(names) => names
.into_iter()
.filter_map(|c| {
c.strip_prefix(crate::lifecycle::AGENT_PREFIX)
.map(str::to_owned)
})
.collect(),
Err(e) => {
return AgentResponse::Err {
message: format!("list containers failed: {e:#}"),
};
}
};
// Walk the full topology and collect every descendant.
let topo = crate::topology::read();
let mut names: Vec<String> = topo
.keys()
.filter(|name| crate::topology::is_descendant_of(name, agent))
.cloned()
.collect();
// Parents before children, then alpha within each tier.
crate::auto_update::topology_sort(&mut names, &topo);
let containers = names
.into_iter()
.map(|name| {
let running = running_set.contains(&name);
hive_sh4re::ContainerInfo { name, running }
})
.collect();
AgentResponse::Containers { containers }
}