hivectl: add hive-wide start/stop verbs

`hivectl stop` brings the whole hive down in one operator action — all
sub-agents plus the ci/forge/gateway/matrix infra containers — and
`hivectl start` brings it back up. Scope flags (--agents, --agent <name>,
--ci, --forge, --gateway, --matrix) narrow the set; a bare invocation
targets everything. hive-c0re never stops itself.

- hive-sh4re: HostRequest::{Stop,Start} + LifecycleScope wire type;
  priv_proto InfraAction + ControlInfraContainer + the
  CONTROLLABLE_INFRA_CONTAINERS allowlist (adds hive-matrix, excludes
  hive-c0re).
- hive-priv: control_infra_container handler (systemctl <verb>
  container@<name>, allowlist-validated root-side).
- hive-c0re: handle_stop / handle_start fan out agents via lifecycle and
  infra via hive-priv; per-target failures are aggregated. Infra
  systemctl routes through hive-priv (the privsep boundary).
- The --graceful flag is threaded through Stop now; the per-agent quiesce
  itself lands with the graceful-agent-stop work.
This commit is contained in:
atlas 2026-06-19 00:30:37 +02:00 committed by mara
commit fbb48ed3ce
6 changed files with 407 additions and 7 deletions

View file

@ -19,7 +19,7 @@ use std::os::unix::process::CommandExt as _;
use std::path::{Path, PathBuf};
use anyhow::{Context as _, Result, bail};
use clap::{Parser, Subcommand};
use clap::{Args, Parser, Subcommand};
use hive_c0re::coordinator::Coordinator;
#[derive(Parser)]
@ -100,6 +100,36 @@ enum Cmd {
#[arg(long)]
fresh: bool,
},
/// Stop containers hive-wide in one operator action. Bare `hivectl
/// stop` stops **everything** — all sub-agents plus the ci, forge,
/// gateway, and matrix infra containers. Narrow it with scope flags:
/// `--agents` (all sub-agents), `--ci` / `--forge` / `--gateway` /
/// `--matrix` (named infra), and `--agent <name>` (repeatable) for
/// specific sub-agents. Flags are additive (e.g. `--agents --matrix`).
/// Requires the hive-c0re daemon (connects to the host admin socket).
/// hive-c0re itself is never stopped — it services the request.
Stop {
#[command(flatten)]
scope: ScopeArgs,
/// Gracefully quiesce each agent (finish the current turn, drain
/// the inbox) before stopping, instead of a hard stop.
#[arg(long)]
graceful: bool,
/// Path to the hive-c0re host admin socket.
#[arg(long, default_value = DEFAULT_HOST_SOCKET)]
socket: PathBuf,
},
/// Start containers hive-wide — the inverse of `hivectl stop`. Bare
/// `hivectl start` starts everything back up; the same scope flags as
/// `stop` narrow it (`--agents`, `--ci`, `--forge`, `--gateway`,
/// `--matrix`, `--agent <name>`). Requires the hive-c0re daemon.
Start {
#[command(flatten)]
scope: ScopeArgs,
/// Path to the hive-c0re host admin socket.
#[arg(long, default_value = DEFAULT_HOST_SOCKET)]
socket: PathBuf,
},
/// Emit the full CLI reference as `CommonMark` to stdout.
///
/// Hidden tooling command (not part of day-to-day operator admin):
@ -111,6 +141,49 @@ enum Cmd {
MarkdownDocs,
}
/// Shared scope flags for `hivectl stop` / `hivectl start`. With no flag
/// set the verb targets **everything** (all sub-agents + every controllable
/// infra container). Setting any flag restricts to the selected classes,
/// additively.
// One bool per selectable container class, each mapping 1:1 to a clap flag;
// orthogonal toggles, not a state machine — hence the bools allow (mirrors
// `hive_sh4re::LifecycleScope`).
#[allow(clippy::struct_excessive_bools)]
#[derive(Args)]
struct ScopeArgs {
/// All sub-agent containers.
#[arg(long)]
agents: bool,
/// A specific sub-agent by name. Repeatable: `--agent a --agent b`.
#[arg(long = "agent", value_name = "NAME")]
agent: Vec<String>,
/// The CI runner container (`hive-ci`).
#[arg(long)]
ci: bool,
/// The forge container (`hive-forge`).
#[arg(long)]
forge: bool,
/// The gateway container (`hive-gateway`).
#[arg(long)]
gateway: bool,
/// The matrix container (`hive-matrix`).
#[arg(long)]
matrix: bool,
}
impl ScopeArgs {
fn to_scope(&self) -> hive_sh4re::LifecycleScope {
hive_sh4re::LifecycleScope {
agents: self.agents,
agent_names: self.agent.clone(),
ci: self.ci,
forge: self.forge,
gateway: self.gateway,
matrix: self.matrix,
}
}
}
#[derive(Subcommand)]
enum ForgeCmd {
/// Create or refresh the Forgejo account + token for `<name>`.
@ -348,6 +421,12 @@ async fn main() -> Result<()> {
AgentsCmd::Restart { name, socket } => agents_restart(&socket, &name).await,
AgentsCmd::RestartAll { socket } => agents_restart_all(&socket).await,
},
Cmd::Stop {
scope,
graceful,
socket,
} => stop(&socket, scope.to_scope(), graceful).await,
Cmd::Start { scope, socket } => start(&socket, scope.to_scope()).await,
Cmd::Choom { name, fresh } => choom(&name, fresh),
Cmd::MarkdownDocs => {
print!("{}", clap_markdown::help_markdown::<Cli>());
@ -804,6 +883,43 @@ async fn agents_restart_all(socket: &Path) -> Result<()> {
Ok(())
}
async fn stop(socket: &Path, scope: hive_sh4re::LifecycleScope, graceful: bool) -> Result<()> {
let resp =
hive_c0re::client::request(socket, hive_sh4re::HostRequest::Stop { scope, graceful })
.await
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
render_lifecycle(&resp, "stopped")
}
async fn start(socket: &Path, scope: hive_sh4re::LifecycleScope) -> Result<()> {
let resp = hive_c0re::client::request(socket, hive_sh4re::HostRequest::Start { scope })
.await
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
render_lifecycle(&resp, "started")
}
/// Render a hive-wide stop/start response: one `<verb>: <name>` line per
/// touched container, then surface any aggregated per-target failure as a
/// non-zero exit. `verb` is the past-tense word printed per item
/// (`stopped` / `started`).
fn render_lifecycle(resp: &hive_sh4re::HostResponse, verb: &str) -> Result<()> {
let items = resp.agents.as_deref().unwrap_or(&[]);
if items.is_empty() {
println!("{verb}: nothing matched the requested scope");
} else {
for item in items {
println!("{verb}: {item}");
}
}
if !resp.ok {
bail!(
"{verb}: {}",
resp.error.as_deref().unwrap_or("unknown error")
);
}
Ok(())
}
/// Reject usernames containing `:` (field separator) or control chars
/// that would corrupt the htpasswd file format.
fn validate_htpasswd_username(username: &str) -> Result<()> {

View file

@ -8,8 +8,8 @@
use anyhow::{Context as _, Result, bail};
use hive_sh4re::priv_proto::{
BindMount, JournalQuery, NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse,
PrivStream,
BindMount, InfraAction, JournalQuery, NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest,
PrivResponse, PrivStream,
};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::net::UnixStream;
@ -295,6 +295,20 @@ pub async fn restart_infra_container(container: &str) -> Result<()> {
.await?)
}
/// Start / stop / restart a hive infrastructure container (`hive-ci`,
/// `hive-gateway`, `hive-forge`, `hive-matrix`) on the host via `systemctl
/// <action> container@<container>.service`. hive-priv re-validates
/// `container` against its root-side allowlist
/// (`CONTROLLABLE_INFRA_CONTAINERS`). Used by the hive-wide `hivectl stop` /
/// `hivectl start` flow.
pub async fn control_infra_container(container: &str, action: InfraAction) -> Result<()> {
ok(call(&PrivRequest::ControlInfraContainer {
container: container.to_owned(),
action,
})
.await?)
}
fn check(resp: PrivResponse) -> Result<(String, String)> {
if resp.ok {
Ok((resp.stdout, resp.stderr))

View file

@ -2,7 +2,8 @@ use std::path::Path;
use std::sync::Arc;
use anyhow::{Context, Result};
use hive_sh4re::{HostRequest, HostResponse};
use hive_sh4re::priv_proto::InfraAction;
use hive_sh4re::{HostRequest, HostResponse, LifecycleScope};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::net::{UnixListener, UnixStream};
@ -93,6 +94,8 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
HostResponse::success()
}
HostRequest::RestartAll => handle_restart_all().await?,
HostRequest::Stop { scope, graceful } => handle_stop(scope, *graceful).await?,
HostRequest::Start { scope } => handle_start(scope).await?,
HostRequest::Destroy { name, purge } => {
actions::destroy(&coord, name, *purge).await?;
HostResponse::success()
@ -199,6 +202,134 @@ async fn handle_restart_all() -> Result<HostResponse> {
}
}
/// Stop the containers a [`LifecycleScope`] selects (`hivectl stop`): the
/// scoped sub-agents, then the scoped infra containers. Agents go down
/// before infra so they're not mid-request against a forge/matrix that's
/// already gone. Per-target failures are aggregated rather than aborting on
/// the first error, mirroring `handle_restart_all`.
async fn handle_stop(scope: &LifecycleScope, graceful: bool) -> Result<HostResponse> {
tracing::info!(?scope, graceful, "stop");
let mut ok_items: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
for agent in scoped_agents(scope).await? {
// TODO(graceful agent stop): when `graceful`, run the per-agent
// quiesce (turn-end → drain → reject new messages) before the kill.
// The flag is threaded through the wire now; the quiesce itself
// lands with the graceful-agent-stop work.
let _ = graceful;
match lifecycle::kill(&agent).await {
Ok(()) => ok_items.push(agent),
Err(e) => {
tracing::warn!(%agent, error = ?e, "stop: agent kill failed");
errors.push(format!("{agent}: {e:#}"));
}
}
}
for container in scoped_infra(scope) {
match crate::priv_client::control_infra_container(container, InfraAction::Stop).await {
Ok(()) => ok_items.push(container.to_owned()),
Err(e) => {
tracing::warn!(%container, error = ?e, "stop: infra stop failed");
errors.push(format!("{container}: {e:#}"));
}
}
}
Ok(finish_lifecycle(ok_items, &errors))
}
/// Start the containers a [`LifecycleScope`] selects (`hivectl start`) —
/// the inverse of [`handle_stop`]. Infra comes up before agents so the
/// agents find forge/matrix/gateway ready. Per-target failures aggregated.
async fn handle_start(scope: &LifecycleScope) -> Result<HostResponse> {
tracing::info!(?scope, "start");
let mut ok_items: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
for container in scoped_infra(scope) {
match crate::priv_client::control_infra_container(container, InfraAction::Start).await {
Ok(()) => ok_items.push(container.to_owned()),
Err(e) => {
tracing::warn!(%container, error = ?e, "start: infra start failed");
errors.push(format!("{container}: {e:#}"));
}
}
}
for agent in scoped_agents(scope).await? {
match lifecycle::start(&agent).await {
Ok(()) => ok_items.push(agent),
Err(e) => {
tracing::warn!(%agent, error = ?e, "start: agent start failed");
errors.push(format!("{agent}: {e:#}"));
}
}
}
Ok(finish_lifecycle(ok_items, &errors))
}
/// Resolve which sub-agent logical names a scope targets: every live
/// container (from `lifecycle::list`) when `agents` is set or the scope is
/// "everything", plus any explicit `agent_names`. Returns de-duplicated
/// logical names with the `h-` container prefix stripped.
async fn scoped_agents(scope: &LifecycleScope) -> Result<Vec<String>> {
use std::collections::BTreeSet;
let mut set: BTreeSet<String> = BTreeSet::new();
if scope.agents || scope.is_everything() {
for c in lifecycle::list().await? {
let logical = c
.strip_prefix(lifecycle::AGENT_PREFIX)
.unwrap_or(&c)
.to_owned();
set.insert(logical);
}
}
for n in &scope.agent_names {
set.insert(n.clone());
}
Ok(set.into_iter().collect())
}
/// Resolve which infra container names a scope targets. An "everything"
/// scope (no flags set) selects all controllable infra; otherwise each set
/// flag maps to its container. Fixed order for deterministic output.
fn scoped_infra(scope: &LifecycleScope) -> Vec<&'static str> {
let everything = scope.is_everything();
let mut out = Vec::new();
if everything || scope.ci {
out.push("hive-ci");
}
if everything || scope.forge {
out.push("hive-forge");
}
if everything || scope.gateway {
out.push("hive-gateway");
}
if everything || scope.matrix {
out.push("hive-matrix");
}
out
}
/// Build the aggregated lifecycle response: `ok` with the touched names when
/// every target succeeded, otherwise `ok: false` with the joined errors and
/// the partial success list (matches `handle_restart_all`).
fn finish_lifecycle(ok_items: Vec<String>, errors: &[String]) -> HostResponse {
if errors.is_empty() {
HostResponse::list(ok_items)
} else {
HostResponse {
ok: false,
error: Some(errors.join("; ")),
agents: Some(ok_items),
approvals: None,
}
}
}
/// Rebuild `name`'s container, notifying the manager of the outcome
/// (success or failure) and kicking the agent's next turn on success.
async fn handle_rebuild(coord: &Arc<Coordinator>, name: &str) -> Result<HostResponse> {

View file

@ -21,9 +21,9 @@ use std::path::{Path, PathBuf};
use anyhow::{Context as _, Result, bail};
use hive_sh4re::priv_proto::{
AGENT_PREFIX, AGENT_STATE_ROOT, BindMount, JournalQuery, MANAGER_NAME, META_DIR,
NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream, PrivStreamLine,
RESTARTABLE_INFRA_CONTAINERS, SIBLING_CONTAINERS,
AGENT_PREFIX, AGENT_STATE_ROOT, BindMount, CONTROLLABLE_INFRA_CONTAINERS, InfraAction,
JournalQuery, MANAGER_NAME, META_DIR, NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest,
PrivResponse, PrivStream, PrivStreamLine, RESTARTABLE_INFRA_CONTAINERS, SIBLING_CONTAINERS,
};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::net::unix::OwnedWriteHalf;
@ -268,6 +268,11 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String,
PrivRequest::RestartInfraContainer { ref container } => {
restart_infra_container(container).await
}
PrivRequest::ControlInfraContainer {
ref container,
action,
} => control_infra_container(container, action).await,
}
}
@ -423,6 +428,37 @@ async fn restart_infra_container(container: &str) -> Result<(String, String)> {
))
}
/// `ControlInfraContainer` — start/stop/restart a hive infrastructure
/// container via `systemctl <verb> container@<container>.service`. The
/// `container` is validated against `CONTROLLABLE_INFRA_CONTAINERS` here,
/// root-side; this is the authoritative allowlist (hive-c0re itself can
/// never appear in it, so a hive-wide stop can't sever the daemon socket
/// the request arrived on).
async fn control_infra_container(container: &str, action: InfraAction) -> Result<(String, String)> {
if !CONTROLLABLE_INFRA_CONTAINERS.contains(&container) {
bail!("container {container:?} is not a controllable hive infra container");
}
let verb = action.systemctl_verb();
let unit = format!("container@{container}.service");
let out = Command::new("systemctl")
.args([verb, &unit])
.output()
.await
.with_context(|| format!("systemctl {verb} {unit}"))?;
if !out.status.success() {
bail!(
"systemctl {verb} {unit} exited {}: {}",
out.status,
String::from_utf8_lossy(&out.stderr).trim()
);
}
tracing::info!(target: "infra-control", "{verb} {unit}");
Ok((
String::from_utf8_lossy(&out.stdout).into_owned(),
String::from_utf8_lossy(&out.stderr).into_owned(),
))
}
/// Shared helper for `WriteAgentForgeToken` and `WriteAgentMatrixToken`.
/// Writes `content` to `AGENT_STATE_ROOT/<agent_name>/state/<filename>`,
/// chowns to the agent user (derived from the state dir's existing owner),

View file

@ -58,6 +58,68 @@ pub enum HostRequest {
child: String,
new_parent: Option<String>,
},
/// Stop managed containers hive-wide in one operator action
/// (`hivectl stop`): agents plus the selected infra containers. `scope`
/// selects which classes; an all-false scope means **everything** (the
/// bare `hivectl stop`). `graceful` runs the per-agent quiesce (graceful
/// agent stop, issue tracker `graceful agent stop`) instead of a hard
/// stop. Agents stop via the lifecycle path; infra containers via the
/// host `container@<name>` units.
Stop {
#[serde(default)]
scope: LifecycleScope,
#[serde(default)]
graceful: bool,
},
/// Start managed containers hive-wide — the inverse of `Stop`
/// (`hivectl start`). Same `scope` semantics (all-false = everything);
/// no graceful flag (start is unconditional).
Start {
#[serde(default)]
scope: LifecycleScope,
},
}
/// Selects which container classes a hive-wide [`HostRequest::Stop`] /
/// [`HostRequest::Start`] touches. An all-false scope means **everything**
/// (the bare `hivectl stop` / `start`); set individual fields to restrict
/// (e.g. only `agents` → just the sub-agent containers). `agents` covers
/// every managed sub-agent container; the rest are the named infra
/// containers (`hive-ci`, `hive-forge`, `hive-gateway`, `hive-matrix`).
//
// A flat bag of independent flag toggles — one bool per selectable
// container class — is exactly the right shape here: each maps 1:1 to a
// `hivectl` `--ci` / `--forge` / `--gateway` / `--matrix` flag, and they're
// orthogonal (any subset is valid), so a state machine or two-variant enums
// would only obscure the mapping. Hence the `struct_excessive_bools` allow.
#[allow(clippy::struct_excessive_bools)]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct LifecycleScope {
/// All sub-agent containers (`--agents`).
#[serde(default)]
pub agents: bool,
/// Specific sub-agents by logical name (`--agent <name>`, repeatable).
/// Additive with the rest of the scope; redundant when `agents` is set
/// (which already covers every sub-agent).
#[serde(default)]
pub agent_names: Vec<String>,
#[serde(default)]
pub ci: bool,
#[serde(default)]
pub forge: bool,
#[serde(default)]
pub gateway: bool,
#[serde(default)]
pub matrix: bool,
}
impl LifecycleScope {
/// True when nothing is explicitly selected — interpreted as "all
/// classes" (the bare `hivectl stop` / `start` with no scope flags).
pub fn is_everything(&self) -> bool {
!(self.agents || self.ci || self.forge || self.gateway || self.matrix)
&& self.agent_names.is_empty()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]

View file

@ -27,6 +27,37 @@ pub const SIBLING_CONTAINERS: &[&str] = &["hive-forge", "hive-matrix", "hive-gat
/// it is the authoritative allowlist regardless of what the caller sends.
pub const RESTARTABLE_INFRA_CONTAINERS: &[&str] = &["hive-ci", "hive-gateway", "hive-forge"];
/// Infra containers hive-c0re may stop/start/restart hive-wide for the
/// `hivectl stop` / `hivectl start` operator flow. Superset of
/// [`RESTARTABLE_INFRA_CONTAINERS`]: it adds `hive-matrix`, because a full
/// stop is a deliberate operator action (unlike the disruptive mid-sync
/// *restart* the `infra_admin` MCP path forbids). `hive-c0re` is still
/// excluded — it runs the daemon servicing the request and must never stop
/// itself. hive-priv re-validates against this list root-side.
pub const CONTROLLABLE_INFRA_CONTAINERS: &[&str] =
&["hive-ci", "hive-gateway", "hive-forge", "hive-matrix"];
/// Lifecycle verb for [`PrivRequest::ControlInfraContainer`]. Maps directly
/// to `systemctl <verb> container@<container>.service`.
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum InfraAction {
Start,
Stop,
Restart,
}
impl InfraAction {
/// The `systemctl` subcommand this action maps to.
pub fn systemctl_verb(self) -> &'static str {
match self {
InfraAction::Start => "start",
InfraAction::Stop => "stop",
InfraAction::Restart => "restart",
}
}
}
/// Host path of the meta flake. The flake ref for agent `<name>` is
/// `{META_DIR}#{name}`, derived by `hive-priv` — never passed over the wire.
pub const META_DIR: &str = "/var/lib/hyperhive/meta";
@ -316,6 +347,16 @@ pub enum PrivRequest {
/// [`RESTARTABLE_INFRA_CONTAINERS`].
container: String,
},
/// Start/stop/restart a hive infrastructure container on the host via
/// `systemctl <action> container@<container>.service`. hive-priv
/// validates `container` against [`CONTROLLABLE_INFRA_CONTAINERS`]
/// root-side. Generalises [`PrivRequest::RestartInfraContainer`] for the
/// hive-wide `hivectl stop` / `hivectl start` operator flow.
ControlInfraContainer {
container: String,
action: InfraAction,
},
}
/// Response from the privileged helper.