diff --git a/hive-c0re/src/bin/hivectl.rs b/hive-c0re/src/bin/hivectl.rs index 9374c004..40a9c81e 100644 --- a/hive-c0re/src/bin/hivectl.rs +++ b/hive-c0re/src/bin/hivectl.rs @@ -19,7 +19,7 @@ use std::os::unix::process::CommandExt as _; use std::path::{Path, PathBuf}; use anyhow::{Context as _, Result, bail}; -use clap::{Parser, Subcommand}; +use clap::{Args, Parser, Subcommand}; use hive_c0re::coordinator::Coordinator; #[derive(Parser)] @@ -100,6 +100,36 @@ enum Cmd { #[arg(long)] fresh: bool, }, + /// Stop containers hive-wide in one operator action. Bare `hivectl + /// stop` stops **everything** — all sub-agents plus the ci, forge, + /// gateway, and matrix infra containers. Narrow it with scope flags: + /// `--agents` (all sub-agents), `--ci` / `--forge` / `--gateway` / + /// `--matrix` (named infra), and `--agent ` (repeatable) for + /// specific sub-agents. Flags are additive (e.g. `--agents --matrix`). + /// Requires the hive-c0re daemon (connects to the host admin socket). + /// hive-c0re itself is never stopped — it services the request. + Stop { + #[command(flatten)] + scope: ScopeArgs, + /// Gracefully quiesce each agent (finish the current turn, drain + /// the inbox) before stopping, instead of a hard stop. + #[arg(long)] + graceful: bool, + /// Path to the hive-c0re host admin socket. + #[arg(long, default_value = DEFAULT_HOST_SOCKET)] + socket: PathBuf, + }, + /// Start containers hive-wide — the inverse of `hivectl stop`. Bare + /// `hivectl start` starts everything back up; the same scope flags as + /// `stop` narrow it (`--agents`, `--ci`, `--forge`, `--gateway`, + /// `--matrix`, `--agent `). Requires the hive-c0re daemon. + Start { + #[command(flatten)] + scope: ScopeArgs, + /// Path to the hive-c0re host admin socket. + #[arg(long, default_value = DEFAULT_HOST_SOCKET)] + socket: PathBuf, + }, /// Emit the full CLI reference as `CommonMark` to stdout. /// /// Hidden tooling command (not part of day-to-day operator admin): @@ -111,6 +141,49 @@ enum Cmd { MarkdownDocs, } +/// Shared scope flags for `hivectl stop` / `hivectl start`. With no flag +/// set the verb targets **everything** (all sub-agents + every controllable +/// infra container). Setting any flag restricts to the selected classes, +/// additively. +// One bool per selectable container class, each mapping 1:1 to a clap flag; +// orthogonal toggles, not a state machine — hence the bools allow (mirrors +// `hive_sh4re::LifecycleScope`). +#[allow(clippy::struct_excessive_bools)] +#[derive(Args)] +struct ScopeArgs { + /// All sub-agent containers. + #[arg(long)] + agents: bool, + /// A specific sub-agent by name. Repeatable: `--agent a --agent b`. + #[arg(long = "agent", value_name = "NAME")] + agent: Vec, + /// The CI runner container (`hive-ci`). + #[arg(long)] + ci: bool, + /// The forge container (`hive-forge`). + #[arg(long)] + forge: bool, + /// The gateway container (`hive-gateway`). + #[arg(long)] + gateway: bool, + /// The matrix container (`hive-matrix`). + #[arg(long)] + matrix: bool, +} + +impl ScopeArgs { + fn to_scope(&self) -> hive_sh4re::LifecycleScope { + hive_sh4re::LifecycleScope { + agents: self.agents, + agent_names: self.agent.clone(), + ci: self.ci, + forge: self.forge, + gateway: self.gateway, + matrix: self.matrix, + } + } +} + #[derive(Subcommand)] enum ForgeCmd { /// Create or refresh the Forgejo account + token for ``. @@ -348,6 +421,12 @@ async fn main() -> Result<()> { AgentsCmd::Restart { name, socket } => agents_restart(&socket, &name).await, AgentsCmd::RestartAll { socket } => agents_restart_all(&socket).await, }, + Cmd::Stop { + scope, + graceful, + socket, + } => stop(&socket, scope.to_scope(), graceful).await, + Cmd::Start { scope, socket } => start(&socket, scope.to_scope()).await, Cmd::Choom { name, fresh } => choom(&name, fresh), Cmd::MarkdownDocs => { print!("{}", clap_markdown::help_markdown::()); @@ -804,6 +883,43 @@ async fn agents_restart_all(socket: &Path) -> Result<()> { Ok(()) } +async fn stop(socket: &Path, scope: hive_sh4re::LifecycleScope, graceful: bool) -> Result<()> { + let resp = + hive_c0re::client::request(socket, hive_sh4re::HostRequest::Stop { scope, graceful }) + .await + .with_context(|| format!("connect to daemon socket {}", socket.display()))?; + render_lifecycle(&resp, "stopped") +} + +async fn start(socket: &Path, scope: hive_sh4re::LifecycleScope) -> Result<()> { + let resp = hive_c0re::client::request(socket, hive_sh4re::HostRequest::Start { scope }) + .await + .with_context(|| format!("connect to daemon socket {}", socket.display()))?; + render_lifecycle(&resp, "started") +} + +/// Render a hive-wide stop/start response: one `: ` line per +/// touched container, then surface any aggregated per-target failure as a +/// non-zero exit. `verb` is the past-tense word printed per item +/// (`stopped` / `started`). +fn render_lifecycle(resp: &hive_sh4re::HostResponse, verb: &str) -> Result<()> { + let items = resp.agents.as_deref().unwrap_or(&[]); + if items.is_empty() { + println!("{verb}: nothing matched the requested scope"); + } else { + for item in items { + println!("{verb}: {item}"); + } + } + if !resp.ok { + bail!( + "{verb}: {}", + resp.error.as_deref().unwrap_or("unknown error") + ); + } + Ok(()) +} + /// Reject usernames containing `:` (field separator) or control chars /// that would corrupt the htpasswd file format. fn validate_htpasswd_username(username: &str) -> Result<()> { diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index 6f38a131..4993d0eb 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -8,8 +8,8 @@ use anyhow::{Context as _, Result, bail}; use hive_sh4re::priv_proto::{ - BindMount, JournalQuery, NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, - PrivStream, + BindMount, InfraAction, JournalQuery, NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, + PrivResponse, PrivStream, }; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::net::UnixStream; @@ -295,6 +295,20 @@ pub async fn restart_infra_container(container: &str) -> Result<()> { .await?) } +/// Start / stop / restart a hive infrastructure container (`hive-ci`, +/// `hive-gateway`, `hive-forge`, `hive-matrix`) on the host via `systemctl +/// container@.service`. hive-priv re-validates +/// `container` against its root-side allowlist +/// (`CONTROLLABLE_INFRA_CONTAINERS`). Used by the hive-wide `hivectl stop` / +/// `hivectl start` flow. +pub async fn control_infra_container(container: &str, action: InfraAction) -> Result<()> { + ok(call(&PrivRequest::ControlInfraContainer { + container: container.to_owned(), + action, + }) + .await?) +} + fn check(resp: PrivResponse) -> Result<(String, String)> { if resp.ok { Ok((resp.stdout, resp.stderr)) diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index 69e643d6..ceb46b3e 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -2,7 +2,8 @@ use std::path::Path; use std::sync::Arc; use anyhow::{Context, Result}; -use hive_sh4re::{HostRequest, HostResponse}; +use hive_sh4re::priv_proto::InfraAction; +use hive_sh4re::{HostRequest, HostResponse, LifecycleScope}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::net::{UnixListener, UnixStream}; @@ -93,6 +94,8 @@ async fn dispatch(req: &HostRequest, coord: Arc) -> HostResponse { HostResponse::success() } HostRequest::RestartAll => handle_restart_all().await?, + HostRequest::Stop { scope, graceful } => handle_stop(scope, *graceful).await?, + HostRequest::Start { scope } => handle_start(scope).await?, HostRequest::Destroy { name, purge } => { actions::destroy(&coord, name, *purge).await?; HostResponse::success() @@ -199,6 +202,134 @@ async fn handle_restart_all() -> Result { } } +/// Stop the containers a [`LifecycleScope`] selects (`hivectl stop`): the +/// scoped sub-agents, then the scoped infra containers. Agents go down +/// before infra so they're not mid-request against a forge/matrix that's +/// already gone. Per-target failures are aggregated rather than aborting on +/// the first error, mirroring `handle_restart_all`. +async fn handle_stop(scope: &LifecycleScope, graceful: bool) -> Result { + tracing::info!(?scope, graceful, "stop"); + let mut ok_items: Vec = Vec::new(); + let mut errors: Vec = Vec::new(); + + for agent in scoped_agents(scope).await? { + // TODO(graceful agent stop): when `graceful`, run the per-agent + // quiesce (turn-end → drain → reject new messages) before the kill. + // The flag is threaded through the wire now; the quiesce itself + // lands with the graceful-agent-stop work. + let _ = graceful; + match lifecycle::kill(&agent).await { + Ok(()) => ok_items.push(agent), + Err(e) => { + tracing::warn!(%agent, error = ?e, "stop: agent kill failed"); + errors.push(format!("{agent}: {e:#}")); + } + } + } + + for container in scoped_infra(scope) { + match crate::priv_client::control_infra_container(container, InfraAction::Stop).await { + Ok(()) => ok_items.push(container.to_owned()), + Err(e) => { + tracing::warn!(%container, error = ?e, "stop: infra stop failed"); + errors.push(format!("{container}: {e:#}")); + } + } + } + + Ok(finish_lifecycle(ok_items, &errors)) +} + +/// Start the containers a [`LifecycleScope`] selects (`hivectl start`) — +/// the inverse of [`handle_stop`]. Infra comes up before agents so the +/// agents find forge/matrix/gateway ready. Per-target failures aggregated. +async fn handle_start(scope: &LifecycleScope) -> Result { + tracing::info!(?scope, "start"); + let mut ok_items: Vec = Vec::new(); + let mut errors: Vec = Vec::new(); + + for container in scoped_infra(scope) { + match crate::priv_client::control_infra_container(container, InfraAction::Start).await { + Ok(()) => ok_items.push(container.to_owned()), + Err(e) => { + tracing::warn!(%container, error = ?e, "start: infra start failed"); + errors.push(format!("{container}: {e:#}")); + } + } + } + + for agent in scoped_agents(scope).await? { + match lifecycle::start(&agent).await { + Ok(()) => ok_items.push(agent), + Err(e) => { + tracing::warn!(%agent, error = ?e, "start: agent start failed"); + errors.push(format!("{agent}: {e:#}")); + } + } + } + + Ok(finish_lifecycle(ok_items, &errors)) +} + +/// Resolve which sub-agent logical names a scope targets: every live +/// container (from `lifecycle::list`) when `agents` is set or the scope is +/// "everything", plus any explicit `agent_names`. Returns de-duplicated +/// logical names with the `h-` container prefix stripped. +async fn scoped_agents(scope: &LifecycleScope) -> Result> { + use std::collections::BTreeSet; + let mut set: BTreeSet = BTreeSet::new(); + if scope.agents || scope.is_everything() { + for c in lifecycle::list().await? { + let logical = c + .strip_prefix(lifecycle::AGENT_PREFIX) + .unwrap_or(&c) + .to_owned(); + set.insert(logical); + } + } + for n in &scope.agent_names { + set.insert(n.clone()); + } + Ok(set.into_iter().collect()) +} + +/// Resolve which infra container names a scope targets. An "everything" +/// scope (no flags set) selects all controllable infra; otherwise each set +/// flag maps to its container. Fixed order for deterministic output. +fn scoped_infra(scope: &LifecycleScope) -> Vec<&'static str> { + let everything = scope.is_everything(); + let mut out = Vec::new(); + if everything || scope.ci { + out.push("hive-ci"); + } + if everything || scope.forge { + out.push("hive-forge"); + } + if everything || scope.gateway { + out.push("hive-gateway"); + } + if everything || scope.matrix { + out.push("hive-matrix"); + } + out +} + +/// Build the aggregated lifecycle response: `ok` with the touched names when +/// every target succeeded, otherwise `ok: false` with the joined errors and +/// the partial success list (matches `handle_restart_all`). +fn finish_lifecycle(ok_items: Vec, errors: &[String]) -> HostResponse { + if errors.is_empty() { + HostResponse::list(ok_items) + } else { + HostResponse { + ok: false, + error: Some(errors.join("; ")), + agents: Some(ok_items), + approvals: None, + } + } +} + /// Rebuild `name`'s container, notifying the manager of the outcome /// (success or failure) and kicking the agent's next turn on success. async fn handle_rebuild(coord: &Arc, name: &str) -> Result { diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 55c69f94..e3a0515b 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -21,9 +21,9 @@ use std::path::{Path, PathBuf}; use anyhow::{Context as _, Result, bail}; use hive_sh4re::priv_proto::{ - AGENT_PREFIX, AGENT_STATE_ROOT, BindMount, JournalQuery, MANAGER_NAME, META_DIR, - NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream, PrivStreamLine, - RESTARTABLE_INFRA_CONTAINERS, SIBLING_CONTAINERS, + AGENT_PREFIX, AGENT_STATE_ROOT, BindMount, CONTROLLABLE_INFRA_CONTAINERS, InfraAction, + JournalQuery, MANAGER_NAME, META_DIR, NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, + PrivResponse, PrivStream, PrivStreamLine, RESTARTABLE_INFRA_CONTAINERS, SIBLING_CONTAINERS, }; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::net::unix::OwnedWriteHalf; @@ -268,6 +268,11 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String, PrivRequest::RestartInfraContainer { ref container } => { restart_infra_container(container).await } + + PrivRequest::ControlInfraContainer { + ref container, + action, + } => control_infra_container(container, action).await, } } @@ -423,6 +428,37 @@ async fn restart_infra_container(container: &str) -> Result<(String, String)> { )) } +/// `ControlInfraContainer` — start/stop/restart a hive infrastructure +/// container via `systemctl container@.service`. The +/// `container` is validated against `CONTROLLABLE_INFRA_CONTAINERS` here, +/// root-side; this is the authoritative allowlist (hive-c0re itself can +/// never appear in it, so a hive-wide stop can't sever the daemon socket +/// the request arrived on). +async fn control_infra_container(container: &str, action: InfraAction) -> Result<(String, String)> { + if !CONTROLLABLE_INFRA_CONTAINERS.contains(&container) { + bail!("container {container:?} is not a controllable hive infra container"); + } + let verb = action.systemctl_verb(); + let unit = format!("container@{container}.service"); + let out = Command::new("systemctl") + .args([verb, &unit]) + .output() + .await + .with_context(|| format!("systemctl {verb} {unit}"))?; + if !out.status.success() { + bail!( + "systemctl {verb} {unit} exited {}: {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + tracing::info!(target: "infra-control", "{verb} {unit}"); + Ok(( + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + )) +} + /// Shared helper for `WriteAgentForgeToken` and `WriteAgentMatrixToken`. /// Writes `content` to `AGENT_STATE_ROOT//state/`, /// chowns to the agent user (derived from the state dir's existing owner), diff --git a/hive-sh4re/src/lib.rs b/hive-sh4re/src/lib.rs index 7308761e..ba3789a7 100644 --- a/hive-sh4re/src/lib.rs +++ b/hive-sh4re/src/lib.rs @@ -58,6 +58,68 @@ pub enum HostRequest { child: String, new_parent: Option, }, + /// Stop managed containers hive-wide in one operator action + /// (`hivectl stop`): agents plus the selected infra containers. `scope` + /// selects which classes; an all-false scope means **everything** (the + /// bare `hivectl stop`). `graceful` runs the per-agent quiesce (graceful + /// agent stop, issue tracker `graceful agent stop`) instead of a hard + /// stop. Agents stop via the lifecycle path; infra containers via the + /// host `container@` units. + Stop { + #[serde(default)] + scope: LifecycleScope, + #[serde(default)] + graceful: bool, + }, + /// Start managed containers hive-wide — the inverse of `Stop` + /// (`hivectl start`). Same `scope` semantics (all-false = everything); + /// no graceful flag (start is unconditional). + Start { + #[serde(default)] + scope: LifecycleScope, + }, +} + +/// Selects which container classes a hive-wide [`HostRequest::Stop`] / +/// [`HostRequest::Start`] touches. An all-false scope means **everything** +/// (the bare `hivectl stop` / `start`); set individual fields to restrict +/// (e.g. only `agents` → just the sub-agent containers). `agents` covers +/// every managed sub-agent container; the rest are the named infra +/// containers (`hive-ci`, `hive-forge`, `hive-gateway`, `hive-matrix`). +// +// A flat bag of independent flag toggles — one bool per selectable +// container class — is exactly the right shape here: each maps 1:1 to a +// `hivectl` `--ci` / `--forge` / `--gateway` / `--matrix` flag, and they're +// orthogonal (any subset is valid), so a state machine or two-variant enums +// would only obscure the mapping. Hence the `struct_excessive_bools` allow. +#[allow(clippy::struct_excessive_bools)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct LifecycleScope { + /// All sub-agent containers (`--agents`). + #[serde(default)] + pub agents: bool, + /// Specific sub-agents by logical name (`--agent `, repeatable). + /// Additive with the rest of the scope; redundant when `agents` is set + /// (which already covers every sub-agent). + #[serde(default)] + pub agent_names: Vec, + #[serde(default)] + pub ci: bool, + #[serde(default)] + pub forge: bool, + #[serde(default)] + pub gateway: bool, + #[serde(default)] + pub matrix: bool, +} + +impl LifecycleScope { + /// True when nothing is explicitly selected — interpreted as "all + /// classes" (the bare `hivectl stop` / `start` with no scope flags). + pub fn is_everything(&self) -> bool { + !(self.agents || self.ci || self.forge || self.gateway || self.matrix) + && self.agent_names.is_empty() + } } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/hive-sh4re/src/priv_proto.rs b/hive-sh4re/src/priv_proto.rs index b23fca72..8265e4ff 100644 --- a/hive-sh4re/src/priv_proto.rs +++ b/hive-sh4re/src/priv_proto.rs @@ -27,6 +27,37 @@ pub const SIBLING_CONTAINERS: &[&str] = &["hive-forge", "hive-matrix", "hive-gat /// it is the authoritative allowlist regardless of what the caller sends. pub const RESTARTABLE_INFRA_CONTAINERS: &[&str] = &["hive-ci", "hive-gateway", "hive-forge"]; +/// Infra containers hive-c0re may stop/start/restart hive-wide for the +/// `hivectl stop` / `hivectl start` operator flow. Superset of +/// [`RESTARTABLE_INFRA_CONTAINERS`]: it adds `hive-matrix`, because a full +/// stop is a deliberate operator action (unlike the disruptive mid-sync +/// *restart* the `infra_admin` MCP path forbids). `hive-c0re` is still +/// excluded — it runs the daemon servicing the request and must never stop +/// itself. hive-priv re-validates against this list root-side. +pub const CONTROLLABLE_INFRA_CONTAINERS: &[&str] = + &["hive-ci", "hive-gateway", "hive-forge", "hive-matrix"]; + +/// Lifecycle verb for [`PrivRequest::ControlInfraContainer`]. Maps directly +/// to `systemctl container@.service`. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InfraAction { + Start, + Stop, + Restart, +} + +impl InfraAction { + /// The `systemctl` subcommand this action maps to. + pub fn systemctl_verb(self) -> &'static str { + match self { + InfraAction::Start => "start", + InfraAction::Stop => "stop", + InfraAction::Restart => "restart", + } + } +} + /// Host path of the meta flake. The flake ref for agent `` is /// `{META_DIR}#{name}`, derived by `hive-priv` — never passed over the wire. pub const META_DIR: &str = "/var/lib/hyperhive/meta"; @@ -316,6 +347,16 @@ pub enum PrivRequest { /// [`RESTARTABLE_INFRA_CONTAINERS`]. container: String, }, + + /// Start/stop/restart a hive infrastructure container on the host via + /// `systemctl container@.service`. hive-priv + /// validates `container` against [`CONTROLLABLE_INFRA_CONTAINERS`] + /// root-side. Generalises [`PrivRequest::RestartInfraContainer`] for the + /// hive-wide `hivectl stop` / `hivectl start` operator flow. + ControlInfraContainer { + container: String, + action: InfraAction, + }, } /// Response from the privileged helper.