vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(hivectl): queue-routed lifecycle verbs with wait + DAG progress

Browse source 
every agent lifecycle verb on the admin socket (rebuild / restart /
restart-all / kill / stop / start) now submits job-queue DAGs and
returns their ids; hivectl polls the new HostRequest::QueueDag and
prints a live node-chain progress line per DAG (fan-out children
included), exiting non-zero on failure — --no-wait opts out. DagView
and the queue wire enums move to hive_sh4re::jobs (wire types live in
the shared crate); the last fused rebuild path (lifecycle::rebuild)
is gone. tracker: #2166
This commit is contained in:
müde 2026-07-06 22:30:49 +02:00
commit b489454dc2
9 changed files with 641 additions and 443 deletions
Download patch file Download diff file Expand all files Collapse all files

220
hive-c0re/src/server.rs
View file

@ -90,21 +90,8 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
tracing::info!(%id, %name, "spawn approval queued");
HostResponse::success()
}
HostRequest::Kill { name } => handle_kill(&coord, name).await?,
HostRequest::Restart { name } => {
tracing::info!(%name, "restart");
// Through the queue: serializes against in-flight
// rebuilds via the agent lease, writes `wanted = Up`,
// and gets the transient/crash-watch suppression the
// direct kill+start lacked. Returns once queued.
crate::job_queue::submit::restart(
&coord,
name,
crate::job_queue::Source::Manual,
"manual restart via hivectl".to_owned(),
);
HostResponse::success()
}
HostRequest::Kill { name } => submit_single(&coord, name, Verb::Kill),
HostRequest::Restart { name } => submit_single(&coord, name, Verb::Restart),
HostRequest::RestartAll => handle_restart_all(&coord).await?,
HostRequest::Stop { scope, graceful } => {
// Resolve the scope to explicit container names at the entry
@ -146,7 +133,17 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
actions::destroy(&coord, name, *purge).await?;
HostResponse::success()
}
HostRequest::Rebuild { name } => handle_rebuild(&coord, name).await?,
HostRequest::Rebuild { name } => submit_single(&coord, name, Verb::Rebuild),
HostRequest::QueueDag { id } => {
// The polled DAG first, then its live fan-out children.
let dags = coord
.job_queue
.snapshot()
.into_iter()
.filter(|d| d.id == *id || d.parent_id == Some(*id))
.collect();
HostResponse::dags(dags)
}
HostRequest::List => HostResponse::list(lifecycle::list().await?),
HostRequest::AgentStatus => {
let rows = crate::container_view::build_all(&coord)
@ -235,19 +232,55 @@ async fn handle_spawn(coord: &Arc<Coordinator>, name: &str) -> Result<HostRespon
Ok(HostResponse::success())
}
/// Kill `name`'s container, unregister its socket, notify the manager.
/// Persists `wanted = Offline` first so reconciles don't undo the kill.
async fn handle_kill(coord: &Arc<Coordinator>, name: &str) -> Result<HostResponse> {
tracing::info!(%name, "kill");
if let Err(e) = coord.power.set(name, crate::power::Wanted::Offline) {
tracing::warn!(%name, error = ?e, "agent_power: set wanted=offline failed");
}
lifecycle::kill(name).await?;
coord.unregister_agent(name);
coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
agent: name.to_owned(),
});
Ok(HostResponse::success())
/// Single-agent queue verbs the admin socket exposes. Each submits the
/// matching DAG (persisting the `wanted` intent, serializing on the
/// agent's lease, with the transient/crash-watch suppression the old
/// direct lifecycle calls lacked) and returns the DAG id for the
/// client's wait loop.
#[derive(Clone, Copy)]
enum Verb {
/// Stop DAG (`wanted = Offline`; Reconcile kills + unregisters +
/// fires `Killed`).
Kill,
/// Restart DAG (`wanted = Up`; mechanical stop + reconcile-start).
Restart,
/// Rebuild DAG — the Swap tail owns the manager `Rebuilt` events +
/// kick, so the CLI path can't drift from the dashboard's.
Rebuild,
}
fn submit_single(coord: &Arc<Coordinator>, name: &str, verb: Verb) -> HostResponse {
use crate::job_queue::{Source, submit};
let id = match verb {
Verb::Kill => {
tracing::info!(%name, "kill");
submit::stop(
coord,
name,
Source::Manual,
"manual kill via hivectl".to_owned(),
)
}
Verb::Restart => {
tracing::info!(%name, "restart");
submit::restart(
coord,
name,
Source::Manual,
"manual restart via hivectl".to_owned(),
)
}
Verb::Rebuild => {
tracing::info!(%name, "rebuild");
submit::rebuild(
coord,
name,
Source::Manual,
"manual rebuild via hivectl".to_owned(),
)
}
};
HostResponse::queued(vec![id])
}
/// Restart every container by submitting one restart DAG per agent —
@ -280,11 +313,13 @@ async fn handle_restart_all(coord: &Arc<Coordinator>) -> Result<HostResponse> {
/// `handle_restart_all`. Callers resolve the [`LifecycleScope`] to these
/// explicit name lists up front — this never sees the "all" flag.
///
/// A `graceful` stop enqueues a `QueueKind::GracefulStop` per agent (signal the
/// harness, run one stop-checkpoint turn, drain, then container stop, with a
/// timeout fallback to a hard stop), mirroring the dashboard `?graceful=1`
/// path. `graceful` applies to agents only - infra containers have no harness
/// turn loop, so they're always hard-stopped.
/// Every agent rides the job queue: a `graceful` stop submits the
/// quiesce DAG (signal → drain → reconcile-stop; all drains overlap),
/// a hard stop a plain stop DAG — both persist `wanted = Offline` and
/// serialize on the agent's lease so nothing races an in-flight
/// rebuild. The response carries the DAG ids so `hivectl` can wait
/// with per-node progress. Infra containers have no harness / lease
/// and stay direct + synchronous.
async fn handle_stop(
coord: &Arc<Coordinator>,
agents: &[String],
@ -294,36 +329,31 @@ async fn handle_stop(
tracing::info!(?agents, ?infra, graceful, "stop");
let mut ok_items: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
let mut queued: Vec<u64> = Vec::new();
for agent in agents {
if graceful {
// Graceful stop: submit the quiesce DAG rather than a hard
// kill. The per-agent lease keeps it from racing an
// in-flight rebuild for the same agent; the cheap Signal
// nodes all fire immediately so every agent's drain
// overlaps. `submit::graceful_stop` also persists
// `wanted = Offline` and emits the queue snapshot.
let reason = if graceful {
"manual via hivectl graceful stop"
} else {
"manual via hivectl stop"
};
let id = if graceful {
crate::job_queue::submit::graceful_stop(
coord,
agent,
crate::job_queue::Source::Manual,
"manual via hivectl graceful stop".to_owned(),
);
ok_items.push(agent.clone());
continue;
}
// Persist the intent even if the kill itself fails — otherwise
// the next boot reconcile would restart the agent.
if let Err(e) = coord.power.set(agent, crate::power::Wanted::Offline) {
tracing::warn!(%agent, error = ?e, "agent_power: set wanted=offline failed");
}
match lifecycle::kill(agent).await {
Ok(()) => ok_items.push(agent.clone()),
Err(e) => {
tracing::warn!(%agent, error = ?e, "stop: agent kill failed");
errors.push(format!("{agent}: {e:#}"));
}
}
reason.to_owned(),
)
} else {
crate::job_queue::submit::stop(
coord,
agent,
crate::job_queue::Source::Manual,
reason.to_owned(),
)
};
queued.push(id);
ok_items.push(agent.clone());
}
for &container in infra {
@ -337,7 +367,9 @@ async fn handle_stop(
}
}
Ok(finish_lifecycle(ok_items, &errors))
let mut resp = finish_lifecycle(ok_items, &errors);
resp.queued_dags = Some(queued);
Ok(resp)
}
/// Start the given `infra` containers then `agents` (`hivectl start`) — the
@ -364,22 +396,23 @@ async fn handle_start(
}
}
let mut queued: Vec<u64> = Vec::new();
for agent in agents {
// Persist the intent even if the start itself fails — the next
// reconcile (boot or queued) retries toward `Up`.
if let Err(e) = coord.power.set(agent, crate::power::Wanted::Up) {
tracing::warn!(%agent, error = ?e, "agent_power: set wanted=up failed");
}
match lifecycle::start(agent).await {
Ok(()) => ok_items.push(agent.clone()),
Err(e) => {
tracing::warn!(%agent, error = ?e, "start: agent start failed");
errors.push(format!("{agent}: {e:#}"));
}
}
// Through the queue: persists `wanted = Up`, upgrades a
// stale-rev start to a full rebuild, and serializes on the
// agent's lease. Ids ride back for hivectl's wait loop.
queued.push(crate::job_queue::submit::start(
coord,
agent,
crate::job_queue::Source::Manual,
"manual via hivectl start".to_owned(),
));
ok_items.push(agent.clone());
}
Ok(finish_lifecycle(ok_items, &errors))
let mut resp = finish_lifecycle(ok_items, &errors);
resp.queued_dags = Some(queued);
Ok(resp)
}
/// Resolve which sub-agent logical names a scope targets: every live
@ -464,48 +497,7 @@ fn finish_lifecycle(ok_items: Vec<String>, errors: &[String]) -> HostResponse {
ok: false,
error: Some(errors.join("; ")),
agents: Some(ok_items),
approvals: None,
urls: None,
agent_statuses: None,
..HostResponse::default()
}
}
}
/// Rebuild `name`'s container, notifying the manager of the outcome
/// (success or failure) and kicking the agent's next turn on success.
async fn handle_rebuild(coord: &Arc<Coordinator>, name: &str) -> Result<HostResponse> {
tracing::info!(%name, "rebuild");
let agent_dir = coord.ensure_runtime(name)?;
let hive = coord.hive_env();
let paths = Coordinator::agent_paths(name, agent_dir);
let result = lifecycle::rebuild(name, &hive, &paths, true, false, &|_| (), &|_| ()).await;
// Mirror auto_update::rebuild_agent — the manager wants to know
// about every rebuild attempt regardless of which surface triggered
// it, especially failures (build error → manager can adjust the
// agent's agent.nix). Without this the admin-socket CLI was a
// notify-gap.
match &result {
Ok(_) => {
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: name.to_owned(),
ok: true,
note: None,
sha: None,
tag: None,
});
// Wake the agent's next turn with the "you were rebuilt"
// hint. Same pattern as auto_update::rebuild_agent and the
// dashboard rebuild path — this is the CLI's equivalent.
coord.kick_agent(name, "container rebuilt");
}
Err(e) => coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: name.to_owned(),
ok: false,
note: Some(format!("{e:#}")),
sha: None,
tag: None,
}),
}
result?;
Ok(HostResponse::success())
}