feat(hivectl): queue-routed lifecycle verbs with wait + DAG progress
every agent lifecycle verb on the admin socket (rebuild / restart / restart-all / kill / stop / start) now submits job-queue DAGs and returns their ids; hivectl polls the new HostRequest::QueueDag and prints a live node-chain progress line per DAG (fan-out children included), exiting non-zero on failure — --no-wait opts out. DagView and the queue wire enums move to hive_sh4re::jobs (wire types live in the shared crate); the last fused rebuild path (lifecycle::rebuild) is gone. tracker: #2166
This commit is contained in:
müde
parent
dc6a37b29a
commit
b489454dc2
9 changed files with 641 additions and 443 deletions
|
|
@ -169,7 +169,10 @@ impl JobQueue {
|
|||
&& d.parent_id == spec.parent_id
|
||||
&& d.approval_id == spec.approval_id
|
||||
&& (d.template != Template::MetaUpdate || d.inputs == spec.inputs)
|
||||
&& PermPayload::same_type(d.perm_payload.as_ref(), spec.perm_payload.as_ref())
|
||||
&& model::perm_payload_same_type(
|
||||
d.perm_payload.as_ref(),
|
||||
spec.perm_payload.as_ref(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,174 +1,37 @@
|
|||
//! Data model for the generic job-DAG queue: templates (what a DAG
|
||||
//! *means*), node kinds (the primitive operations), dependency edges,
|
||||
//! states, and the wire-facing `DagView` / `NodeView` snapshot shapes.
|
||||
//! Data model for the generic job-DAG queue: node kinds (the primitive
|
||||
//! operations), dependency edges, and the runtime `Dag` / `Node` store.
|
||||
//! The serialized *views* — `DagView` / `NodeView` plus the `Template`
|
||||
//! / `Source` / `State` / `PermPayload` wire enums — live in
|
||||
//! `hive_sh4re::jobs` (wire types belong to the shared crate) and are
|
||||
//! re-exported here for the queue's internal use.
|
||||
//!
|
||||
//! Two levels: the **DAG** is the unit of dedup / cancel /
|
||||
//! approval-resolution and the dashboard group; the **node** is the
|
||||
//! unit of scheduling / execution / build-log / step label. See
|
||||
//! `docs/coordinator.md::Job queue` for the full design.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
pub use hive_sh4re::jobs::{DagView, NodeId, NodeView, PermPayload, Source, State, Template};
|
||||
use serde::Serialize;
|
||||
|
||||
/// What a DAG *means* — the request-level shape. Wire strings match the
|
||||
/// old `QueueKind` values (serialized as the `kind` field on `DagView`)
|
||||
/// so the dashboard's glyph map keeps working.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Template {
|
||||
/// Rebuild one agent's container: `Prebuild → StopForUpdate → Swap
|
||||
/// → Reconcile` (the tail `Reconcile` runs after `Swap` terminal,
|
||||
/// ok *or* fail — the recovery-start).
|
||||
Rebuild,
|
||||
/// Bump meta flake locks: one `MetaLock` node; child `Rebuild`
|
||||
/// DAGs fan out on completion for every affected agent.
|
||||
MetaUpdate,
|
||||
/// First-deploy spawn (approval-driven): `Create → WriteDropin →
|
||||
/// Reconcile`.
|
||||
Spawn,
|
||||
/// Reserved for a future destroy integration — kept so the wire
|
||||
/// shape doesn't need to change later.
|
||||
#[allow(dead_code, reason = "wire shape — routed by a future PR")]
|
||||
Destroy,
|
||||
/// Boot-time config sweep: one `MetaLock` (hyperhive input,
|
||||
/// non-fatal) node; stale agents' `Rebuild` DAGs fan out on
|
||||
/// completion.
|
||||
StartupSweep,
|
||||
/// `StopForUpdate → Reconcile` with `wanted` set to `Up` at submit
|
||||
/// time — a mechanical stop + start, like the old
|
||||
/// `lifecycle::restart`, regardless of prior intent drift.
|
||||
Restart,
|
||||
/// `WritePermFile → Prebuild → StopForUpdate → Swap → Reconcile` —
|
||||
/// perm-file commit followed by the rebuild subgraph.
|
||||
PermChange,
|
||||
/// `Signal → Drain → Reconcile` with `wanted` set to `Offline` at
|
||||
/// submit time: quiesce the harness, await the drain (bounded),
|
||||
/// then the tail `Reconcile` performs the actual container stop.
|
||||
GracefulStop,
|
||||
/// Single `Reconcile` with `wanted` set to `Up` at submit time.
|
||||
Start,
|
||||
/// Single `Reconcile` with `wanted` set to `Offline` at submit time.
|
||||
Stop,
|
||||
/// Single `Reconcile` with `wanted` untouched — boot-time converge
|
||||
/// of observed state to the persisted intent.
|
||||
Reconcile,
|
||||
/// Dedup compares the perm *type*, not the value — a tool-groups
|
||||
/// change and a capabilities change for the same agent are distinct
|
||||
/// operations that must not collapse.
|
||||
pub(super) fn perm_payload_same_type(a: Option<&PermPayload>, b: Option<&PermPayload>) -> bool {
|
||||
matches!(
|
||||
(a, b),
|
||||
(
|
||||
Some(PermPayload::ToolGroups { .. }),
|
||||
Some(PermPayload::ToolGroups { .. })
|
||||
) | (
|
||||
Some(PermPayload::Capabilities { .. }),
|
||||
Some(PermPayload::Capabilities { .. })
|
||||
) | (
|
||||
Some(PermPayload::Combined { .. }),
|
||||
Some(PermPayload::Combined { .. })
|
||||
) | (None, None)
|
||||
)
|
||||
}
|
||||
|
||||
impl Template {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Template::Rebuild => "rebuild",
|
||||
Template::MetaUpdate => "meta_update",
|
||||
Template::Spawn => "spawn",
|
||||
Template::Destroy => "destroy",
|
||||
Template::StartupSweep => "startup_sweep",
|
||||
Template::Restart => "restart",
|
||||
Template::PermChange => "perm_change",
|
||||
Template::GracefulStop => "graceful_stop",
|
||||
Template::Start => "start",
|
||||
Template::Stop => "stop",
|
||||
Template::Reconcile => "reconcile",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the submit request originated. Same variants + wire strings
|
||||
/// as the old `QueueSource` — drives the "why" chip on the dashboard.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Source {
|
||||
/// Operator action (dashboard button, CLI, manager tool).
|
||||
Manual,
|
||||
/// Cascade child of a `MetaUpdate` DAG's fan-out; `parent_id`
|
||||
/// points back at the originating meta-update.
|
||||
MetaUpdate,
|
||||
/// Boot-time submission (sweep parent, boot reconciles).
|
||||
AutoUpdate,
|
||||
/// Cascade child of a `StartupSweep` DAG's fan-out.
|
||||
StartupSweep,
|
||||
/// Crash recovery path (future use).
|
||||
#[allow(dead_code, reason = "wire shape — used by a future feature")]
|
||||
CrashRecover,
|
||||
/// Operator approved a pending `Approval` row; `approval_id` on
|
||||
/// the DAG points back at the source row.
|
||||
Approval,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Source::Manual => "manual",
|
||||
Source::MetaUpdate => "meta_update",
|
||||
Source::AutoUpdate => "auto_update",
|
||||
Source::StartupSweep => "startup_sweep",
|
||||
Source::CrashRecover => "crash_recover",
|
||||
Source::Approval => "approval",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Lifecycle state of a node — and, rolled up, of a DAG. Same wire
|
||||
/// strings as the old `QueueState`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum State {
|
||||
Queued,
|
||||
Running,
|
||||
Done,
|
||||
Failed,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl State {
|
||||
pub fn is_terminal(self) -> bool {
|
||||
matches!(self, State::Done | State::Failed | State::Cancelled)
|
||||
}
|
||||
}
|
||||
|
||||
/// Kind-specific payload for `Template::PermChange` DAGs. Carried on
|
||||
/// the DAG (not the node) so dedup can compare the perm *type*
|
||||
/// discriminant. Identical to the old `rebuild_queue::PermPayload`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum PermPayload {
|
||||
/// Set the tool groups for one agent (`tool-groups.json`).
|
||||
ToolGroups { groups: Vec<String> },
|
||||
/// Set the capabilities for one agent (`capabilities.json`).
|
||||
Capabilities { caps: Vec<String> },
|
||||
/// Set both perm-types in one entry — the batch
|
||||
/// `POST /api/permissions` path. `None` leaves that file untouched;
|
||||
/// the worker commits whichever are present in one git commit,
|
||||
/// then rebuilds once.
|
||||
Combined {
|
||||
groups: Option<Vec<String>>,
|
||||
caps: Option<Vec<String>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl PermPayload {
|
||||
/// Dedup compares the perm *type*, not the value — a tool-groups
|
||||
/// change and a capabilities change for the same agent are
|
||||
/// distinct operations that must not collapse.
|
||||
pub fn same_type(a: Option<&PermPayload>, b: Option<&PermPayload>) -> bool {
|
||||
matches!(
|
||||
(a, b),
|
||||
(
|
||||
Some(PermPayload::ToolGroups { .. }),
|
||||
Some(PermPayload::ToolGroups { .. })
|
||||
) | (
|
||||
Some(PermPayload::Capabilities { .. }),
|
||||
Some(PermPayload::Capabilities { .. })
|
||||
) | (
|
||||
Some(PermPayload::Combined { .. }),
|
||||
Some(PermPayload::Combined { .. })
|
||||
) | (None, None)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Node id, unique within its DAG (dense small ints assigned by the
|
||||
/// template builders).
|
||||
pub type NodeId = u32;
|
||||
|
||||
/// When a dependency edge is considered satisfied.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
|
|
@ -427,58 +290,6 @@ impl Dag {
|
|||
}
|
||||
}
|
||||
|
||||
/// Wire shape of one node inside a `DagView`.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct NodeView {
|
||||
pub id: NodeId,
|
||||
/// Flattened `NodeKind` tag ("prebuild", "swap", …).
|
||||
pub kind: &'static str,
|
||||
/// Ids of the nodes this one waits for.
|
||||
pub deps: Vec<NodeId>,
|
||||
pub state: State,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub step: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub build_log_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub finished_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Wire shape of a DAG, serialized onto `RebuildQueueChanged` and the
|
||||
/// `/api/state` snapshot. DAG-level fields mirror the old `QueueEntry`
|
||||
/// names (`kind` = template string, roll-up `state`); everything
|
||||
/// per-node — step labels, build-log links, errors, timestamps —
|
||||
/// appears exactly once, inside `nodes`.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct DagView {
|
||||
pub id: u64,
|
||||
pub agent: String,
|
||||
/// Template wire string — same values the old `kind` field used.
|
||||
pub kind: Template,
|
||||
/// Roll-up state (see [`Dag::rollup`]).
|
||||
pub state: State,
|
||||
pub source: Source,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub parent_id: Option<u64>,
|
||||
pub reason: String,
|
||||
pub enqueued_at: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub finished_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub inputs: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub approval_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub perm_payload: Option<PermPayload>,
|
||||
pub nodes: Vec<NodeView>,
|
||||
}
|
||||
|
||||
impl Dag {
|
||||
pub fn view(&self) -> DagView {
|
||||
let started_at = self.nodes.iter().filter_map(|n| n.started_at).min();
|
||||
|
|
@ -506,7 +317,7 @@ impl Dag {
|
|||
.iter()
|
||||
.map(|n| NodeView {
|
||||
id: n.id,
|
||||
kind: n.kind.as_str(),
|
||||
kind: n.kind.as_str().to_owned(),
|
||||
deps: n.deps.iter().map(|d| d.on).collect(),
|
||||
state: n.state,
|
||||
step: n.step.clone(),
|
||||
|
|
|
|||
Loading…
Reference in a new issue