feat(hivectl): queue-routed lifecycle verbs with wait + DAG progress
every agent lifecycle verb on the admin socket (rebuild / restart / restart-all / kill / stop / start) now submits job-queue DAGs and returns their ids; hivectl polls the new HostRequest::QueueDag and prints a live node-chain progress line per DAG (fan-out children included), exiting non-zero on failure — --no-wait opts out. DagView and the queue wire enums move to hive_sh4re::jobs (wire types live in the shared crate); the last fused rebuild path (lifecycle::rebuild) is gone. tracker: #2166
This commit is contained in:
parent
dc6a37b29a
commit
b489454dc2
9 changed files with 641 additions and 443 deletions
191
hive-sh4re/src/jobs.rs
Normal file
191
hive-sh4re/src/jobs.rs
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
//! Wire shapes of hive-c0re's job-DAG queue: what a queued job looks
|
||||
//! like on the dashboard SSE channel (`rebuild_queue_changed`), the
|
||||
//! `/api/state.rebuild_queue` snapshot, and the host admin socket's
|
||||
//! `QueueDag` polling surface (`hivectl`'s wait/progress loop). The
|
||||
//! queue *internals* — node kinds, dependency edges, scheduling state —
|
||||
//! live in `hive-c0re::job_queue`; these are the serialized views it
|
||||
//! produces. Semantics: `docs/coordinator.md::Job queue`.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// What a DAG *means* — the request-level shape. Wire strings match
|
||||
/// the pre-DAG queue's `kind` values so dashboards key off the same
|
||||
/// tags.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Template {
|
||||
/// Rebuild one agent's container (prebuild → stop → profile-swap →
|
||||
/// reconcile).
|
||||
Rebuild,
|
||||
/// Bump meta flake locks; child `Rebuild` DAGs fan out on
|
||||
/// completion for every affected agent.
|
||||
MetaUpdate,
|
||||
/// First-deploy spawn (approval-driven).
|
||||
Spawn,
|
||||
/// Reserved for a future destroy integration.
|
||||
Destroy,
|
||||
/// Boot-time config sweep (hyperhive lock bump + stale-agent
|
||||
/// rebuild fan-out).
|
||||
StartupSweep,
|
||||
/// Mechanical stop + converge to `wanted = Up` (a restart).
|
||||
Restart,
|
||||
/// Perm-file commit followed by the rebuild subgraph.
|
||||
PermChange,
|
||||
/// Quiesce the harness, drain, then stop (`wanted = Offline`).
|
||||
GracefulStop,
|
||||
/// Converge to `wanted = Up`.
|
||||
Start,
|
||||
/// Converge to `wanted = Offline`.
|
||||
Stop,
|
||||
/// Bare converge of observed power state to the persisted intent
|
||||
/// (boot reconcile).
|
||||
Reconcile,
|
||||
}
|
||||
|
||||
impl Template {
|
||||
#[must_use]
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Template::Rebuild => "rebuild",
|
||||
Template::MetaUpdate => "meta_update",
|
||||
Template::Spawn => "spawn",
|
||||
Template::Destroy => "destroy",
|
||||
Template::StartupSweep => "startup_sweep",
|
||||
Template::Restart => "restart",
|
||||
Template::PermChange => "perm_change",
|
||||
Template::GracefulStop => "graceful_stop",
|
||||
Template::Start => "start",
|
||||
Template::Stop => "stop",
|
||||
Template::Reconcile => "reconcile",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the submit request originated — drives the "why" chip on the
|
||||
/// dashboard.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Source {
|
||||
/// Operator action (dashboard button, CLI, manager tool).
|
||||
Manual,
|
||||
/// Cascade child of a `MetaUpdate` DAG's fan-out.
|
||||
MetaUpdate,
|
||||
/// Boot-time submission (sweep parent, boot reconciles).
|
||||
AutoUpdate,
|
||||
/// Cascade child of a `StartupSweep` DAG's fan-out.
|
||||
StartupSweep,
|
||||
/// Crash recovery path (future use).
|
||||
CrashRecover,
|
||||
/// Operator approved a pending `Approval` row; `approval_id` on
|
||||
/// the DAG points back at the source row.
|
||||
Approval,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
#[must_use]
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Source::Manual => "manual",
|
||||
Source::MetaUpdate => "meta_update",
|
||||
Source::AutoUpdate => "auto_update",
|
||||
Source::StartupSweep => "startup_sweep",
|
||||
Source::CrashRecover => "crash_recover",
|
||||
Source::Approval => "approval",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Lifecycle state of a node — and, rolled up, of a DAG.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum State {
|
||||
Queued,
|
||||
Running,
|
||||
Done,
|
||||
Failed,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl State {
|
||||
#[must_use]
|
||||
pub fn is_terminal(self) -> bool {
|
||||
matches!(self, State::Done | State::Failed | State::Cancelled)
|
||||
}
|
||||
}
|
||||
|
||||
/// Kind-specific payload for `Template::PermChange` DAGs.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum PermPayload {
|
||||
/// Set the tool groups for one agent (`tool-groups.json`).
|
||||
ToolGroups { groups: Vec<String> },
|
||||
/// Set the capabilities for one agent (`capabilities.json`).
|
||||
Capabilities { caps: Vec<String> },
|
||||
/// Set both perm-types in one entry — the batch
|
||||
/// `POST /api/permissions` path. `None` leaves that file untouched;
|
||||
/// present fields commit together and rebuild once.
|
||||
Combined {
|
||||
groups: Option<Vec<String>>,
|
||||
caps: Option<Vec<String>>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Node id, unique within its DAG.
|
||||
pub type NodeId = u32;
|
||||
|
||||
/// One node of a queued DAG, as serialized. Step labels, build-log
|
||||
/// links, errors, and timestamps are per-node; the DAG-level `state`
|
||||
/// is a roll-up.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NodeView {
|
||||
pub id: NodeId,
|
||||
/// Node primitive tag: `"prebuild"`, `"stop_for_update"`,
|
||||
/// `"swap"`, `"create"`, `"meta_lock"`, `"reconcile"`, `"signal"`,
|
||||
/// `"drain"`, `"write_dropin"`, `"write_perm_file"`,
|
||||
/// `"approval_deploy"`.
|
||||
pub kind: String,
|
||||
/// Ids of the nodes this one waits for.
|
||||
#[serde(default)]
|
||||
pub deps: Vec<NodeId>,
|
||||
pub state: State,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub step: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub build_log_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub finished_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// A queued/running/recent DAG. DAG-level fields mirror the pre-DAG
|
||||
/// `QueueEntry` names (`kind` = template string, roll-up `state`);
|
||||
/// everything per-node appears exactly once, inside `nodes`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DagView {
|
||||
pub id: u64,
|
||||
pub agent: String,
|
||||
/// Template wire string — same values the old `kind` field used.
|
||||
pub kind: Template,
|
||||
/// Roll-up: `failed` if any node failed, else `running` /
|
||||
/// `queued` / `cancelled` / `done`.
|
||||
pub state: State,
|
||||
pub source: Source,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub parent_id: Option<u64>,
|
||||
pub reason: String,
|
||||
pub enqueued_at: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub finished_at: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub inputs: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub approval_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub perm_payload: Option<PermPayload>,
|
||||
pub nodes: Vec<NodeView>,
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ use chrono::{DateTime, Utc};
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub mod assets;
|
||||
pub mod jobs;
|
||||
pub mod paths;
|
||||
pub mod priv_proto;
|
||||
pub mod wire_time;
|
||||
|
|
@ -69,6 +70,10 @@ pub enum HostRequest {
|
|||
/// matrix GUI disabled). Backs `hivectl open` + the federation
|
||||
/// peer-config block (which reads the bare `domain`).
|
||||
Urls,
|
||||
/// Fetch one job-queue DAG (plus its live fan-out children, linked
|
||||
/// via `parent_id`) by id — the polling surface behind `hivectl`'s
|
||||
/// wait/progress loop. Result: [`HostResponse::dags`].
|
||||
QueueDag { id: u64 },
|
||||
/// List pending approval requests.
|
||||
Pending,
|
||||
/// Approve a pending request by id; the action runs immediately.
|
||||
|
|
@ -170,7 +175,7 @@ pub struct HiveUrls {
|
|||
pub matrix: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct HostResponse {
|
||||
pub ok: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
|
|
@ -188,6 +193,16 @@ pub struct HostResponse {
|
|||
/// request kind.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub agent_statuses: Option<Vec<AgentStatusRow>>,
|
||||
/// Ids of the job-queue DAGs this request submitted (rebuild /
|
||||
/// restart / power ops). Clients poll them via
|
||||
/// [`HostRequest::QueueDag`]; `None` for non-submitting requests.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub queued_dags: Option<Vec<u64>>,
|
||||
/// `QueueDag` result — the requested DAG followed by its live
|
||||
/// fan-out children ([`jobs::DagView`]). Empty when the DAG has
|
||||
/// been evicted from the queue's history tail.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub dags: Option<Vec<jobs::DagView>>,
|
||||
}
|
||||
|
||||
/// One row in the approval queue. `commit_ref` is overloaded per
|
||||
|
|
@ -298,11 +313,7 @@ impl HostResponse {
|
|||
pub fn success() -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
error: None,
|
||||
agents: None,
|
||||
approvals: None,
|
||||
urls: None,
|
||||
agent_statuses: None,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -311,10 +322,7 @@ impl HostResponse {
|
|||
Self {
|
||||
ok: false,
|
||||
error: Some(message.into()),
|
||||
agents: None,
|
||||
approvals: None,
|
||||
urls: None,
|
||||
agent_statuses: None,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -322,11 +330,8 @@ impl HostResponse {
|
|||
pub fn list(agents: Vec<String>) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
error: None,
|
||||
agents: Some(agents),
|
||||
approvals: None,
|
||||
urls: None,
|
||||
agent_statuses: None,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -334,11 +339,8 @@ impl HostResponse {
|
|||
pub fn pending(approvals: Vec<Approval>) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
error: None,
|
||||
agents: None,
|
||||
approvals: Some(approvals),
|
||||
urls: None,
|
||||
agent_statuses: None,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -347,11 +349,8 @@ impl HostResponse {
|
|||
pub fn urls(urls: HiveUrls) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
error: None,
|
||||
agents: None,
|
||||
approvals: None,
|
||||
urls: Some(urls),
|
||||
agent_statuses: None,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -360,11 +359,29 @@ impl HostResponse {
|
|||
pub fn agent_statuses(rows: Vec<AgentStatusRow>) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
error: None,
|
||||
agents: None,
|
||||
approvals: None,
|
||||
urls: None,
|
||||
agent_statuses: Some(rows),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// A request that submitted job-queue DAGs — carries their ids for
|
||||
/// the client's wait/progress loop.
|
||||
#[must_use]
|
||||
pub fn queued(ids: Vec<u64>) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
queued_dags: Some(ids),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// `QueueDag` result — the polled DAG + its live children.
|
||||
#[must_use]
|
||||
pub fn dags(dags: Vec<jobs::DagView>) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
dags: Some(dags),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue