vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(hive-c0re): replace rebuild queue with generic job-DAG queue

Browse source 
jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap,
reconcile, signal, drain, ...) driven by one scheduler with N build
slots + per-agent lifecycle leases. per-agent power intent (wanted
up/offline) is durable in agent_power.sqlite; Reconcile nodes converge
observed state to it. kills the graceful-stop watcher thread, the
deferred-start follow-up, and the cascade pre-enqueue (fan-out on
MetaLock completion instead). tracker: #2166
This commit is contained in:
müde 2026-07-06 20:13:14 +02:00
commit 7946e03fde
25 changed files with 3673 additions and 2731 deletions
Download patch file Download diff file Expand all files Collapse all files

81
hive-c0re/src/dashboard/lifecycle_ops.rs
View file

@ -1,10 +1,12 @@
//! Container lifecycle endpoints for the dashboard.
//!
//! Rebuild / restart / start / stop (hard + graceful) / update-all all
//! enqueue onto the rebuild queue, so each shows a visible queued→running
//! transient on the dashboard — a direct sub-second start/stop only flashed
//! the badge; destroy delegates to `actions::destroy` (optionally
//! purging).
//! submit DAGs to the job queue (`job_queue::submit`), so each shows a
//! visible queued→running transient on the dashboard — a direct
//! sub-second start/stop only flashed the badge. Start/stop also
//! persist the agent's `wanted` power intent before submitting; the
//! DAG's `Reconcile` converges to it. Destroy delegates to
//! `actions::destroy` (optionally purging).
use axum::{
extract::{Form, Path as AxumPath, Query, State},
@ -23,6 +25,7 @@ pub(super) struct KillParams {
}
use super::{AppState, error_response, guard_agent_name, strip_container_prefix};
use crate::job_queue::{Source, submit};
use crate::{actions, lifecycle};
pub(super) async fn post_rebuild(
@ -33,14 +36,12 @@ pub(super) async fn post_rebuild(
if let Some(reject) = guard_agent_name(&state, &logical).await {
return reject;
}
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
logical,
crate::rebuild_queue::QueueSource::Manual,
submit::rebuild(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard ↻ R3BU1LD button".to_owned(),
None,
);
state.coord.emit_rebuild_queue_snapshot();
(StatusCode::OK, "ok").into_response()
}
@ -54,19 +55,17 @@ pub(super) async fn post_kill(
return reject;
}
if params.graceful {
// Graceful stop: enqueue the quiesce orchestration (signal the harness
// → one stop-checkpoint turn → drain → container stop, with a timeout
// fallback to a hard stop). Serialised through the rebuild queue so it
// can't race an in-flight rebuild for the same agent, and its per-step
// progress surfaces on the queue snapshot + build log.
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::GracefulStop,
logical,
crate::rebuild_queue::QueueSource::Manual,
// Graceful stop: submit the quiesce DAG (signal the harness →
// one stop-checkpoint turn → drain → container stop, with a
// timeout fallback to a hard stop). The agent's lifecycle
// lease keeps it from racing an in-flight rebuild for the same
// agent, and per-node progress surfaces on the queue snapshot.
submit::graceful_stop(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard graceful stop".to_owned(),
None,
);
state.coord.emit_rebuild_queue_snapshot();
return (StatusCode::OK, "ok").into_response();
}
// Manager is stoppable from the dashboard like any other
@ -79,14 +78,12 @@ pub(super) async fn post_kill(
// `socket_server.rs::ManagerRequest::Kill` stays in place: a
// manager calling Kill on its own container is self-suicide
// mid-call, not a legitimate operator action.
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Stop,
logical,
crate::rebuild_queue::QueueSource::Manual,
submit::stop(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard stop".to_owned(),
None,
);
state.coord.emit_rebuild_queue_snapshot();
(StatusCode::OK, "ok").into_response()
}
@ -98,14 +95,12 @@ pub(super) async fn post_restart(
if let Some(reject) = guard_agent_name(&state, &logical).await {
return reject;
}
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Restart,
logical,
crate::rebuild_queue::QueueSource::Manual,
submit::restart(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard ↺ R3START button".to_owned(),
None,
);
state.coord.emit_rebuild_queue_snapshot();
(StatusCode::OK, "ok").into_response()
}
@ -117,14 +112,12 @@ pub(super) async fn post_start(
if let Some(reject) = guard_agent_name(&state, &logical).await {
return reject;
}
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Start,
logical,
crate::rebuild_queue::QueueSource::Manual,
submit::start(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard start".to_owned(),
None,
);
state.coord.emit_rebuild_queue_snapshot();
(StatusCode::OK, "ok").into_response()
}
@ -137,15 +130,13 @@ pub(super) async fn post_update_all(State(state): State<AppState>) -> Response {
else {
continue;
};
state.coord.rebuild_queue.enqueue(
crate::rebuild_queue::QueueKind::Rebuild,
logical,
crate::rebuild_queue::QueueSource::Manual,
submit::rebuild(
&state.coord,
&logical,
Source::Manual,
"manual via dashboard 🌀 UPDATE ALL".to_owned(),
None,
);
}
state.coord.emit_rebuild_queue_snapshot();
(StatusCode::OK, "ok").into_response()
}

46
hive-c0re/src/dashboard/permissions.rs
View file

@ -128,18 +128,19 @@ pub(super) async fn post_tool_groups(
return Err(ProblemDetails::from_status_code(StatusCode::BAD_REQUEST)
.with_detail(format!("invalid tool-groups for {logical}: {e}")));
}
// Enqueue a PermChange so the JSON file write is serialised through
// the FIFO worker. Prevents concurrent batch-apply actions for
// different agents from racing on the shared tool-groups.json.
state.coord.rebuild_queue.enqueue_with_perm(
logical.clone(),
crate::rebuild_queue::QueueSource::Manual,
// Submit a PermChange DAG: the JSON file write commits under
// META_LOCK inside the WritePermFile node, so concurrent
// batch-apply actions for different agents never race on the
// shared tool-groups.json.
crate::job_queue::submit::perm_change(
&state.coord,
&logical,
crate::job_queue::Source::Manual,
"tool-group change via permissions UI".to_owned(),
crate::rebuild_queue::PermPayload::ToolGroups {
crate::job_queue::PermPayload::ToolGroups {
groups: body.groups.clone(),
},
);
state.coord.emit_rebuild_queue_snapshot();
tracing::info!(agent = %logical, groups = ?body.groups, "operator: set tool-groups via dashboard");
Ok((StatusCode::OK, "ok").into_response())
}
@ -214,18 +215,19 @@ pub(super) async fn post_capabilities(
.with_detail(format!("unknown capability: {cap}")));
}
}
// Enqueue a PermChange so the JSON file write is serialised through
// the FIFO worker. Prevents concurrent batch-apply actions for
// different agents from racing on the shared capabilities.json.
state.coord.rebuild_queue.enqueue_with_perm(
logical.clone(),
crate::rebuild_queue::QueueSource::Manual,
// Submit a PermChange DAG: the JSON file write commits under
// META_LOCK inside the WritePermFile node, so concurrent
// batch-apply actions for different agents never race on the
// shared capabilities.json.
crate::job_queue::submit::perm_change(
&state.coord,
&logical,
crate::job_queue::Source::Manual,
"capability change via dashboard".to_owned(),
crate::rebuild_queue::PermPayload::Capabilities {
crate::job_queue::PermPayload::Capabilities {
caps: body.caps.clone(),
},
);
state.coord.emit_rebuild_queue_snapshot();
tracing::info!(agent = %logical, caps = ?body.caps, "operator: set capabilities via dashboard");
Ok((StatusCode::OK, "ok").into_response())
}
@ -298,17 +300,17 @@ pub(super) async fn post_permissions(
));
}
}
// Phase 2 — enqueue one combined PermChange per affected agent.
// Phase 2 — submit one combined PermChange DAG per affected agent.
for (logical, groups, caps) in staged {
state.coord.rebuild_queue.enqueue_with_perm(
logical.clone(),
crate::rebuild_queue::QueueSource::Manual,
crate::job_queue::submit::perm_change(
&state.coord,
&logical,
crate::job_queue::Source::Manual,
"batch permission change via permissions UI".to_owned(),
crate::rebuild_queue::PermPayload::Combined { groups, caps },
crate::job_queue::PermPayload::Combined { groups, caps },
);
tracing::info!(agent = %logical, "operator: batch perm change via dashboard");
}
state.coord.emit_rebuild_queue_snapshot();
Ok((StatusCode::OK, "ok").into_response())
}

19
hive-c0re/src/dashboard/schedules.rs
View file

@ -115,20 +115,19 @@ pub(super) async fn post_schedule_fire_now(
}
}
/// `POST /api/rebuild-queue/{id}/cancel` — drop a `Queued` entry
/// from the rebuild queue. Refuses `Running` / terminal
/// entries: an in-flight rebuild owns the agent's nix store +
/// nixos-container update lock and can't be safely interrupted
/// from the queue side. Always returns 200; the body is
/// `{"cancelled": true}` on a successful flip from Queued →
/// Cancelled, `{"cancelled": false}` when the row was Running /
/// terminal / gone. On success a fresh `RebuildQueueChanged`
/// snapshot fires so the row's state flip surfaces live.
/// `POST /api/rebuild-queue/{id}/cancel` — drop a still-fully-queued
/// DAG from the job queue. Refuses `Running` / terminal DAGs: an
/// in-flight node owns the agent's nix store + nixos-container update
/// lock and can't be safely interrupted from the queue side. Always
/// returns 200; the body is `{"cancelled": true}` on a successful
/// flip to Cancelled, `{"cancelled": false}` when the DAG was
/// Running / terminal / gone. On success a fresh `RebuildQueueChanged`
/// snapshot fires so the state flip surfaces live.
pub(super) async fn post_rebuild_queue_cancel(
State(state): State<AppState>,
AxumPath(id): AxumPath<u64>,
) -> Response {
let cancelled = state.coord.rebuild_queue.cancel(id);
let cancelled = state.coord.job_queue.cancel(id);
if cancelled {
state.coord.emit_rebuild_queue_snapshot();
axum::Json(serde_json::json!({"cancelled": true})).into_response()