hive-c0re: remove the approvals subsystem
Nothing submits an approval since scheduled prompts left hive-c0re, so the
queue only acted on rows written before that.
- hive-c0re: the approvals store and its migrations, approve/deny/
resolve actions, the ResolveApproval tail node (and meta_update's
approval_id), POST /api/approve and /api/deny, the approval views on
/api/state, the approval_resolved dashboard event, the orphan GC,
destroy/purge failing pending rows, and the approval rows in
get_loose_ends. DestroyBookkeeping loses the purge flag, which only
worded the approval-failure note.
- The CancelLooseEnd request goes: its only hive-c0re arm was the
approval one. cancel_loose_end handles reminder and todo in the harness.
- Wire types: hive_sh4re::approvals (ReminderStats moves to inbox),
LooseEnd::Approval, CancelLooseEndKind, HelperEvent::ApprovalResolved,
ToolGroup::Approvals, HostRequest::{Pending,Approve,Deny} and
HostResponse.approvals. HOST_SOCK_VERSION goes to 2 for the removed
field.
- hivectl loses the approvals verbs; docs/tools/hivectl-cli.md is
regenerated.
- The system prompt drops approval_resolved.
- The approvals table stays in broker.sqlite, unread, the same way the
operator_questions table was left when its store was removed.
- An `approvals` entry still in tool-groups.json is skipped with a warning
by ToolGroup::parse_list, like any unknown group.
- docs/agent-lifecycle/approvals.md is renamed to config-changes.md so the
code's doc pointers resolve; its content follows in the docs commit.
This commit is contained in:
parent
99bd912ae0
commit
58ef299c3c
30 changed files with 153 additions and 1828 deletions
|
|
@ -6,53 +6,10 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context as _, Result, bail};
|
||||
use hive_sh4re::approvals::{ApprovalKind, ApprovalStatus};
|
||||
use hive_sh4re::manager::HelperEvent;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::lifecycle;
|
||||
|
||||
/// Approve a pending request. Marks the approval row durably, then submits
|
||||
/// the work to the job queue so the dashboard POST returns immediately while
|
||||
/// the long-running pipeline runs off-thread.
|
||||
///
|
||||
/// Dispatch:
|
||||
/// - `UpdateMetaInputs` → a `MetaUpdate` DAG (fan-out on completion)
|
||||
///
|
||||
/// Every queued kind resolves its approval row via
|
||||
/// [`resolve_approval_dag`] when the DAG settles terminal.
|
||||
pub fn approve(coord: &Arc<Coordinator>, id: i64) -> Result<()> {
|
||||
let approval = coord.approvals.mark_approved(id)?;
|
||||
tracing::info!(
|
||||
%approval.id,
|
||||
%approval.agent,
|
||||
kind = ?approval.kind,
|
||||
%approval.commit_ref,
|
||||
"approval: dispatching",
|
||||
);
|
||||
match approval.kind {
|
||||
ApprovalKind::UpdateMetaInputs => {
|
||||
// Inputs JSON-encoded into commit_ref by the manager's
|
||||
// submit path — surface them on the DAG so the dashboard
|
||||
// can show *which* inputs are about to bump. The cascade
|
||||
// rebuilds fan out when the lock bump lands (so they build
|
||||
// against the post-bump lock, and a failed bump fans out
|
||||
// nothing).
|
||||
let inputs: Vec<String> =
|
||||
serde_json::from_str(&approval.commit_ref).unwrap_or_default();
|
||||
let inserted = coord.job_queue.insert_job(|b| {
|
||||
crate::job_queue::templates::meta_update(b, inputs, Some(id));
|
||||
Vec::new()
|
||||
});
|
||||
if let Err(e) = inserted {
|
||||
return Err(e.context("insert meta-update dag"));
|
||||
}
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `applied/main`, and whether `target` descends from it. Fast-forwarding
|
||||
/// `main` to a `target` that does not is a rewind: it drops every commit on
|
||||
/// `main` that `target` lacks.
|
||||
|
|
@ -149,90 +106,6 @@ pub(crate) async fn post_rev_deploy_failure_to_pr(agent: &str, rev: &str, err: &
|
|||
}
|
||||
}
|
||||
|
||||
/// Resolve an approval row from how its DAG's work ended — the body of the
|
||||
/// [`NodeKind::ResolveApproval`] tail node. Every approval-carrying template
|
||||
/// resolves here: the pipeline is ordinary queue nodes,
|
||||
/// so the work's terminal state is the authoritative outcome and there's no
|
||||
/// in-node resolution to skip around.
|
||||
///
|
||||
/// `outcome` is the one the calling node was built to report — a template emits
|
||||
/// one tail per outcome, so success, failure and cancel each arrive here from
|
||||
/// their own node rather than from one node branching.
|
||||
///
|
||||
/// [`NodeKind::ResolveApproval`]: crate::job_queue::NodeKind::ResolveApproval
|
||||
pub(crate) fn resolve_approval_dag(
|
||||
coord: &Arc<Coordinator>,
|
||||
approval_id: i64,
|
||||
outcome: crate::job_queue::TerminalState,
|
||||
error: Option<&str>,
|
||||
) {
|
||||
use crate::job_queue::TerminalState;
|
||||
let approval = match coord.approvals.get(approval_id) {
|
||||
Ok(Some(a)) => a,
|
||||
Ok(None) => {
|
||||
tracing::warn!(approval_id, "approval dag terminal: row no longer exists");
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(approval_id, error = ?e, "approval dag terminal: row read failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let result: Result<()> = match outcome {
|
||||
TerminalState::Done => Ok(()),
|
||||
// `Skipped` never reaches here — no template emits a tail for it — but it
|
||||
// reads the same to an operator either way: the work did not happen.
|
||||
TerminalState::Cancelled | TerminalState::Skipped => {
|
||||
Err(anyhow::anyhow!("cancelled before completion"))
|
||||
}
|
||||
TerminalState::Failed => Err(anyhow::anyhow!("{}", error.unwrap_or("job dag failed"))),
|
||||
};
|
||||
if let Err(e) = finish_approval(coord, &approval, result) {
|
||||
tracing::warn!(approval_id, error = ?e, "approval dag resolved with failure");
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_approval(
|
||||
coord: &Coordinator,
|
||||
approval: &hive_sh4re::approvals::Approval,
|
||||
result: Result<()>,
|
||||
) -> Result<()> {
|
||||
let (status, note, ok) = match &result {
|
||||
Ok(()) => (ApprovalStatus::Approved, None, true),
|
||||
Err(e) => {
|
||||
let note = format!("{e:#}");
|
||||
let _ = coord.approvals.mark_failed(approval.id, ¬e);
|
||||
(ApprovalStatus::Failed, Some(note), false)
|
||||
}
|
||||
};
|
||||
coord.notify_submitter(
|
||||
approval.id,
|
||||
&HelperEvent::ApprovalResolved {
|
||||
id: approval.id,
|
||||
agent: approval.agent.to_string(),
|
||||
commit_ref: approval.commit_ref.clone(),
|
||||
status,
|
||||
note: note.clone(),
|
||||
},
|
||||
);
|
||||
// Phase 5b: also fire on the dashboard event channel so the
|
||||
// browser moves the row out of pending into history without a
|
||||
// snapshot refetch. `approved` rows that succeed get the
|
||||
// approval's logged resolved_at indirectly via `Utc::now()`;
|
||||
// failures already wrote it via mark_failed above.
|
||||
let approval_kind = <&str>::from(approval.kind);
|
||||
let status_str = if ok { "approved" } else { "failed" };
|
||||
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
||||
id: approval.id,
|
||||
agent: approval.agent.as_str(),
|
||||
approval_kind,
|
||||
status: status_str,
|
||||
note: note.clone(),
|
||||
description: approval.description.clone(),
|
||||
});
|
||||
result
|
||||
}
|
||||
|
||||
/// Fast-forward `applied/main` to `target` and sync the working tree.
|
||||
///
|
||||
/// Compare-and-swap, not a bare set: `main` must still be `expected_main`, the
|
||||
|
|
@ -284,36 +157,6 @@ pub fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) {
|
|||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
|
||||
pub fn deny(coord: &Coordinator, id: i64, note: Option<&str>) -> Result<()> {
|
||||
let approval = coord.approvals.get(id)?;
|
||||
coord.approvals.mark_denied(id, note)?;
|
||||
tracing::info!(%id, note, "approval denied");
|
||||
if let Some(a) = approval {
|
||||
let approval_kind = <&str>::from(a.kind);
|
||||
let description = a.description.clone();
|
||||
let agent_owned = a.agent.clone();
|
||||
coord.notify_submitter(
|
||||
a.id,
|
||||
&HelperEvent::ApprovalResolved {
|
||||
id: a.id,
|
||||
agent: a.agent.to_string(),
|
||||
commit_ref: a.commit_ref,
|
||||
status: ApprovalStatus::Denied,
|
||||
note: note.map(String::from),
|
||||
},
|
||||
);
|
||||
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
||||
id,
|
||||
agent: agent_owned.as_str(),
|
||||
approval_kind,
|
||||
status: "denied",
|
||||
note: note.map(String::from),
|
||||
description,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{RevAdvance, advance_applied_main};
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ use std::sync::{Arc, Mutex};
|
|||
use anyhow::{Context, Result};
|
||||
use tokio::sync::{broadcast, watch};
|
||||
|
||||
use crate::approvals::Approvals;
|
||||
use crate::broker::Broker;
|
||||
use crate::container_view::{self, ContainerView};
|
||||
use crate::dashboard_events::DashboardEvent;
|
||||
|
|
@ -30,7 +29,6 @@ const LAST_STOPPED_RUNNING_KEY: &str = "last_stopped_running";
|
|||
|
||||
pub struct Coordinator {
|
||||
pub broker: Arc<Broker>,
|
||||
pub approvals: Arc<Approvals>,
|
||||
/// Scheduled-prompt rows, read-only: the dashboard lists them so the
|
||||
/// operator can recreate them with `swarmctl schedule`. Nothing on
|
||||
/// the hive writes or fires them.
|
||||
|
|
@ -170,7 +168,7 @@ pub struct Coordinator {
|
|||
/// Unified wire-facing event channel feeding the dashboard SSE
|
||||
/// stream. Carries broker messages (mirrored from `broker.subscribe`
|
||||
/// by the forwarder task in `main.rs`) and dashboard-only mutation
|
||||
/// events (approval added/resolved, etc.).
|
||||
/// events (container state, rebuild queue, etc.).
|
||||
/// Snapshot endpoints capture `event_seq` before reading state so
|
||||
/// the client can dedupe its buffered live traffic against the
|
||||
/// snapshot.
|
||||
|
|
@ -413,19 +411,6 @@ impl Drop for MetaUpdateGuard {
|
|||
}
|
||||
}
|
||||
|
||||
/// Field-named payload for [`Coordinator::emit_approval_resolved`].
|
||||
/// Mirrors the `ApprovalResolved` dashboard-event fields. `agent`
|
||||
/// borrows from the caller; `approval_kind` / `status` are
|
||||
/// compile-time constants.
|
||||
pub struct ApprovalResolved<'a> {
|
||||
pub id: i64,
|
||||
pub agent: &'a str,
|
||||
pub approval_kind: &'static str,
|
||||
pub status: &'static str,
|
||||
pub note: Option<String>,
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
/// The two ways [`Coordinator::set_paused_by_name`] can fail. See its own
|
||||
/// doc comment for why this is a distinct type rather than one
|
||||
/// `anyhow::Error` (`Write` already wraps one, `BadName` never needs to).
|
||||
|
|
@ -467,7 +452,6 @@ impl Coordinator {
|
|||
agent_io_weight,
|
||||
} = env;
|
||||
let broker = Broker::open(db_path).context("open broker")?;
|
||||
let approvals = Approvals::open(db_path).context("open approvals")?;
|
||||
let scheduled_prompts = crate::scheduled_prompts::ScheduledPrompts::open(db_path)
|
||||
.context("open scheduled_prompts")?;
|
||||
// BuildLogs wants a directory (it picks its own `build_logs.sqlite`
|
||||
|
|
@ -488,7 +472,6 @@ impl Coordinator {
|
|||
let (shutdown_tx, _) = watch::channel(false);
|
||||
Ok(Self {
|
||||
broker: Arc::new(broker),
|
||||
approvals: Arc::new(approvals),
|
||||
scheduled_prompts: Arc::new(scheduled_prompts),
|
||||
build_logs,
|
||||
hyperhive_flake,
|
||||
|
|
@ -730,41 +713,6 @@ impl Coordinator {
|
|||
self.meta_updates_active.load(Ordering::SeqCst) > 0
|
||||
}
|
||||
|
||||
/// Emit `ApprovalResolved` after `mark_approved` / `mark_denied` /
|
||||
/// `mark_failed` lands. `resolved_at` is stamped from the system
|
||||
/// clock here so call sites don't repeat the conversion; if you
|
||||
/// already have an authoritative timestamp from the db update,
|
||||
/// the tiny skew between "row updated" and "event emitted" is
|
||||
/// presentation-only and doesn't matter to clients.
|
||||
///
|
||||
/// Takes [`ApprovalResolved`] rather than a positional arg list so
|
||||
/// the seven fields are named at every call site.
|
||||
pub fn emit_approval_resolved(&self, ev: ApprovalResolved<'_>) {
|
||||
let ApprovalResolved {
|
||||
id,
|
||||
agent,
|
||||
approval_kind,
|
||||
status,
|
||||
note,
|
||||
description,
|
||||
} = ev;
|
||||
let resolved_at = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| i64::try_from(d.as_secs()).ok())
|
||||
.unwrap_or(0);
|
||||
self.emit_dashboard_event(DashboardEvent::ApprovalResolved {
|
||||
seq: self.next_seq(),
|
||||
id,
|
||||
agent: agent.to_owned(),
|
||||
approval_kind,
|
||||
status,
|
||||
resolved_at: hive_sh4re::wire_time::from_secs(resolved_at),
|
||||
note,
|
||||
description,
|
||||
});
|
||||
}
|
||||
|
||||
/// Rebuild the per-container snapshot, diff it against the last
|
||||
/// one cached on `self`, and emit one
|
||||
/// `DashboardEvent::ContainerStateChanged` per added/changed row
|
||||
|
|
@ -844,7 +792,7 @@ impl Coordinator {
|
|||
.with_context(|| format!("create agent dir {}", agent_dir.display()))?;
|
||||
let socket_path = Self::socket_path(name);
|
||||
// Hand the full Coordinator to the per-agent socket — it needs
|
||||
// more than just the broker (approvals, ...).
|
||||
// more than just the broker.
|
||||
let socket = socket_server::start(name, &socket_path, self.clone())?;
|
||||
self.agents.lock().unwrap().insert(name.to_owned(), socket);
|
||||
Ok(agent_dir)
|
||||
|
|
@ -1094,28 +1042,6 @@ impl Coordinator {
|
|||
self.notify_agent(hive_sh4re::manager::MANAGER_AGENT, event);
|
||||
}
|
||||
|
||||
/// Route an approval-scoped helper event to the agent that submitted
|
||||
/// approval `approval_id` (the authenticated socket caller at submit
|
||||
/// time). Legacy rows with no recorded submitter — and any lookup
|
||||
/// failure — fall back to the root agent, preserving the prior
|
||||
/// always-root behaviour.
|
||||
pub fn notify_submitter(&self, approval_id: i64, event: &hive_sh4re::manager::HelperEvent) {
|
||||
let target = self.submitter_or_manager(approval_id);
|
||||
self.notify_agent(&target, event);
|
||||
}
|
||||
|
||||
/// Shared resolution for "which agent should hear about approval
|
||||
/// `approval_id`" — the authenticated socket caller at submit time,
|
||||
/// falling back to the manager for legacy rows with no recorded
|
||||
/// submitter (or any lookup failure). Used by both `notify_submitter`
|
||||
/// and `push_todo_submitter` so the fallback rule lives in one place.
|
||||
fn submitter_or_manager(&self, approval_id: i64) -> String {
|
||||
self.approvals
|
||||
.submitter_of(approval_id)
|
||||
.unwrap_or_default()
|
||||
.unwrap_or_else(|| hive_sh4re::manager::MANAGER_AGENT.to_owned())
|
||||
}
|
||||
|
||||
/// Push a `HelperEvent` into an arbitrary agent's inbox. Encoded
|
||||
/// the same way as `notify_manager` (sender = `SYSTEM_SENDER`,
|
||||
/// body = JSON-encoded event) — e.g. `ContainerCrash`, `NeedsUpdate`.
|
||||
|
|
|
|||
|
|
@ -1,104 +0,0 @@
|
|||
//! Approval endpoints for the dashboard.
|
||||
//!
|
||||
//! Approve/deny actions plus the orphan-approval GC sweep used by the
|
||||
//! `/api/state` builder.
|
||||
|
||||
use axum::{
|
||||
extract::{Form, Path as AxumPath, State},
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use hive_sh4re::approvals::Approval;
|
||||
use serde::Deserialize;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use super::{AppState, error_response};
|
||||
use crate::actions;
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
/// Approve a pending approval row.
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/api/approve/{id}",
|
||||
params(("id" = i64, Path, description = "approval row id")),
|
||||
responses(
|
||||
(status = 200, description = "approved", body = String),
|
||||
(status = 500, description = "approve failed"),
|
||||
),
|
||||
tag = "approvals"
|
||||
)]
|
||||
pub(super) async fn post_approve(
|
||||
State(state): State<AppState>,
|
||||
AxumPath(id): AxumPath<i64>,
|
||||
) -> Response {
|
||||
match actions::approve(&state.coord, id) {
|
||||
// 200 instead of 303 — `actions::approve` fires
|
||||
// `ApprovalResolved` (success path) or the eventual failure
|
||||
// event, both of which the dashboard's derived store applies
|
||||
// live. The matching form carries `data-no-refresh`.
|
||||
Ok(()) => (StatusCode::OK, "ok").into_response(),
|
||||
Err(e) => error_response(&format!("approve {id} failed: {e:#}")),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default, ToSchema)]
|
||||
pub(super) struct DenyForm {
|
||||
#[serde(default)]
|
||||
note: Option<String>,
|
||||
}
|
||||
|
||||
/// Deny a pending approval row, with an optional
|
||||
/// note (form field `note`).
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/api/deny/{id}",
|
||||
params(("id" = i64, Path, description = "approval row id")),
|
||||
request_body(content = DenyForm, content_type = "application/x-www-form-urlencoded"),
|
||||
responses(
|
||||
(status = 200, description = "denied", body = String),
|
||||
(status = 500, description = "deny failed"),
|
||||
),
|
||||
tag = "approvals"
|
||||
)]
|
||||
pub(super) async fn post_deny(
|
||||
State(state): State<AppState>,
|
||||
AxumPath(id): AxumPath<i64>,
|
||||
Form(form): Form<DenyForm>,
|
||||
) -> Response {
|
||||
let note = form
|
||||
.note
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
match actions::deny(&state.coord, id, note) {
|
||||
Ok(()) => (StatusCode::OK, "ok").into_response(),
|
||||
Err(e) => error_response(&format!("deny {id} failed: {e:#}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Filter out approvals whose agent state dir was wiped out from under us
|
||||
/// (e.g. by a test script's cleanup). Marks them failed so they fall out of
|
||||
/// `pending` on next render.
|
||||
pub(super) fn gc_orphans(coord: &Coordinator, approvals: Vec<Approval>) -> Vec<Approval> {
|
||||
approvals
|
||||
.into_iter()
|
||||
.filter(|a| {
|
||||
if Coordinator::agent_proposed_dir(&a.agent).exists() {
|
||||
true
|
||||
} else {
|
||||
let note = "agent state dir missing";
|
||||
let _ = coord.approvals.mark_failed(a.id, note);
|
||||
tracing::info!(id = a.id, agent = %a.agent, "auto-failed orphan approval");
|
||||
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
||||
id: a.id,
|
||||
agent: a.agent.as_str(),
|
||||
approval_kind: <&str>::from(a.kind),
|
||||
status: "failed",
|
||||
note: Some(note.to_owned()),
|
||||
description: a.description.clone(),
|
||||
});
|
||||
false
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
|
@ -103,11 +103,8 @@ pub(super) async fn post_kill(
|
|||
}
|
||||
// Manager is stoppable from the dashboard like any other
|
||||
// agent. The host's dashboard server keeps running (it's
|
||||
// hive-c0re, not the manager container), per-agent approvals
|
||||
// submitted by other sub-agents still process through the
|
||||
// host-side approval queue without the manager up, and
|
||||
// operator-driven meta-input updates work from the dashboard
|
||||
// either way.
|
||||
// hive-c0re, not the manager container), and operator-driven
|
||||
// meta-input updates work from the dashboard either way.
|
||||
if let Err(e) =
|
||||
crate::job_queue::power::stop_many(&state.coord, std::slice::from_ref(&logical), false)
|
||||
.await
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ pub(super) async fn post_meta_update(
|
|||
.coord
|
||||
.job_queue
|
||||
.insert_job(|b| {
|
||||
crate::job_queue::templates::meta_update(b, inputs, None);
|
||||
crate::job_queue::templates::meta_update(b, inputs);
|
||||
Vec::new()
|
||||
})
|
||||
.expect("template-declared shapes are acyclic");
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
//! Hyperhive dashboard. Lists managed containers (with deep-links to each
|
||||
//! container's web UI), pending approvals (with unified diff vs the applied
|
||||
//! repo, plus approve/deny buttons), and the manager.
|
||||
//! container's web UI) and the manager.
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
|
@ -35,7 +34,6 @@ use crate::lifecycle;
|
|||
tags(
|
||||
(name = "health", description = "hive-wide liveness/readiness probes"),
|
||||
(name = "journal", description = "container + host journal reads"),
|
||||
(name = "approvals", description = "approve/deny pending approval rows"),
|
||||
(name = "build_logs", description = "build log headers, full rows, and raw text downloads"),
|
||||
(name = "lifecycle_ops", description = "agent container lifecycle: rebuild/restart/start/stop/pause/limits"),
|
||||
(name = "meta_inputs", description = "bulk flake-input update for the meta flake"),
|
||||
|
|
@ -49,7 +47,6 @@ use crate::lifecycle;
|
|||
)]
|
||||
struct ApiDoc;
|
||||
|
||||
mod approvals;
|
||||
mod build_logs;
|
||||
// The single validated identifier type — homed in `hive-host-sock` (the crate
|
||||
// owning agent-path facts) so every dashboard path-param validates through the
|
||||
|
|
@ -142,8 +139,6 @@ pub async fn serve(port: u16, coord: Arc<Coordinator>) -> Result<()> {
|
|||
.routes(routes!(permissions::delete_agent_permissions))
|
||||
.routes(routes!(schedules::api_schedules))
|
||||
.routes(routes!(schedules::post_rebuild_queue_cancel))
|
||||
.routes(routes!(approvals::post_approve))
|
||||
.routes(routes!(approvals::post_deny))
|
||||
.routes(routes!(lifecycle_ops::post_destroy))
|
||||
.routes(routes!(lifecycle_ops::post_kill))
|
||||
.routes(routes!(lifecycle_ops::post_restart))
|
||||
|
|
@ -360,8 +355,6 @@ mod router_build_probe {
|
|||
.routes(routes!(permissions::delete_agent_permissions))
|
||||
.routes(routes!(schedules::api_schedules))
|
||||
.routes(routes!(schedules::post_rebuild_queue_cancel))
|
||||
.routes(routes!(approvals::post_approve))
|
||||
.routes(routes!(approvals::post_deny))
|
||||
.routes(routes!(lifecycle_ops::post_destroy))
|
||||
.routes(routes!(lifecycle_ops::post_kill))
|
||||
.routes(routes!(lifecycle_ops::post_restart))
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@ use axum::{
|
|||
sse::{Event, KeepAlive, Sse},
|
||||
},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use hive_sh4re::approvals::Approval;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio_stream::wrappers::BroadcastStream;
|
||||
use tokio_stream::{Stream, StreamExt};
|
||||
|
|
@ -24,7 +22,7 @@ use crate::container_view::ContainerView;
|
|||
|
||||
use super::meta_inputs::{MetaInputView, read_meta_inputs};
|
||||
use super::tombstones::{TombstoneView, build_tombstone_views};
|
||||
use super::{AppState, approvals, error_response, scan_validated_paths};
|
||||
use super::{AppState, error_response, scan_validated_paths};
|
||||
|
||||
#[allow(clippy::struct_excessive_bools)]
|
||||
#[derive(Serialize)]
|
||||
|
|
@ -48,10 +46,6 @@ pub(super) struct StateSnapshot {
|
|||
any_stale: bool,
|
||||
containers: Vec<ContainerView>,
|
||||
transients: Vec<TransientView>,
|
||||
approvals: Vec<ApprovalView>,
|
||||
/// Last 30 resolved approvals (approved / denied / failed), newest-
|
||||
/// first. Drives the "history" tab on the approvals section.
|
||||
approval_history: Vec<ApprovalHistoryView>,
|
||||
/// State dirs (config history + claude creds + /state/ notes) that
|
||||
/// survive after a destroy-without-purge. The operator can re-spawn
|
||||
/// with the same name to resume, or PURG3 to wipe them.
|
||||
|
|
@ -70,8 +64,8 @@ pub(super) struct StateSnapshot {
|
|||
/// `MetaUpdateRunning` event.
|
||||
meta_update_running: bool,
|
||||
/// Whether the hive-forge container is up. When true the dashboard
|
||||
/// links each container's config + each approval's commit into the
|
||||
/// forge's `agent-configs` repos.
|
||||
/// links each container's config into the forge's `agent-configs`
|
||||
/// repos.
|
||||
forge_present: bool,
|
||||
/// Whether `hive-gateway` is in front of this dashboard. Sourced
|
||||
/// from the `HIVE_GATEWAY_ENABLED` env var, which the c0re NixOS
|
||||
|
|
@ -126,60 +120,6 @@ struct TransientView {
|
|||
secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ApprovalHistoryView {
|
||||
id: i64,
|
||||
agent: String,
|
||||
kind: &'static str,
|
||||
/// First 12 chars of the manager-supplied ref. None when it is empty.
|
||||
sha_short: Option<String>,
|
||||
/// `approved` / `denied` / `failed`.
|
||||
status: &'static str,
|
||||
/// RFC 3339 UTC. Renders as a relative time on the dashboard.
|
||||
resolved_at: DateTime<Utc>,
|
||||
/// Operator-supplied deny reason (for `denied`) or build error
|
||||
/// (for `failed`). None on `approved`.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
note: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ApprovalView {
|
||||
id: i64,
|
||||
agent: String,
|
||||
kind: &'static str,
|
||||
/// Manager-supplied description shown on the approval card.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
description: Option<String>,
|
||||
/// Raw `commit_ref` payload for `UpdateMetaInputs` (JSON-encoded
|
||||
/// `Vec<String>` of input names; `"[]"` = all inputs). The
|
||||
/// frontend parses this to render a human-readable card body.
|
||||
commit_ref: String,
|
||||
/// RFC 3339 UTC time the approval was queued. Rendered as a
|
||||
/// relative time on the card so the operator can spot a stale
|
||||
/// request.
|
||||
requested_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Replace silent `.unwrap_or_default()` on the data sources behind
|
||||
/// `/api/state` so that whichever query degrades surfaces in journald
|
||||
/// instead of leaving the operator staring at an empty list. The
|
||||
/// dashboard still degrades to a sensible default value; the warn
|
||||
/// is just the diagnostic breadcrumb the old code swallowed.
|
||||
fn log_default<T, E>(what: &str, result: std::result::Result<T, E>) -> T
|
||||
where
|
||||
T: Default,
|
||||
E: std::fmt::Debug,
|
||||
{
|
||||
match result {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(target: "api_state", source = %what, error = ?e, "snapshot source failed; using default");
|
||||
T::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Window over which container crashes count toward the `agents_crashing`
|
||||
/// banner warning. Wide enough that a crash-looping container (restarted
|
||||
/// by `Restart=on-failure` every few seconds) keeps the warning lit
|
||||
|
|
@ -189,12 +129,11 @@ const CRASH_WARNING_WINDOW: std::time::Duration = std::time::Duration::from_mins
|
|||
|
||||
/// Cold-load snapshot of the whole dashboard.
|
||||
///
|
||||
/// Includes the roster, approvals (+ history),
|
||||
/// tombstones, job queue, meta inputs, and more. Live clients then
|
||||
/// follow `/api/dashboard/stream` (SSE) for incremental updates keyed
|
||||
/// off `seq`.
|
||||
/// Includes the roster, tombstones, job queue, meta inputs, and more. Live
|
||||
/// clients then follow `/api/dashboard/stream` (SSE) for incremental updates
|
||||
/// keyed off `seq`.
|
||||
// `StateSnapshot` is a large tree of nested view types (`ContainerView`,
|
||||
// `ApprovalView`, ...) with no `ToSchema` anywhere in that
|
||||
// `TransientView`, ...) with no `ToSchema` anywhere in that
|
||||
// graph; wiring it up is a schema-modelling project of its own, well past
|
||||
// "annotate what's reachable". `serde_json::Value` placeholder for now —
|
||||
// see the batch report.
|
||||
|
|
@ -252,19 +191,7 @@ pub(super) async fn api_state(
|
|||
let containers = state.coord.containers_snapshot().await;
|
||||
let any_stale = containers.iter().any(|c| c.needs_update);
|
||||
let transient_snapshot = state.coord.transient_snapshot();
|
||||
let pending_approvals = approvals::gc_orphans(
|
||||
&state.coord,
|
||||
log_default("approvals.pending", state.coord.approvals.pending()),
|
||||
);
|
||||
let transients = build_transient_views(&containers, &transient_snapshot);
|
||||
let approvals = build_approval_views(pending_approvals);
|
||||
let approval_history = log_default(
|
||||
"approvals.recent_resolved",
|
||||
state.coord.approvals.recent_resolved(30),
|
||||
)
|
||||
.into_iter()
|
||||
.map(history_view)
|
||||
.collect();
|
||||
let tombstones = build_tombstone_views(&state.coord, &containers);
|
||||
let port_conflicts = build_port_conflicts(&containers);
|
||||
|
||||
|
|
@ -287,8 +214,6 @@ pub(super) async fn api_state(
|
|||
any_stale,
|
||||
containers,
|
||||
transients,
|
||||
approvals,
|
||||
approval_history,
|
||||
meta_inputs: read_meta_inputs(),
|
||||
meta_update_running: state.coord.meta_update_in_progress(),
|
||||
tombstones,
|
||||
|
|
@ -375,50 +300,6 @@ fn build_transient_views(
|
|||
out
|
||||
}
|
||||
|
||||
/// Project a resolved sqlite row into the lean shape the dashboard
|
||||
/// history tab consumes — no `diff_html` (rendering 30 of them
|
||||
/// per /api/state poll would mean 30 git diffs per refresh).
|
||||
fn history_view(a: Approval) -> ApprovalHistoryView {
|
||||
let displayed = a.commit_ref.as_str();
|
||||
let sha_short = if displayed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(displayed[..displayed.len().min(12)].to_owned())
|
||||
};
|
||||
let status = match a.status {
|
||||
hive_sh4re::approvals::ApprovalStatus::Approved => "approved",
|
||||
hive_sh4re::approvals::ApprovalStatus::Denied => "denied",
|
||||
hive_sh4re::approvals::ApprovalStatus::Failed => "failed",
|
||||
hive_sh4re::approvals::ApprovalStatus::Cancelled => "cancelled",
|
||||
// Pending shouldn't appear in recent_resolved, but be defensive.
|
||||
hive_sh4re::approvals::ApprovalStatus::Pending => "pending",
|
||||
};
|
||||
let kind = <&str>::from(a.kind);
|
||||
ApprovalHistoryView {
|
||||
id: a.id,
|
||||
agent: a.agent.to_string(),
|
||||
kind,
|
||||
sha_short,
|
||||
status,
|
||||
resolved_at: a.resolved_at.unwrap_or_default(),
|
||||
note: a.note,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_approval_views(approvals: Vec<Approval>) -> Vec<ApprovalView> {
|
||||
approvals
|
||||
.into_iter()
|
||||
.map(|a| ApprovalView {
|
||||
id: a.id,
|
||||
agent: a.agent.to_string(),
|
||||
kind: <&str>::from(a.kind),
|
||||
description: a.description,
|
||||
commit_ref: a.commit_ref,
|
||||
requested_at: a.requested_at,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// `/api/jobq/graph` query string — the generic jobq/wire query shape (any
|
||||
/// host serving a `GraphWire` projection over HTTP takes the same one; see
|
||||
/// `hive_jobq_wire::parse_states`, which does the actual parsing this side
|
||||
|
|
|
|||
|
|
@ -189,13 +189,6 @@ pub(super) async fn post_purge_tombstone(
|
|||
errors.push(format!("{}: {e}", dir.display()));
|
||||
}
|
||||
}
|
||||
if let Err(e) = state
|
||||
.coord
|
||||
.approvals
|
||||
.fail_pending_for_agent(name.as_str(), "agent state purged")
|
||||
{
|
||||
tracing::warn!(%name, error = ?e, "approvals: failing pending on tombstone purge failed");
|
||||
}
|
||||
if errors.is_empty() {
|
||||
tracing::info!(%name, "tombstone purged");
|
||||
// Fire the post-purge tombstones snapshot so dashboards
|
||||
|
|
|
|||
|
|
@ -48,25 +48,6 @@ pub enum DashboardEvent {
|
|||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
file_refs: Vec<String>,
|
||||
},
|
||||
/// A pending approval transitioned to a terminal state
|
||||
/// (approved / denied / failed). Clients move the row out of the
|
||||
/// pending list and into history.
|
||||
///
|
||||
/// The approval's own kind (`"update_meta_inputs"`) lives
|
||||
/// on `approval_kind` rather than `kind` because the latter is taken
|
||||
/// by the serde tag identifying which `DashboardEvent` variant
|
||||
/// this is.
|
||||
ApprovalResolved {
|
||||
seq: u64,
|
||||
id: i64,
|
||||
agent: String,
|
||||
approval_kind: &'static str,
|
||||
/// `"approved"` / `"denied"` / `"failed"`.
|
||||
status: &'static str,
|
||||
resolved_at: DateTime<Utc>,
|
||||
note: Option<String>,
|
||||
description: Option<String>,
|
||||
},
|
||||
/// A lifecycle action started for an agent (spawn / start / stop
|
||||
/// / restart / rebuild / destroy). Clients render a spinner next
|
||||
/// to the row; the client computes "seconds in this state"
|
||||
|
|
@ -109,7 +90,7 @@ pub enum DashboardEvent {
|
|||
/// Fired by `Coordinator::rescan_containers_and_emit`, which diffs
|
||||
/// a fresh `nixos-container list`–derived snapshot against the
|
||||
/// last one cached on the coordinator. Mutation sites (lifecycle
|
||||
/// endpoints, `actions::destroy` / approve, `crash_watch`'s poll loop)
|
||||
/// endpoints, `actions::destroy`, `crash_watch`'s poll loop)
|
||||
/// call the rescan after their work lands.
|
||||
ContainerStateChanged { seq: u64, container: ContainerView },
|
||||
/// A container that was in the previous snapshot is gone. Clients
|
||||
|
|
@ -157,7 +138,7 @@ pub enum DashboardEvent {
|
|||
///
|
||||
/// Kept as an event rather than deleted in favour of polling because
|
||||
/// push-on-change is what every other live surface here does
|
||||
/// (transients, container state, approvals), and a poll loop would be
|
||||
/// (transients, container state), and a poll loop would be
|
||||
/// slower for the same information.
|
||||
RebuildQueueChanged { seq: u64 },
|
||||
/// Full snapshot of capability grants (per-agent `Vec<cap_name>`).
|
||||
|
|
@ -214,7 +195,6 @@ impl DashboardEvent {
|
|||
match self {
|
||||
DashboardEvent::Sent { .. } => "sent",
|
||||
DashboardEvent::Delivered { .. } => "delivered",
|
||||
DashboardEvent::ApprovalResolved { .. } => "approval_resolved",
|
||||
DashboardEvent::TransientSet { .. } => "transient_set",
|
||||
DashboardEvent::TransientCleared { .. } => "transient_cleared",
|
||||
DashboardEvent::ContainerStateChanged { .. } => "container_state_changed",
|
||||
|
|
@ -271,16 +251,6 @@ mod tests {
|
|||
in_reply_to: None,
|
||||
file_refs: Vec::new(),
|
||||
},
|
||||
DashboardEvent::ApprovalResolved {
|
||||
seq: 1,
|
||||
id: 1,
|
||||
agent: "x".into(),
|
||||
approval_kind: "update_meta_inputs",
|
||||
status: "approved",
|
||||
resolved_at: hive_sh4re::wire_time::from_secs(0),
|
||||
note: None,
|
||||
description: None,
|
||||
},
|
||||
DashboardEvent::TransientSet {
|
||||
seq: 1,
|
||||
name: "x".into(),
|
||||
|
|
|
|||
|
|
@ -368,7 +368,7 @@ pub async fn fetch_config_main_into_applied(name: &str) -> bool {
|
|||
/// already exists and just needs to catch up: force-sets the local ref to
|
||||
/// match live main rather than requiring a fast-forward or any review
|
||||
/// gate — there is no PR to review on this path, so nothing to check
|
||||
/// ancestry against. Contrast the approval-deploy flow's CAS'd,
|
||||
/// ancestry against. Contrast the merged-PR deploy flow's CAS'd,
|
||||
/// ancestry-gated advance for a reviewed PR merge, which this is not.
|
||||
///
|
||||
/// Best-effort, same shape as [`fetch_config_main_into_applied`]: returns
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ use std::sync::{Arc, Mutex, OnceLock};
|
|||
|
||||
use anyhow::{Context as _, Result};
|
||||
|
||||
use hive_jobq::{NodeId, TerminalState};
|
||||
use hive_jobq::NodeId;
|
||||
|
||||
use super::model::NodeKind;
|
||||
use crate::coordinator::Coordinator;
|
||||
|
|
@ -79,8 +79,8 @@ pub(super) async fn run_node(
|
|||
run_purge_state(agent).await;
|
||||
Ok(())
|
||||
}
|
||||
NodeKind::DestroyBookkeeping { purge, .. } => {
|
||||
run_destroy_bookkeeping(coord, agent, *purge).await;
|
||||
NodeKind::DestroyBookkeeping { .. } => {
|
||||
run_destroy_bookkeeping(coord, agent).await;
|
||||
Ok(())
|
||||
}
|
||||
NodeKind::MetaLock {
|
||||
|
|
@ -121,13 +121,6 @@ pub(super) async fn run_node(
|
|||
// takes it directly instead of re-matching the kind behind a `bail!`
|
||||
// that could never fire.
|
||||
NodeKind::WritePermFile { payload, .. } => run_write_perm_file(coord, agent, payload).await,
|
||||
NodeKind::ResolveApproval {
|
||||
approval_id,
|
||||
outcome,
|
||||
} => {
|
||||
run_resolve_approval(coord, coord.job_queue.root_of(id), *approval_id, *outcome);
|
||||
Ok(())
|
||||
}
|
||||
NodeKind::EmitRebuilt { ok, .. } => {
|
||||
run_emit_rebuilt(coord, agent, coord.job_queue.root_of(id), *ok).await;
|
||||
Ok(())
|
||||
|
|
@ -266,25 +259,6 @@ async fn run_wanted_pull(coord: &Arc<Coordinator>) -> Result<()> {
|
|||
crate::workers::wanted::pull(coord).await
|
||||
}
|
||||
|
||||
/// Resolve the DAG's approval row the way this node's own `outcome` says.
|
||||
///
|
||||
/// Nothing is inspected: a template emits one of these per outcome, each edged to
|
||||
/// accept only that one, so *which* node the scheduler let run already is the
|
||||
/// answer. Best-effort — a resolution failure is logged inside
|
||||
/// [`crate::actions::resolve_approval_dag`], never surfaced as a node failure,
|
||||
/// since the work already happened and failing the tail would only misreport it.
|
||||
fn run_resolve_approval(
|
||||
coord: &Arc<Coordinator>,
|
||||
dag_id: Option<u64>,
|
||||
approval_id: i64,
|
||||
outcome: TerminalState,
|
||||
) {
|
||||
let reason = (outcome == TerminalState::Failed)
|
||||
.then(|| dag_id.and_then(|dag| coord.job_queue.first_error(dag)))
|
||||
.flatten();
|
||||
crate::actions::resolve_approval_dag(coord, approval_id, outcome, reason.as_deref());
|
||||
}
|
||||
|
||||
/// Rerender the meta flake from whatever containers still exist on disk.
|
||||
/// Idempotent — a no-op when nothing changed. Lives here because the destroy
|
||||
/// tail is its only caller; it moved with `destroy` when that became a DAG.
|
||||
|
|
@ -349,23 +323,13 @@ async fn run_purge_state(agent: &str) {
|
|||
/// none of this is undoable — a failed meta sync or power-store write is a
|
||||
/// bookkeeping drift to log, not a reason to red a DAG whose container is
|
||||
/// already gone.
|
||||
async fn run_destroy_bookkeeping(coord: &Arc<Coordinator>, agent: &str, purge: bool) {
|
||||
async fn run_destroy_bookkeeping(coord: &Arc<Coordinator>, agent: &str) {
|
||||
// Meta flake: drop the agent's input + nixosConfiguration so a future spawn
|
||||
// under the same name re-seeds cleanly, and so the meta lock doesn't
|
||||
// reference a vanished applied repo.
|
||||
if let Err(e) = sync_meta_after_lifecycle(coord).await {
|
||||
tracing::warn!(error = ?e, %agent, "meta sync after destroy failed");
|
||||
}
|
||||
if let Err(e) = coord.approvals.fail_pending_for_agent(
|
||||
agent,
|
||||
if purge {
|
||||
"agent purged"
|
||||
} else {
|
||||
"agent destroyed"
|
||||
},
|
||||
) {
|
||||
tracing::warn!(%agent, error = ?e, "approvals: failing pending on destroy failed");
|
||||
}
|
||||
// Drop the durable power intent — a future agent of the same name seeds
|
||||
// fresh from its observed state.
|
||||
if let Err(e) = coord.power.remove(agent) {
|
||||
|
|
|
|||
|
|
@ -15,11 +15,10 @@
|
|||
//! parent axis (a root's rolled-up state *is* its subtree's), so membership is
|
||||
//! a graph walk with no host-side side-tables. The lease is owned by a subtree
|
||||
//! root and borrowed by its descendants (continuity);
|
||||
//! - terminal work is an ordinary **tail node**
|
||||
//! ([`NodeKind::ResolveApproval`] / [`NodeKind::EmitRebuilt`]) that the builder
|
||||
//! appends in [`templates`], edged onto the job's group roots by the outcome it
|
||||
//! reports. Templates emit one tail per outcome and the graph runs exactly one,
|
||||
//! so nothing branches at runtime.
|
||||
//! - terminal work is an ordinary **tail node** ([`NodeKind::EmitRebuilt`])
|
||||
//! that the builder appends in [`templates`], edged onto the job's group
|
||||
//! roots by the outcome it reports. Templates emit one tail per outcome and
|
||||
//! the graph runs exactly one, so nothing branches at runtime.
|
||||
//!
|
||||
//! The queue is runtime-only (no persistence): an empty graph on boot; desired
|
||||
//! state is re-derived by the reconcile sweep. A single scheduler task
|
||||
|
|
@ -44,7 +43,6 @@ use hive_jobq::{Graph, NodeId};
|
|||
use hive_jobq_wire::{GraphNode, GraphWire};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
pub use hive_jobq::TerminalState;
|
||||
pub use model::{NodeKind, PermPayload, State};
|
||||
use resource::Resource;
|
||||
|
||||
|
|
@ -222,9 +220,7 @@ impl JobQueue {
|
|||
///
|
||||
/// **Nodes that explicitly observe cancellation are spared** — a node whose
|
||||
/// edge names [`hive_jobq::TerminalState::Cancelled`] is asking to run when
|
||||
/// the work it follows was dropped, which is exactly what an approval tail
|
||||
/// needs: cancel the work, and the tail still fires to resolve the approval
|
||||
/// row rather than leaving it dangling forever.
|
||||
/// the work it follows was dropped, so it stays pending and runs.
|
||||
///
|
||||
/// Nothing is special-cased by node kind. `AFTER_ANY` deliberately does *not*
|
||||
/// accept `Cancelled`, so an ordinary weak-edged step (rebuild's `Reconcile`,
|
||||
|
|
|
|||
|
|
@ -15,8 +15,6 @@
|
|||
pub use hive_host_sock::jobs::{PermPayload, State};
|
||||
use serde::Serialize;
|
||||
|
||||
use hive_jobq::TerminalState;
|
||||
|
||||
/// The primitive operations — each kind maps to one executor fn in
|
||||
/// `exec.rs`, a thin wrapper over existing `lifecycle.rs` / `meta.rs`
|
||||
/// code. Concurrency is gated by two resource classes (see
|
||||
|
|
@ -61,7 +59,7 @@ pub enum NodeKind {
|
|||
/// `remove_dir_all`) plus its state and applied dirs.
|
||||
PurgeState { agent: String },
|
||||
/// The post-destroy bookkeeping tail.
|
||||
DestroyBookkeeping { agent: String, purge: bool },
|
||||
DestroyBookkeeping { agent: String },
|
||||
/// Meta flake lock bump. `sweep = false`: `meta::lock_update` with this
|
||||
/// node's own `inputs`; `sweep = true`: `meta::lock_update_hyperhive`
|
||||
/// (boot sweep, `inputs` empty). See _Notable collapses_ for how the
|
||||
|
|
@ -105,17 +103,6 @@ pub enum NodeKind {
|
|||
/// Group root of a rebuild subtree — the **brace**, scoped to one
|
||||
/// agent. Why braces exist and what they cost: _Braces_.
|
||||
AgentWindow { agent: String },
|
||||
/// Tail node of an approval-carrying DAG (a meta-inputs update): resolve
|
||||
/// the approval row from how the work ended.
|
||||
ResolveApproval {
|
||||
approval_id: i64,
|
||||
/// Which outcome this node reports. A template emits **one per
|
||||
/// outcome**, each edged to accept only that one, so exactly one is
|
||||
/// ever runnable. The `Cancelled` one is also the node
|
||||
/// [`super::JobQueue::cancel`] spares, since its edge is the only
|
||||
/// one that accepts a dropped dependency.
|
||||
outcome: TerminalState,
|
||||
},
|
||||
/// Tail node of a rebuild / perm-change: emit this agent's `Rebuilt`
|
||||
/// manager event. One node per agent *and* per outcome; no cancel
|
||||
/// variant (a cancelled DAG never ran, so there is no rebuild to
|
||||
|
|
@ -175,8 +162,8 @@ impl hive_jobq_wire::WireNode for NodeKind {
|
|||
|
||||
impl NodeKind {
|
||||
/// The agent this node targets, or `""` for agentless kinds
|
||||
/// ([`NodeKind::MetaLock`] on the `hyperhive` pseudo-agent,
|
||||
/// [`NodeKind::ResolveApproval`] which acts on an approval row).
|
||||
/// ([`NodeKind::MetaLock`] on the `hyperhive` pseudo-agent, and the
|
||||
/// swarm-wide sweeps and pulls).
|
||||
#[must_use]
|
||||
pub fn agent(&self) -> &str {
|
||||
match self {
|
||||
|
|
@ -203,7 +190,6 @@ impl NodeKind {
|
|||
| NodeKind::EmitRebuilt { agent, .. }
|
||||
| NodeKind::SetWanted { agent, .. } => agent,
|
||||
NodeKind::MetaLock { .. }
|
||||
| NodeKind::ResolveApproval { .. }
|
||||
| NodeKind::ForgeSweep
|
||||
| NodeKind::MatrixSweep
|
||||
| NodeKind::KnowledgePull
|
||||
|
|
|
|||
|
|
@ -10,10 +10,9 @@
|
|||
//! label tracks the DAG's progress (signal → swap → reconcile) instead of
|
||||
//! repeating one intent the template declared before any of it started.
|
||||
//!
|
||||
//! Per-DAG terminal work (approval resolution, `Rebuilt`) is not drained here:
|
||||
//! it runs as the DAG's focused terminal node (`ResolveApproval` /
|
||||
//! `EmitRebuilt`), dispatched through `exec::run_node` like any other node once
|
||||
//! the DAG settles.
|
||||
//! Per-DAG terminal work (`Rebuilt`) is not drained here: it runs as the DAG's
|
||||
//! focused terminal node (`EmitRebuilt`), dispatched through `exec::run_node`
|
||||
//! like any other node once the DAG settles.
|
||||
//!
|
||||
//! In-DAG growth (a `MetaLock` growing rebuild subgraphs, a `Reconcile` fanning
|
||||
//! its `Start`/`Stop`) is declared onto the builder each node is handed, and
|
||||
|
|
@ -55,7 +54,7 @@ type TransientSeen = HashMap<(String, String), bool>;
|
|||
/// `nixos-container update`, which is itself declarative). That's a property
|
||||
/// of the node set, not of the queue — it holds today but isn't enforced by
|
||||
/// the type system, so it's worth re-checking whenever a new `NodeKind` is
|
||||
/// added. The side-effect tails (`EmitRebuilt`, `ResolveApproval`) sit
|
||||
/// added. The side-effect tail (`EmitRebuilt`) sits
|
||||
/// closest to the edge: dropping or re-running one affects something outside
|
||||
/// the graph (today, a missed or duplicated notification) rather than
|
||||
/// reconverging silently.
|
||||
|
|
|
|||
|
|
@ -20,8 +20,6 @@
|
|||
//! `lifecycle::is_running` read), so [`super::power`] assembles them out of
|
||||
//! the primitives this module exports ([`rebuild_nodes`]).
|
||||
|
||||
use hive_jobq::TerminalState;
|
||||
|
||||
use super::model::{NodeKind, PermPayload};
|
||||
use super::resource::Resource;
|
||||
use super::{Handle, JobBuilder};
|
||||
|
|
@ -57,26 +55,6 @@ fn emit_rebuilt_tails(builder: &JobBuilder, agent: &str, roots: &[Handle<'_>]) {
|
|||
);
|
||||
}
|
||||
|
||||
/// The approval-resolving tails for an approval-carrying DAG: one per outcome of
|
||||
/// the DAG's single group-root `root`, each accepting only its own.
|
||||
///
|
||||
/// The `Cancelled` node is what keeps a dropped approval DAG from dangling its
|
||||
/// row forever — its edge is the only one [`super::JobQueue::cancel`] spares.
|
||||
fn resolve_approval_tails(builder: &JobBuilder, approval_id: i64, root: Handle<'_>) {
|
||||
for outcome in [
|
||||
TerminalState::Done,
|
||||
TerminalState::Failed,
|
||||
TerminalState::Cancelled,
|
||||
] {
|
||||
let _ = builder
|
||||
.node(NodeKind::ResolveApproval {
|
||||
approval_id,
|
||||
outcome,
|
||||
})
|
||||
.on_outcome(root, &[outcome]);
|
||||
}
|
||||
}
|
||||
|
||||
/// Declare one rebuild subgraph per agent onto the builder a running
|
||||
/// [`NodeKind::MetaLock`] was handed.
|
||||
///
|
||||
|
|
@ -349,9 +327,6 @@ pub fn rebuild(builder: &JobBuilder, agent: &str, relock: bool) -> Vec<hive_jobq
|
|||
/// borrow it. A failure cancel-cascades the rest — unlike rebuild there's no
|
||||
/// recovery-reconcile (nothing to converge if the container was never created).
|
||||
///
|
||||
/// No approval tail: the operator authorised the creation at swarm level, and
|
||||
/// the deploy request carries that authorisation.
|
||||
///
|
||||
/// Returns the group root so a caller can wait on the whole subtree.
|
||||
pub fn first_deploy(builder: &JobBuilder, agent: &str) -> Vec<hive_jobq::NodeGuid> {
|
||||
let a = || agent.to_owned();
|
||||
|
|
@ -423,7 +398,7 @@ pub fn destroy(builder: &JobBuilder, agent: &str, purge: bool) {
|
|||
destroy
|
||||
};
|
||||
let _tail = builder
|
||||
.node(NodeKind::DestroyBookkeeping { agent: a(), purge })
|
||||
.node(NodeKind::DestroyBookkeeping { agent: a() })
|
||||
.part_of(stop)
|
||||
.after_ok(last);
|
||||
}
|
||||
|
|
@ -458,8 +433,8 @@ pub fn perm_change(builder: &JobBuilder, agent: &str, payload: PermPayload) {
|
|||
/// so the "hyperhive" pseudo-agent gets no pill), giving each cascade agent
|
||||
/// crash-watch suppression during its `Swap` — the property the old child
|
||||
/// `Rebuild` DAGs carried via their own transient.
|
||||
pub fn meta_update(builder: &JobBuilder, inputs: Vec<String>, approval_id: Option<i64>) {
|
||||
let lock = builder
|
||||
pub fn meta_update(builder: &JobBuilder, inputs: Vec<String>) {
|
||||
let _lock = builder
|
||||
.node(NodeKind::MetaLock {
|
||||
sweep: false,
|
||||
fanout: None,
|
||||
|
|
@ -467,14 +442,6 @@ pub fn meta_update(builder: &JobBuilder, inputs: Vec<String>, approval_id: Optio
|
|||
})
|
||||
.needs(Resource::BuildSlot)
|
||||
.needs(Resource::MetaWindow);
|
||||
// The bump itself has no side effect, so an operator-driven one ends
|
||||
// at the `MetaLock`; an approval-driven one still has its row to
|
||||
// resolve and gets the per-outcome tails edged onto that single
|
||||
// group-root — whose roll-up covers the rebuild subgraphs `MetaLock`
|
||||
// grows into itself.
|
||||
if let Some(approval_id) = approval_id {
|
||||
resolve_approval_tails(builder, approval_id, lock);
|
||||
}
|
||||
}
|
||||
|
||||
// The boot is assembled inline in `workers/auto_update.rs::submit_boot_tree`
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
|
||||
use super::model::NodeKind;
|
||||
use super::*;
|
||||
use hive_jobq::TerminalState;
|
||||
|
||||
/// Insert a declared job, naming nothing — the shape assertions read the whole
|
||||
/// graph. A test that needs a handle calls `q.insert` directly and names the
|
||||
|
|
@ -101,10 +102,9 @@ struct Declared {
|
|||
|
||||
/// Render a dep's outcome set as the outcomes it actually accepts.
|
||||
///
|
||||
/// ⚠️ Spelled out rather than bucketed into `ok` / `any` / other. The first
|
||||
/// version of this did bucket, and a template's three `ResolveApproval` tails —
|
||||
/// which differ ONLY in their accepted outcomes — all rendered as `"other"`.
|
||||
/// A helper that prints two structurally different nodes identically turns an
|
||||
/// ⚠️ Spelled out rather than bucketed into `ok` / `any` / other: tails that
|
||||
/// differ ONLY in their accepted outcomes would render identically, and a
|
||||
/// helper that prints two structurally different nodes identically turns an
|
||||
/// assertion into a tautology.
|
||||
fn when_tag(when: hive_jobq::DepWhen) -> String {
|
||||
[
|
||||
|
|
@ -194,19 +194,6 @@ fn pending_kinds_for(q: &JobQueue, agent: &str) -> Vec<&'static str> {
|
|||
pending_kinds_filtered(q, &|kind: &NodeKind| kind.agent() == agent)
|
||||
}
|
||||
|
||||
/// The payloads of every node still `Pending`, for the cases where *which* of a
|
||||
/// family of same-kind nodes survived is the assertion — a template emits one
|
||||
/// tail per outcome and they differ only in what they carry.
|
||||
fn pending_payloads(q: &JobQueue) -> Vec<NodeKind> {
|
||||
let sched = q.sched().lock().expect("job_queue mutex poisoned");
|
||||
let graph = sched.graph();
|
||||
graph
|
||||
.nodes()
|
||||
.filter(|n| n.state == State::Pending)
|
||||
.map(|n| n.payload.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn pending_kinds_filtered(q: &JobQueue, keep: &dyn Fn(&NodeKind) -> bool) -> Vec<&'static str> {
|
||||
let sched = q.sched().lock().expect("job_queue mutex poisoned");
|
||||
let graph = sched.graph();
|
||||
|
|
@ -1189,43 +1176,6 @@ fn cancelled_power_op_runs_no_compensating_node() {
|
|||
}
|
||||
}
|
||||
|
||||
// ---- terminal reporting + lease release ----
|
||||
|
||||
/// A DAG cancelled while fully queued must still **run its tail**, or a queued
|
||||
/// approval DAG cancelled by the operator would dangle its approval forever.
|
||||
///
|
||||
/// This is the load-bearing case for sparing tails in [`JobQueue::cancel`]: the
|
||||
/// work nodes all cancel, but `ResolveApproval` is weak-edged, so a `Cancelled`
|
||||
/// dep satisfies its edge and it becomes claimable instead of being cancelled
|
||||
/// along with everything else. It reads `Cancelled` off its own deps and resolves
|
||||
/// the approval as "cancelled before completion".
|
||||
#[test]
|
||||
fn cancelled_dag_still_runs_its_approval_tail() {
|
||||
let q = JobQueue::new(1);
|
||||
insert(&q, |builder| {
|
||||
templates::meta_update(builder, Vec::new(), Some(7));
|
||||
});
|
||||
// Found by kind, not returned: `meta_update` names nothing, because the
|
||||
// approval row is how an operator follows it.
|
||||
let lock = node_of(&q, "meta_lock").get();
|
||||
assert!(q.cancel(lock), "fully-queued dag cancels");
|
||||
// The `Cancelled` tail is the only node whose edge accepts a dropped
|
||||
// dependency, so it is the only one `cancel` spares — and *which* tail
|
||||
// survives names how the approval row is about to be resolved.
|
||||
let spared = pending_payloads(&q);
|
||||
assert!(
|
||||
matches!(
|
||||
spared.as_slice(),
|
||||
[NodeKind::ResolveApproval {
|
||||
approval_id: 7,
|
||||
outcome: TerminalState::Cancelled
|
||||
}]
|
||||
),
|
||||
"only the cancelled-outcome tail is spared, got {spared:?}"
|
||||
);
|
||||
assert_eq!(state_of(&q, lock), State::Cancelled);
|
||||
}
|
||||
|
||||
// ---- history ----
|
||||
//
|
||||
// The node → build-log link is no longer queue state: the log row carries
|
||||
|
|
|
|||
|
|
@ -364,7 +364,7 @@ async fn set_nspawn_flags(
|
|||
|
||||
// Own state, harness, and config dirs — same for every agent including
|
||||
// the manager. Config is RO: an agent must not edit its own config; changes
|
||||
// only ever flow through the approval queue.
|
||||
// only ever flow through a config PR on the forge.
|
||||
binds.push(BindMount {
|
||||
host_path: notes_dir.to_string_lossy().into_owned(),
|
||||
container_path: format!("/agents/{agent_name}/state"),
|
||||
|
|
|
|||
|
|
@ -1,77 +1,34 @@
|
|||
//! Loose-ends aggregator. Walks the `approvals` table once per call and
|
||||
//! assembles a `Vec<LooseEnd>` for a single agent (`for_agent`) — always
|
||||
//! the caller's own. `Request::GetLooseEnds` from either the agent or
|
||||
//! manager socket lands here so the age-seconds derivation stays in one
|
||||
//! place. Reminders are agent-local (in-container
|
||||
//! `hive-agent::reminders` store) and never sourced from here. The
|
||||
//! `ask`/`answer` MCP tools, their wire protocol (`hive-c0re::questions`,
|
||||
//! `stores::operator_questions`), and the operator dashboard's questions
|
||||
//! pane have all been removed entirely — this file never had a
|
||||
//! `Question` loose-end path to begin with (it only ever emitted
|
||||
//! `PendingMessages`/`Approval`), so nothing here changed shape when
|
||||
//! that removal landed.
|
||||
//! Loose-ends aggregator. Assembles a `Vec<LooseEnd>` for a single agent
|
||||
//! (`for_agent`) — always the caller's own. `Request::GetLooseEnds` from
|
||||
//! either the agent or manager socket lands here. Reminders and todos are
|
||||
//! agent-local (in-container `hive-agent` stores) and never sourced from
|
||||
//! here.
|
||||
//!
|
||||
//! Call frequency is low (an agent doing self-introspection between
|
||||
//! turns), so the sweep happens fresh every time — no caching, no
|
||||
//! mutation events. If the sweep ever shows up in a profile, the sqlite
|
||||
//! queries already filter on the same index (`idx_approvals_pending`)
|
||||
//! that the dashboard uses, so the bottleneck would be json
|
||||
//! (de)serialisation, not the read.
|
||||
//! mutation events.
|
||||
|
||||
use anyhow::Result;
|
||||
use chrono::Utc;
|
||||
use hive_sh4re::inbox::{LooseEnd, saturating_age};
|
||||
use hive_sh4re::inbox::LooseEnd;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
/// Open threads pending against `agent`:
|
||||
/// - undelivered inbox messages this agent still owes itself a `recv`
|
||||
/// for (only when the count is non-zero);
|
||||
/// - pending approvals where this agent is the submitter (a parent
|
||||
/// agent with the `approvals` group submits for its children; the
|
||||
/// root submits for top-level agents). Legacy rows with no recorded
|
||||
/// submitter count as the root's.
|
||||
///
|
||||
/// Ordered `pending_messages` (when non-zero) → approvals within the
|
||||
/// returned vector. Within each kind, source-of-truth ordering (sqlite's
|
||||
/// `pending()` query returns newest-first within its index).
|
||||
/// Open threads pending against `agent`: undelivered inbox messages this
|
||||
/// agent still owes itself a `recv` for, only when the count is non-zero.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Propagates errors from `count_pending` and the pending-approval
|
||||
/// sqlite query.
|
||||
/// Propagates errors from `count_pending`.
|
||||
pub fn for_agent(coord: &Coordinator, agent: &str) -> Result<Vec<LooseEnd>> {
|
||||
let now = Utc::now().timestamp();
|
||||
let mut out = Vec::new();
|
||||
// Undelivered inbox messages this agent still owes itself a `recv`
|
||||
// for. Surfaced first (most actionable) and only when non-zero so a
|
||||
// clean inbox doesn't add noise. The wake-message that drove the
|
||||
// current turn is already delivered, so this counts only what's
|
||||
// genuinely still queued.
|
||||
// Only emitted when non-zero so a clean inbox doesn't add noise. The
|
||||
// wake-message that drove the current turn is already delivered, so
|
||||
// this counts only what's genuinely still queued.
|
||||
let pending_messages = coord.broker.count_pending(agent)?;
|
||||
if pending_messages > 0 {
|
||||
out.push(LooseEnd::PendingMessages {
|
||||
count: pending_messages,
|
||||
});
|
||||
}
|
||||
// Show each pending approval to the agent that submitted it. The
|
||||
// submitter column is NULL for rows predating it; those count as
|
||||
// operator-initiated (no agent tracking predates the column).
|
||||
for a in coord.approvals.pending()? {
|
||||
let submitter = coord
|
||||
.approvals
|
||||
.submitter_of(a.id)?
|
||||
.unwrap_or_else(|| "operator".to_owned());
|
||||
if submitter != agent {
|
||||
continue;
|
||||
}
|
||||
out.push(LooseEnd::Approval {
|
||||
id: a.id,
|
||||
agent: a.agent.to_string(),
|
||||
commit_ref: a.commit_ref,
|
||||
description: a.description,
|
||||
age_seconds: saturating_age(now, a.requested_at.timestamp()),
|
||||
});
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ mod workers;
|
|||
|
||||
pub(crate) use agent_config::{capabilities, limits, resource_limits, tool_groups, topology};
|
||||
pub(crate) use stats::{container_stats, hive_stats, host_stats, otel_metrics, warnings};
|
||||
pub(crate) use stores::{approvals, broker, build_logs, db, power, scheduled_prompts};
|
||||
pub(crate) use stores::{broker, build_logs, db, power, scheduled_prompts};
|
||||
pub(crate) use workers::{agent_sockets, auto_update, crash_watch, knowledge, mcp_sockets};
|
||||
|
||||
use coordinator::{Coordinator, HiveEnv, ServeConfig};
|
||||
|
|
@ -436,7 +436,7 @@ async fn cmd_serve(
|
|||
// build-slot count + per-agent leases inside the queue, not from
|
||||
// multiple workers — cheap nodes (graceful signals, drains,
|
||||
// reconciles) overlap nix-heavy ones structurally. Call sites
|
||||
// (auto_update, dashboard, manager, approval handler) submit DAGs
|
||||
// (auto_update, dashboard, manager) submit DAGs
|
||||
// instead of awaiting lifecycle work inline. See `job_queue/`.
|
||||
{
|
||||
let q_coord = coord.clone();
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
//! exports one `nixosConfiguration` per agent and drives the system-wide
|
||||
//! deploy audit trail. Flow (`sync_agents`, `lock_update_for_rebuild`,
|
||||
//! `lock_update_hyperhive`):
|
||||
//! `docs/agent-lifecycle/approvals.md::Meta flake`.
|
||||
//! `docs/agent-lifecycle/config-changes.md::Meta flake`.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue