Compare commits
41 changed files with 644 additions and 1469 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -1795,7 +1795,6 @@ dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"hive-priv-sock",
|
"hive-priv-sock",
|
||||||
"libc",
|
"libc",
|
||||||
"serde",
|
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
|
|
||||||
|
|
@ -204,7 +204,6 @@
|
||||||
system
|
system
|
||||||
treefmt-eval
|
treefmt-eval
|
||||||
;
|
;
|
||||||
inherit (nixpkgs.lib) nixosSystem;
|
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -80,19 +80,6 @@ pub struct RemindArgs {
|
||||||
pub file_path: Option<String>,
|
pub file_path: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// MCP tool args for `compact`.
|
|
||||||
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
|
|
||||||
pub struct CompactArgs {
|
|
||||||
/// Optional wake-up prompt. When set and the compact actually runs
|
|
||||||
/// (gated on context usage — see the tool description), the harness
|
|
||||||
/// drives one synthetic follow-up turn with this string as its body
|
|
||||||
/// as soon as compaction finishes, so you don't have to wait for the
|
|
||||||
/// next external event to continue. Omit for a fire-and-forget compact
|
|
||||||
/// with no follow-up.
|
|
||||||
#[serde(default)]
|
|
||||||
pub wake_prompt: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
// -----------------------------------------------------------------------------
|
// -----------------------------------------------------------------------------
|
||||||
// Privileged tool arg types (lifecycle, approvals, scheduling, diagnostics)
|
// Privileged tool arg types (lifecycle, approvals, scheduling, diagnostics)
|
||||||
// -----------------------------------------------------------------------------
|
// -----------------------------------------------------------------------------
|
||||||
|
|
|
||||||
|
|
@ -25,10 +25,10 @@ mod render;
|
||||||
|
|
||||||
pub use args::{
|
pub use args::{
|
||||||
AckUntilArgs, AgentGetLooseEndsArgs, AnswerArgs, AskArgs, CancelLooseEndArgs,
|
AckUntilArgs, AgentGetLooseEndsArgs, AnswerArgs, AskArgs, CancelLooseEndArgs,
|
||||||
CancelScheduleArgs, CompactArgs, CreateRepoArgs, EditScheduleArgs, FireScheduleNowArgs,
|
CancelScheduleArgs, CreateRepoArgs, EditScheduleArgs, FireScheduleNowArgs, GetAgentMetaArgs,
|
||||||
GetAgentMetaArgs, GetHostJournalArgs, GetLogsArgs, KillArgs, MarkTodosDoneArgs, RecvArgs,
|
GetHostJournalArgs, GetLogsArgs, KillArgs, MarkTodosDoneArgs, RecvArgs, RemindArgs,
|
||||||
RemindArgs, RequestInitConfigArgs, RequestSchedulePromptArgs, RestartArgs, SendArgs,
|
RequestInitConfigArgs, RequestSchedulePromptArgs, RestartArgs, SendArgs, SetStatusArgs,
|
||||||
SetStatusArgs, StartArgs, UpdateArgs, UpdateMetaInputsArgs,
|
StartArgs, UpdateArgs, UpdateMetaInputsArgs,
|
||||||
};
|
};
|
||||||
pub use render::{annotate_retries, format_ack, format_agent_meta, format_recv};
|
pub use render::{annotate_retries, format_ack, format_agent_meta, format_recv};
|
||||||
|
|
||||||
|
|
@ -657,19 +657,11 @@ impl AgentServer {
|
||||||
that the call is refused with an explanation and has no effect. On a pass, \
|
that the call is refused with an explanation and has no effect. On a pass, \
|
||||||
queues compaction for the end of the current turn (same deferred mechanism \
|
queues compaction for the end of the current turn (same deferred mechanism \
|
||||||
the dashboard button uses, so it never races a live claude process); the \
|
the dashboard button uses, so it never races a live claude process); the \
|
||||||
usual pre-compaction notes-checkpoint turn still fires first. Pass \
|
usual pre-compaction notes-checkpoint turn still fires first. No args."
|
||||||
`wake_prompt` to have the harness drive one synthetic follow-up turn with \
|
|
||||||
that body as soon as compaction finishes — without it you just go idle \
|
|
||||||
waiting for the next external event, same as ending a turn normally."
|
|
||||||
)]
|
)]
|
||||||
async fn compact(&self, Parameters(args): Parameters<CompactArgs>) -> String {
|
async fn compact(&self) -> String {
|
||||||
let log = format!("{args:?}");
|
run_tool_envelope("compact", String::new(), async move {
|
||||||
run_tool_envelope("compact", log, async move {
|
match dial_agent_socket(&hive_agent_sock::Request::Compact).await {
|
||||||
match dial_agent_socket(&hive_agent_sock::Request::Compact {
|
|
||||||
wake_prompt: args.wake_prompt,
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Some(hive_agent_sock::Response::Ok) => {
|
Some(hive_agent_sock::Response::Ok) => {
|
||||||
"compact queued — will run at the end of the current turn".to_owned()
|
"compact queued — will run at the end of the current turn".to_owned()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -117,12 +117,7 @@ pub enum Request {
|
||||||
/// explains why and takes no action. On a pass, queues the same
|
/// explains why and takes no action. On a pass, queues the same
|
||||||
/// deferred `compact_pending` flag the operator's button sets (consumed
|
/// deferred `compact_pending` flag the operator's button sets (consumed
|
||||||
/// at the next turn boundary), so it never races a live claude process.
|
/// at the next turn boundary), so it never races a live claude process.
|
||||||
/// `wake_prompt`, when set, is driven as a synthetic follow-up turn once
|
Compact,
|
||||||
/// the compaction actually finishes — the dashboard button's own
|
|
||||||
/// requests go through `Bus::request_compact` directly with `None`, not
|
|
||||||
/// through this variant, since a human watching the dashboard isn't
|
|
||||||
/// waiting on a wake.
|
|
||||||
Compact { wake_prompt: Option<String> },
|
|
||||||
/// Mirror an outstanding question this agent asked (`ask()` succeeded).
|
/// Mirror an outstanding question this agent asked (`ask()` succeeded).
|
||||||
/// `target` is who it's waiting on (`"operator"` when asked with
|
/// `target` is who it's waiting on (`"operator"` when asked with
|
||||||
/// `to: None`). Part of the questions-mirror increment — see
|
/// `to: None`). Part of the questions-mirror increment — see
|
||||||
|
|
|
||||||
|
|
@ -260,14 +260,6 @@ pub enum TurnState {
|
||||||
Compacting,
|
Compacting,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One pending `/compact` request (see `Bus::request_compact`).
|
|
||||||
/// `wake_prompt` is what to drive as a synthetic follow-up turn once the
|
|
||||||
/// compaction actually finishes, if anything.
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct CompactRequest {
|
|
||||||
pub wake_prompt: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct Bus {
|
pub struct Bus {
|
||||||
tx: Arc<broadcast::Sender<BusEvent>>,
|
tx: Arc<broadcast::Sender<BusEvent>>,
|
||||||
|
|
@ -315,23 +307,7 @@ pub struct Bus {
|
||||||
/// One-shot: run `/compact` after the next turn ends. Consumed at the end
|
/// One-shot: run `/compact` after the next turn ends. Consumed at the end
|
||||||
/// of the current/next turn by `turn::drive_turn`. Deferring to the turn
|
/// of the current/next turn by `turn::drive_turn`. Deferring to the turn
|
||||||
/// boundary keeps compaction from racing a live claude process mid-turn.
|
/// boundary keeps compaction from racing a live claude process mid-turn.
|
||||||
/// `Some(request)` when a compact is pending; `request.wake_prompt` is
|
compact_pending: Arc<AtomicBool>,
|
||||||
/// what to drive as a synthetic follow-up turn once the compaction
|
|
||||||
/// actually completes (`None` = pending but no follow-up wake wanted,
|
|
||||||
/// e.g. the operator dashboard's `/compact` button). `None` = no compact
|
|
||||||
/// pending. Wrapped in [`CompactRequest`] rather than
|
|
||||||
/// `Option<Option<String>>` (clippy pedantic's `option_option` lint,
|
|
||||||
/// and the named field reads clearer at call sites than a bare nested
|
|
||||||
/// `Option`) so "pending" and "what to wake with" can never desync.
|
|
||||||
compact_pending: Arc<Mutex<Option<CompactRequest>>>,
|
|
||||||
/// One-shot, written by `turn::drive_turn`/`turn::run_pending_compact`
|
|
||||||
/// right after a compaction they served finishes, when that compact's
|
|
||||||
/// request carried a `wake_prompt`. Read once by the `hive-agent` serve
|
|
||||||
/// loop after either call site to decide whether to drive a synthetic
|
|
||||||
/// follow-up turn. Separate from `compact_pending`: by the time this is
|
|
||||||
/// set, the compact has already run and that flag has already been
|
|
||||||
/// cleared by `take_compact`.
|
|
||||||
post_compact_wake: Arc<Mutex<Option<String>>>,
|
|
||||||
/// Current fresh-claude-session id (FK to `sessions.id`). Set by the
|
/// Current fresh-claude-session id (FK to `sessions.id`). Set by the
|
||||||
/// bin loop after minting a session row on a fresh start; stamped onto
|
/// bin loop after minting a session row on a fresh start; stamped onto
|
||||||
/// every `turn_stats` row until the next fresh session. `None` before
|
/// every `turn_stats` row until the next fresh session. `None` before
|
||||||
|
|
@ -421,8 +397,7 @@ impl Bus {
|
||||||
last_cost_usage: Arc::new(Mutex::new(None)),
|
last_cost_usage: Arc::new(Mutex::new(None)),
|
||||||
rate_limited: Arc::new(AtomicBool::new(was_rate_limited)),
|
rate_limited: Arc::new(AtomicBool::new(was_rate_limited)),
|
||||||
session_reset_pending: Arc::new(AtomicBool::new(false)),
|
session_reset_pending: Arc::new(AtomicBool::new(false)),
|
||||||
compact_pending: Arc::new(Mutex::new(None)),
|
compact_pending: Arc::new(AtomicBool::new(false)),
|
||||||
post_compact_wake: Arc::new(Mutex::new(None)),
|
|
||||||
session_id: Arc::new(Mutex::new(None)),
|
session_id: Arc::new(Mutex::new(None)),
|
||||||
fresh_session: Arc::new(AtomicBool::new(false)),
|
fresh_session: Arc::new(AtomicBool::new(false)),
|
||||||
tool_calls: Arc::new(Mutex::new(std::collections::HashMap::new())),
|
tool_calls: Arc::new(Mutex::new(std::collections::HashMap::new())),
|
||||||
|
|
@ -463,40 +438,16 @@ impl Bus {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Request a compaction after the next turn ends (deferred to the turn
|
/// Request a compaction after the next turn ends (deferred to the turn
|
||||||
/// boundary). Idempotent — a second request before the first is
|
/// boundary). Idempotent.
|
||||||
/// serviced just overwrites `wake_prompt` with the latest ask. `Some
|
pub fn request_compact(&self) {
|
||||||
/// (wake_prompt)` schedules a synthetic follow-up turn (driven with
|
self.compact_pending.store(true, Ordering::SeqCst);
|
||||||
/// `wake_prompt` as its body) once the compaction actually completes;
|
|
||||||
/// `None` requests a plain compact with no follow-up wake (the operator
|
|
||||||
/// dashboard's `/compact` button).
|
|
||||||
pub fn request_compact(&self, wake_prompt: Option<String>) {
|
|
||||||
*self.compact_pending.lock().unwrap() = Some(CompactRequest { wake_prompt });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Take + clear the compact one-shot. `Some(request)` means
|
/// Take + clear the compact one-shot. Returns true iff `drive_turn` should
|
||||||
/// `drive_turn`/`run_pending_compact` should compact now —
|
/// compact at the end of this turn.
|
||||||
/// `request.wake_prompt` is what to pass to `set_post_compact_wake` once
|
|
||||||
/// that compaction finishes. `None` means no compact is pending.
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn take_compact(&self) -> Option<CompactRequest> {
|
pub fn take_compact(&self) -> bool {
|
||||||
self.compact_pending.lock().unwrap().take()
|
self.compact_pending.swap(false, Ordering::SeqCst)
|
||||||
}
|
|
||||||
|
|
||||||
/// Record that a just-finished compaction should drive a synthetic
|
|
||||||
/// follow-up turn with `prompt` as its body. Called by
|
|
||||||
/// `turn::drive_turn`/`turn::run_pending_compact` right after the
|
|
||||||
/// compaction they served (whose `take_compact()` returned a request
|
|
||||||
/// with `wake_prompt: Some(prompt)`) completes.
|
|
||||||
pub fn set_post_compact_wake(&self, prompt: String) {
|
|
||||||
*self.post_compact_wake.lock().unwrap() = Some(prompt);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Take + clear the post-compact wake one-shot. The serve loop calls
|
|
||||||
/// this after either compact call site to decide whether to
|
|
||||||
/// synthesize a follow-up turn.
|
|
||||||
#[must_use]
|
|
||||||
pub fn take_post_compact_wake(&self) -> Option<String> {
|
|
||||||
self.post_compact_wake.lock().unwrap().take()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Mark that the current turn started a fresh claude session.
|
/// Mark that the current turn started a fresh claude session.
|
||||||
|
|
|
||||||
|
|
@ -240,21 +240,6 @@ fn synthetic_todo_message(stern: bool) -> hive_sh4re::inbox::DeliveredMessage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Synthetic message that drives the follow-up turn when a `/compact` call
|
|
||||||
/// (self-requested via the `compact` MCP tool's `wake_prompt` arg) finishes
|
|
||||||
/// and asked to be woken. Mirrors `synthetic_todo_message`'s "no broker row"
|
|
||||||
/// sentinel shape (`id = 0`) — the compact tool call itself is the durable
|
|
||||||
/// record that a wake was requested, not a broker message.
|
|
||||||
fn post_compact_wake_message(prompt: String) -> hive_sh4re::inbox::DeliveredMessage {
|
|
||||||
hive_sh4re::inbox::DeliveredMessage {
|
|
||||||
from: "compact".into(),
|
|
||||||
body: prompt,
|
|
||||||
id: 0,
|
|
||||||
redelivered: false,
|
|
||||||
in_reply_to: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Synthetic message that drives the single stop-checkpoint turn when c0re
|
/// Synthetic message that drives the single stop-checkpoint turn when c0re
|
||||||
/// signals a graceful stop. The agent gets one final turn to flush durable
|
/// signals a graceful stop. The agent gets one final turn to flush durable
|
||||||
/// `/state` before the container is stopped; new inbound is already fenced.
|
/// `/state` before the container is stopped; new inbound is already fenced.
|
||||||
|
|
@ -787,18 +772,8 @@ async fn serve_loop<S: Surface>(
|
||||||
let compacted = turn::run_pending_compact(files, &bus, &session).await;
|
let compacted = turn::run_pending_compact(files, &bus, &session).await;
|
||||||
if !compacted {
|
if !compacted {
|
||||||
tokio::time::sleep(interval).await;
|
tokio::time::sleep(interval).await;
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
// The compact that just ran may have carried a wake prompt
|
continue;
|
||||||
// (the agent's own `compact` tool, not the operator button —
|
|
||||||
// see `Bus::request_compact`). If so, drive it as a synthetic
|
|
||||||
// turn right now instead of looping back to `recv_next` and
|
|
||||||
// waiting for the next external event.
|
|
||||||
let Some(prompt) = bus.take_post_compact_wake() else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
tracing::debug!("post-compact wake queued, driving synthetic follow-up turn");
|
|
||||||
post_compact_wake_message(prompt)
|
|
||||||
}
|
}
|
||||||
RecvOutcome::TransportError => {
|
RecvOutcome::TransportError => {
|
||||||
// `recv_next` already logged the detail; just retry.
|
// `recv_next` already logged the detail; just retry.
|
||||||
|
|
@ -827,83 +802,32 @@ async fn serve_loop<S: Surface>(
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let turn_ctx = TurnCtx {
|
|
||||||
socket,
|
|
||||||
bus: &bus,
|
|
||||||
stats: stats.as_ref(),
|
|
||||||
files,
|
|
||||||
session: &session,
|
|
||||||
interrupted: &interrupted,
|
|
||||||
login_state: &login_state,
|
|
||||||
claude_dir: &claude_dir,
|
|
||||||
interval,
|
|
||||||
};
|
|
||||||
drive_turn_and_wake_chain::<S>(&turn_ctx, next, &mut todo_miss_streak).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Loop-invariant turn-driving context for `drive_turn_and_wake_chain`,
|
|
||||||
/// threaded as one bundle instead of double-digit positional args
|
|
||||||
/// (clippy's `too_many_arguments`). Everything here is constant for the
|
|
||||||
/// lifetime of one `serve_loop` call; only the message to drive and the
|
|
||||||
/// todo-miss streak vary per turn and stay as separate params.
|
|
||||||
struct TurnCtx<'a> {
|
|
||||||
socket: &'a Path,
|
|
||||||
bus: &'a Bus,
|
|
||||||
stats: Option<&'a TurnStats>,
|
|
||||||
files: &'a turn::TurnFiles,
|
|
||||||
session: &'a turn::AgentSession,
|
|
||||||
interrupted: &'a Arc<std::sync::atomic::AtomicBool>,
|
|
||||||
login_state: &'a Arc<Mutex<LoginState>>,
|
|
||||||
claude_dir: &'a Path,
|
|
||||||
interval: Duration,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Drive `next`, then keep driving synthetic follow-up turns for as long as
|
|
||||||
/// a compact that just ran carries a wake prompt
|
|
||||||
/// (`Bus::take_post_compact_wake`) — see `serve_loop`'s comment at the call
|
|
||||||
/// site for why this loops in place instead of returning to the outer
|
|
||||||
/// `select!`/`recv_next`. Ordinarily runs exactly one iteration; only chains
|
|
||||||
/// further if the follow-up turn itself requests another woken compact.
|
|
||||||
/// Split out of `serve_loop` purely to keep that function under clippy's
|
|
||||||
/// line limit.
|
|
||||||
async fn drive_turn_and_wake_chain<S: Surface>(
|
|
||||||
ctx: &TurnCtx<'_>,
|
|
||||||
mut next: hive_sh4re::inbox::DeliveredMessage,
|
|
||||||
todo_miss_streak: &mut u32,
|
|
||||||
) {
|
|
||||||
loop {
|
|
||||||
let ctrl = handle_turn::<S>(
|
let ctrl = handle_turn::<S>(
|
||||||
ctx.socket,
|
socket,
|
||||||
ctx.bus,
|
&bus,
|
||||||
ctx.stats,
|
stats.as_ref(),
|
||||||
ctx.files,
|
files,
|
||||||
ctx.session,
|
&session,
|
||||||
next,
|
next,
|
||||||
ctx.interrupted,
|
&interrupted,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
apply_todo_wake_checked(ctrl.todo_wake_checked, todo_miss_streak, ctx.bus);
|
apply_todo_wake_checked(ctrl.todo_wake_checked, &mut todo_miss_streak, &bus);
|
||||||
if ctrl.auth_failed {
|
if ctrl.auth_failed {
|
||||||
*ctx.login_state.lock().unwrap() = LoginState::NeedsLogin;
|
*login_state.lock().unwrap() = LoginState::NeedsLogin;
|
||||||
// Baseline the resume check on *this instant*, not on a
|
// Baseline the resume check on *this instant*, not on a
|
||||||
// directory snapshot taken after `wait_for_login` starts
|
// directory snapshot taken after `wait_for_login` starts
|
||||||
// polling — closes the race where a login lands between the
|
// polling — closes the race where a login lands between the
|
||||||
// 401 and the first poll. See `wait_for_login`'s doc comment.
|
// 401 and the first poll. See `wait_for_login`'s doc comment.
|
||||||
login::wait_for_login(
|
login::wait_for_login(
|
||||||
ctx.claude_dir,
|
&claude_dir,
|
||||||
ctx.login_state.clone(),
|
login_state.clone(),
|
||||||
ctx.bus,
|
&bus,
|
||||||
u64::try_from(ctx.interval.as_millis()).unwrap_or(2000),
|
u64::try_from(interval.as_millis()).unwrap_or(2000),
|
||||||
std::time::SystemTime::now(),
|
std::time::SystemTime::now(),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
let Some(prompt) = ctx.bus.take_post_compact_wake() else {
|
|
||||||
break;
|
|
||||||
};
|
|
||||||
tracing::debug!("post-compact wake queued mid-turn-flow, driving synthetic follow-up turn");
|
|
||||||
next = post_compact_wake_message(prompt);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -242,7 +242,7 @@ fn dispatch(
|
||||||
} => record_answering_question(questions, id, &asker, &question),
|
} => record_answering_question(questions, id, &asker, &question),
|
||||||
Request::ClearQuestion { id } => clear_question(questions, id),
|
Request::ClearQuestion { id } => clear_question(questions, id),
|
||||||
Request::ListQuestions => list_questions(questions),
|
Request::ListQuestions => list_questions(questions),
|
||||||
Request::Compact { wake_prompt } => compact(bus, wake_prompt),
|
Request::Compact => compact(bus),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -458,10 +458,8 @@ fn mark_todos_done(store: &Todos, ids: &[i64]) -> Response {
|
||||||
/// the agent's own MCP tool instead of the dashboard, and refuses below
|
/// the agent's own MCP tool instead of the dashboard, and refuses below
|
||||||
/// [`COMPACT_MIN_USAGE_FRACTION`] instead of always honouring the request —
|
/// [`COMPACT_MIN_USAGE_FRACTION`] instead of always honouring the request —
|
||||||
/// an agent can call this speculatively, a human clicking the dashboard
|
/// an agent can call this speculatively, a human clicking the dashboard
|
||||||
/// button already made the judgment call. `wake_prompt`, when set, is
|
/// button already made the judgment call.
|
||||||
/// forwarded to `Bus::request_compact` so the turn loop drives a synthetic
|
fn compact(bus: &Bus) -> Response {
|
||||||
/// follow-up turn once the compaction actually finishes.
|
|
||||||
fn compact(bus: &Bus, wake_prompt: Option<String>) -> Response {
|
|
||||||
let Some(usage) = bus.last_ctx_usage() else {
|
let Some(usage) = bus.last_ctx_usage() else {
|
||||||
return Response::Err {
|
return Response::Err {
|
||||||
message: "compact refused: no completed turn yet — nothing to compact".to_owned(),
|
message: "compact refused: no completed turn yet — nothing to compact".to_owned(),
|
||||||
|
|
@ -489,16 +487,9 @@ fn compact(bus: &Bus, wake_prompt: Option<String>) -> Response {
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
let will_wake = wake_prompt.is_some();
|
bus.request_compact();
|
||||||
bus.request_compact(wake_prompt);
|
|
||||||
bus.emit(crate::events::LiveEvent::Note {
|
bus.emit(crate::events::LiveEvent::Note {
|
||||||
text: if will_wake {
|
text: "agent: self-requested /compact — running at the end of the current turn".into(),
|
||||||
"agent: self-requested /compact (with wake prompt) — running at the end of the \
|
|
||||||
current turn"
|
|
||||||
.into()
|
|
||||||
} else {
|
|
||||||
"agent: self-requested /compact — running at the end of the current turn".into()
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
Response::Ok
|
Response::Ok
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -374,18 +374,15 @@ pub async fn drive_turn(
|
||||||
archive_session(bus);
|
archive_session(bus);
|
||||||
return Err(TurnError::PromptTooLong);
|
return Err(TurnError::PromptTooLong);
|
||||||
}
|
}
|
||||||
// Operator `/compact` (`POST /api/compact`) or an agent's own `compact`
|
// Operator `/compact` (`POST /api/compact`) deferred to the turn boundary:
|
||||||
// MCP tool call, deferred to the turn boundary: run it now that the turn
|
// run it now that the turn is done, so it works mid-turn rather than only
|
||||||
// is done, so it works mid-turn rather than only when the agent is idle.
|
// when the agent is idle. Only on a healthy turn — no point spawning a
|
||||||
// Only on a healthy turn — no point spawning a compaction after a
|
// compaction after a rate-limited / auth-failed / crashed one.
|
||||||
// rate-limited / auth-failed / crashed one. `is_ok()` first: `take_compact()`
|
// `is_ok()` first: `take_compact()` clears the flag, so it must only fire
|
||||||
// clears the flag, so it must only fire when the compaction will actually
|
// when the compaction will actually run. On an unhealthy turn
|
||||||
// run. On an unhealthy turn (rate-limited / auth-failed / failed) the flag
|
// (rate-limited / auth-failed / failed) the flag is left set for the next
|
||||||
// is left set for the next turn or the idle `run_pending_compact` to
|
// turn or the idle `run_pending_compact` to service — not silently eaten.
|
||||||
// service — not silently eaten.
|
if outcome.is_ok() && bus.take_compact() {
|
||||||
if outcome.is_ok()
|
|
||||||
&& let Some(request) = bus.take_compact()
|
|
||||||
{
|
|
||||||
bus.emit(LiveEvent::Note {
|
bus.emit(LiveEvent::Note {
|
||||||
text: "operator: /compact — running at turn end".into(),
|
text: "operator: /compact — running at turn end".into(),
|
||||||
});
|
});
|
||||||
|
|
@ -393,14 +390,6 @@ pub async fn drive_turn(
|
||||||
// does; the serve loop resets to `Idle` once this turn returns.
|
// does; the serve loop resets to `Idle` once this turn returns.
|
||||||
bus.set_state(crate::events::TurnState::Compacting);
|
bus.set_state(crate::events::TurnState::Compacting);
|
||||||
let _ = session.compact(&config, &sink).await;
|
let _ = session.compact(&config, &sink).await;
|
||||||
// If the compact call asked to be woken (the agent's own `compact`
|
|
||||||
// tool with a `wake_prompt`), stash it — the serve loop reads it
|
|
||||||
// back after this turn returns and drives a synthetic follow-up
|
|
||||||
// turn, so a self-requested compact provably doesn't strand the
|
|
||||||
// agent idle waiting for the next external event.
|
|
||||||
if let Some(prompt) = request.wake_prompt {
|
|
||||||
bus.set_post_compact_wake(prompt);
|
|
||||||
}
|
|
||||||
return Ok(true);
|
return Ok(true);
|
||||||
}
|
}
|
||||||
outcome
|
outcome
|
||||||
|
|
@ -511,13 +500,11 @@ pub fn emit_turn_end(bus: &Bus, outcome: &TurnOutcome) {
|
||||||
/// so a queued `/compact` runs even when no turn is driving. (The in-flight
|
/// so a queued `/compact` runs even when no turn is driving. (The in-flight
|
||||||
/// case is handled at the end of [`drive_turn`].) Resume-only via
|
/// case is handled at the end of [`drive_turn`].) Resume-only via
|
||||||
/// [`InfiniteSession::compact`]: a missing session is a harmless no-op. Returns
|
/// [`InfiniteSession::compact`]: a missing session is a harmless no-op. Returns
|
||||||
/// `true` if a compaction ran; the serve loop follows up with
|
/// `true` if a compaction ran.
|
||||||
/// `Bus::take_post_compact_wake` to see whether a synthetic follow-up turn
|
|
||||||
/// should run (set below when the compact request carried a `wake_prompt`).
|
|
||||||
pub async fn run_pending_compact(files: &TurnFiles, bus: &Bus, session: &AgentSession) -> bool {
|
pub async fn run_pending_compact(files: &TurnFiles, bus: &Bus, session: &AgentSession) -> bool {
|
||||||
let Some(request) = bus.take_compact() else {
|
if !bus.take_compact() {
|
||||||
return false;
|
return false;
|
||||||
};
|
}
|
||||||
bus.emit(LiveEvent::Note {
|
bus.emit(LiveEvent::Note {
|
||||||
text: "operator: /compact — running on idle session".into(),
|
text: "operator: /compact — running on idle session".into(),
|
||||||
});
|
});
|
||||||
|
|
@ -533,9 +520,6 @@ pub async fn run_pending_compact(files: &TurnFiles, bus: &Bus, session: &AgentSe
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
bus.set_state(crate::events::TurnState::Idle);
|
bus.set_state(crate::events::TurnState::Idle);
|
||||||
if let Some(prompt) = request.wake_prompt {
|
|
||||||
bus.set_post_compact_wake(prompt);
|
|
||||||
}
|
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ use axum::{
|
||||||
http::StatusCode,
|
http::StatusCode,
|
||||||
response::{IntoResponse, Response},
|
response::{IntoResponse, Response},
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::Deserialize;
|
||||||
|
|
||||||
use super::{AppState, SigintOutcome, error_response};
|
use super::{AppState, SigintOutcome, error_response};
|
||||||
|
|
||||||
|
|
@ -80,10 +80,7 @@ pub(super) async fn post_cancel_turn(State(state): State<AppState>) -> Response
|
||||||
/// claude process) rather than only when the agent is idle. Returns 200
|
/// claude process) rather than only when the agent is idle. Returns 200
|
||||||
/// immediately; the compaction stream lands in the live panel when it runs.
|
/// immediately; the compaction stream lands in the live panel when it runs.
|
||||||
pub(super) async fn post_compact(State(state): State<AppState>) -> Response {
|
pub(super) async fn post_compact(State(state): State<AppState>) -> Response {
|
||||||
// No wake prompt: the operator is watching the dashboard, not waiting on
|
state.bus.request_compact();
|
||||||
// an inbox message — `request_compact`'s wake-prompt arg exists for the
|
|
||||||
// agent's own `compact` MCP tool (`todo_server.rs::compact`).
|
|
||||||
state.bus.request_compact(None);
|
|
||||||
state.bus.emit(crate::events::LiveEvent::Note {
|
state.bus.emit(crate::events::LiveEvent::Note {
|
||||||
text: "operator: /compact queued — runs at the end of the current turn".into(),
|
text: "operator: /compact queued — runs at the end of the current turn".into(),
|
||||||
});
|
});
|
||||||
|
|
@ -205,10 +202,5 @@ pub(super) async fn post_mark_todos_done(Form(form): Form<MarkTodosDoneForm>) ->
|
||||||
acked += count;
|
acked += count;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
axum::Json(MarkTodosDoneBody { acked }).into_response()
|
axum::Json(serde_json::json!({ "acked": acked })).into_response()
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
struct MarkTodosDoneBody {
|
|
||||||
acked: u64,
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
use axum::extract::State;
|
use axum::extract::State;
|
||||||
use axum::response::{IntoResponse, Response};
|
use axum::response::{IntoResponse, Response};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::Deserialize;
|
||||||
|
|
||||||
use super::AppState;
|
use super::AppState;
|
||||||
|
|
||||||
|
|
@ -40,11 +40,6 @@ async fn fetch_reminder_stats(window_secs: u64) -> Option<hive_sh4re::approvals:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
struct TodosBody {
|
|
||||||
todos: Vec<hive_sh4re::inbox::LooseEnd>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `GET /api/todos` — snapshot of this agent's local todos (loose-ends v2).
|
/// `GET /api/todos` — snapshot of this agent's local todos (loose-ends v2).
|
||||||
///
|
///
|
||||||
/// Connects to the in-agent harness socket (`HIVE_AGENT_SOCKET`) and calls
|
/// Connects to the in-agent harness socket (`HIVE_AGENT_SOCKET`) and calls
|
||||||
|
|
@ -59,5 +54,5 @@ pub(super) async fn api_todos() -> Response {
|
||||||
Some(hive_agent_sock::Response::LooseEnds { loose_ends }) => loose_ends,
|
Some(hive_agent_sock::Response::LooseEnds { loose_ends }) => loose_ends,
|
||||||
_ => Vec::new(),
|
_ => Vec::new(),
|
||||||
};
|
};
|
||||||
axum::Json(TodosBody { todos }).into_response()
|
axum::Json(serde_json::json!({ "todos": todos })).into_response()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,23 +5,11 @@ use std::convert::Infallible;
|
||||||
use axum::Json;
|
use axum::Json;
|
||||||
use axum::extract::{Query, State};
|
use axum::extract::{Query, State};
|
||||||
use axum::response::sse::{Event, KeepAlive, Sse};
|
use axum::response::sse::{Event, KeepAlive, Sse};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::Deserialize;
|
||||||
use tokio_stream::{Stream, StreamExt, wrappers::BroadcastStream};
|
use tokio_stream::{Stream, StreamExt, wrappers::BroadcastStream};
|
||||||
|
|
||||||
use super::AppState;
|
use super::AppState;
|
||||||
|
|
||||||
/// Response body for `GET /api/events/history`. `seq` is omitted from the
|
|
||||||
/// wire entirely on a paginated (non-initial) load — matches the old
|
|
||||||
/// `json!` shape, which only ever set the `"seq"` key when `Some`.
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub(super) struct EventsHistoryBody {
|
|
||||||
events: Vec<crate::events::StoredEvent>,
|
|
||||||
min_id: Option<i64>,
|
|
||||||
has_more: bool,
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
seq: Option<u64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Query params for the paginated history endpoint.
|
/// Query params for the paginated history endpoint.
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
pub(super) struct HistoryParams {
|
pub(super) struct HistoryParams {
|
||||||
|
|
@ -35,7 +23,7 @@ pub(super) struct HistoryParams {
|
||||||
pub(super) async fn events_history(
|
pub(super) async fn events_history(
|
||||||
State(state): State<AppState>,
|
State(state): State<AppState>,
|
||||||
Query(params): Query<HistoryParams>,
|
Query(params): Query<HistoryParams>,
|
||||||
) -> Json<EventsHistoryBody> {
|
) -> Json<serde_json::Value> {
|
||||||
use crate::events::HISTORY_CAPACITY;
|
use crate::events::HISTORY_CAPACITY;
|
||||||
let limit = params.limit.unwrap_or(100).min(HISTORY_CAPACITY);
|
let limit = params.limit.unwrap_or(100).min(HISTORY_CAPACITY);
|
||||||
let before = params.before;
|
let before = params.before;
|
||||||
|
|
@ -63,12 +51,15 @@ pub(super) async fn events_history(
|
||||||
se
|
se
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
Json(EventsHistoryBody {
|
let mut resp = serde_json::json!({
|
||||||
events,
|
"events": events,
|
||||||
min_id,
|
"min_id": min_id,
|
||||||
has_more,
|
"has_more": has_more,
|
||||||
seq,
|
});
|
||||||
})
|
if let Some(s) = seq {
|
||||||
|
resp["seq"] = serde_json::json!(s);
|
||||||
|
}
|
||||||
|
Json(resp)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn events_stream(
|
pub(super) async fn events_stream(
|
||||||
|
|
|
||||||
|
|
@ -899,20 +899,99 @@ pub async fn run_finalize_deploy(coord: &Arc<Coordinator>, approval_id: i64) ->
|
||||||
/// imperative infra that `auto_update::ensure_root_agent` recreates on the
|
/// imperative infra that `auto_update::ensure_root_agent` recreates on the
|
||||||
/// next hive-c0re startup if absent, so destroying it is transient rather
|
/// next hive-c0re startup if absent, so destroying it is transient rather
|
||||||
/// than something to refuse at the API.
|
/// than something to refuse at the API.
|
||||||
///
|
pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Result<()> {
|
||||||
/// Submits the teardown DAG and returns — it does not wait for the container to
|
|
||||||
/// go away. Same contract as every other lifecycle op (`rebuild`, `kill`,
|
|
||||||
/// `restart`, `start`): the queue owns the work, the caller gets an
|
|
||||||
/// acknowledgement. Progress is visible as real nodes on the dashboard.
|
|
||||||
pub fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) {
|
|
||||||
tracing::info!(%name, purge, "destroy");
|
tracing::info!(%name, purge, "destroy");
|
||||||
if let Err(e) = coord.job_queue.insert_job(|b| {
|
// Guard auto-clears on the success path's final scope exit and on
|
||||||
crate::job_queue::templates::destroy(b, name, purge);
|
// every early-return / cancellation along the way.
|
||||||
Vec::new()
|
// Destroy has no queue node behind it, so nothing in the graph says this
|
||||||
}) {
|
// container is going away on purpose — without this the crash watcher
|
||||||
tracing::error!(agent = %name, error = ?e, "destroy: insert failed");
|
// reports every destroy as a crash and the manager tries to recover it.
|
||||||
|
let guard = coord.suppress_crash_watch(name);
|
||||||
|
lifecycle::destroy(name).await?;
|
||||||
|
coord.unregister_agent(name);
|
||||||
|
let runtime = crate::paths::agent_runtime_dir(name);
|
||||||
|
if runtime.exists() {
|
||||||
|
let _ = std::fs::remove_dir_all(&runtime);
|
||||||
}
|
}
|
||||||
coord.emit_rebuild_queue_snapshot();
|
if purge {
|
||||||
|
// The state root may be a btrfs subvolume: a subvolume root
|
||||||
|
// can't be removed with rmdir/`remove_dir_all`, so delete it via
|
||||||
|
// hive-priv (root) first. No-op for plain-dir agents — the loop below
|
||||||
|
// then handles the plain-dir state root plus the applied dir.
|
||||||
|
if let Err(e) = crate::priv_client::delete_agent_subvolume(name).await {
|
||||||
|
tracing::warn!(error = ?e, %name, "purge: delete state subvolume failed");
|
||||||
|
}
|
||||||
|
// A malformed name can't have a persistent state tree (the state dir
|
||||||
|
// is only ever created under a validated Ident), so its removal is a
|
||||||
|
// no-op — skip the state-dir sweep and just clear the applied dir.
|
||||||
|
let state_dir = hive_types::Ident::parse(name)
|
||||||
|
.ok()
|
||||||
|
.map(|id| crate::paths::agent_state_dir(&id));
|
||||||
|
for dir in state_dir
|
||||||
|
.into_iter()
|
||||||
|
.chain([crate::paths::applied_dir(name)])
|
||||||
|
{
|
||||||
|
if dir.exists()
|
||||||
|
&& let Err(e) = std::fs::remove_dir_all(&dir)
|
||||||
|
{
|
||||||
|
tracing::warn!(error = ?e, dir = %dir.display(), "purge: remove failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Meta flake: drop the agent's input + nixosConfiguration so a
|
||||||
|
// future spawn under the same name re-seeds cleanly, and so the
|
||||||
|
// meta lock doesn't reference a vanished applied repo. Log + keep
|
||||||
|
// going on failure — destroy already succeeded at the
|
||||||
|
// nixos-container level, the meta repo is just bookkeeping.
|
||||||
|
if let Err(e) = sync_meta_after_lifecycle(coord).await {
|
||||||
|
tracing::warn!(error = ?e, %name, "meta sync after destroy failed");
|
||||||
|
}
|
||||||
|
let _ = coord.approvals.fail_pending_for_agent(
|
||||||
|
name,
|
||||||
|
if purge {
|
||||||
|
"agent purged"
|
||||||
|
} else {
|
||||||
|
"agent destroyed"
|
||||||
|
},
|
||||||
|
);
|
||||||
|
// Drop the durable power intent — a future agent of the same name
|
||||||
|
// seeds fresh from its observed state.
|
||||||
|
if let Err(e) = coord.power.remove(name) {
|
||||||
|
tracing::warn!(%name, error = ?e, "agent_power: remove on destroy failed");
|
||||||
|
}
|
||||||
|
drop(guard);
|
||||||
|
let _ = coord
|
||||||
|
.push_todo(
|
||||||
|
hive_sh4re::manager::MANAGER_AGENT,
|
||||||
|
"core",
|
||||||
|
Some(format!("destroyed:{name}")),
|
||||||
|
format!("agent '{name}' destroyed"),
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
// Container row disappeared — rescan so the dashboard fires
|
||||||
|
// `ContainerRemoved` for the gone row, then emit the
|
||||||
|
// tombstones snapshot (gained one on destroy, lost one on
|
||||||
|
// purge — recompute either way).
|
||||||
|
coord.rescan_containers_and_emit().await;
|
||||||
|
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
||||||
|
// Re-emit the schedules snapshot: the rescan above refreshed the live
|
||||||
|
// roster, so any schedule that still targets the just-destroyed agent
|
||||||
|
// now drops that ghost column live (no page reload needed).
|
||||||
|
coord.emit_schedules_snapshot();
|
||||||
|
// Update tmpfiles.d to remove the destroyed agent's dirs from the
|
||||||
|
// boot-time pre-creation list. Best-effort: failure is logged only.
|
||||||
|
tokio::spawn(lifecycle::sync_tmpfiles());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rerender the meta flake from whatever containers still exist on
|
||||||
|
/// disk. Called after lifecycle ops that change the agent set (today:
|
||||||
|
/// destroy). Idempotent — a no-op when nothing changed.
|
||||||
|
async fn sync_meta_after_lifecycle(coord: &Coordinator) -> Result<()> {
|
||||||
|
let agents = lifecycle::agents_for_meta_listing().await?;
|
||||||
|
crate::meta::sync_agents(&coord.hive_env(), &agents).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn deny(coord: &Coordinator, id: i64, note: Option<&str>) -> Result<()> {
|
pub fn deny(coord: &Coordinator, id: i64, note: Option<&str>) -> Result<()> {
|
||||||
|
|
|
||||||
|
|
@ -116,6 +116,13 @@ pub struct Coordinator {
|
||||||
/// is never injected into containers.
|
/// is never injected into containers.
|
||||||
pub model_prices: crate::hive_stats::PriceTable,
|
pub model_prices: crate::hive_stats::PriceTable,
|
||||||
agents: Mutex<HashMap<String, AgentSocket>>,
|
agents: Mutex<HashMap<String, AgentSocket>>,
|
||||||
|
/// Agents whose lifecycle action (currently just spawn) is in flight.
|
||||||
|
/// Read by the dashboard to render a spinner; cleared when the action
|
||||||
|
/// resolves (success or failure).
|
||||||
|
/// Agents whose container is being taken down by work with **no queue node
|
||||||
|
/// behind it** (destroy, migration), so the crash watcher must not report
|
||||||
|
/// the disappearance as a crash. Not a pill — see [`CrashWatchSuppression`].
|
||||||
|
crash_suppressed: Mutex<HashSet<String>>,
|
||||||
/// Tombstone for transients that have JUST been cleared. The
|
/// Tombstone for transients that have JUST been cleared. The
|
||||||
/// crash watcher polls every 10s and would race the
|
/// crash watcher polls every 10s and would race the
|
||||||
/// drop-clears-immediately path of `TransientGuard`: an operator
|
/// drop-clears-immediately path of `TransientGuard`: an operator
|
||||||
|
|
@ -137,7 +144,9 @@ pub struct Coordinator {
|
||||||
/// live and both clear. Keyed by agent, the last clear *overwrites* the
|
/// live and both clear. Keyed by agent, the last clear *overwrites* the
|
||||||
/// others: a `Prebuild` (`deliberate_stop = false`) landing after a
|
/// others: a `Prebuild` (`deliberate_stop = false`) landing after a
|
||||||
/// `StopForUpdate` (`true`) leaves the tombstone reading `false`, and the
|
/// `StopForUpdate` (`true`) leaves the tombstone reading `false`, and the
|
||||||
/// crash watcher then reports an intentional stop as a **crash**.
|
/// crash watcher then reports an intentional stop as a **crash**. The
|
||||||
|
/// out-of-band suppression guard, which has no node behind it, uses
|
||||||
|
/// [`NO_NODE_LABEL`].
|
||||||
recent_transient: Mutex<HashMap<(String, String), (bool, std::time::Instant)>>,
|
recent_transient: Mutex<HashMap<(String, String), (bool, std::time::Instant)>>,
|
||||||
/// Timestamps of recent unexpected container crashes, keyed by agent.
|
/// Timestamps of recent unexpected container crashes, keyed by agent.
|
||||||
/// Fed by `crash_watch` each time it classifies a stop as a crash (so
|
/// Fed by `crash_watch` each time it classifies a stop as a crash (so
|
||||||
|
|
@ -391,6 +400,57 @@ fn fold_tombstones_by_agent<'a>(
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Tombstone label for work with **no queue node behind it** — the
|
||||||
|
/// out-of-band operations (destroy, migration) that hold a
|
||||||
|
/// [`CrashWatchGuard`] instead of appearing in the derived transient set.
|
||||||
|
///
|
||||||
|
/// [`Coordinator::recent_transient`] is keyed by `(agent, label)` so concurrent
|
||||||
|
/// pills can't overwrite each other's `deliberate_stop`; a guard has no node and
|
||||||
|
/// therefore no node label, so it needs one of its own. Angle-bracketed to keep
|
||||||
|
/// it out of the `NodeKind::as_str` namespace — no node can ever render this.
|
||||||
|
const NO_NODE_LABEL: &str = "<no-node>";
|
||||||
|
|
||||||
|
/// RAII handle returned by [`Coordinator::suppress_crash_watch`]. While held,
|
||||||
|
/// the crash watcher treats this container disappearing as **expected**.
|
||||||
|
///
|
||||||
|
/// This is *not* a dashboard pill. Transients are derived from running queue
|
||||||
|
/// nodes and nothing stores them. But destroy and migration take a container
|
||||||
|
/// down without a node behind them, so nothing in the graph says the
|
||||||
|
/// disappearance was intended — and without that, `crash_watch` fires a
|
||||||
|
/// `ContainerCrash` for every destroy and every migrated agent, and the manager
|
||||||
|
/// tries to "recover" containers that were removed on purpose.
|
||||||
|
///
|
||||||
|
/// It is held rather than stamped once because
|
||||||
|
/// [`crate::workers::crash_watch`]'s grace window is finite and these
|
||||||
|
/// operations are not: a long destroy would outlive a single tombstone. The
|
||||||
|
/// tombstone is stamped on drop, covering the poll that lands just after.
|
||||||
|
///
|
||||||
|
/// Goes away entirely once destroy + migration are real queue nodes.
|
||||||
|
#[must_use = "suppression lasts as long as the guard; bind it for the operation's duration \
|
||||||
|
(`let _guard = coord.suppress_crash_watch(...)`). An unbound call drops it \
|
||||||
|
immediately and the very next poll can report a deliberate stop as a crash."]
|
||||||
|
pub struct CrashWatchSuppression {
|
||||||
|
coord: Arc<Coordinator>,
|
||||||
|
name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for CrashWatchSuppression {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.coord
|
||||||
|
.crash_suppressed
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.remove(&self.name);
|
||||||
|
// Tombstone the release so the next poll — which may land in the
|
||||||
|
// window between the container going away and this guard dropping —
|
||||||
|
// still reads the stop as deliberate.
|
||||||
|
self.coord.recent_transient.lock().unwrap().insert(
|
||||||
|
(self.name.clone(), NO_NODE_LABEL.to_owned()),
|
||||||
|
(true, std::time::Instant::now()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// RAII guard for the `meta-update` in-progress flag, held for the
|
/// RAII guard for the `meta-update` in-progress flag, held for the
|
||||||
/// duration of a `run_meta_update` background task. Created by
|
/// duration of a `run_meta_update` background task. Created by
|
||||||
/// `Coordinator::meta_update_guard`. Drop decrements the active-run
|
/// `Coordinator::meta_update_guard`. Drop decrements the active-run
|
||||||
|
|
@ -526,6 +586,7 @@ impl Coordinator {
|
||||||
agent_io_weight,
|
agent_io_weight,
|
||||||
model_prices,
|
model_prices,
|
||||||
agents: Mutex::new(HashMap::new()),
|
agents: Mutex::new(HashMap::new()),
|
||||||
|
crash_suppressed: Mutex::new(HashSet::new()),
|
||||||
recent_transient: Mutex::new(HashMap::new()),
|
recent_transient: Mutex::new(HashMap::new()),
|
||||||
recent_crashes: Mutex::new(HashMap::new()),
|
recent_crashes: Mutex::new(HashMap::new()),
|
||||||
graceful_stop_pending: Mutex::new(HashSet::new()),
|
graceful_stop_pending: Mutex::new(HashSet::new()),
|
||||||
|
|
@ -1239,16 +1300,46 @@ impl Coordinator {
|
||||||
map.iter().map(|(k, v)| (k.clone(), v.len())).collect()
|
map.iter().map(|(k, v)| (k.clone(), v.len())).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Tell the crash watcher that `name`'s container is going down **on
|
||||||
|
/// purpose**, for the lifetime of the returned guard. See
|
||||||
|
/// [`CrashWatchSuppression`] for why this exists at all.
|
||||||
|
///
|
||||||
|
/// Only for the operations with no queue node behind them. Anything the
|
||||||
|
/// job queue runs answers this from the node itself
|
||||||
|
/// ([`crate::job_queue::NodeKind::takes_container_down`]) and must not come
|
||||||
|
/// through here.
|
||||||
|
///
|
||||||
|
/// The guard's `Drop` runs even on task cancellation, so an aborted HTTP
|
||||||
|
/// request or a panic mid-destroy can't leave a container permanently
|
||||||
|
/// exempt from crash reporting.
|
||||||
|
pub fn suppress_crash_watch(self: &Arc<Self>, name: &str) -> CrashWatchSuppression {
|
||||||
|
self.crash_suppressed
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.insert(name.to_owned());
|
||||||
|
CrashWatchSuppression {
|
||||||
|
coord: self.clone(),
|
||||||
|
name: name.to_owned(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a no-node operation is currently taking this container down.
|
||||||
|
#[must_use]
|
||||||
|
pub fn crash_watch_suppressed(&self, name: &str) -> bool {
|
||||||
|
self.crash_suppressed.lock().unwrap().contains(name)
|
||||||
|
}
|
||||||
|
|
||||||
/// Every live transient, keyed by agent.
|
/// Every live transient, keyed by agent.
|
||||||
///
|
///
|
||||||
/// **Derived on read, stored nowhere.** Straight off the running graph, so
|
/// **Derived on read, stored nowhere.** Straight off the running graph, so
|
||||||
/// there is no cached copy to go stale, leak, or disagree with what is
|
/// there is no cached copy to go stale, leak, or disagree with what is
|
||||||
/// actually running.
|
/// actually running.
|
||||||
///
|
///
|
||||||
/// Migration is the last operation with no queue node behind it, so it
|
/// Work with no queue node behind it (destroy, migration) therefore shows
|
||||||
/// shows **no pill** — there is nothing in the graph to derive one from.
|
/// **no pill** — there is nothing in the graph to derive one from. Its
|
||||||
/// It gets one for free once it becomes real nodes, the way destroy did.
|
/// crash-watch suppression is a separate, narrower thing
|
||||||
///
|
/// ([`Coordinator::suppress_crash_watch`]); the pill comes back for free
|
||||||
|
/// once those become real nodes.
|
||||||
/// ⚠️ **A `Vec` per agent, not one entry.** `running_transients` tests
|
/// ⚠️ **A `Vec` per agent, not one entry.** `running_transients` tests
|
||||||
/// status alone, so a lease-exempt `Prebuild` for `a` and a lease-holding
|
/// status alone, so a lease-exempt `Prebuild` for `a` and a lease-holding
|
||||||
/// `StopForUpdate` for `a` are both live pills. Collapsing them to one
|
/// `StopForUpdate` for `a` are both live pills. Collapsing them to one
|
||||||
|
|
@ -1476,13 +1567,8 @@ impl Coordinator {
|
||||||
crate::paths::agent_runtime_dir(name).join("mcp.sock")
|
crate::paths::agent_runtime_dir(name).join("mcp.sock")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The *proposed* config repo: where a config change lands before it is
|
/// Manager-editable proposed config repo. Bind-mounted into the manager
|
||||||
/// applied, and what an approved deploy promotes into `applied_dir`.
|
/// container as `/agents/<name>/config/`.
|
||||||
///
|
|
||||||
/// **Not bind-mounted into any container.** An agent that edits a config
|
|
||||||
/// clones it from the forge itself; `/agents/<name>/config` shows the
|
|
||||||
/// applied (deployed) tree instead — see `config_bind_source` in
|
|
||||||
/// `lifecycle/host_config.rs`.
|
|
||||||
pub fn agent_proposed_dir(name: &hive_types::Ident) -> PathBuf {
|
pub fn agent_proposed_dir(name: &hive_types::Ident) -> PathBuf {
|
||||||
crate::paths::agent_state_dir(name).join("config")
|
crate::paths::agent_state_dir(name).join("config")
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -28,20 +28,19 @@ use utoipa::ToSchema;
|
||||||
|
|
||||||
use crate::host_stats::ServerWarning;
|
use crate::host_stats::ServerWarning;
|
||||||
|
|
||||||
#[derive(Serialize, ToSchema)]
|
|
||||||
struct LiveBody {
|
|
||||||
status: &'static str,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Liveness. Always `200`; no further checks.
|
/// Liveness. Always `200`; no further checks.
|
||||||
#[utoipa::path(
|
#[utoipa::path(
|
||||||
get,
|
get,
|
||||||
path = "/health/live",
|
path = "/health/live",
|
||||||
responses((status = 200, description = "process is up", body = LiveBody)),
|
responses((status = 200, description = "process is up", body = serde_json::Value)),
|
||||||
tag = "health"
|
tag = "health"
|
||||||
)]
|
)]
|
||||||
pub(super) async fn get_health_live() -> Response {
|
pub(super) async fn get_health_live() -> Response {
|
||||||
(StatusCode::OK, axum::Json(LiveBody { status: "ok" })).into_response()
|
(
|
||||||
|
StatusCode::OK,
|
||||||
|
axum::Json(serde_json::json!({ "status": "ok" })),
|
||||||
|
)
|
||||||
|
.into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, ToSchema)]
|
#[derive(Serialize, ToSchema)]
|
||||||
|
|
|
||||||
|
|
@ -436,9 +436,10 @@ pub(super) struct DestroyForm {
|
||||||
params(("name" = String, Path, description = "agent name")),
|
params(("name" = String, Path, description = "agent name")),
|
||||||
request_body(content = DestroyForm, content_type = "application/x-www-form-urlencoded"),
|
request_body(content = DestroyForm, content_type = "application/x-www-form-urlencoded"),
|
||||||
responses(
|
responses(
|
||||||
(status = 200, description = "destroy queued", body = String),
|
(status = 200, description = "destroyed", body = String),
|
||||||
(status = 400, description = "bad agent name"),
|
(status = 400, description = "bad agent name"),
|
||||||
(status = 404, description = "no such agent"),
|
(status = 404, description = "no such agent"),
|
||||||
|
(status = 500, description = "destroy failed"),
|
||||||
),
|
),
|
||||||
tag = "lifecycle_ops"
|
tag = "lifecycle_ops"
|
||||||
)]
|
)]
|
||||||
|
|
@ -452,10 +453,11 @@ pub(super) async fn post_destroy(
|
||||||
}
|
}
|
||||||
// Checkbox semantics: any non-empty value (axum sends "on") = purge.
|
// Checkbox semantics: any non-empty value (axum sends "on") = purge.
|
||||||
let purge = form.purge.as_deref().is_some_and(|v| !v.is_empty());
|
let purge = form.purge.as_deref().is_some_and(|v| !v.is_empty());
|
||||||
// Submit-and-return, like every other lifecycle endpoint here. The
|
// `actions::destroy` rescans the container list on success, so the
|
||||||
// container rescan now runs in the DAG's bookkeeping tail, so
|
// `ContainerRemoved` event lands before we return 200. The matching
|
||||||
// `ContainerRemoved` arrives *after* this 200 rather than before it — the
|
// form carries `data-no-refresh`.
|
||||||
// row disappears when the event lands, same as a rebuild's does.
|
match actions::destroy(&state.coord, &name, purge).await {
|
||||||
actions::destroy(&state.coord, &name, purge);
|
Ok(()) => (StatusCode::OK, "ok").into_response(),
|
||||||
(StatusCode::OK, "ok").into_response()
|
Err(e) => error_response(&format!("destroy {name} failed: {e:#}")),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,41 +8,26 @@ use axum::{
|
||||||
http::StatusCode,
|
http::StatusCode,
|
||||||
response::{IntoResponse, Response},
|
response::{IntoResponse, Response},
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::Deserialize;
|
||||||
use utoipa::{IntoParams, ToSchema};
|
use utoipa::{IntoParams, ToSchema};
|
||||||
|
|
||||||
use super::{AppState, Ident, error_response, scan_validated_paths};
|
use super::{AppState, Ident, error_response, scan_validated_paths};
|
||||||
use crate::audit_log::AuditEntry;
|
|
||||||
use crate::container_stats::ContainerResource;
|
use crate::container_stats::ContainerResource;
|
||||||
use crate::hive_stats::HiveStats;
|
use crate::hive_stats::HiveStats;
|
||||||
|
|
||||||
#[derive(Serialize, ToSchema)]
|
|
||||||
pub(super) struct OperatorInboxItem {
|
|
||||||
id: i64,
|
|
||||||
from: String,
|
|
||||||
body: String,
|
|
||||||
at: chrono::DateTime<chrono::Utc>,
|
|
||||||
in_reply_to: Option<i64>,
|
|
||||||
file_refs: Vec<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize, ToSchema)]
|
|
||||||
pub(super) struct OperatorInboxBody {
|
|
||||||
messages: Vec<OperatorInboxItem>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Unread operator-directed messages for the dashboard's Y3R C4LL inbox.
|
/// Unread operator-directed messages for the dashboard's Y3R C4LL inbox.
|
||||||
///
|
///
|
||||||
/// Returns messages addressed to `"operator"` that haven't been
|
/// Returns messages addressed to `"operator"` that haven't been
|
||||||
/// acked yet (the operator clears them via the existing
|
/// acked yet (the operator clears them via the existing
|
||||||
/// `POST /api/agent/operator/mark-all-read`). Newest-first; path-shaped
|
/// `POST /api/agent/operator/mark-all-read`). Newest-first; path-shaped
|
||||||
/// tokens are validated so the client renders file links like the
|
/// tokens are validated so the client renders file links like the
|
||||||
/// terminal does.
|
/// terminal does. Shape: `{ "messages": [{ id, from, body, at,
|
||||||
|
/// in_reply_to, file_refs }] }`.
|
||||||
#[utoipa::path(
|
#[utoipa::path(
|
||||||
get,
|
get,
|
||||||
path = "/api/operator-inbox",
|
path = "/api/operator-inbox",
|
||||||
responses(
|
responses(
|
||||||
(status = 200, description = "unread operator-directed messages", body = OperatorInboxBody),
|
(status = 200, description = "unread operator-directed messages", body = serde_json::Value),
|
||||||
(status = 500, description = "broker read failed"),
|
(status = 500, description = "broker read failed"),
|
||||||
),
|
),
|
||||||
tag = "misc_api"
|
tag = "misc_api"
|
||||||
|
|
@ -55,7 +40,7 @@ pub(super) async fn api_operator_inbox(State(state): State<AppState>) -> Respons
|
||||||
.unread_for_recipient("operator", INBOX_LIMIT)
|
.unread_for_recipient("operator", INBOX_LIMIT)
|
||||||
{
|
{
|
||||||
Ok(messages) => {
|
Ok(messages) => {
|
||||||
let messages: Vec<OperatorInboxItem> = messages
|
let items: Vec<serde_json::Value> = messages
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter_map(|m| {
|
.filter_map(|m| {
|
||||||
let crate::broker::MessageEvent::Sent {
|
let crate::broker::MessageEvent::Sent {
|
||||||
|
|
@ -70,17 +55,17 @@ pub(super) async fn api_operator_inbox(State(state): State<AppState>) -> Respons
|
||||||
return None;
|
return None;
|
||||||
};
|
};
|
||||||
let file_refs = scan_validated_paths(&body);
|
let file_refs = scan_validated_paths(&body);
|
||||||
Some(OperatorInboxItem {
|
Some(serde_json::json!({
|
||||||
id,
|
"id": id,
|
||||||
from,
|
"from": from,
|
||||||
at: hive_sh4re::wire_time::from_secs(at),
|
"body": body,
|
||||||
body,
|
"at": hive_sh4re::wire_time::from_secs(at),
|
||||||
in_reply_to,
|
"in_reply_to": in_reply_to,
|
||||||
file_refs,
|
"file_refs": file_refs,
|
||||||
})
|
}))
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
axum::Json(OperatorInboxBody { messages }).into_response()
|
axum::Json(serde_json::json!({ "messages": items })).into_response()
|
||||||
}
|
}
|
||||||
Err(e) => error_response(&format!("operator-inbox failed: {e:#}")),
|
Err(e) => error_response(&format!("operator-inbox failed: {e:#}")),
|
||||||
}
|
}
|
||||||
|
|
@ -129,23 +114,18 @@ pub(super) async fn api_container_resources() -> Response {
|
||||||
axum::Json(crate::container_stats::gather().await).into_response()
|
axum::Json(crate::container_stats::gather().await).into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, ToSchema)]
|
|
||||||
pub(super) struct AuditLogBody {
|
|
||||||
entries: Vec<AuditEntry>,
|
|
||||||
total: i64,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Most-recent agent-initiated privileged-action
|
/// Most-recent agent-initiated privileged-action
|
||||||
/// audit entries, newest first (server-clamped to 500).
|
/// audit entries, newest first (server-clamped to 500).
|
||||||
///
|
///
|
||||||
/// Backs the operator dashboard's audit view. `total` lets the UI show
|
/// Backs the operator dashboard's audit view. Returns
|
||||||
|
/// `{ "entries": [AuditEntry…], "total": N }` so the UI can show
|
||||||
/// "latest 500 of N" rather than silently capping. `ts_unix` is in
|
/// "latest 500 of N" rather than silently capping. `ts_unix` is in
|
||||||
/// **seconds**.
|
/// **seconds**.
|
||||||
#[utoipa::path(
|
#[utoipa::path(
|
||||||
get,
|
get,
|
||||||
path = "/api/audit-log",
|
path = "/api/audit-log",
|
||||||
responses(
|
responses(
|
||||||
(status = 200, description = "recent audit entries + total count", body = AuditLogBody),
|
(status = 200, description = "recent audit entries + total count", body = serde_json::Value),
|
||||||
(status = 500, description = "sqlite read failed"),
|
(status = 500, description = "sqlite read failed"),
|
||||||
),
|
),
|
||||||
tag = "misc_api"
|
tag = "misc_api"
|
||||||
|
|
@ -160,12 +140,7 @@ pub(super) async fn api_audit_log(State(state): State<AppState>) -> Response {
|
||||||
Ok(n) => n,
|
Ok(n) => n,
|
||||||
Err(e) => return error_response(&format!("audit-log count: {e:#}")),
|
Err(e) => return error_response(&format!("audit-log count: {e:#}")),
|
||||||
};
|
};
|
||||||
axum::Json(AuditLogBody { entries, total }).into_response()
|
axum::Json(serde_json::json!({ "entries": entries, "total": total })).into_response()
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize, ToSchema)]
|
|
||||||
pub(super) struct MarkAllReadBody {
|
|
||||||
marked: u64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Operator-driven "clear this agent's inbox" — backs the side-panel
|
/// Operator-driven "clear this agent's inbox" — backs the side-panel
|
||||||
|
|
@ -173,14 +148,14 @@ pub(super) struct MarkAllReadBody {
|
||||||
///
|
///
|
||||||
/// Marks every message addressed to the agent as acked (backfilling
|
/// Marks every message addressed to the agent as acked (backfilling
|
||||||
/// `delivered_at` for any still-pending rows so vacuum can collect
|
/// `delivered_at` for any still-pending rows so vacuum can collect
|
||||||
/// them). `marked` lets the frontend show "cleared N messages"
|
/// them). Returns `{ "marked": N }` so the frontend can show "cleared
|
||||||
/// feedback without an extra fetch.
|
/// N messages" feedback without an extra fetch.
|
||||||
#[utoipa::path(
|
#[utoipa::path(
|
||||||
post,
|
post,
|
||||||
path = "/api/agent/{name}/mark-all-read",
|
path = "/api/agent/{name}/mark-all-read",
|
||||||
params(("name" = String, Path, description = "agent name")),
|
params(("name" = String, Path, description = "agent name")),
|
||||||
responses(
|
responses(
|
||||||
(status = 200, description = "count of messages marked read", body = MarkAllReadBody),
|
(status = 200, description = "count of messages marked read", body = serde_json::Value),
|
||||||
(status = 400, description = "bad agent name"),
|
(status = 400, description = "bad agent name"),
|
||||||
(status = 500, description = "broker write failed"),
|
(status = 500, description = "broker write failed"),
|
||||||
),
|
),
|
||||||
|
|
@ -197,9 +172,9 @@ pub(super) async fn post_mark_all_read(
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
match state.coord.broker.mark_all_read(name.as_str()) {
|
match state.coord.broker.mark_all_read(name.as_str()) {
|
||||||
Ok(marked) => {
|
Ok(n) => {
|
||||||
tracing::info!(%name, marked, "operator marked all messages read");
|
tracing::info!(%name, marked = n, "operator marked all messages read");
|
||||||
axum::Json(MarkAllReadBody { marked }).into_response()
|
axum::Json(serde_json::json!({ "marked": n })).into_response()
|
||||||
}
|
}
|
||||||
Err(e) => error_response(&format!("mark-all-read {name} failed: {e:#}")),
|
Err(e) => error_response(&format!("mark-all-read {name} failed: {e:#}")),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -18,16 +18,6 @@ use crate::scheduled_prompts_worker::FireNowReport;
|
||||||
|
|
||||||
use super::{AppState, error_problem, error_response};
|
use super::{AppState, error_problem, error_response};
|
||||||
|
|
||||||
#[derive(serde::Serialize, utoipa::ToSchema)]
|
|
||||||
pub(super) struct NewScheduleBody {
|
|
||||||
id: i64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(serde::Serialize, utoipa::ToSchema)]
|
|
||||||
pub(super) struct CancelResultBody {
|
|
||||||
cancelled: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Snapshot of every schedule for the
|
/// Snapshot of every schedule for the
|
||||||
/// scheduled-prompts tab.
|
/// scheduled-prompts tab.
|
||||||
///
|
///
|
||||||
|
|
@ -84,7 +74,7 @@ pub(super) async fn api_schedules(State(state): State<AppState>) -> Response {
|
||||||
// `api_schedules` above.
|
// `api_schedules` above.
|
||||||
request_body(content = serde_json::Value, description = "SchedulePromptPayload wire shape"),
|
request_body(content = serde_json::Value, description = "SchedulePromptPayload wire shape"),
|
||||||
responses(
|
responses(
|
||||||
(status = 200, description = "created; body carries the new row id", body = NewScheduleBody),
|
(status = 200, description = "created; body carries the new row id", body = serde_json::Value),
|
||||||
(status = 400, description = "no targets, empty body, or interval_seconds == 0"),
|
(status = 400, description = "no targets, empty body, or interval_seconds == 0"),
|
||||||
(status = 500, description = "submit failed"),
|
(status = 500, description = "submit failed"),
|
||||||
),
|
),
|
||||||
|
|
@ -118,7 +108,7 @@ pub(super) async fn post_schedule_new(
|
||||||
match state.coord.scheduled_prompts.submit(&new) {
|
match state.coord.scheduled_prompts.submit(&new) {
|
||||||
Ok(id) => {
|
Ok(id) => {
|
||||||
state.coord.emit_schedules_snapshot();
|
state.coord.emit_schedules_snapshot();
|
||||||
Ok(axum::Json(NewScheduleBody { id }).into_response())
|
Ok(axum::Json(serde_json::json!({"id": id})).into_response())
|
||||||
}
|
}
|
||||||
Err(e) => Err(error_problem(&format!("schedule submit: {e:#}"))),
|
Err(e) => Err(error_problem(&format!("schedule submit: {e:#}"))),
|
||||||
}
|
}
|
||||||
|
|
@ -187,7 +177,7 @@ pub(super) async fn post_schedule_fire_now(
|
||||||
post,
|
post,
|
||||||
path = "/api/rebuild-queue/{id}/cancel",
|
path = "/api/rebuild-queue/{id}/cancel",
|
||||||
params(("id" = u64, Path, description = "job-queue node id (a DAG's root cancels the group)")),
|
params(("id" = u64, Path, description = "job-queue node id (a DAG's root cancels the group)")),
|
||||||
responses((status = 200, description = "whether the DAG was cancelled", body = CancelResultBody)),
|
responses((status = 200, description = "whether the DAG was cancelled", body = serde_json::Value)),
|
||||||
tag = "schedules"
|
tag = "schedules"
|
||||||
)]
|
)]
|
||||||
pub(super) async fn post_rebuild_queue_cancel(
|
pub(super) async fn post_rebuild_queue_cancel(
|
||||||
|
|
@ -198,9 +188,9 @@ pub(super) async fn post_rebuild_queue_cancel(
|
||||||
// Any terminal side effect is the DAG's own spared tail node, which the
|
// Any terminal side effect is the DAG's own spared tail node, which the
|
||||||
// scheduler picks up on its next pass — nothing to fire from here.
|
// scheduler picks up on its next pass — nothing to fire from here.
|
||||||
state.coord.emit_rebuild_queue_snapshot();
|
state.coord.emit_rebuild_queue_snapshot();
|
||||||
axum::Json(CancelResultBody { cancelled: true }).into_response()
|
axum::Json(serde_json::json!({"cancelled": true})).into_response()
|
||||||
} else {
|
} else {
|
||||||
axum::Json(CancelResultBody { cancelled: false }).into_response()
|
axum::Json(serde_json::json!({"cancelled": false})).into_response()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -726,18 +726,6 @@ pub(super) async fn jobq_rollup(
|
||||||
axum::Json(state.coord.job_queue.state_rollup())
|
axum::Json(state.coord.job_queue.state_rollup())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Response body for `/api/dashboard/history`. No `ToSchema` — its
|
|
||||||
/// `events` field wraps [`crate::dashboard_events::DashboardEvent`],
|
|
||||||
/// which doesn't derive `ToSchema` either (a large enum with many
|
|
||||||
/// variants; see that type's doc comment for why annotating it is
|
|
||||||
/// out of scope here). The `responses(...)` doc below spells out the
|
|
||||||
/// shape in prose instead of a `body = ...` reference.
|
|
||||||
#[derive(Serialize)]
|
|
||||||
struct DashboardHistoryBody {
|
|
||||||
seq: u64,
|
|
||||||
events: Vec<crate::dashboard_events::DashboardEvent>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[utoipa::path(
|
#[utoipa::path(
|
||||||
get,
|
get,
|
||||||
path = "/api/dashboard/history",
|
path = "/api/dashboard/history",
|
||||||
|
|
@ -810,7 +798,7 @@ pub(super) async fn dashboard_history(State(state): State<AppState>) -> Response
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
axum::Json(DashboardHistoryBody { seq, events }).into_response()
|
axum::Json(serde_json::json!({ "seq": seq, "events": events })).into_response()
|
||||||
}
|
}
|
||||||
Err(e) => error_response(&format!("dashboard/history failed: {e:#}")),
|
Err(e) => error_response(&format!("dashboard/history failed: {e:#}")),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -73,15 +73,6 @@ pub(super) async fn run_node(
|
||||||
NodeKind::RebuildBookkeeping { .. } => run_rebuild_bookkeeping(coord, agent).await,
|
NodeKind::RebuildBookkeeping { .. } => run_rebuild_bookkeeping(coord, agent).await,
|
||||||
NodeKind::Provision { .. } => run_provision(coord, agent).await,
|
NodeKind::Provision { .. } => run_provision(coord, agent).await,
|
||||||
NodeKind::Create { .. } => run_create(agent).await,
|
NodeKind::Create { .. } => run_create(agent).await,
|
||||||
NodeKind::DestroyContainer { .. } => run_destroy_container(coord, agent).await,
|
|
||||||
NodeKind::PurgeState { .. } => {
|
|
||||||
run_purge_state(agent).await;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
NodeKind::DestroyBookkeeping { purge, .. } => {
|
|
||||||
run_destroy_bookkeeping(coord, agent, *purge).await;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
NodeKind::MetaLock {
|
NodeKind::MetaLock {
|
||||||
sweep,
|
sweep,
|
||||||
fanout,
|
fanout,
|
||||||
|
|
@ -172,112 +163,6 @@ async fn run_resolve_approval(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Rerender the meta flake from whatever containers still exist on disk.
|
|
||||||
/// Idempotent — a no-op when nothing changed. Lives here because the destroy
|
|
||||||
/// tail is its only caller; it moved with `destroy` when that became a DAG.
|
|
||||||
async fn sync_meta_after_lifecycle(coord: &Coordinator) -> Result<()> {
|
|
||||||
let agents = crate::lifecycle::agents_for_meta_listing().await?;
|
|
||||||
crate::meta::sync_agents(&coord.hive_env(), &agents).await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `nixos-container destroy`, then drop the agent from the roster and clear
|
|
||||||
/// its ephemeral runtime dir (the mcp socket, which does not survive a restart
|
|
||||||
/// anyway).
|
|
||||||
///
|
|
||||||
/// The only fallible step is the destroy itself: once the container is gone the
|
|
||||||
/// un-registration cannot meaningfully fail, and returning early would strand
|
|
||||||
/// the roster claiming an agent that no longer exists.
|
|
||||||
async fn run_destroy_container(coord: &Arc<Coordinator>, agent: &str) -> Result<()> {
|
|
||||||
crate::lifecycle::destroy(agent).await?;
|
|
||||||
coord.unregister_agent(agent);
|
|
||||||
let runtime = crate::paths::agent_runtime_dir(agent);
|
|
||||||
if runtime.exists() {
|
|
||||||
let _ = std::fs::remove_dir_all(&runtime);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The `purge = true` half: wipe the agent's persistent trees.
|
|
||||||
///
|
|
||||||
/// Every step is best-effort-with-a-warning rather than fatal, and that is
|
|
||||||
/// deliberate — the container is already destroyed by the time this runs, so
|
|
||||||
/// failing the node would leave the operator with a half-purged agent and a red
|
|
||||||
/// DAG, when what they can actually act on is the log line naming the path.
|
|
||||||
async fn run_purge_state(agent: &str) {
|
|
||||||
// The state root may be a btrfs subvolume: a subvolume root can't be
|
|
||||||
// removed with rmdir/`remove_dir_all`, so delete it via hive-priv (root)
|
|
||||||
// first. No-op for plain-dir agents — the loop below then handles the
|
|
||||||
// plain-dir state root plus the applied dir.
|
|
||||||
if let Err(e) = crate::priv_client::delete_agent_subvolume(agent).await {
|
|
||||||
tracing::warn!(error = ?e, %agent, "purge: delete state subvolume failed");
|
|
||||||
}
|
|
||||||
// A malformed name can't have a persistent state tree (the state dir is
|
|
||||||
// only ever created under a validated Ident), so its removal is a no-op —
|
|
||||||
// skip the state-dir sweep and just clear the applied dir.
|
|
||||||
let state_dir = hive_types::Ident::parse(agent)
|
|
||||||
.ok()
|
|
||||||
.map(|id| crate::paths::agent_state_dir(&id));
|
|
||||||
for dir in state_dir
|
|
||||||
.into_iter()
|
|
||||||
.chain([crate::paths::applied_dir(agent)])
|
|
||||||
{
|
|
||||||
if dir.exists()
|
|
||||||
&& let Err(e) = std::fs::remove_dir_all(&dir)
|
|
||||||
{
|
|
||||||
tracing::warn!(error = ?e, dir = %dir.display(), "purge: remove failed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Post-destroy bookkeeping. Infallible by construction: every step is
|
|
||||||
/// warn-and-continue, because the destroy it follows has already succeeded and
|
|
||||||
/// none of this is undoable — a failed meta sync or power-store write is a
|
|
||||||
/// bookkeeping drift to log, not a reason to red a DAG whose container is
|
|
||||||
/// already gone.
|
|
||||||
async fn run_destroy_bookkeeping(coord: &Arc<Coordinator>, agent: &str, purge: bool) {
|
|
||||||
// Meta flake: drop the agent's input + nixosConfiguration so a future spawn
|
|
||||||
// under the same name re-seeds cleanly, and so the meta lock doesn't
|
|
||||||
// reference a vanished applied repo.
|
|
||||||
if let Err(e) = sync_meta_after_lifecycle(coord).await {
|
|
||||||
tracing::warn!(error = ?e, %agent, "meta sync after destroy failed");
|
|
||||||
}
|
|
||||||
let _ = coord.approvals.fail_pending_for_agent(
|
|
||||||
agent,
|
|
||||||
if purge {
|
|
||||||
"agent purged"
|
|
||||||
} else {
|
|
||||||
"agent destroyed"
|
|
||||||
},
|
|
||||||
);
|
|
||||||
// Drop the durable power intent — a future agent of the same name seeds
|
|
||||||
// fresh from its observed state.
|
|
||||||
if let Err(e) = coord.power.remove(agent) {
|
|
||||||
tracing::warn!(%agent, error = ?e, "agent_power: remove on destroy failed");
|
|
||||||
}
|
|
||||||
let _ = coord
|
|
||||||
.push_todo(
|
|
||||||
hive_sh4re::manager::MANAGER_AGENT,
|
|
||||||
"core",
|
|
||||||
Some(format!("destroyed:{agent}")),
|
|
||||||
format!("agent '{agent}' destroyed"),
|
|
||||||
None,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
// Container row disappeared — rescan so the dashboard fires
|
|
||||||
// `ContainerRemoved` for the gone row, then emit the tombstones snapshot
|
|
||||||
// (gained one on destroy, lost one on purge — recompute either way).
|
|
||||||
coord.rescan_containers_and_emit().await;
|
|
||||||
crate::dashboard::emit_tombstones_snapshot(coord).await;
|
|
||||||
// Re-emit the schedules snapshot: the rescan above refreshed the live
|
|
||||||
// roster, so any schedule that still targets the just-destroyed agent now
|
|
||||||
// drops that ghost column live (no page reload needed).
|
|
||||||
coord.emit_schedules_snapshot();
|
|
||||||
// Update tmpfiles.d to remove the destroyed agent's dirs from the boot-time
|
|
||||||
// pre-creation list. Best-effort: failure is logged only.
|
|
||||||
tokio::spawn(crate::lifecycle::sync_tmpfiles());
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Emit this agent's rebuild-complete todo. `ok` is not computed — it is which
|
/// Emit this agent's rebuild-complete todo. `ok` is not computed — it is which
|
||||||
/// of the tail pair the graph let run. The failure note comes from the DAG's
|
/// of the tail pair the graph let run. The failure note comes from the DAG's
|
||||||
/// first failing node, since the branch knows *that* it failed but not *why*.
|
/// first failing node, since the branch knows *that* it failed but not *why*.
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue