hive-c0re: graceful agent stop — quiesce harness, flush state, then stop

This commit is contained in:
damocles 2026-06-19 08:43:08 +02:00 committed by mara
commit 03ea5d601b
8 changed files with 202 additions and 2 deletions

View file

@ -173,6 +173,22 @@ fn synthetic_continue() -> hive_sh4re::DeliveredMessage {
}
}
/// Synthetic message that drives the single stop-checkpoint turn when c0re
/// signals a graceful stop. The agent gets one final turn to flush durable
/// `/state` before the container is stopped; new inbound is already fenced.
fn graceful_stop_message() -> hive_sh4re::DeliveredMessage {
hive_sh4re::DeliveredMessage {
from: "graceful-stop".into(),
body: "You are being gracefully stopped — the container will shut down after this turn, \
and new inbound messages are already fenced. Flush anything worth keeping to your \
durable /state files now, then end your turn. Do not start new long-running work."
.into(),
id: 0,
redelivered: false,
in_reply_to: None,
}
}
// ---------- surface trait ----------
/// What a `Recv` long-poll returned. Decoupled from the per-role
@ -188,6 +204,10 @@ enum RecvOutcome {
/// retries; the surface impl is responsible for tracing the
/// detail before returning this.
TransportError,
/// c0re signalled a graceful stop for this agent. The serve loop runs
/// one stop-checkpoint turn (flush durable `/state`), reports
/// `GracefulStopComplete`, and exits so the container can be stopped.
GracefulStop,
}
/// Wire surface abstraction. `AgentSurface` is the only impl — the trait
@ -211,6 +231,12 @@ trait Surface {
/// Either field is `None` when the underlying request errors.
fn post_turn_counts(socket: &Path) -> impl Future<Output = (Option<u64>, Option<u64>)>;
/// Tell c0re the graceful-stop checkpoint is done and the harness is
/// exiting its serve loop (fire-and-forget; logs on error). Lets the
/// `GracefulStop` orchestration stop the container without waiting out
/// its timeout fallback.
fn graceful_stop_complete(socket: &Path) -> impl Future<Output = ()>;
/// Send a message addressed to `<parent>` (broker resolves the
/// sentinel via `topology::parent_of` at delivery time; root
/// agents/manager fall through to operator).
@ -259,6 +285,18 @@ impl Surface for AgentSurface {
}
}
async fn graceful_stop_complete(socket: &Path) {
match client::request::<_, AgentResponse>(socket, &AgentRequest::GracefulStopComplete).await
{
Ok(AgentResponse::Ok) => {}
Ok(AgentResponse::Err { message }) => {
tracing::warn!(%message, "graceful_stop_complete rejected by broker");
}
Ok(other) => tracing::warn!(?other, "graceful_stop_complete unexpected response"),
Err(e) => tracing::warn!(error = ?e, "graceful_stop_complete transport error"),
}
}
async fn inbox_unread(socket: &Path) -> u64 {
match client::request::<_, AgentResponse>(socket, &AgentRequest::Status).await {
Ok(AgentResponse::Status { unread }) => unread,
@ -318,6 +356,7 @@ impl Surface for AgentSurface {
RecvOutcome::Message(first)
}
Ok(AgentResponse::Messages { .. }) => RecvOutcome::Empty,
Ok(AgentResponse::GracefulStop) => RecvOutcome::GracefulStop,
Ok(AgentResponse::Err { message }) => {
tracing::warn!(%message, "recv error");
RecvOutcome::TransportError
@ -473,6 +512,26 @@ async fn serve_loop<S: Surface>(
// No backoff: the long-poll wait is itself the throttle.
continue;
}
RecvOutcome::GracefulStop => {
// c0re fenced our inbox and wants a clean stop. Run one
// checkpoint turn so the agent flushes durable /state,
// report completion, then exit the loop → the harness
// process ends and the container can be stopped.
tracing::info!(
"graceful stop signalled — running stop-checkpoint turn, then exiting"
);
let _ = handle_turn::<S>(
socket,
&bus,
stats.as_ref(),
files,
&turn_lock,
graceful_stop_message(),
)
.await;
S::graceful_stop_complete(socket).await;
return Ok(());
}
},
};
let ctrl = handle_turn::<S>(socket, &bus, stats.as_ref(), files, &turn_lock, next).await;