hive-c0re: graceful agent stop — quiesce harness, flush state, then stop
This commit is contained in:
parent
bf65345b83
commit
03ea5d601b
8 changed files with 202 additions and 2 deletions
|
|
@ -173,6 +173,22 @@ fn synthetic_continue() -> hive_sh4re::DeliveredMessage {
|
|||
}
|
||||
}
|
||||
|
||||
/// Synthetic message that drives the single stop-checkpoint turn when c0re
|
||||
/// signals a graceful stop. The agent gets one final turn to flush durable
|
||||
/// `/state` before the container is stopped; new inbound is already fenced.
|
||||
fn graceful_stop_message() -> hive_sh4re::DeliveredMessage {
|
||||
hive_sh4re::DeliveredMessage {
|
||||
from: "graceful-stop".into(),
|
||||
body: "You are being gracefully stopped — the container will shut down after this turn, \
|
||||
and new inbound messages are already fenced. Flush anything worth keeping to your \
|
||||
durable /state files now, then end your turn. Do not start new long-running work."
|
||||
.into(),
|
||||
id: 0,
|
||||
redelivered: false,
|
||||
in_reply_to: None,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- surface trait ----------
|
||||
|
||||
/// What a `Recv` long-poll returned. Decoupled from the per-role
|
||||
|
|
@ -188,6 +204,10 @@ enum RecvOutcome {
|
|||
/// retries; the surface impl is responsible for tracing the
|
||||
/// detail before returning this.
|
||||
TransportError,
|
||||
/// c0re signalled a graceful stop for this agent. The serve loop runs
|
||||
/// one stop-checkpoint turn (flush durable `/state`), reports
|
||||
/// `GracefulStopComplete`, and exits so the container can be stopped.
|
||||
GracefulStop,
|
||||
}
|
||||
|
||||
/// Wire surface abstraction. `AgentSurface` is the only impl — the trait
|
||||
|
|
@ -211,6 +231,12 @@ trait Surface {
|
|||
/// Either field is `None` when the underlying request errors.
|
||||
fn post_turn_counts(socket: &Path) -> impl Future<Output = (Option<u64>, Option<u64>)>;
|
||||
|
||||
/// Tell c0re the graceful-stop checkpoint is done and the harness is
|
||||
/// exiting its serve loop (fire-and-forget; logs on error). Lets the
|
||||
/// `GracefulStop` orchestration stop the container without waiting out
|
||||
/// its timeout fallback.
|
||||
fn graceful_stop_complete(socket: &Path) -> impl Future<Output = ()>;
|
||||
|
||||
/// Send a message addressed to `<parent>` (broker resolves the
|
||||
/// sentinel via `topology::parent_of` at delivery time; root
|
||||
/// agents/manager fall through to operator).
|
||||
|
|
@ -259,6 +285,18 @@ impl Surface for AgentSurface {
|
|||
}
|
||||
}
|
||||
|
||||
async fn graceful_stop_complete(socket: &Path) {
|
||||
match client::request::<_, AgentResponse>(socket, &AgentRequest::GracefulStopComplete).await
|
||||
{
|
||||
Ok(AgentResponse::Ok) => {}
|
||||
Ok(AgentResponse::Err { message }) => {
|
||||
tracing::warn!(%message, "graceful_stop_complete rejected by broker");
|
||||
}
|
||||
Ok(other) => tracing::warn!(?other, "graceful_stop_complete unexpected response"),
|
||||
Err(e) => tracing::warn!(error = ?e, "graceful_stop_complete transport error"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn inbox_unread(socket: &Path) -> u64 {
|
||||
match client::request::<_, AgentResponse>(socket, &AgentRequest::Status).await {
|
||||
Ok(AgentResponse::Status { unread }) => unread,
|
||||
|
|
@ -318,6 +356,7 @@ impl Surface for AgentSurface {
|
|||
RecvOutcome::Message(first)
|
||||
}
|
||||
Ok(AgentResponse::Messages { .. }) => RecvOutcome::Empty,
|
||||
Ok(AgentResponse::GracefulStop) => RecvOutcome::GracefulStop,
|
||||
Ok(AgentResponse::Err { message }) => {
|
||||
tracing::warn!(%message, "recv error");
|
||||
RecvOutcome::TransportError
|
||||
|
|
@ -473,6 +512,26 @@ async fn serve_loop<S: Surface>(
|
|||
// No backoff: the long-poll wait is itself the throttle.
|
||||
continue;
|
||||
}
|
||||
RecvOutcome::GracefulStop => {
|
||||
// c0re fenced our inbox and wants a clean stop. Run one
|
||||
// checkpoint turn so the agent flushes durable /state,
|
||||
// report completion, then exit the loop → the harness
|
||||
// process ends and the container can be stopped.
|
||||
tracing::info!(
|
||||
"graceful stop signalled — running stop-checkpoint turn, then exiting"
|
||||
);
|
||||
let _ = handle_turn::<S>(
|
||||
socket,
|
||||
&bus,
|
||||
stats.as_ref(),
|
||||
files,
|
||||
&turn_lock,
|
||||
graceful_stop_message(),
|
||||
)
|
||||
.await;
|
||||
S::graceful_stop_complete(socket).await;
|
||||
return Ok(());
|
||||
}
|
||||
},
|
||||
};
|
||||
let ctrl = handle_turn::<S>(socket, &bus, stats.as_ref(), files, &turn_lock, next).await;
|
||||
|
|
|
|||
Loading…
Reference in a new issue