hive-c0re: graceful agent stop — quiesce harness, flush state, then stop

hive-c0re/src/agent_server.rs

@@ -185,6 +185,13 @@ pub(crate) async fn dispatch_shared(
         }
         hive_sh4re::Request::AckTurn => handle_ack_turn(coord, agent),
         hive_sh4re::Request::RequeueInflight => handle_requeue_inflight(coord, agent),
+        hive_sh4re::Request::GracefulStopComplete => {
+            // Harness drained + is exiting: clear the fence so the
+            // `GracefulStop` orchestration (which polls this flag) proceeds
+            // to stop the container without waiting out its timeout.
+            coord.clear_graceful_stop(agent);
+            hive_sh4re::Response::Ok
+        }
         hive_sh4re::Request::GetHostJournal {
             unit,
             container,
@@ -221,6 +228,15 @@ async fn handle_recv(
     wait_seconds: Option<u64>,
     max: Option<u32>,
 ) -> hive_sh4re::Response {
+    // Graceful-stop fence: while a graceful stop is pending for this agent,
+    // return `GracefulStop` instead of polling the broker. The harness runs
+    // one stop-checkpoint turn then exits; new sends keep queueing in the
+    // broker for the agent's next start. Checked before the (blocking) poll
+    // so a flag set between polls is seen on the next Recv — the orchestration
+    // also fires a transient wake to break an in-flight long-poll.
+    if coord.is_graceful_stop_pending(agent) {
+        return hive_sh4re::Response::GracefulStop;
+    }
     let cap = max.unwrap_or(1).min(RECV_BATCH_MAX) as usize;
     match coord
         .broker

hive-c0re/src/coordinator.rs

@@ -125,6 +125,12 @@ pub struct Coordinator {
     /// the dashboard's `agents_crashing` banner warning via
     /// `recent_crash_counts`, which prunes entries older than its window.
     recent_crashes: Mutex<HashMap<String, Vec<std::time::Instant>>>,
+    /// Agents with a graceful stop in progress. Set by the `GracefulStop`
+    /// orchestration; read by `agent_server::handle_recv`, which returns
+    /// `Response::GracefulStop` (instead of polling the broker) while an
+    /// agent is in this set — the inbound fence. Cleared when the agent
+    /// reports `GracefulStopComplete` or the container is stopped.
+    graceful_stop_pending: Mutex<HashSet<String>>,
     /// Unified wire-facing event channel feeding the dashboard SSE
     /// stream. Carries broker messages (mirrored from `broker.subscribe`
     /// by the forwarder task in `main.rs`) and dashboard-only mutation
@@ -450,6 +456,7 @@ impl Coordinator {
             transient: Mutex::new(HashMap::new()),
             recent_transient: Mutex::new(HashMap::new()),
             recent_crashes: Mutex::new(HashMap::new()),
+            graceful_stop_pending: Mutex::new(HashSet::new()),
             dashboard_events,
             event_seq: AtomicU64::new(0),
             meta_updates_active: AtomicU64::new(0),
@@ -1128,6 +1135,28 @@ impl Coordinator {
         }
     }
 
+    /// Mark `name` as having a graceful stop in progress. While set,
+    /// `agent_server::handle_recv` returns `Response::GracefulStop` for
+    /// this agent instead of polling the broker (the inbound fence).
+    pub fn mark_graceful_stop(&self, name: &str) {
+        self.graceful_stop_pending
+            .lock()
+            .unwrap()
+            .insert(name.to_owned());
+    }
+
+    /// Whether a graceful stop is pending for `name`.
+    #[must_use]
+    pub fn is_graceful_stop_pending(&self, name: &str) -> bool {
+        self.graceful_stop_pending.lock().unwrap().contains(name)
+    }
+
+    /// Clear the graceful-stop flag for `name` (agent reported
+    /// `GracefulStopComplete`, or the stop finished / was abandoned).
+    pub fn clear_graceful_stop(&self, name: &str) {
+        self.graceful_stop_pending.lock().unwrap().remove(name);
+    }
+
     /// Set of agents whose transient was cleared within the last
     /// `grace` seconds — i.e. agents the operator just acted on,
     /// whose stop the crash watcher should NOT classify as a crash.

hive-c0re/src/dashboard/lifecycle_ops.rs

@@ -6,12 +6,21 @@
 //! spinner); destroy delegates to `actions::destroy` (optionally purging).
 
 use axum::{
-    extract::{Form, Path as AxumPath, State},
+    extract::{Form, Path as AxumPath, Query, State},
     http::StatusCode,
     response::{IntoResponse, Response},
 };
 use serde::Deserialize;
 
+/// Query params for `post_kill`. `?graceful=1` routes to the graceful-stop
+/// orchestration (quiesce the harness, flush `/state`, then container stop)
+/// instead of an immediate hard stop. Defaults false → today's hard kill.
+#[derive(Deserialize)]
+pub(super) struct KillParams {
+    #[serde(default)]
+    graceful: bool,
+}
+
 use super::{AppState, error_response, guard_agent_name, strip_container_prefix};
 use crate::{actions, lifecycle};
 
@@ -75,11 +84,28 @@ where
 pub(super) async fn post_kill(
     State(state): State<AppState>,
     AxumPath(name): AxumPath<String>,
+    Query(params): Query<KillParams>,
 ) -> Response {
     let logical = strip_container_prefix(&name);
     if let Some(reject) = guard_agent_name(&state, &logical).await {
         return reject;
     }
+    if params.graceful {
+        // Graceful stop: enqueue the quiesce orchestration (signal the harness
+        // → one stop-checkpoint turn → drain → container stop, with a timeout
+        // fallback to a hard stop). Serialised through the rebuild queue so it
+        // can't race an in-flight rebuild for the same agent, and its per-step
+        // progress surfaces on the queue snapshot + build log.
+        state.coord.rebuild_queue.enqueue(
+            crate::rebuild_queue::QueueKind::GracefulStop,
+            logical,
+            crate::rebuild_queue::QueueSource::Manual,
+            "manual via dashboard graceful stop".to_owned(),
+            None,
+        );
+        state.coord.emit_rebuild_queue_snapshot();
+        return (StatusCode::OK, "ok").into_response();
+    }
     // Manager is stoppable from the dashboard like any other
     // agent. The host's dashboard server keeps running (it's
     // hive-c0re, not the manager container), per-agent approvals

hive-c0re/src/rebuild_queue.rs

@@ -42,6 +42,10 @@ pub enum QueueKind {
     /// Serialised through the queue so concurrent dashboard batch-apply
     /// actions for different agents never race on the shared JSON file.
     PermChange,
+    /// Gracefully stop a container: signal the harness to run one
+    /// stop-checkpoint turn (flush durable `/state`), wait for it to drain,
+    /// then `nixos-container stop`. Falls back to a hard stop on timeout.
+    GracefulStop,
 }
 
 impl QueueKind {
@@ -54,6 +58,7 @@ impl QueueKind {
             QueueKind::StartupSweep => "startup_sweep",
             QueueKind::Restart => "restart",
             QueueKind::PermChange => "perm_change",
+            QueueKind::GracefulStop => "graceful_stop",
         }
     }
 }
@@ -662,6 +667,12 @@ impl RebuildQueue {
 /// signal the worker exits after its current entry finishes; pending
 /// `Queued` entries are dropped (they'll either be replayed by the
 /// startup sweep on next boot or left for an operator to re-queue).
+/// Max time the `GracefulStop` worker waits for the harness to run its
+/// stop-checkpoint turn + drain before falling back to a hard container stop.
+/// Generous — a checkpoint turn can take a while — but bounded so a wedged
+/// agent never blocks the stop indefinitely.
+const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3);
+
 pub async fn run_worker(coord: std::sync::Arc<crate::coordinator::Coordinator>) {
     let mut shutdown = coord.shutdown_rx();
     loop {
@@ -833,9 +844,49 @@ async fn dispatch(
                 crate::auto_update::current_flake_rev(&coord.hyperhive_flake).unwrap_or_default();
             crate::auto_update::rebuild_agent(coord, name, &current_rev, Some(entry.id)).await
         }
+        (QueueKind::GracefulStop, _) => run_graceful_stop(coord, entry).await,
     }
 }
 
+/// Run one `GracefulStop` entry: signal the harness to quiesce (it returns
+/// `GracefulStop` on its next `Recv`, runs one stop-checkpoint turn to flush
+/// durable `/state`, then exits), wait for it to drain — bounded by
+/// `GRACEFUL_STOP_TIMEOUT` so a wedged agent can't block forever — then stop
+/// the container with the same teardown as a plain kill.
+async fn run_graceful_stop(
+    coord: &std::sync::Arc<crate::coordinator::Coordinator>,
+    entry: &QueueEntry,
+) -> anyhow::Result<()> {
+    let name = &entry.agent;
+    let _guard = coord.transient_guard(name, crate::coordinator::TransientKind::Stopping);
+    // Signal the harness; the kick breaks an idle long-poll so it's seen promptly.
+    coord.set_queue_step(Some(entry.id), "graceful stop: signalling agent");
+    coord.mark_graceful_stop(name);
+    coord.kick_agent(name, "graceful stop requested");
+    // Wait for the harness to drain (it clears the flag via `GracefulStopComplete`)
+    // or fall back to a hard stop after the timeout. The single queue worker is
+    // intentionally held for the duration — graceful stops are infrequent.
+    coord.set_queue_step(Some(entry.id), "graceful stop: waiting for agent to drain");
+    let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT;
+    while coord.is_graceful_stop_pending(name) {
+        if std::time::Instant::now() >= deadline {
+            tracing::warn!(agent = %name, "graceful stop: drain timed out — hard-stopping");
+            break;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+    }
+    coord.clear_graceful_stop(name);
+    // Stop the container — same teardown as a plain kill.
+    coord.set_queue_step(Some(entry.id), "nixos-container stop");
+    crate::lifecycle::kill(name).await?;
+    coord.unregister_agent(name);
+    coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
+        agent: name.clone(),
+    });
+    coord.rescan_containers_and_emit().await;
+    Ok(())
+}
+
 /// Run one `MetaUpdate` entry: bump the meta flake's locks for the
 /// requested inputs, then enqueue a cascade of `Rebuild` entries
 /// (with `parent_id` set to this entry's id) for every agent affected