hive-c0re: graceful agent stop — quiesce harness, flush state, then stop

hive-c0re/src/rebuild_queue.rs

@@ -42,6 +42,10 @@ pub enum QueueKind {
     /// Serialised through the queue so concurrent dashboard batch-apply
     /// actions for different agents never race on the shared JSON file.
     PermChange,
+    /// Gracefully stop a container: signal the harness to run one
+    /// stop-checkpoint turn (flush durable `/state`), wait for it to drain,
+    /// then `nixos-container stop`. Falls back to a hard stop on timeout.
+    GracefulStop,
 }
 
 impl QueueKind {
@@ -54,6 +58,7 @@ impl QueueKind {
             QueueKind::StartupSweep => "startup_sweep",
             QueueKind::Restart => "restart",
             QueueKind::PermChange => "perm_change",
+            QueueKind::GracefulStop => "graceful_stop",
         }
     }
 }
@@ -662,6 +667,12 @@ impl RebuildQueue {
 /// signal the worker exits after its current entry finishes; pending
 /// `Queued` entries are dropped (they'll either be replayed by the
 /// startup sweep on next boot or left for an operator to re-queue).
+/// Max time the `GracefulStop` worker waits for the harness to run its
+/// stop-checkpoint turn + drain before falling back to a hard container stop.
+/// Generous — a checkpoint turn can take a while — but bounded so a wedged
+/// agent never blocks the stop indefinitely.
+const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3);
+
 pub async fn run_worker(coord: std::sync::Arc<crate::coordinator::Coordinator>) {
     let mut shutdown = coord.shutdown_rx();
     loop {
@@ -833,9 +844,49 @@ async fn dispatch(
                 crate::auto_update::current_flake_rev(&coord.hyperhive_flake).unwrap_or_default();
             crate::auto_update::rebuild_agent(coord, name, &current_rev, Some(entry.id)).await
         }
+        (QueueKind::GracefulStop, _) => run_graceful_stop(coord, entry).await,
     }
 }
 
+/// Run one `GracefulStop` entry: signal the harness to quiesce (it returns
+/// `GracefulStop` on its next `Recv`, runs one stop-checkpoint turn to flush
+/// durable `/state`, then exits), wait for it to drain — bounded by
+/// `GRACEFUL_STOP_TIMEOUT` so a wedged agent can't block forever — then stop
+/// the container with the same teardown as a plain kill.
+async fn run_graceful_stop(
+    coord: &std::sync::Arc<crate::coordinator::Coordinator>,
+    entry: &QueueEntry,
+) -> anyhow::Result<()> {
+    let name = &entry.agent;
+    let _guard = coord.transient_guard(name, crate::coordinator::TransientKind::Stopping);
+    // Signal the harness; the kick breaks an idle long-poll so it's seen promptly.
+    coord.set_queue_step(Some(entry.id), "graceful stop: signalling agent");
+    coord.mark_graceful_stop(name);
+    coord.kick_agent(name, "graceful stop requested");
+    // Wait for the harness to drain (it clears the flag via `GracefulStopComplete`)
+    // or fall back to a hard stop after the timeout. The single queue worker is
+    // intentionally held for the duration — graceful stops are infrequent.
+    coord.set_queue_step(Some(entry.id), "graceful stop: waiting for agent to drain");
+    let deadline = std::time::Instant::now() + GRACEFUL_STOP_TIMEOUT;
+    while coord.is_graceful_stop_pending(name) {
+        if std::time::Instant::now() >= deadline {
+            tracing::warn!(agent = %name, "graceful stop: drain timed out — hard-stopping");
+            break;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+    }
+    coord.clear_graceful_stop(name);
+    // Stop the container — same teardown as a plain kill.
+    coord.set_queue_step(Some(entry.id), "nixos-container stop");
+    crate::lifecycle::kill(name).await?;
+    coord.unregister_agent(name);
+    coord.notify_manager(&hive_sh4re::HelperEvent::Killed {
+        agent: name.clone(),
+    });
+    coord.rescan_containers_and_emit().await;
+    Ok(())
+}
+
 /// Run one `MetaUpdate` entry: bump the meta flake's locks for the
 /// requested inputs, then enqueue a cascade of `Rebuild` entries
 /// (with `parent_id` set to this entry's id) for every agent affected