hive-c0re/hivectl/hive-agent: pause as a job-queue DAG node (closes #3056)

hive-c0re/src/job_queue/exec.rs

@@ -24,6 +24,15 @@ use crate::power::{ReconcileAction, reconcile_action};
 /// N × this timeout.
 pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3);
 
+/// Max time `PauseDrain` waits for the harness to report
+/// `PauseAcknowledged` before giving up and resolving anyway (the
+/// marker itself — not this node — is what actually gates the turn
+/// loop, so "giving up" costs nothing but a slightly-late dashboard
+/// badge). Same ceiling as `GRACEFUL_STOP_TIMEOUT` — no reason for the
+/// two to diverge yet, but aliased under its own name so a future
+/// change to one doesn't silently retune the other.
+const PAUSE_ACK_TIMEOUT: std::time::Duration = GRACEFUL_STOP_TIMEOUT;
+
 /// Run one claimed node to completion. Called from a task the
 /// scheduler spawns per claim; the `Result` (stringified) becomes the
 /// node's terminal state.
@@ -97,6 +106,8 @@ pub(super) async fn run_node(
             Ok(())
         }
         NodeKind::Drain { .. } => run_drain(coord, agent).await,
+        NodeKind::PauseSignal { .. } => run_pause_signal(coord, agent).await,
+        NodeKind::PauseDrain { .. } => run_pause_drain(coord, agent).await,
         NodeKind::WriteDropin { .. } => run_write_dropin(coord, agent).await,
         // The payload rides the node and is destructured here, so the executor
         // takes it directly instead of re-matching the kind behind a `bail!`
@@ -490,6 +501,54 @@ async fn run_drain(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
     Ok(())
 }
 
+/// Write the pause marker + mark `pause_pending`, no kick (unlike
+/// `run_signal`) — the harness's own between-turns poll (`PAUSE_POLL`,
+/// 1s default) is already responsive enough, and `run_signal`'s kick
+/// message ("you were just (re)started") would be actively misleading
+/// here.
+///
+/// Skips marking `pause_pending` (the marker write still happens,
+/// harmlessly idempotent either way) if the agent is already paused:
+/// the harness reports `PauseAcknowledged` only on the marker's
+/// `false → true` edge, so re-pausing an already-paused agent produces
+/// no edge for `run_pause_drain` to observe — marking pending here
+/// would just burn its timeout every time an operator re-confirms a
+/// pause that already took.
+async fn run_pause_signal(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
+    let agent = hive_types::Ident::parse(name)
+        .map_err(|e| anyhow::anyhow!("invalid agent name for pause {name:?}: {e}"))?;
+    let already_paused = Coordinator::is_paused(&agent);
+    Coordinator::set_paused(&agent, true).await?;
+    if !already_paused {
+        coord.mark_pause_pending(name);
+    }
+    // Same pattern `run_start`/`run_stop` use: refresh the dashboard's
+    // view right after the state change so the paused badge flips
+    // immediately instead of waiting on the next periodic rescan.
+    coord.rescan_containers_and_emit().await;
+    Ok(())
+}
+
+/// Await the harness reporting `PauseAcknowledged`, bounded by
+/// `PAUSE_ACK_TIMEOUT`. Resolves ok either way, mirroring `run_drain` —
+/// pausing is best-effort from the queue's perspective; the marker
+/// (not this node) is what actually gates the harness's turn loop, so
+/// a timed-out wait doesn't leave the agent un-paused, just leaves the
+/// dashboard's "pausing…" badge running a little longer than it needed
+/// to.
+async fn run_pause_drain(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
+    let deadline = std::time::Instant::now() + PAUSE_ACK_TIMEOUT;
+    while coord.is_pause_pending(name) {
+        if std::time::Instant::now() >= deadline {
+            tracing::warn!(agent = %name, "pause: ack wait timed out — marker is set regardless");
+            break;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+    }
+    coord.clear_pause_pending(name);
+    Ok(())
+}
+
 /// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
 async fn run_write_dropin(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
     // write_dropins only needs the path value to build AgentPaths; the