hive-c0re/hivectl/hive-agent: pause as a job-queue DAG node (closes #3056)
This commit is contained in:
parent
a92f7351d9
commit
20a7a21053
13 changed files with 316 additions and 27 deletions
|
|
@ -382,7 +382,11 @@ Park this agent's turn loop, leaving the container running.
|
||||||
|
|
||||||
The harness stops driving turns but keeps serving its web UI and MCP daemons, so the container, its mounts and its warm caches stay up while it burns no tokens. Inbox messages queue unacked and the backlog drains on `resume`. Sticky: it survives a restart, and pausing a stopped agent makes it come up paused.
|
The harness stops driving turns but keeps serving its web UI and MCP daemons, so the container, its mounts and its warm caches stay up while it burns no tokens. Inbox messages queue unacked and the backlog drains on `resume`. Sticky: it survives a restart, and pausing a stopped agent makes it come up paused.
|
||||||
|
|
||||||
**Usage:** `hivectl agent pause`
|
**Usage:** `hivectl agent pause [OPTIONS]`
|
||||||
|
|
||||||
|
###### **Options:**
|
||||||
|
|
||||||
|
* `--no-wait` — Return immediately after the pause DAG is queued instead of waiting for the harness to acknowledge it
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -317,6 +317,12 @@ trait Surface {
|
||||||
/// its timeout fallback.
|
/// its timeout fallback.
|
||||||
fn graceful_stop_complete(socket: &Path) -> impl Future<Output = ()>;
|
fn graceful_stop_complete(socket: &Path) -> impl Future<Output = ()>;
|
||||||
|
|
||||||
|
/// Tell c0re the harness's own pause-marker check (between turns) just
|
||||||
|
/// saw the marker appear (fire-and-forget; logs on error). Lets the
|
||||||
|
/// pause DAG's drain node resolve without waiting out its timeout
|
||||||
|
/// fallback. Same shape as `graceful_stop_complete`.
|
||||||
|
fn pause_acknowledged(socket: &Path) -> impl Future<Output = ()>;
|
||||||
|
|
||||||
/// Send a message addressed to `<parent>` (broker resolves the
|
/// Send a message addressed to `<parent>` (broker resolves the
|
||||||
/// sentinel via `topology::parent_of` at delivery time; root
|
/// sentinel via `topology::parent_of` at delivery time; root
|
||||||
/// agents/manager fall through to operator).
|
/// agents/manager fall through to operator).
|
||||||
|
|
@ -368,6 +374,10 @@ impl Surface for AgentSurface {
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn pause_acknowledged(socket: &Path) {
|
||||||
|
fire_and_forget(socket, Request::PauseAcknowledged, "pause_acknowledged").await;
|
||||||
|
}
|
||||||
|
|
||||||
async fn inbox_unread(socket: &Path) -> u64 {
|
async fn inbox_unread(socket: &Path) -> u64 {
|
||||||
match hive_sock_client::request::<_, Response>(
|
match hive_sock_client::request::<_, Response>(
|
||||||
socket,
|
socket,
|
||||||
|
|
@ -700,6 +710,12 @@ async fn serve_loop<S: Surface>(
|
||||||
text: "paused: turn loop parked, messages will queue".into(),
|
text: "paused: turn loop parked, messages will queue".into(),
|
||||||
});
|
});
|
||||||
was_paused = true;
|
was_paused = true;
|
||||||
|
// Fire-and-forget: this is the same "no turn in flight"
|
||||||
|
// moment `GracefulStopComplete` reports at, and needs no
|
||||||
|
// extra tracking for the same reason — the check above is
|
||||||
|
// already between-turns only. Lets the pause DAG's drain
|
||||||
|
// node resolve immediately instead of timing out.
|
||||||
|
S::pause_acknowledged(socket).await;
|
||||||
}
|
}
|
||||||
tokio::time::sleep(PAUSE_POLL).await;
|
tokio::time::sleep(PAUSE_POLL).await;
|
||||||
continue;
|
continue;
|
||||||
|
|
|
||||||
|
|
@ -162,6 +162,14 @@ pub struct Coordinator {
|
||||||
/// agent is in this set — the inbound fence. Cleared when the agent
|
/// agent is in this set — the inbound fence. Cleared when the agent
|
||||||
/// reports `GracefulStopComplete` or the container is stopped.
|
/// reports `GracefulStopComplete` or the container is stopped.
|
||||||
graceful_stop_pending: Mutex<HashSet<String>>,
|
graceful_stop_pending: Mutex<HashSet<String>>,
|
||||||
|
/// Agents with a pause acknowledgement in progress — set by the pause
|
||||||
|
/// DAG's signal node, cleared when the agent reports
|
||||||
|
/// `PauseAcknowledged` (or the drain node's timeout fallback fires).
|
||||||
|
/// Same shape as `graceful_stop_pending`, one set per concern rather
|
||||||
|
/// than reusing it: a pause and a graceful stop are independent
|
||||||
|
/// orchestrations that can be in flight for different agents
|
||||||
|
/// simultaneously.
|
||||||
|
pause_pending: Mutex<HashSet<String>>,
|
||||||
/// Logical agent names that were running at the last broad-scope
|
/// Logical agent names that were running at the last broad-scope
|
||||||
/// `hivectl stop`. A subsequent broad-scope `hivectl start` restores
|
/// `hivectl stop`. A subsequent broad-scope `hivectl start` restores
|
||||||
/// only this set (intersected with the requested scope) rather than
|
/// only this set (intersected with the requested scope) rather than
|
||||||
|
|
@ -582,6 +590,7 @@ impl Coordinator {
|
||||||
recent_transient: Mutex::new(HashMap::new()),
|
recent_transient: Mutex::new(HashMap::new()),
|
||||||
recent_crashes: Mutex::new(HashMap::new()),
|
recent_crashes: Mutex::new(HashMap::new()),
|
||||||
graceful_stop_pending: Mutex::new(HashSet::new()),
|
graceful_stop_pending: Mutex::new(HashSet::new()),
|
||||||
|
pause_pending: Mutex::new(HashSet::new()),
|
||||||
last_stopped_running: Mutex::new(None),
|
last_stopped_running: Mutex::new(None),
|
||||||
dashboard_events,
|
dashboard_events,
|
||||||
event_seq: AtomicU64::new(0),
|
event_seq: AtomicU64::new(0),
|
||||||
|
|
@ -1188,6 +1197,26 @@ impl Coordinator {
|
||||||
self.graceful_stop_pending.lock().unwrap().remove(name);
|
self.graceful_stop_pending.lock().unwrap().remove(name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Mark `name` as having a pause acknowledgement in progress. Set by
|
||||||
|
/// the pause DAG's signal node; the drain node polls
|
||||||
|
/// `is_pause_pending` until the agent reports back or the timeout
|
||||||
|
/// fallback fires.
|
||||||
|
pub fn mark_pause_pending(&self, name: &str) {
|
||||||
|
self.pause_pending.lock().unwrap().insert(name.to_owned());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a pause acknowledgement is pending for `name`.
|
||||||
|
#[must_use]
|
||||||
|
pub fn is_pause_pending(&self, name: &str) -> bool {
|
||||||
|
self.pause_pending.lock().unwrap().contains(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Clear the pause-pending flag for `name` (agent reported
|
||||||
|
/// `PauseAcknowledged`, or the drain node's wait finished / timed out).
|
||||||
|
pub fn clear_pause_pending(&self, name: &str) {
|
||||||
|
self.pause_pending.lock().unwrap().remove(name);
|
||||||
|
}
|
||||||
|
|
||||||
/// Record the set of agents that were running at a broad-scope
|
/// Record the set of agents that were running at a broad-scope
|
||||||
/// `hivectl stop`, so the next broad-scope `start` restores exactly
|
/// `hivectl stop`, so the next broad-scope `start` restores exactly
|
||||||
/// this set. See the `last_stopped_running` field doc. Persists a
|
/// this set. See the `last_stopped_running` field doc. Persists a
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,15 @@ use crate::power::{ReconcileAction, reconcile_action};
|
||||||
/// N × this timeout.
|
/// N × this timeout.
|
||||||
pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3);
|
pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(3);
|
||||||
|
|
||||||
|
/// Max time `PauseDrain` waits for the harness to report
|
||||||
|
/// `PauseAcknowledged` before giving up and resolving anyway (the
|
||||||
|
/// marker itself — not this node — is what actually gates the turn
|
||||||
|
/// loop, so "giving up" costs nothing but a slightly-late dashboard
|
||||||
|
/// badge). Same ceiling as `GRACEFUL_STOP_TIMEOUT` — no reason for the
|
||||||
|
/// two to diverge yet, but aliased under its own name so a future
|
||||||
|
/// change to one doesn't silently retune the other.
|
||||||
|
const PAUSE_ACK_TIMEOUT: std::time::Duration = GRACEFUL_STOP_TIMEOUT;
|
||||||
|
|
||||||
/// Run one claimed node to completion. Called from a task the
|
/// Run one claimed node to completion. Called from a task the
|
||||||
/// scheduler spawns per claim; the `Result` (stringified) becomes the
|
/// scheduler spawns per claim; the `Result` (stringified) becomes the
|
||||||
/// node's terminal state.
|
/// node's terminal state.
|
||||||
|
|
@ -97,6 +106,8 @@ pub(super) async fn run_node(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
NodeKind::Drain { .. } => run_drain(coord, agent).await,
|
NodeKind::Drain { .. } => run_drain(coord, agent).await,
|
||||||
|
NodeKind::PauseSignal { .. } => run_pause_signal(coord, agent).await,
|
||||||
|
NodeKind::PauseDrain { .. } => run_pause_drain(coord, agent).await,
|
||||||
NodeKind::WriteDropin { .. } => run_write_dropin(coord, agent).await,
|
NodeKind::WriteDropin { .. } => run_write_dropin(coord, agent).await,
|
||||||
// The payload rides the node and is destructured here, so the executor
|
// The payload rides the node and is destructured here, so the executor
|
||||||
// takes it directly instead of re-matching the kind behind a `bail!`
|
// takes it directly instead of re-matching the kind behind a `bail!`
|
||||||
|
|
@ -490,6 +501,54 @@ async fn run_drain(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Write the pause marker + mark `pause_pending`, no kick (unlike
|
||||||
|
/// `run_signal`) — the harness's own between-turns poll (`PAUSE_POLL`,
|
||||||
|
/// 1s default) is already responsive enough, and `run_signal`'s kick
|
||||||
|
/// message ("you were just (re)started") would be actively misleading
|
||||||
|
/// here.
|
||||||
|
///
|
||||||
|
/// Skips marking `pause_pending` (the marker write still happens,
|
||||||
|
/// harmlessly idempotent either way) if the agent is already paused:
|
||||||
|
/// the harness reports `PauseAcknowledged` only on the marker's
|
||||||
|
/// `false → true` edge, so re-pausing an already-paused agent produces
|
||||||
|
/// no edge for `run_pause_drain` to observe — marking pending here
|
||||||
|
/// would just burn its timeout every time an operator re-confirms a
|
||||||
|
/// pause that already took.
|
||||||
|
async fn run_pause_signal(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
|
||||||
|
let agent = hive_types::Ident::parse(name)
|
||||||
|
.map_err(|e| anyhow::anyhow!("invalid agent name for pause {name:?}: {e}"))?;
|
||||||
|
let already_paused = Coordinator::is_paused(&agent);
|
||||||
|
Coordinator::set_paused(&agent, true).await?;
|
||||||
|
if !already_paused {
|
||||||
|
coord.mark_pause_pending(name);
|
||||||
|
}
|
||||||
|
// Same pattern `run_start`/`run_stop` use: refresh the dashboard's
|
||||||
|
// view right after the state change so the paused badge flips
|
||||||
|
// immediately instead of waiting on the next periodic rescan.
|
||||||
|
coord.rescan_containers_and_emit().await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Await the harness reporting `PauseAcknowledged`, bounded by
|
||||||
|
/// `PAUSE_ACK_TIMEOUT`. Resolves ok either way, mirroring `run_drain` —
|
||||||
|
/// pausing is best-effort from the queue's perspective; the marker
|
||||||
|
/// (not this node) is what actually gates the harness's turn loop, so
|
||||||
|
/// a timed-out wait doesn't leave the agent un-paused, just leaves the
|
||||||
|
/// dashboard's "pausing…" badge running a little longer than it needed
|
||||||
|
/// to.
|
||||||
|
async fn run_pause_drain(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
|
||||||
|
let deadline = std::time::Instant::now() + PAUSE_ACK_TIMEOUT;
|
||||||
|
while coord.is_pause_pending(name) {
|
||||||
|
if std::time::Instant::now() >= deadline {
|
||||||
|
tracing::warn!(agent = %name, "pause: ack wait timed out — marker is set regardless");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||||
|
}
|
||||||
|
coord.clear_pause_pending(name);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
|
/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
|
||||||
async fn run_write_dropin(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
|
async fn run_write_dropin(coord: &Arc<Coordinator>, name: &str) -> Result<()> {
|
||||||
// write_dropins only needs the path value to build AgentPaths; the
|
// write_dropins only needs the path value to build AgentPaths; the
|
||||||
|
|
|
||||||
|
|
@ -119,6 +119,16 @@ pub enum NodeKind {
|
||||||
/// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the
|
/// `GRACEFUL_STOP_TIMEOUT`. Resolves ok either way — the
|
||||||
/// downstream `Reconcile` performs the actual stop.
|
/// downstream `Reconcile` performs the actual stop.
|
||||||
Drain { agent: String },
|
Drain { agent: String },
|
||||||
|
/// Write the pause marker (`Coordinator::set_paused`) + mark
|
||||||
|
/// `pause_pending`. No kick, unlike `Signal` — the harness's own
|
||||||
|
/// between-turns poll (`PAUSE_POLL`, 1s default) is already
|
||||||
|
/// responsive enough, and `Signal`'s kick-message body ("you were
|
||||||
|
/// just (re)started") would be actively misleading here.
|
||||||
|
PauseSignal { agent: String },
|
||||||
|
/// Await the harness reporting `PauseAcknowledged`, bounded by a
|
||||||
|
/// timeout. Resolves ok either way — pausing is best-effort from
|
||||||
|
/// the queue's perspective, same as `Drain`.
|
||||||
|
PauseDrain { agent: String },
|
||||||
/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
|
/// `set_nspawn_flags` + `set_resource_limits` + daemon-reload.
|
||||||
WriteDropin { agent: String },
|
WriteDropin { agent: String },
|
||||||
/// Commit `tool-groups.json` / `capabilities.json` per its `payload`
|
/// Commit `tool-groups.json` / `capabilities.json` per its `payload`
|
||||||
|
|
@ -338,6 +348,8 @@ impl NodeKind {
|
||||||
NodeKind::StopForUpdate { .. } => "stop_for_update",
|
NodeKind::StopForUpdate { .. } => "stop_for_update",
|
||||||
NodeKind::Signal { .. } => "signal",
|
NodeKind::Signal { .. } => "signal",
|
||||||
NodeKind::Drain { .. } => "drain",
|
NodeKind::Drain { .. } => "drain",
|
||||||
|
NodeKind::PauseSignal { .. } => "pause_signal",
|
||||||
|
NodeKind::PauseDrain { .. } => "pause_drain",
|
||||||
NodeKind::WriteDropin { .. } => "write_dropin",
|
NodeKind::WriteDropin { .. } => "write_dropin",
|
||||||
NodeKind::WritePermFile { .. } => "write_perm_file",
|
NodeKind::WritePermFile { .. } => "write_perm_file",
|
||||||
NodeKind::Reparent { .. } => "reparent",
|
NodeKind::Reparent { .. } => "reparent",
|
||||||
|
|
@ -372,6 +384,8 @@ impl NodeKind {
|
||||||
| NodeKind::StopForUpdate { agent }
|
| NodeKind::StopForUpdate { agent }
|
||||||
| NodeKind::Signal { agent }
|
| NodeKind::Signal { agent }
|
||||||
| NodeKind::Drain { agent }
|
| NodeKind::Drain { agent }
|
||||||
|
| NodeKind::PauseSignal { agent }
|
||||||
|
| NodeKind::PauseDrain { agent }
|
||||||
| NodeKind::WriteDropin { agent }
|
| NodeKind::WriteDropin { agent }
|
||||||
| NodeKind::WritePermFile { agent, .. }
|
| NodeKind::WritePermFile { agent, .. }
|
||||||
| NodeKind::DeployWindow { agent, .. }
|
| NodeKind::DeployWindow { agent, .. }
|
||||||
|
|
|
||||||
|
|
@ -327,3 +327,31 @@ pub async fn stop_many(
|
||||||
coord.emit_rebuild_queue_snapshot();
|
coord.emit_rebuild_queue_snapshot();
|
||||||
Ok(ids)
|
Ok(ids)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Declare the pause DAG for `agents` — one `PauseSignal → PauseDrain`
|
||||||
|
/// pair (see [`super::templates::pause_quiesce`]) per agent, independent
|
||||||
|
/// roots on their own agent lease so a whole-hive pause overlaps rather
|
||||||
|
/// than serialising.
|
||||||
|
pub(crate) fn pause_nodes(builder: &JobBuilder, agents: &[String]) -> Vec<hive_jobq::NodeGuid> {
|
||||||
|
agents
|
||||||
|
.iter()
|
||||||
|
.map(|agent| super::templates::pause_quiesce(builder, agent).guid())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pause `agents` in a **single** DAG. Unlike `stop`/`start`/`restart`,
|
||||||
|
/// this needs no live-state read first — `Coordinator::set_paused`
|
||||||
|
/// works on a stopped container too (the marker is sticky), so there's
|
||||||
|
/// no `running`/`stale` branch to resolve async before building.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Propagates a graph-insert error.
|
||||||
|
pub async fn pause_many(
|
||||||
|
coord: &Arc<Coordinator>,
|
||||||
|
agents: &[String],
|
||||||
|
) -> anyhow::Result<Vec<NodeId>> {
|
||||||
|
let targets: Vec<String> = agents.to_vec();
|
||||||
|
let ids = coord.job_queue.insert_job(|b| pause_nodes(b, &targets))?;
|
||||||
|
coord.emit_rebuild_queue_snapshot();
|
||||||
|
Ok(ids)
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -137,6 +137,46 @@ pub(crate) fn quiesce<'a>(builder: &'a JobBuilder, agent: &str, brace: Handle<'a
|
||||||
.after_ok(signal)
|
.after_ok(signal)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The pause quiesce pair — `PauseSignal` then `PauseDrain`: write the
|
||||||
|
/// pause marker, then wait for the harness to acknowledge it. Returns
|
||||||
|
/// the **group root** — the brace, not the tail. Unlike [`quiesce`],
|
||||||
|
/// nothing chains onto `PauseDrain` (there's no downstream
|
||||||
|
/// `Reconcile`-shaped node the way a stop has one), so the handle a
|
||||||
|
/// caller actually needs is the brace whose roll-up covers the whole
|
||||||
|
/// pair, same as [`super::power`]'s `stop_chain` returning `wanted`'s
|
||||||
|
/// guid rather than `quiesce`'s own returned `Drain` handle.
|
||||||
|
///
|
||||||
|
/// Unlike [`quiesce`], there's no natural resource-holding head to
|
||||||
|
/// borrow a `brace` from — pausing isn't a `wanted`-state transition,
|
||||||
|
/// so there's no `SetWanted`-shaped parent the way `stop_chain` has
|
||||||
|
/// one. This is exactly the case [`NodeKind::AgentWindow`] exists for
|
||||||
|
/// (see the module header's _brace_ paragraph): a pure-resource-holder
|
||||||
|
/// root with no work of its own, so `PauseSignal`/`PauseDrain` can be
|
||||||
|
/// plain siblings under it, both borrowing its lease via `part_of`.
|
||||||
|
///
|
||||||
|
/// ⚠️ `PauseSignal` can *not* hold the lease itself with `PauseDrain`
|
||||||
|
/// nested under it (`.part_of(signal)`) — that was the first shape
|
||||||
|
/// tried here, and `hive_jobq` rejects it at insert: `PauseDrain`
|
||||||
|
/// depending on `PauseSignal` via `after_ok` while also being its
|
||||||
|
/// *child* reaches outside `PauseDrain`'s own group (its parent, not a
|
||||||
|
/// sibling) — "an edge must stay within the depender's own group".
|
||||||
|
/// `AgentWindow` as a separate, actual brace is what makes them
|
||||||
|
/// siblings instead.
|
||||||
|
pub(crate) fn pause_quiesce<'a>(builder: &'a JobBuilder, agent: &str) -> Handle<'a> {
|
||||||
|
let a = || agent.to_owned();
|
||||||
|
let brace = builder
|
||||||
|
.node(NodeKind::AgentWindow { agent: a() })
|
||||||
|
.needs(Resource::Agent(a()));
|
||||||
|
let signal = builder
|
||||||
|
.node(NodeKind::PauseSignal { agent: a() })
|
||||||
|
.part_of(brace);
|
||||||
|
let _drain = builder
|
||||||
|
.node(NodeKind::PauseDrain { agent: a() })
|
||||||
|
.part_of(brace)
|
||||||
|
.after_ok(signal);
|
||||||
|
brace
|
||||||
|
}
|
||||||
|
|
||||||
/// The group-roots a [`rebuild_nodes`] subgraph exposes to its caller: what a
|
/// The group-roots a [`rebuild_nodes`] subgraph exposes to its caller: what a
|
||||||
/// tail node edges onto, and what a follow-up node waits for.
|
/// tail node edges onto, and what a follow-up node waits for.
|
||||||
///
|
///
|
||||||
|
|
|
||||||
|
|
@ -1587,6 +1587,47 @@ fn graceful_stop_shape_signal_drain_reconcile() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pause_shape_signal_drain() {
|
||||||
|
let q = JobQueue::new(1);
|
||||||
|
insert(&q, |builder| {
|
||||||
|
power::pause_nodes(builder, &["agent-a".to_owned()]);
|
||||||
|
});
|
||||||
|
assert_eq!(
|
||||||
|
declared_shape(&q),
|
||||||
|
vec![
|
||||||
|
// Unlike the stop quiesce pair (which hangs under an existing
|
||||||
|
// `set_wanted` head), pausing has no natural parent to reuse, so
|
||||||
|
// this shape declares its own `AgentWindow` brace — the pair are
|
||||||
|
// plain siblings under it, not a signal-holds-the-lease-itself
|
||||||
|
// shape (that was tried first and rejected at insert: a child
|
||||||
|
// can't `after_ok` its own parent — see `pause_quiesce`'s doc
|
||||||
|
// comment for the exact error).
|
||||||
|
row("agent_window", None, &[]),
|
||||||
|
row("pause_signal", Some("agent_window"), &[]),
|
||||||
|
row(
|
||||||
|
"pause_drain",
|
||||||
|
Some("agent_window"),
|
||||||
|
&[("pause_signal", "done")]
|
||||||
|
),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
declared_resources(&q, node_of(&q, "agent_window")),
|
||||||
|
vec![Resource::Agent("agent-a".to_owned())],
|
||||||
|
"the brace holds the lease for the whole pair"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
[
|
||||||
|
declared_resources(&q, node_of(&q, "pause_signal")),
|
||||||
|
declared_resources(&q, node_of(&q, "pause_drain")),
|
||||||
|
],
|
||||||
|
[vec![], vec![]],
|
||||||
|
"the pause pair borrows the brace's lease and declares nothing itself \
|
||||||
|
— same shape the stop quiesce pair uses"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn spawn_shape_provision_create_dropin_reconcile() {
|
fn spawn_shape_provision_create_dropin_reconcile() {
|
||||||
let q = JobQueue::new(1);
|
let q = JobQueue::new(1);
|
||||||
|
|
|
||||||
|
|
@ -329,24 +329,43 @@ async fn handle_spawn(coord: &Arc<Coordinator>, name: &str) -> Result<HostRespon
|
||||||
/// `hivectl pause|resume` / the dashboard toggle: write or remove the
|
/// `hivectl pause|resume` / the dashboard toggle: write or remove the
|
||||||
/// agent's pause marker.
|
/// agent's pause marker.
|
||||||
///
|
///
|
||||||
/// Deliberately not a lifecycle DAG. There's no container operation to
|
/// **Resume** stays synchronous, direct marker removal — there's
|
||||||
/// sequence — it's one marker file, and the harness picks it up on its
|
/// nothing to acknowledge (a resumed agent just starts driving turns
|
||||||
/// next poll — so queueing it would only add latency and a lease. That
|
/// again on its own next poll, no handshake needed), and nobody has
|
||||||
/// also means it works on a stopped agent: the marker is sticky, so the
|
/// asked resume to wait.
|
||||||
/// agent comes up paused.
|
///
|
||||||
|
/// **Pause** rides the job queue (`PauseSignal → PauseDrain`, see
|
||||||
|
/// `job_queue::power::pause_many`) instead of writing the marker
|
||||||
|
/// synchronously here: pausing wants the same "confirmed, not just
|
||||||
|
/// requested" signal a graceful stop gets from its `Signal → Drain`
|
||||||
|
/// pair, and the DAG's agent lease is what stops a pause from racing
|
||||||
|
/// an in-flight rebuild/stop of the same agent — a synchronous write
|
||||||
|
/// had no such protection. Still works on a stopped agent (the queued
|
||||||
|
/// `PauseSignal` writes the same sticky marker either way, container
|
||||||
|
/// running or not).
|
||||||
async fn handle_set_paused(
|
async fn handle_set_paused(
|
||||||
coord: &std::sync::Arc<Coordinator>,
|
coord: &std::sync::Arc<Coordinator>,
|
||||||
name: &hive_types::Ident,
|
name: &hive_types::Ident,
|
||||||
paused: bool,
|
paused: bool,
|
||||||
) -> HostResponse {
|
) -> HostResponse {
|
||||||
if let Err(e) = Coordinator::set_paused(name, paused).await {
|
if !paused {
|
||||||
return HostResponse::error(format!("set paused={paused} for {name}: {e}"));
|
if let Err(e) = Coordinator::set_paused(name, false).await {
|
||||||
|
return HostResponse::error(format!("set paused=false for {name}: {e}"));
|
||||||
|
}
|
||||||
|
tracing::info!(%name, "agent pause marker cleared");
|
||||||
|
// Refresh the dashboard's view so the paused badge flips without
|
||||||
|
// waiting for the next periodic rescan.
|
||||||
|
coord.rescan_containers_and_emit().await;
|
||||||
|
return HostResponse::success();
|
||||||
|
}
|
||||||
|
match crate::job_queue::power::pause_many(coord, std::slice::from_ref(&name.to_string())).await
|
||||||
|
{
|
||||||
|
Ok(ids) => {
|
||||||
|
tracing::info!(%name, "agent pause queued");
|
||||||
|
HostResponse::queued(ids.into_iter().map(hive_jobq::NodeId::get).collect())
|
||||||
|
}
|
||||||
|
Err(e) => HostResponse::error(format!("queue pause for {name}: {e}")),
|
||||||
}
|
}
|
||||||
tracing::info!(%name, paused, "agent pause marker updated");
|
|
||||||
// Refresh the dashboard's view so the paused badge flips without
|
|
||||||
// waiting for the next periodic rescan.
|
|
||||||
coord.rescan_containers_and_emit().await;
|
|
||||||
HostResponse::success()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Collect per-agent status rows for `hivectl status` and the dashboard.
|
/// Collect per-agent status rows for `hivectl status` and the dashboard.
|
||||||
|
|
|
||||||
|
|
@ -254,6 +254,13 @@ pub(crate) async fn dispatch_shared(
|
||||||
coord.clear_graceful_stop(agent);
|
coord.clear_graceful_stop(agent);
|
||||||
hive_core_agent_sock::Response::Ok
|
hive_core_agent_sock::Response::Ok
|
||||||
}
|
}
|
||||||
|
hive_core_agent_sock::Request::PauseAcknowledged => {
|
||||||
|
// Harness saw its own pause marker flip between turns: clear
|
||||||
|
// the fence so the pause DAG's drain node (which polls this
|
||||||
|
// flag) resolves without waiting out its timeout.
|
||||||
|
coord.clear_pause_pending(agent);
|
||||||
|
hive_core_agent_sock::Response::Ok
|
||||||
|
}
|
||||||
hive_core_agent_sock::Request::GetHostJournal {
|
hive_core_agent_sock::Request::GetHostJournal {
|
||||||
unit,
|
unit,
|
||||||
container,
|
container,
|
||||||
|
|
|
||||||
|
|
@ -138,6 +138,11 @@ pub enum Request {
|
||||||
/// serve loop now." Lets the `GracefulStop` orchestration stop the
|
/// serve loop now." Lets the `GracefulStop` orchestration stop the
|
||||||
/// container immediately instead of waiting out its timeout fallback.
|
/// container immediately instead of waiting out its timeout fallback.
|
||||||
GracefulStopComplete,
|
GracefulStopComplete,
|
||||||
|
/// Harness → c0re: "the `Pause` marker check I run between turns
|
||||||
|
/// (never mid-turn) just flipped `false → true`." Lets the pause
|
||||||
|
/// DAG's drain node resolve immediately instead of waiting out its
|
||||||
|
/// timeout fallback — same shape as `GracefulStopComplete`.
|
||||||
|
PauseAcknowledged,
|
||||||
/// *(capability-gated: `read_host_journal`)* Fetch recent lines
|
/// *(capability-gated: `read_host_journal`)* Fetch recent lines
|
||||||
/// from the host journal. Filters are all optional; omitting all
|
/// from the host journal. Filters are all optional; omitting all
|
||||||
/// returns the last `lines` entries from the global journal.
|
/// returns the last `lines` entries from the global journal.
|
||||||
|
|
|
||||||
|
|
@ -60,7 +60,19 @@ async fn agents_start(socket: &Path, name: &str, paused: bool) -> Result<()> {
|
||||||
}
|
}
|
||||||
if paused {
|
if paused {
|
||||||
let already_running = agent_running(socket, name).await?;
|
let already_running = agent_running(socket, name).await?;
|
||||||
set_paused(socket, name, true).await?;
|
// `no_wait = true`: correctness here doesn't come from waiting —
|
||||||
|
// the pause DAG's `AgentWindow` brace and the `Start` request's
|
||||||
|
// `SetWanted` head both declare `Resource::Agent`, and this call's
|
||||||
|
// `insert_job` has already returned (so the pause DAG's claim is
|
||||||
|
// registered) before `Start` is even sent, so the lease itself
|
||||||
|
// orders the marker write ahead of the container boot. Waiting for
|
||||||
|
// the *pause DAG* to finish
|
||||||
|
// would be actively wrong when the agent isn't running yet: its
|
||||||
|
// `PauseDrain` can't be acked until the harness is up to see the
|
||||||
|
// marker, which is the very thing `Start` (below) is about to
|
||||||
|
// cause — waiting here would just block for `PAUSE_ACK_TIMEOUT`
|
||||||
|
// for an ack that can only land after this call returns.
|
||||||
|
set_paused(socket, name, true, true).await?;
|
||||||
if already_running {
|
if already_running {
|
||||||
eprintln!("'{name}' is already running — paused in place, not (re)started");
|
eprintln!("'{name}' is already running — paused in place, not (re)started");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
|
|
@ -201,16 +213,23 @@ pub(crate) async fn agents_list(socket: &Path, json: bool) -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `hivectl agent <name> pause|resume` — flip the agent's pause marker. Not a
|
/// `hivectl agent <name> pause|resume` — flip the agent's pause marker.
|
||||||
/// DAG, so there's nothing to wait on: the daemon writes the marker and
|
///
|
||||||
/// the harness picks it up on its next poll.
|
/// **Resume** is not a DAG — nothing to wait on: the daemon writes the
|
||||||
|
/// marker directly and the harness picks it up on its next poll.
|
||||||
|
///
|
||||||
|
/// **Pause** rides the job queue (`PauseSignal → PauseDrain`) instead —
|
||||||
|
/// see `handle_set_paused`'s doc comment on the daemon side for why. This
|
||||||
|
/// fn's `paused` branch prints once the DAG is *queued*, then optionally
|
||||||
|
/// waits for the harness's ack via `wait_for_nodes` (skippable with
|
||||||
|
/// `no_wait`, same knob `agents_restart` exposes).
|
||||||
///
|
///
|
||||||
/// Checks existence first, same as `agents_start` above — the daemon side
|
/// Checks existence first, same as `agents_start` above — the daemon side
|
||||||
/// (`Coordinator::set_paused`) writes the marker file unconditionally via
|
/// (`Coordinator::set_paused`) writes the marker file unconditionally via
|
||||||
/// the priv-helper and has no notion of "no such agent", so without this
|
/// the priv-helper and has no notion of "no such agent", so without this
|
||||||
/// check `hivectl agent typo-name resume` would exit 0 and print
|
/// check `hivectl agent typo-name resume` would exit 0 and print
|
||||||
/// `resumed: typo-name` for a name that was never a real agent.
|
/// `resumed: typo-name` for a name that was never a real agent.
|
||||||
async fn set_paused(socket: &Path, name: &str, paused: bool) -> Result<()> {
|
async fn set_paused(socket: &Path, name: &str, paused: bool, no_wait: bool) -> Result<()> {
|
||||||
if !crate::util::agent_exists(socket, name).await? {
|
if !crate::util::agent_exists(socket, name).await? {
|
||||||
bail!(
|
bail!(
|
||||||
"no such agent: '{name}' (no state dir under {}/)",
|
"no such agent: '{name}' (no state dir under {}/)",
|
||||||
|
|
@ -226,16 +245,19 @@ async fn set_paused(socket: &Path, name: &str, paused: bool) -> Result<()> {
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
||||||
if resp.ok {
|
if !resp.ok {
|
||||||
let verb = if paused { "paused" } else { "resumed" };
|
|
||||||
println!("{verb}: {name}");
|
|
||||||
Ok(())
|
|
||||||
} else {
|
|
||||||
bail!(
|
bail!(
|
||||||
"{} {name}: {}",
|
"{} {name}: {}",
|
||||||
if paused { "pause" } else { "resume" },
|
if paused { "pause" } else { "resume" },
|
||||||
resp.error.as_deref().unwrap_or("unknown error")
|
resp.error.as_deref().unwrap_or("unknown error")
|
||||||
)
|
);
|
||||||
|
}
|
||||||
|
if paused {
|
||||||
|
println!("pause queued: {name}");
|
||||||
|
wait_for_nodes(socket, resp.queued_dags.unwrap_or_default(), no_wait).await
|
||||||
|
} else {
|
||||||
|
println!("resumed: {name}");
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -245,8 +267,8 @@ async fn set_paused(socket: &Path, name: &str, paused: bool) -> Result<()> {
|
||||||
pub(crate) async fn run_agent(socket: &Path, name: &str, cmd: AgentCmd) -> Result<()> {
|
pub(crate) async fn run_agent(socket: &Path, name: &str, cmd: AgentCmd) -> Result<()> {
|
||||||
match cmd {
|
match cmd {
|
||||||
AgentCmd::Restart { no_wait } => agents_restart(socket, name, no_wait).await,
|
AgentCmd::Restart { no_wait } => agents_restart(socket, name, no_wait).await,
|
||||||
AgentCmd::Pause => set_paused(socket, name, true).await,
|
AgentCmd::Pause { no_wait } => set_paused(socket, name, true, no_wait).await,
|
||||||
AgentCmd::Resume => set_paused(socket, name, false).await,
|
AgentCmd::Resume => set_paused(socket, name, false, false).await,
|
||||||
AgentCmd::Start { paused } => agents_start(socket, name, paused).await,
|
AgentCmd::Start { paused } => agents_start(socket, name, paused).await,
|
||||||
AgentCmd::Create => {
|
AgentCmd::Create => {
|
||||||
let name = crate::util::parse_ident(name)?;
|
let name = crate::util::parse_ident(name)?;
|
||||||
|
|
|
||||||
|
|
@ -465,7 +465,12 @@ pub enum AgentCmd {
|
||||||
/// up while it burns no tokens. Inbox messages queue unacked and the
|
/// up while it burns no tokens. Inbox messages queue unacked and the
|
||||||
/// backlog drains on `resume`. Sticky: it survives a restart, and
|
/// backlog drains on `resume`. Sticky: it survives a restart, and
|
||||||
/// pausing a stopped agent makes it come up paused.
|
/// pausing a stopped agent makes it come up paused.
|
||||||
Pause,
|
Pause {
|
||||||
|
/// Return immediately after the pause DAG is queued instead of
|
||||||
|
/// waiting for the harness to acknowledge it.
|
||||||
|
#[arg(long)]
|
||||||
|
no_wait: bool,
|
||||||
|
},
|
||||||
/// Resume this paused agent — it drains whatever queued up while parked.
|
/// Resume this paused agent — it drains whatever queued up while parked.
|
||||||
Resume,
|
Resume,
|
||||||
/// Start this EXISTING agent container. Fails immediately if `name`
|
/// Start this EXISTING agent container. Fails immediately if `name`
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue