hive-c0re/hivectl/hive-agent: pause as a job-queue DAG node (closes #3056)
This commit is contained in:
parent
a92f7351d9
commit
20a7a21053
13 changed files with 316 additions and 27 deletions
|
|
@ -60,7 +60,19 @@ async fn agents_start(socket: &Path, name: &str, paused: bool) -> Result<()> {
|
|||
}
|
||||
if paused {
|
||||
let already_running = agent_running(socket, name).await?;
|
||||
set_paused(socket, name, true).await?;
|
||||
// `no_wait = true`: correctness here doesn't come from waiting —
|
||||
// the pause DAG's `AgentWindow` brace and the `Start` request's
|
||||
// `SetWanted` head both declare `Resource::Agent`, and this call's
|
||||
// `insert_job` has already returned (so the pause DAG's claim is
|
||||
// registered) before `Start` is even sent, so the lease itself
|
||||
// orders the marker write ahead of the container boot. Waiting for
|
||||
// the *pause DAG* to finish
|
||||
// would be actively wrong when the agent isn't running yet: its
|
||||
// `PauseDrain` can't be acked until the harness is up to see the
|
||||
// marker, which is the very thing `Start` (below) is about to
|
||||
// cause — waiting here would just block for `PAUSE_ACK_TIMEOUT`
|
||||
// for an ack that can only land after this call returns.
|
||||
set_paused(socket, name, true, true).await?;
|
||||
if already_running {
|
||||
eprintln!("'{name}' is already running — paused in place, not (re)started");
|
||||
return Ok(());
|
||||
|
|
@ -201,16 +213,23 @@ pub(crate) async fn agents_list(socket: &Path, json: bool) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// `hivectl agent <name> pause|resume` — flip the agent's pause marker. Not a
|
||||
/// DAG, so there's nothing to wait on: the daemon writes the marker and
|
||||
/// the harness picks it up on its next poll.
|
||||
/// `hivectl agent <name> pause|resume` — flip the agent's pause marker.
|
||||
///
|
||||
/// **Resume** is not a DAG — nothing to wait on: the daemon writes the
|
||||
/// marker directly and the harness picks it up on its next poll.
|
||||
///
|
||||
/// **Pause** rides the job queue (`PauseSignal → PauseDrain`) instead —
|
||||
/// see `handle_set_paused`'s doc comment on the daemon side for why. This
|
||||
/// fn's `paused` branch prints once the DAG is *queued*, then optionally
|
||||
/// waits for the harness's ack via `wait_for_nodes` (skippable with
|
||||
/// `no_wait`, same knob `agents_restart` exposes).
|
||||
///
|
||||
/// Checks existence first, same as `agents_start` above — the daemon side
|
||||
/// (`Coordinator::set_paused`) writes the marker file unconditionally via
|
||||
/// the priv-helper and has no notion of "no such agent", so without this
|
||||
/// check `hivectl agent typo-name resume` would exit 0 and print
|
||||
/// `resumed: typo-name` for a name that was never a real agent.
|
||||
async fn set_paused(socket: &Path, name: &str, paused: bool) -> Result<()> {
|
||||
async fn set_paused(socket: &Path, name: &str, paused: bool, no_wait: bool) -> Result<()> {
|
||||
if !crate::util::agent_exists(socket, name).await? {
|
||||
bail!(
|
||||
"no such agent: '{name}' (no state dir under {}/)",
|
||||
|
|
@ -226,16 +245,19 @@ async fn set_paused(socket: &Path, name: &str, paused: bool) -> Result<()> {
|
|||
)
|
||||
.await
|
||||
.with_context(|| format!("connect to daemon socket {}", socket.display()))?;
|
||||
if resp.ok {
|
||||
let verb = if paused { "paused" } else { "resumed" };
|
||||
println!("{verb}: {name}");
|
||||
Ok(())
|
||||
} else {
|
||||
if !resp.ok {
|
||||
bail!(
|
||||
"{} {name}: {}",
|
||||
if paused { "pause" } else { "resume" },
|
||||
resp.error.as_deref().unwrap_or("unknown error")
|
||||
)
|
||||
);
|
||||
}
|
||||
if paused {
|
||||
println!("pause queued: {name}");
|
||||
wait_for_nodes(socket, resp.queued_dags.unwrap_or_default(), no_wait).await
|
||||
} else {
|
||||
println!("resumed: {name}");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -245,8 +267,8 @@ async fn set_paused(socket: &Path, name: &str, paused: bool) -> Result<()> {
|
|||
pub(crate) async fn run_agent(socket: &Path, name: &str, cmd: AgentCmd) -> Result<()> {
|
||||
match cmd {
|
||||
AgentCmd::Restart { no_wait } => agents_restart(socket, name, no_wait).await,
|
||||
AgentCmd::Pause => set_paused(socket, name, true).await,
|
||||
AgentCmd::Resume => set_paused(socket, name, false).await,
|
||||
AgentCmd::Pause { no_wait } => set_paused(socket, name, true, no_wait).await,
|
||||
AgentCmd::Resume => set_paused(socket, name, false, false).await,
|
||||
AgentCmd::Start { paused } => agents_start(socket, name, paused).await,
|
||||
AgentCmd::Create => {
|
||||
let name = crate::util::parse_ident(name)?;
|
||||
|
|
|
|||
|
|
@ -465,7 +465,12 @@ pub enum AgentCmd {
|
|||
/// up while it burns no tokens. Inbox messages queue unacked and the
|
||||
/// backlog drains on `resume`. Sticky: it survives a restart, and
|
||||
/// pausing a stopped agent makes it come up paused.
|
||||
Pause,
|
||||
Pause {
|
||||
/// Return immediately after the pause DAG is queued instead of
|
||||
/// waiting for the harness to acknowledge it.
|
||||
#[arg(long)]
|
||||
no_wait: bool,
|
||||
},
|
||||
/// Resume this paused agent — it drains whatever queued up while parked.
|
||||
Resume,
|
||||
/// Start this EXISTING agent container. Fails immediately if `name`
|
||||
|
|
|
|||
Loading…
Reference in a new issue