hive-c0re: reconcile declared agents periodically, with backoff
This commit is contained in:
parent
7468aa4cff
commit
b8c07a5492
2 changed files with 516 additions and 9 deletions
|
|
@ -224,6 +224,36 @@ impl JobQueue {
|
|||
Ok(Some(named))
|
||||
}
|
||||
|
||||
/// Queue a lone `Reconcile` for `agent` unless any queued or running node
|
||||
/// already targets it.
|
||||
///
|
||||
/// `None` means the agent is busy: an update, create, destroy, power op or
|
||||
/// earlier reconcile is live, and the work in flight converges it (or
|
||||
/// decides not to). The check and the insert share one lock, so a caller
|
||||
/// racing another insert cannot queue behind a node it did not see. The
|
||||
/// shape is the down-agent restart: a `Reconcile` on the agent's lease,
|
||||
/// which reads the agent's power intent when it runs.
|
||||
///
|
||||
/// # Errors
|
||||
/// Propagates a graph-insert error.
|
||||
pub fn insert_reconcile_if_idle(&self, agent: &str) -> anyhow::Result<Option<Vec<NodeId>>> {
|
||||
let mut inner = self.lock();
|
||||
let busy = inner
|
||||
.graph()
|
||||
.nodes()
|
||||
.any(|n| !n.state.is_terminal() && n.payload.agent() == agent);
|
||||
if busy {
|
||||
return Ok(None);
|
||||
}
|
||||
let targets = [(agent.to_owned(), false)];
|
||||
let named = inner
|
||||
.insert_job(None, |b| power::restart_nodes(b, &targets, false))
|
||||
.map_err(|e| anyhow::anyhow!("job_queue: graph insert failed: {e}"))?;
|
||||
drop(inner);
|
||||
self.notify.notify_one();
|
||||
Ok(Some(named))
|
||||
}
|
||||
|
||||
/// The scheduler itself, for `hive_jobq`'s run-loop seam
|
||||
/// (`Scheduler::claim_next`), which takes exactly this type.
|
||||
///
|
||||
|
|
|
|||
|
|
@ -1556,6 +1556,62 @@ fn a_rebuild_is_live_for_its_own_agent_only() {
|
|||
assert!(!q.rebuild_live("bob"));
|
||||
}
|
||||
|
||||
/// The periodic reconcile tick queues one lone `Reconcile` on the agent's lease
|
||||
/// and nothing for an agent any live node already targets — whichever kind
|
||||
/// of work it is — while other agents and finished work do not block it.
|
||||
#[test]
|
||||
fn an_idle_agent_gets_one_reconcile_and_a_busy_one_none() {
|
||||
let q = JobQueue::new(1);
|
||||
let first = q
|
||||
.insert_reconcile_if_idle("alice")
|
||||
.expect("insert")
|
||||
.expect("control: an idle agent gets a reconcile");
|
||||
assert_eq!(
|
||||
declared_shape(&q)
|
||||
.iter()
|
||||
.filter(|d| d.kind == "reconcile")
|
||||
.count(),
|
||||
1,
|
||||
"exactly one Reconcile node"
|
||||
);
|
||||
assert_eq!(
|
||||
declared_resources_of_kind(&q, "reconcile"),
|
||||
vec![vec![Resource::Agent("alice".to_owned())]],
|
||||
"it holds the agent's lease"
|
||||
);
|
||||
|
||||
assert!(
|
||||
q.insert_reconcile_if_idle("alice")
|
||||
.expect("insert")
|
||||
.is_none(),
|
||||
"the live reconcile blocks another"
|
||||
);
|
||||
assert!(
|
||||
q.insert_reconcile_if_idle("bob").expect("insert").is_some(),
|
||||
"control: another agent is independent"
|
||||
);
|
||||
|
||||
// Work that is not a reconcile blocks too.
|
||||
insert(&q, |b| {
|
||||
let _ = rebuild(b, "carol");
|
||||
});
|
||||
assert!(
|
||||
q.insert_reconcile_if_idle("carol")
|
||||
.expect("insert")
|
||||
.is_none(),
|
||||
"a rebuild in flight blocks it"
|
||||
);
|
||||
|
||||
// A finished reconcile does not.
|
||||
assert!(q.cancel(first[0].get()), "cancel the queued reconcile");
|
||||
assert!(
|
||||
q.insert_reconcile_if_idle("alice")
|
||||
.expect("insert")
|
||||
.is_some(),
|
||||
"a terminal node does not count as busy"
|
||||
);
|
||||
}
|
||||
|
||||
/// Agents whose running closure differs from the declaration are rebuilt in one
|
||||
/// sweep. Each rebuild's window holds a build slot, so `buildSlots` caps how
|
||||
/// many swap at once, however many agents the sweep rebuilds.
|
||||
|
|
|
|||
Loading…
Reference in a new issue