Watch
0
0
Fork
You've already forked hyperhive
0

hive-c0re: reconcile declared agents periodically, with backoff

This commit is contained in:
atlas 2026-11-14 20:52:31 +01:00 • committed by mara
commit b8c07a5492
2 changed files with 516 additions and 9 deletions

View file

@ -224,6 +224,36 @@ impl JobQueue {
Ok(Some(named))
}
/// Queue a lone `Reconcile` for `agent` unless any queued or running node
/// already targets it.
///
/// `None` means the agent is busy: an update, create, destroy, power op or
/// earlier reconcile is live, and the work in flight converges it (or
/// decides not to). The check and the insert share one lock, so a caller
/// racing another insert cannot queue behind a node it did not see. The
/// shape is the down-agent restart: a `Reconcile` on the agent's lease,
/// which reads the agent's power intent when it runs.
///
/// # Errors
/// Propagates a graph-insert error.
pub fn insert_reconcile_if_idle(&self, agent: &str) -> anyhow::Result<Option<Vec<NodeId>>> {
let mut inner = self.lock();
let busy = inner
.graph()
.nodes()
.any(|n| !n.state.is_terminal() && n.payload.agent() == agent);
if busy {
return Ok(None);
}
let targets = [(agent.to_owned(), false)];
let named = inner
.insert_job(None, |b| power::restart_nodes(b, &targets, false))
.map_err(|e| anyhow::anyhow!("job_queue: graph insert failed: {e}"))?;
drop(inner);
self.notify.notify_one();
Ok(Some(named))
}
/// The scheduler itself, for `hive_jobq`'s run-loop seam
/// (`Scheduler::claim_next`), which takes exactly this type.
///

View file

@ -1556,6 +1556,62 @@ fn a_rebuild_is_live_for_its_own_agent_only() {
assert!(!q.rebuild_live("bob"));
}
/// The periodic reconcile tick queues one lone `Reconcile` on the agent's lease
/// and nothing for an agent any live node already targets — whichever kind
/// of work it is — while other agents and finished work do not block it.
#[test]
fn an_idle_agent_gets_one_reconcile_and_a_busy_one_none() {
let q = JobQueue::new(1);
let first = q
.insert_reconcile_if_idle("alice")
.expect("insert")
.expect("control: an idle agent gets a reconcile");
assert_eq!(
declared_shape(&q)
.iter()
.filter(|d| d.kind == "reconcile")
.count(),
1,
"exactly one Reconcile node"
);
assert_eq!(
declared_resources_of_kind(&q, "reconcile"),
vec![vec![Resource::Agent("alice".to_owned())]],
"it holds the agent's lease"
);
assert!(
q.insert_reconcile_if_idle("alice")
.expect("insert")
.is_none(),
"the live reconcile blocks another"
);
assert!(
q.insert_reconcile_if_idle("bob").expect("insert").is_some(),
"control: another agent is independent"
);
// Work that is not a reconcile blocks too.
insert(&q, |b| {
let _ = rebuild(b, "carol");
});
assert!(
q.insert_reconcile_if_idle("carol")
.expect("insert")
.is_none(),
"a rebuild in flight blocks it"
);
// A finished reconcile does not.
assert!(q.cancel(first[0].get()), "cancel the queued reconcile");
assert!(
q.insert_reconcile_if_idle("alice")
.expect("insert")
.is_some(),
"a terminal node does not count as busy"
);
}
/// Agents whose running closure differs from the declaration are rebuilt in one
/// sweep. Each rebuild's window holds a build slot, so `buildSlots` caps how
/// many swap at once, however many agents the sweep rebuilds.