hive-c0re re-converges wanted state whenever its watch attaches
pull waits (bounded) for the swarm queue connection instead of failing at boot while the client is still connecting. The watch runs one pull after every attach, including re-attaches, and a failed attach or ended stream is retried with a bounded backoff instead of waiting for the next boot. A first deploy is not queued again while one for the agent is queued or running. An agent whose first deploy was lost to a c0re restart is now deployed on the next boot, or the next watch re-attach, without anyone re-declaring it.
This commit is contained in:
parent
d989636d98
commit
784507f554
5 changed files with 395 additions and 70 deletions
|
|
@ -322,14 +322,18 @@ async fn handle_deploy_request(
|
|||
coord
|
||||
.job_queue
|
||||
.insert_job(|b| crate::job_queue::templates::rebuild(b, &agent, true))
|
||||
.map(Some)
|
||||
} else {
|
||||
queue_first_deploy(coord, &agent)
|
||||
};
|
||||
match inserted {
|
||||
Ok(_) => {
|
||||
Ok(Some(_)) => {
|
||||
tracing::info!(%agent, known, "swarm events: deploy requested by the swarm, queued");
|
||||
coord.emit_rebuild_queue_snapshot();
|
||||
}
|
||||
Ok(None) => {
|
||||
tracing::info!(%agent, "swarm events: first deploy already queued; not queueing another");
|
||||
}
|
||||
Err(e) => tracing::warn!(%agent, error = %e, "swarm events: queueing the deploy failed"),
|
||||
}
|
||||
}
|
||||
|
|
@ -377,16 +381,17 @@ async fn apply_rev(agent: &str, rev: &str) -> bool {
|
|||
/// Setting the row up front closes that window. A second caller open-coding
|
||||
/// the insert would lose exactly that, and the agent would come up stopped for
|
||||
/// no visible reason.
|
||||
///
|
||||
/// `Ok(None)` when a first deploy for the agent is already queued or running:
|
||||
/// the pull, the watch and the deploy event can all see the same declaration.
|
||||
pub(crate) fn queue_first_deploy(
|
||||
coord: &std::sync::Arc<crate::coordinator::Coordinator>,
|
||||
agent: &str,
|
||||
) -> Result<Vec<hive_jobq::NodeId>> {
|
||||
) -> Result<Option<Vec<hive_jobq::NodeId>>> {
|
||||
if let Err(e) = coord.power.set(agent, crate::power::Wanted::Up) {
|
||||
tracing::warn!(%agent, error = ?e, "agent_power: seed on swarm first-deploy failed");
|
||||
}
|
||||
coord
|
||||
.job_queue
|
||||
.insert_job(|b| crate::job_queue::templates::first_deploy(b, agent))
|
||||
coord.job_queue.insert_first_deploy_once(agent)
|
||||
}
|
||||
|
||||
/// Offer one snapshot: this hive's current readiness, under its own key.
|
||||
|
|
|
|||
Loading…
Reference in a new issue