defer start-after-rebuild to the fast lane so the build lane frees up (closes #2094)

This commit is contained in:
damocles 2026-07-01 23:37:13 +02:00
commit b191858366
6 changed files with 162 additions and 61 deletions

View file

@ -73,6 +73,13 @@ pub fn agent_config_pending(name: &str, deployed_sha: Option<&str>) -> bool {
/// rebuilds, where re-locking would revert the bump the cascade just
/// committed (see `lifecycle::rebuild`).
///
/// `defer_start_source` is `Some(source)` for queue-dispatched rebuilds:
/// instead of holding the serialized build lane through the container
/// boot, the start-after-rebuild is enqueued as a fast-lane `Start`
/// entry (grouped under this rebuild via `parent_id`, same split as the
/// graceful-stop follow-up). Pass `None` for direct callers to keep the
/// start inline.
///
/// # Errors
///
/// Propagates errors from `coord.ensure_runtime` and `lifecycle::rebuild`.
@ -82,6 +89,7 @@ pub async fn rebuild_agent(
current_rev: &str,
queue_entry_id: Option<u64>,
relock: bool,
defer_start_source: Option<crate::rebuild_queue::QueueSource>,
) -> Result<()> {
tracing::info!(%name, rev = %current_rev, "rebuild agent");
let agent_dir = coord
@ -99,6 +107,7 @@ pub async fn rebuild_agent(
&hive,
&paths,
relock,
defer_start_source.is_some(),
&|step| coord.set_queue_step(queue_entry_id, step),
&|log_id| {
if let Some(qid) = queue_entry_id
@ -111,10 +120,33 @@ pub async fn rebuild_agent(
.await;
drop(guard);
match &result {
Ok(()) => {
Ok(needs_start) => {
if let Err(e) = std::fs::write(rev_marker_path(name), current_rev) {
tracing::warn!(%name, error = ?e, "write rev marker failed");
}
// Deferred start: hand the container boot to the fast lane so
// this build-lane entry completes now and the next queued
// rebuild's nix build overlaps with the boot. `parent_id`
// groups the follow-up under this rebuild on the dashboard —
// same split the graceful-stop path uses for its container
// stop. A start failure surfaces on the Start entry (with
// the cold-start fallback) instead of failing the rebuild.
if *needs_start && let Some(source) = defer_start_source {
coord
.rebuild_queue
.enqueue_full(crate::rebuild_queue::FullEnqueue {
kind: crate::rebuild_queue::QueueKind::Start,
agent: name.to_owned(),
source,
reason: format!("start after rebuild of {name}"),
parent_id: queue_entry_id,
inputs: Vec::new(),
approval_id: None,
perm_payload: None,
depends_on: Vec::new(),
});
coord.emit_rebuild_queue_snapshot();
}
coord.notify_manager(&hive_sh4re::HelperEvent::Rebuilt {
agent: name.to_owned(),
ok: true,
@ -161,7 +193,7 @@ pub async fn rebuild_agent(
coord.rescan_containers_and_emit().await;
}
}
result
result.map(|_| ())
}
/// Whether this hive is "ruthless" — running with no root/manager agent at
@ -210,7 +242,7 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
);
let coord_clone = coord.clone();
if let Err(e) =
rebuild_agent(&coord_clone, MANAGER_NAME, rev.as_str(), None, true).await
rebuild_agent(&coord_clone, MANAGER_NAME, rev.as_str(), None, true, None).await
{
tracing::warn!(error = ?e, "manager migration rebuild failed");
}