vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

defer start-after-rebuild to the fast lane so the build lane frees up (closes #2094)

Browse source
This commit is contained in:
damocles 2026-07-01 23:37:13 +02:00
commit b191858366
6 changed files with 162 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

109
hive-c0re/src/lifecycle.rs
View file

@ -372,6 +372,50 @@ pub async fn start(name: &str) -> Result<()> {
priv_run("start", name).await
}
/// Start with the cold-start fallback: when a plain start fails (the
/// activation-error shape), retry once via stop + kill + start before
/// giving up. Used by the queue's fast-lane `Start` handler and the
/// inline start-after-rebuild path.
/// See `docs/coordinator.md::Cold-start fallback`.
///
/// # Errors
///
/// Propagates the retry's start error (annotated with the original
/// failure) when the fallback also fails.
pub async fn start_with_fallback(name: &str) -> Result<()> {
validate(name)?;
if let Err(start_err) = priv_run("start", name).await {
let container = container_name(name);
tracing::warn!(
container = %container,
error = %start_err,
"start failed (possible activation error); retrying via stop + kill + start"
);
priv_run("stop", name).await.unwrap_or_else(|e| {
tracing::warn!(
container = %container,
error = %e,
"stop before cold-start retry failed (ignored)"
);
});
priv_run("kill", name).await.unwrap_or_else(|e| {
tracing::warn!(
container = %container,
error = %e,
"kill before cold-start retry failed (ignored)"
);
});
priv_run("start", name).await.map_err(|e| {
anyhow::anyhow!(
"cold-start fallback also failed: {e:#} \
(original start error: {start_err:#})"
)
})
} else {
Ok(())
}
}
/// Stop + start without regenerating any config. For "kick the container"
/// without touching the flake or nspawn flags.
pub async fn restart(name: &str) -> Result<()> {
@ -421,14 +465,19 @@ pub async fn destroy(name: &str) -> Result<()> {
///
/// Propagates errors from meta-flake sync / lock-update and the
/// `nixos-container` apply + restart shellouts.
///
/// Returns `true` when `defer_start` suppressed the start-after-update —
/// the caller owns bringing the container back up (see
/// [`rebuild_no_meta`]).
pub async fn rebuild(
name: &str,
hive: &HiveEnv,
paths: &AgentPaths,
relock: bool,
defer_start: bool,
on_step: &(dyn Fn(&str) + Send + Sync),
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
) -> Result<()> {
) -> Result<bool> {
// Sync the meta flake (idempotent — no-op when the rendered
// flake matches disk) so a manual rebuild from the dashboard
// can also recover from a divergent meta repo (e.g. an agent
@ -449,7 +498,7 @@ pub async fn rebuild(
if relock {
crate::meta::lock_update_for_rebuild(name).await?;
}
rebuild_no_meta(name, hive, paths, on_step, on_build_log_id).await
rebuild_no_meta(name, hive, paths, defer_start, on_step, on_build_log_id).await
}
/// Container-level rebuild without touching the meta repo. Callers
@ -467,13 +516,22 @@ pub async fn rebuild(
/// the `nixos-container update` log row opens, before the actual update
/// command starts. Callers can use this to link the queue entry to the log
/// for live streaming. Pass `&|_| ()` when not needed.
///
/// `defer_start` skips the start-after-update for a previously-running
/// container and returns `true` instead, so a queue-side caller can hand
/// the (potentially slow) container boot to the fast lane rather than
/// holding the serialized build lane through it. With `defer_start =
/// false` the start (with cold-start fallback) runs inline as before and
/// the return value is always `false`. The spawn path always starts
/// inline — a freshly-created container boots as part of provisioning.
pub async fn rebuild_no_meta(
name: &str,
hive: &HiveEnv,
paths: &AgentPaths,
defer_start: bool,
on_step: &(dyn Fn(&str) + Send + Sync),
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
) -> Result<()> {
) -> Result<bool> {
validate(name)?;
if let Some(other) = port_collision(name).await {
bail!(
@ -528,42 +586,16 @@ pub async fn rebuild_no_meta(
}
update_result?;
if was_running {
// Cold-start fallback on activation errors.
// See `docs/coordinator.md::Cold-start fallback`.
on_step("nixos-container start");
if let Err(start_err) = priv_run("start", name).await {
tracing::warn!(
container = %container,
error = %start_err,
"start after rebuild failed (possible activation error); \
retrying via stop + kill + start"
);
priv_run("stop", name).await.unwrap_or_else(|e| {
tracing::warn!(
container = %container,
error = %e,
"stop before cold-start retry failed (ignored)"
);
});
priv_run("kill", name).await.unwrap_or_else(|e| {
tracing::warn!(
container = %container,
error = %e,
"kill before cold-start retry failed (ignored)"
);
});
priv_run("start", name).await.map_err(|e| {
anyhow::anyhow!(
"cold-start fallback also failed: {e:#} \
(original start error: {start_err:#})"
)
})
} else {
Ok(())
if defer_start {
// The caller re-queues the start on the fast lane so the
// build lane is freed for the next entry instead of
// waiting out the container boot here.
return Ok(true);
}
} else {
Ok(())
on_step("nixos-container start");
start_with_fallback(name).await?;
}
Ok(false)
} else {
// Spawn path: create is atomic, no prebuild needed.
// See `docs/coordinator.md::Spawn path`.
@ -579,7 +611,8 @@ pub async fn rebuild_no_meta(
set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
systemd_daemon_reload().await?;
on_step("nixos-container start");
priv_run("start", name).await
priv_run("start", name).await?;
Ok(false)
}
}