feat(hive-c0re): replace rebuild queue with generic job-DAG queue

jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap,
reconcile, signal, drain, ...) driven by one scheduler with N build
slots + per-agent lifecycle leases. per-agent power intent (wanted
up/offline) is durable in agent_power.sqlite; Reconcile nodes converge
observed state to it. kills the graceful-stop watcher thread, the
deferred-start follow-up, and the cascade pre-enqueue (fan-out on
MetaLock completion instead). tracker: #2166
This commit is contained in:
müde 2026-07-06 20:13:14 +02:00
commit 7946e03fde
25 changed files with 3673 additions and 2731 deletions

View file

@ -260,6 +260,16 @@ async fn port_collision(self_name: &str) -> Option<String> {
}
pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
create_container(name, hive, paths).await?;
write_dropins(name, hive, paths).await?;
priv_run("start", name).await
}
/// First-spawn provisioning + `nixos-container create`, without the
/// drop-in write or the start — the job queue's `Create` node.
/// `spawn` composes this with `write_dropins` + start for direct
/// callers (root-agent bootstrap).
pub async fn create_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
validate(name)?;
if let Some(other) = port_collision(name).await {
bail!(
@ -277,8 +287,17 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>
// ref resolves.
let agents = agents_after_spawn(name).await?;
crate::meta::sync_agents(hive, &agents).await?;
priv_run("create", name).await
}
/// Re-apply the per-container host-side config: nspawn flags (bind
/// mounts etc.), the systemd resource-limits drop-in, and a daemon
/// reload so both take effect on the next unit (re)start. Idempotent —
/// the job queue's `WriteDropin` node, also folded into every `Swap`
/// (rebuild is the reconcile verb).
pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
validate(name)?;
let container = container_name(name);
priv_run("create", name).await?;
set_nspawn_flags(
&container,
&paths.agent_dir,
@ -287,8 +306,42 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>
)
.await?;
set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
systemd_daemon_reload().await?;
priv_run("start", name).await
systemd_daemon_reload().await
}
/// Rebuild-path preamble shared by the job queue's `Prebuild` node and
/// `rebuild_no_meta`: fail fast on a port collision, then make sure
/// the applied repo + state dirs exist. Container untouched.
pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> {
validate(name)?;
if let Some(other) = port_collision(name).await {
bail!(
"port {} is already taken by '{other}' — rename one of them and retry",
agent_web_port(name)
);
}
setup_applied(&paths.applied_dir, None, name).await?;
ensure_agent_state_subvolume(name).await?;
ensure_claude_dir(&paths.claude_dir)?;
ensure_state_dir(&paths.notes_dir)?;
Ok(())
}
/// Profile-swap for an existing, stopped container: re-apply the
/// drop-ins, then `nixos-container update`. The job queue's `Swap`
/// node. Requires the container stopped (the queue's `StopForUpdate`
/// upstream); does NOT start it — the DAG's tail `Reconcile` owns
/// bringing the agent back to its wanted power state.
pub async fn swap_update(
name: &str,
hive: &HiveEnv,
paths: &AgentPaths,
on_step: &(dyn Fn(&str) + Send + Sync),
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
) -> Result<()> {
write_dropins(name, hive, paths).await?;
on_step("nixos-container update");
priv_run_inner("update", name, Some(on_build_log_id)).await
}
/// Build the `AgentSpec` list for the meta flake from `nixos-container
@ -532,35 +585,16 @@ pub async fn rebuild_no_meta(
on_step: &(dyn Fn(&str) + Send + Sync),
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
) -> Result<bool> {
validate(name)?;
if let Some(other) = port_collision(name).await {
bail!(
"port {} is already taken by '{other}' — rename one of them and retry",
agent_web_port(name)
);
}
setup_applied(&paths.applied_dir, None, name).await?;
ensure_agent_state_subvolume(name).await?;
ensure_claude_dir(&paths.claude_dir)?;
ensure_state_dir(&paths.notes_dir)?;
let container = container_name(name);
prepare_rebuild_dirs(name, paths).await?;
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
if container_exists(name).await {
// Rebuild strategy: stop-before-update + pre-build.
// See `docs/coordinator.md::Container lifecycle`.
let was_running = is_running(name).await;
set_nspawn_flags(
&container,
&paths.agent_dir,
&paths.claude_dir,
&paths.notes_dir,
)
.await?;
set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
systemd_daemon_reload().await?;
write_dropins(name, hive, paths).await?;
if was_running {
on_step("nix build");
prebuild_toplevel(name, &flake_ref).await?;
prebuild_toplevel(name, &flake_ref, &|_| ()).await?;
on_step("nixos-container stop");
priv_run("stop", name).await?;
}
@ -601,15 +635,7 @@ pub async fn rebuild_no_meta(
// See `docs/coordinator.md::Spawn path`.
on_step("nixos-container create");
priv_run("create", name).await?;
set_nspawn_flags(
&container,
&paths.agent_dir,
&paths.claude_dir,
&paths.notes_dir,
)
.await?;
set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
systemd_daemon_reload().await?;
write_dropins(name, hive, paths).await?;
on_step("nixos-container start");
priv_run("start", name).await?;
Ok(false)
@ -622,7 +648,15 @@ pub async fn rebuild_no_meta(
/// is untouched. See `docs/coordinator.md::Rebuild path` for why
/// the prebuild happens before stop, and `docs/coordinator.md::Prebuild
/// attr path` for why the explicit nixosConfigurations attr is required.
async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
///
/// `on_build_log_id` fires with the `build_logs` row id as soon as the
/// row opens, so queue-side callers can link their node to the live
/// stream. Pass `&|_| ()` when not needed.
pub async fn prebuild_toplevel(
name: &str,
flake_ref: &str,
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
) -> Result<()> {
use tokio::io::{AsyncBufReadExt, BufReader};
// Split `<root>#<name>` so we can re-emit with the explicit
// `nixosConfigurations.<name>` segment. The flake_ref shape is
@ -663,6 +697,9 @@ async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
})
.ok()
});
if let Some(id) = log_id {
on_build_log_id(id);
}
let mut child = Command::new("nix")
.args(&args)