feat(hive-c0re): replace rebuild queue with generic job-DAG queue
jobs are now DAGs of primitive nodes (prebuild, stop-for-update, swap, reconcile, signal, drain, ...) driven by one scheduler with N build slots + per-agent lifecycle leases. per-agent power intent (wanted up/offline) is durable in agent_power.sqlite; Reconcile nodes converge observed state to it. kills the graceful-stop watcher thread, the deferred-start follow-up, and the cascade pre-enqueue (fan-out on MetaLock completion instead). tracker: #2166
This commit is contained in:
parent
79a3993def
commit
7946e03fde
25 changed files with 3673 additions and 2731 deletions
|
|
@ -260,6 +260,16 @@ async fn port_collision(self_name: &str) -> Option<String> {
|
|||
}
|
||||
|
||||
pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
||||
create_container(name, hive, paths).await?;
|
||||
write_dropins(name, hive, paths).await?;
|
||||
priv_run("start", name).await
|
||||
}
|
||||
|
||||
/// First-spawn provisioning + `nixos-container create`, without the
|
||||
/// drop-in write or the start — the job queue's `Create` node.
|
||||
/// `spawn` composes this with `write_dropins` + start for direct
|
||||
/// callers (root-agent bootstrap).
|
||||
pub async fn create_container(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
||||
validate(name)?;
|
||||
if let Some(other) = port_collision(name).await {
|
||||
bail!(
|
||||
|
|
@ -277,8 +287,17 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>
|
|||
// ref resolves.
|
||||
let agents = agents_after_spawn(name).await?;
|
||||
crate::meta::sync_agents(hive, &agents).await?;
|
||||
priv_run("create", name).await
|
||||
}
|
||||
|
||||
/// Re-apply the per-container host-side config: nspawn flags (bind
|
||||
/// mounts etc.), the systemd resource-limits drop-in, and a daemon
|
||||
/// reload so both take effect on the next unit (re)start. Idempotent —
|
||||
/// the job queue's `WriteDropin` node, also folded into every `Swap`
|
||||
/// (rebuild is the reconcile verb).
|
||||
pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
||||
validate(name)?;
|
||||
let container = container_name(name);
|
||||
priv_run("create", name).await?;
|
||||
set_nspawn_flags(
|
||||
&container,
|
||||
&paths.agent_dir,
|
||||
|
|
@ -287,8 +306,42 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>
|
|||
)
|
||||
.await?;
|
||||
set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
|
||||
systemd_daemon_reload().await?;
|
||||
priv_run("start", name).await
|
||||
systemd_daemon_reload().await
|
||||
}
|
||||
|
||||
/// Rebuild-path preamble shared by the job queue's `Prebuild` node and
|
||||
/// `rebuild_no_meta`: fail fast on a port collision, then make sure
|
||||
/// the applied repo + state dirs exist. Container untouched.
|
||||
pub async fn prepare_rebuild_dirs(name: &str, paths: &AgentPaths) -> Result<()> {
|
||||
validate(name)?;
|
||||
if let Some(other) = port_collision(name).await {
|
||||
bail!(
|
||||
"port {} is already taken by '{other}' — rename one of them and retry",
|
||||
agent_web_port(name)
|
||||
);
|
||||
}
|
||||
setup_applied(&paths.applied_dir, None, name).await?;
|
||||
ensure_agent_state_subvolume(name).await?;
|
||||
ensure_claude_dir(&paths.claude_dir)?;
|
||||
ensure_state_dir(&paths.notes_dir)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Profile-swap for an existing, stopped container: re-apply the
|
||||
/// drop-ins, then `nixos-container update`. The job queue's `Swap`
|
||||
/// node. Requires the container stopped (the queue's `StopForUpdate`
|
||||
/// upstream); does NOT start it — the DAG's tail `Reconcile` owns
|
||||
/// bringing the agent back to its wanted power state.
|
||||
pub async fn swap_update(
|
||||
name: &str,
|
||||
hive: &HiveEnv,
|
||||
paths: &AgentPaths,
|
||||
on_step: &(dyn Fn(&str) + Send + Sync),
|
||||
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
|
||||
) -> Result<()> {
|
||||
write_dropins(name, hive, paths).await?;
|
||||
on_step("nixos-container update");
|
||||
priv_run_inner("update", name, Some(on_build_log_id)).await
|
||||
}
|
||||
|
||||
/// Build the `AgentSpec` list for the meta flake from `nixos-container
|
||||
|
|
@ -532,35 +585,16 @@ pub async fn rebuild_no_meta(
|
|||
on_step: &(dyn Fn(&str) + Send + Sync),
|
||||
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
|
||||
) -> Result<bool> {
|
||||
validate(name)?;
|
||||
if let Some(other) = port_collision(name).await {
|
||||
bail!(
|
||||
"port {} is already taken by '{other}' — rename one of them and retry",
|
||||
agent_web_port(name)
|
||||
);
|
||||
}
|
||||
setup_applied(&paths.applied_dir, None, name).await?;
|
||||
ensure_agent_state_subvolume(name).await?;
|
||||
ensure_claude_dir(&paths.claude_dir)?;
|
||||
ensure_state_dir(&paths.notes_dir)?;
|
||||
let container = container_name(name);
|
||||
prepare_rebuild_dirs(name, paths).await?;
|
||||
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
|
||||
if container_exists(name).await {
|
||||
// Rebuild strategy: stop-before-update + pre-build.
|
||||
// See `docs/coordinator.md::Container lifecycle`.
|
||||
let was_running = is_running(name).await;
|
||||
set_nspawn_flags(
|
||||
&container,
|
||||
&paths.agent_dir,
|
||||
&paths.claude_dir,
|
||||
&paths.notes_dir,
|
||||
)
|
||||
.await?;
|
||||
set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
|
||||
systemd_daemon_reload().await?;
|
||||
write_dropins(name, hive, paths).await?;
|
||||
if was_running {
|
||||
on_step("nix build");
|
||||
prebuild_toplevel(name, &flake_ref).await?;
|
||||
prebuild_toplevel(name, &flake_ref, &|_| ()).await?;
|
||||
on_step("nixos-container stop");
|
||||
priv_run("stop", name).await?;
|
||||
}
|
||||
|
|
@ -601,15 +635,7 @@ pub async fn rebuild_no_meta(
|
|||
// See `docs/coordinator.md::Spawn path`.
|
||||
on_step("nixos-container create");
|
||||
priv_run("create", name).await?;
|
||||
set_nspawn_flags(
|
||||
&container,
|
||||
&paths.agent_dir,
|
||||
&paths.claude_dir,
|
||||
&paths.notes_dir,
|
||||
)
|
||||
.await?;
|
||||
set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?;
|
||||
systemd_daemon_reload().await?;
|
||||
write_dropins(name, hive, paths).await?;
|
||||
on_step("nixos-container start");
|
||||
priv_run("start", name).await?;
|
||||
Ok(false)
|
||||
|
|
@ -622,7 +648,15 @@ pub async fn rebuild_no_meta(
|
|||
/// is untouched. See `docs/coordinator.md::Rebuild path` for why
|
||||
/// the prebuild happens before stop, and `docs/coordinator.md::Prebuild
|
||||
/// attr path` for why the explicit nixosConfigurations attr is required.
|
||||
async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
|
||||
///
|
||||
/// `on_build_log_id` fires with the `build_logs` row id as soon as the
|
||||
/// row opens, so queue-side callers can link their node to the live
|
||||
/// stream. Pass `&|_| ()` when not needed.
|
||||
pub async fn prebuild_toplevel(
|
||||
name: &str,
|
||||
flake_ref: &str,
|
||||
on_build_log_id: &(dyn Fn(i64) + Send + Sync),
|
||||
) -> Result<()> {
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
// Split `<root>#<name>` so we can re-emit with the explicit
|
||||
// `nixosConfigurations.<name>` segment. The flake_ref shape is
|
||||
|
|
@ -663,6 +697,9 @@ async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
|
|||
})
|
||||
.ok()
|
||||
});
|
||||
if let Some(id) = log_id {
|
||||
on_build_log_id(id);
|
||||
}
|
||||
|
||||
let mut child = Command::new("nix")
|
||||
.args(&args)
|
||||
|
|
|
|||
Loading…
Reference in a new issue