diff --git a/docs/coordinator.md b/docs/coordinator.md index 5063356e..421e0312 100644 --- a/docs/coordinator.md +++ b/docs/coordinator.md @@ -113,6 +113,67 @@ Key operations: --- +## Container lifecycle (`lifecycle.rs`) + +Every container operation ultimately calls into `lifecycle.rs`. Two paths exist: +**rebuild** (existing container) and **spawn** (first-time creation). + +### Rebuild path (existing container) + +Goal: apply the new system profile and any `EXTRA_NSPAWN_FLAGS` / drop-in changes +in a single start, with minimum downtime. + +`nixos-container update` only runs `systemctl reload container@` when the +container is already up (per `isContainerRunning` in `nixos-container.pl`). Stopping +first turns `update` into a boot-style operation: it builds + `nix-env --set`s the +new profile and skips the in-container `switch-to-configuration`. The subsequent +`start` then applies both the new profile and any `EXTRA_NSPAWN_FLAGS` changes in +one go, rather than the double-bounce a live `update` would trigger. + +Sequence for a running container: + +1. `prebuild_toplevel` — build the new `system.build.toplevel` **before** stopping. + The container keeps serving the previous generation while eval + fetch + build + happen out-of-band. `nixos-container update` then finds the result cached and + skips straight to the profile-swap. Build failures surface here, before the + running container is touched. +2. `nixos-container stop` — bring the container down. +3. `nixos-container update --flake meta#` — profile-swap (near-instant after + the prebuild). +4. `nixos-container start` — boot into the new generation; the in-container + activation script transitions old → new. + +If the container is already stopped, step 1 is skipped (no downtime to shave — no +point evaluating the flake twice). + +### Cold-start fallback + +`start` after `update` can exit non-zero when packages are **removed** between +generations: the old-generation activation script references units that no longer +exist in the new closure, causing systemd to exit non-zero. The container may be +half-started at that point. + +Fallback: `stop` (graceful SIGTERM drain) → `kill` (SIGKILL any lingering processes) +→ `start` (clean cold-start, no generation transition, new activation runs cleanly). +Both errors are preserved and surfaced if the cold-start also fails. + +### Spawn path (new container) + +For a first-time `create`, `nixos-container create` is atomic: if the build fails, +no container record is left to clean up. A separate prebuild would just duplicate +the eval, so it's skipped. Sequence: `create --flake meta#` → write nspawn +flags → `systemctl daemon-reload` → `start`. + +### Prebuild attr path + +`nix build` does not auto-resolve `meta#` against `nixosConfigurations` the +way `nixos-container` does internally. The explicit attr path +`#nixosConfigurations..config.system.build.toplevel` is required; +using the bare `meta#` ref would make nix look in `packages`, `legacyPackages`, +or the flake root directly — none of which exist in the rendered meta flake. + +--- + ## See also - `docs/approvals.md` — approval flow + scheduled prompts diff --git a/hive-c0re/src/lifecycle.rs b/hive-c0re/src/lifecycle.rs index 9dc9d080..ea8f3eb7 100644 --- a/hive-c0re/src/lifecycle.rs +++ b/hive-c0re/src/lifecycle.rs @@ -408,38 +408,13 @@ pub async fn rebuild_no_meta( let container = container_name(name); let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display()); if container_exists(name).await { - // Existing container: preserve the prior running state across - // rebuild, and apply both the new system profile - // AND any `/etc/nixos-containers/.conf` / drop-in changes - // in a single start rather than `update`'s reload-then-outer- - // restart double-bounce. - // - // `nixos-container update` only runs `systemctl reload - // container@` when the container is up (per the - // `isContainerRunning` check in nixos-container.pl), so - // stopping first makes `update` boot-style: build + nix-env - // --set the new profile, skip the in-container - // switch-to-configuration, let the next `start` apply both - // the new profile and the new EXTRA_NSPAWN_FLAGS in one go. - // If the container was already stopped, `update` builds + sets - // the profile and we leave it stopped. + // Rebuild strategy: stop-before-update + pre-build. + // See `docs/coordinator.md::Container lifecycle`. let was_running = is_running(name).await; set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?; set_resource_limits(&container)?; systemd_daemon_reload().await?; if was_running { - // Pre-build the system toplevel **before** stopping the - // running container so the agent keeps serving its - // previous generation while the eval + fetch + build - // happens out-of-band. `nixos-container update` then - // finds the toplevel cached and skips straight to the - // profile-swap + restart — downtime collapses to that - // window only. Build failures surface here, before we - // touch the container. - // - // When the container is already stopped there's no - // downtime to shave — let `update` do the build inline - // rather than evaluating the flake twice for nothing. on_step("nix build"); prebuild_toplevel(name, &flake_ref).await?; on_step("nixos-container stop"); @@ -448,18 +423,8 @@ pub async fn rebuild_no_meta( on_step("nixos-container update"); run(&["update", &container, "--flake", &flake_ref]).await?; if was_running { - // Normal path: start into the new generation. The activation - // script runs inside the container to transition old → new. - // This can fail when packages are removed between generations — - // the old-generation activation references units that no longer - // exist in the new closure, causing systemd to exit non-zero. - // - // Fallback: stop + kill + start (cold-start). The activation - // script can fail when packages are removed between generations — - // `start` exits non-zero but the container may be half-started. - // `stop` requests a graceful SIGTERM drain; `kill` then SIGKILLs - // any lingering processes so the next `start` enters a clean state - // without a generation transition, letting the activation succeed. + // Cold-start fallback on activation errors. + // See `docs/coordinator.md::Cold-start fallback`. on_step("nixos-container start"); if let Err(start_err) = run(&["start", &container]).await { tracing::warn!( @@ -494,11 +459,8 @@ pub async fn rebuild_no_meta( Ok(()) } } else { - // First spawn: no running container, no downtime to shave. - // `nixos-container create` builds + creates atomically — if - // the build fails, no container record is left around to - // clean up — so a pre-build adds nothing but a duplicate - // eval. + // Spawn path: create is atomic, no prebuild needed. + // See `docs/coordinator.md::Spawn path`. on_step("nixos-container create"); run(&["create", &container, "--flake", &flake_ref]).await?; set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?; @@ -509,29 +471,12 @@ pub async fn rebuild_no_meta( } } -/// Pre-build the agent's `system.build.toplevel` derivation against -/// `meta#` so the subsequent `nixos-container update` / -/// `create` finds the result already in the store. The container -/// itself is untouched — this is purely a store-warming pass. -/// -/// Streams nix's stdout to INFO and stderr to WARN like the -/// `nixos-container` shellouts so progress shows up in journald as -/// it happens. `--no-link` keeps us from littering the working -/// directory with `result` symlinks. Per-derivation cost: pure -/// cache hit when nothing changed (handful of seconds for the -/// eval), expensive only on the rebuild that actually has work. -/// -/// Attr path is `#nixosConfigurations..config. -/// system.build.toplevel` — `nix build` won't auto-resolve the bare -/// `` against `nixosConfigurations` like `nixos-container` does -/// internally, so we have to spell the path out explicitly. Falling -/// back to `meta#` (the shape `nixos-container update --flake -/// meta#` uses) makes nix look for `packages..`, -/// `legacyPackages..`, or `` at the flake root — -/// none of which exist in the rendered meta flake. -/// -/// Returns the same error shape as the other nixos-container -/// helpers so callers can use `?` without translation. +/// Pre-build `system.build.toplevel` against `meta#` so the +/// subsequent `nixos-container update` finds the result cached and +/// skips straight to the profile-swap. Store-warming only — container +/// is untouched. See `docs/coordinator.md::Rebuild path` for why +/// the prebuild happens before stop, and `docs/coordinator.md::Prebuild +/// attr path` for why the explicit nixosConfigurations attr is required. async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> { use tokio::io::{AsyncBufReadExt, BufReader}; // Split `#` so we can re-emit with the explicit