docs: add container lifecycle section to coordinator.md, trim lifecycle.rs prose
This commit is contained in:
damocles
parent
f20432430d
commit
88cdab411e
2 changed files with 73 additions and 67 deletions
|
|
@ -113,6 +113,67 @@ Key operations:
|
|||
|
||||
---
|
||||
|
||||
## Container lifecycle (`lifecycle.rs`)
|
||||
|
||||
Every container operation ultimately calls into `lifecycle.rs`. Two paths exist:
|
||||
**rebuild** (existing container) and **spawn** (first-time creation).
|
||||
|
||||
### Rebuild path (existing container)
|
||||
|
||||
Goal: apply the new system profile and any `EXTRA_NSPAWN_FLAGS` / drop-in changes
|
||||
in a single start, with minimum downtime.
|
||||
|
||||
`nixos-container update` only runs `systemctl reload container@<c>` when the
|
||||
container is already up (per `isContainerRunning` in `nixos-container.pl`). Stopping
|
||||
first turns `update` into a boot-style operation: it builds + `nix-env --set`s the
|
||||
new profile and skips the in-container `switch-to-configuration`. The subsequent
|
||||
`start` then applies both the new profile and any `EXTRA_NSPAWN_FLAGS` changes in
|
||||
one go, rather than the double-bounce a live `update` would trigger.
|
||||
|
||||
Sequence for a running container:
|
||||
|
||||
1. `prebuild_toplevel` — build the new `system.build.toplevel` **before** stopping.
|
||||
The container keeps serving the previous generation while eval + fetch + build
|
||||
happen out-of-band. `nixos-container update` then finds the result cached and
|
||||
skips straight to the profile-swap. Build failures surface here, before the
|
||||
running container is touched.
|
||||
2. `nixos-container stop` — bring the container down.
|
||||
3. `nixos-container update --flake meta#<name>` — profile-swap (near-instant after
|
||||
the prebuild).
|
||||
4. `nixos-container start` — boot into the new generation; the in-container
|
||||
activation script transitions old → new.
|
||||
|
||||
If the container is already stopped, step 1 is skipped (no downtime to shave — no
|
||||
point evaluating the flake twice).
|
||||
|
||||
### Cold-start fallback
|
||||
|
||||
`start` after `update` can exit non-zero when packages are **removed** between
|
||||
generations: the old-generation activation script references units that no longer
|
||||
exist in the new closure, causing systemd to exit non-zero. The container may be
|
||||
half-started at that point.
|
||||
|
||||
Fallback: `stop` (graceful SIGTERM drain) → `kill` (SIGKILL any lingering processes)
|
||||
→ `start` (clean cold-start, no generation transition, new activation runs cleanly).
|
||||
Both errors are preserved and surfaced if the cold-start also fails.
|
||||
|
||||
### Spawn path (new container)
|
||||
|
||||
For a first-time `create`, `nixos-container create` is atomic: if the build fails,
|
||||
no container record is left to clean up. A separate prebuild would just duplicate
|
||||
the eval, so it's skipped. Sequence: `create --flake meta#<name>` → write nspawn
|
||||
flags → `systemctl daemon-reload` → `start`.
|
||||
|
||||
### Prebuild attr path
|
||||
|
||||
`nix build` does not auto-resolve `meta#<name>` against `nixosConfigurations` the
|
||||
way `nixos-container` does internally. The explicit attr path
|
||||
`<flake-root>#nixosConfigurations.<name>.config.system.build.toplevel` is required;
|
||||
using the bare `meta#<name>` ref would make nix look in `packages`, `legacyPackages`,
|
||||
or the flake root directly — none of which exist in the rendered meta flake.
|
||||
|
||||
---
|
||||
|
||||
## See also
|
||||
|
||||
- `docs/approvals.md` — approval flow + scheduled prompts
|
||||
|
|
|
|||
|
|
@ -408,38 +408,13 @@ pub async fn rebuild_no_meta(
|
|||
let container = container_name(name);
|
||||
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
|
||||
if container_exists(name).await {
|
||||
// Existing container: preserve the prior running state across
|
||||
// rebuild, and apply both the new system profile
|
||||
// AND any `/etc/nixos-containers/<c>.conf` / drop-in changes
|
||||
// in a single start rather than `update`'s reload-then-outer-
|
||||
// restart double-bounce.
|
||||
//
|
||||
// `nixos-container update` only runs `systemctl reload
|
||||
// container@<c>` when the container is up (per the
|
||||
// `isContainerRunning` check in nixos-container.pl), so
|
||||
// stopping first makes `update` boot-style: build + nix-env
|
||||
// --set the new profile, skip the in-container
|
||||
// switch-to-configuration, let the next `start` apply both
|
||||
// the new profile and the new EXTRA_NSPAWN_FLAGS in one go.
|
||||
// If the container was already stopped, `update` builds + sets
|
||||
// the profile and we leave it stopped.
|
||||
// Rebuild strategy: stop-before-update + pre-build.
|
||||
// See `docs/coordinator.md::Container lifecycle`.
|
||||
let was_running = is_running(name).await;
|
||||
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
|
||||
set_resource_limits(&container)?;
|
||||
systemd_daemon_reload().await?;
|
||||
if was_running {
|
||||
// Pre-build the system toplevel **before** stopping the
|
||||
// running container so the agent keeps serving its
|
||||
// previous generation while the eval + fetch + build
|
||||
// happens out-of-band. `nixos-container update` then
|
||||
// finds the toplevel cached and skips straight to the
|
||||
// profile-swap + restart — downtime collapses to that
|
||||
// window only. Build failures surface here, before we
|
||||
// touch the container.
|
||||
//
|
||||
// When the container is already stopped there's no
|
||||
// downtime to shave — let `update` do the build inline
|
||||
// rather than evaluating the flake twice for nothing.
|
||||
on_step("nix build");
|
||||
prebuild_toplevel(name, &flake_ref).await?;
|
||||
on_step("nixos-container stop");
|
||||
|
|
@ -448,18 +423,8 @@ pub async fn rebuild_no_meta(
|
|||
on_step("nixos-container update");
|
||||
run(&["update", &container, "--flake", &flake_ref]).await?;
|
||||
if was_running {
|
||||
// Normal path: start into the new generation. The activation
|
||||
// script runs inside the container to transition old → new.
|
||||
// This can fail when packages are removed between generations —
|
||||
// the old-generation activation references units that no longer
|
||||
// exist in the new closure, causing systemd to exit non-zero.
|
||||
//
|
||||
// Fallback: stop + kill + start (cold-start). The activation
|
||||
// script can fail when packages are removed between generations —
|
||||
// `start` exits non-zero but the container may be half-started.
|
||||
// `stop` requests a graceful SIGTERM drain; `kill` then SIGKILLs
|
||||
// any lingering processes so the next `start` enters a clean state
|
||||
// without a generation transition, letting the activation succeed.
|
||||
// Cold-start fallback on activation errors.
|
||||
// See `docs/coordinator.md::Cold-start fallback`.
|
||||
on_step("nixos-container start");
|
||||
if let Err(start_err) = run(&["start", &container]).await {
|
||||
tracing::warn!(
|
||||
|
|
@ -494,11 +459,8 @@ pub async fn rebuild_no_meta(
|
|||
Ok(())
|
||||
}
|
||||
} else {
|
||||
// First spawn: no running container, no downtime to shave.
|
||||
// `nixos-container create` builds + creates atomically — if
|
||||
// the build fails, no container record is left around to
|
||||
// clean up — so a pre-build adds nothing but a duplicate
|
||||
// eval.
|
||||
// Spawn path: create is atomic, no prebuild needed.
|
||||
// See `docs/coordinator.md::Spawn path`.
|
||||
on_step("nixos-container create");
|
||||
run(&["create", &container, "--flake", &flake_ref]).await?;
|
||||
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
|
||||
|
|
@ -509,29 +471,12 @@ pub async fn rebuild_no_meta(
|
|||
}
|
||||
}
|
||||
|
||||
/// Pre-build the agent's `system.build.toplevel` derivation against
|
||||
/// `meta#<name>` so the subsequent `nixos-container update` /
|
||||
/// `create` finds the result already in the store. The container
|
||||
/// itself is untouched — this is purely a store-warming pass.
|
||||
///
|
||||
/// Streams nix's stdout to INFO and stderr to WARN like the
|
||||
/// `nixos-container` shellouts so progress shows up in journald as
|
||||
/// it happens. `--no-link` keeps us from littering the working
|
||||
/// directory with `result` symlinks. Per-derivation cost: pure
|
||||
/// cache hit when nothing changed (handful of seconds for the
|
||||
/// eval), expensive only on the rebuild that actually has work.
|
||||
///
|
||||
/// Attr path is `<flake-root>#nixosConfigurations.<name>.config.
|
||||
/// system.build.toplevel` — `nix build` won't auto-resolve the bare
|
||||
/// `<name>` against `nixosConfigurations` like `nixos-container` does
|
||||
/// internally, so we have to spell the path out explicitly. Falling
|
||||
/// back to `meta#<name>` (the shape `nixos-container update --flake
|
||||
/// meta#<name>` uses) makes nix look for `packages.<system>.<name>`,
|
||||
/// `legacyPackages.<system>.<name>`, or `<name>` at the flake root —
|
||||
/// none of which exist in the rendered meta flake.
|
||||
///
|
||||
/// Returns the same error shape as the other nixos-container
|
||||
/// helpers so callers can use `?` without translation.
|
||||
/// Pre-build `system.build.toplevel` against `meta#<name>` so the
|
||||
/// subsequent `nixos-container update` finds the result cached and
|
||||
/// skips straight to the profile-swap. Store-warming only — container
|
||||
/// is untouched. See `docs/coordinator.md::Rebuild path` for why
|
||||
/// the prebuild happens before stop, and `docs/coordinator.md::Prebuild
|
||||
/// attr path` for why the explicit nixosConfigurations attr is required.
|
||||
async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
// Split `<root>#<name>` so we can re-emit with the explicit
|
||||
|
|
|
|||
Loading…
Reference in a new issue