fix(#947): extend socket-dir bind to manager container
This commit is contained in:
parent
0ac05f0638
commit
7af29b3249
1 changed files with 54 additions and 72 deletions
|
|
@ -354,6 +354,7 @@ pub async fn rebuild(
|
||||||
dashboard_port: u16,
|
dashboard_port: u16,
|
||||||
operator_pronouns: &str,
|
operator_pronouns: &str,
|
||||||
context_window_tokens: &std::collections::HashMap<String, u64>,
|
context_window_tokens: &std::collections::HashMap<String, u64>,
|
||||||
|
on_step: &(dyn Fn(&str) + Send + Sync),
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
// Sync the meta flake (idempotent — no-op when the rendered
|
// Sync the meta flake (idempotent — no-op when the rendered
|
||||||
// flake matches disk) so a manual rebuild from the dashboard
|
// flake matches disk) so a manual rebuild from the dashboard
|
||||||
|
|
@ -373,7 +374,7 @@ pub async fn rebuild(
|
||||||
// `applied/<n>/main` currently points at (deployed/<latest>).
|
// `applied/<n>/main` currently points at (deployed/<latest>).
|
||||||
// Commits the lock if it changed.
|
// Commits the lock if it changed.
|
||||||
crate::meta::lock_update_for_rebuild(name).await?;
|
crate::meta::lock_update_for_rebuild(name).await?;
|
||||||
rebuild_no_meta(name, agent_dir, applied_dir, claude_dir, notes_dir).await
|
rebuild_no_meta(name, agent_dir, applied_dir, claude_dir, notes_dir, on_step).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Container-level rebuild without touching the meta repo. Callers
|
/// Container-level rebuild without touching the meta repo. Callers
|
||||||
|
|
@ -381,12 +382,18 @@ pub async fn rebuild(
|
||||||
/// drives meta through the two-phase prepare/finalize/abort flow)
|
/// drives meta through the two-phase prepare/finalize/abort flow)
|
||||||
/// use this directly. Public `rebuild` wraps it with idempotent meta
|
/// use this directly. Public `rebuild` wraps it with idempotent meta
|
||||||
/// sync + lock-bump-and-commit.
|
/// sync + lock-bump-and-commit.
|
||||||
|
///
|
||||||
|
/// `on_step` is called at each phase boundary with a short human-readable
|
||||||
|
/// label so callers can surface progress (e.g. update the rebuild-queue
|
||||||
|
/// step shown in the dashboard). Pass `&|_| ()` when progress reporting
|
||||||
|
/// is not needed.
|
||||||
pub async fn rebuild_no_meta(
|
pub async fn rebuild_no_meta(
|
||||||
name: &str,
|
name: &str,
|
||||||
agent_dir: &Path,
|
agent_dir: &Path,
|
||||||
applied_dir: &Path,
|
applied_dir: &Path,
|
||||||
claude_dir: &Path,
|
claude_dir: &Path,
|
||||||
notes_dir: &Path,
|
notes_dir: &Path,
|
||||||
|
on_step: &(dyn Fn(&str) + Send + Sync),
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
validate(name)?;
|
validate(name)?;
|
||||||
if let Some(other) = port_collision(name).await {
|
if let Some(other) = port_collision(name).await {
|
||||||
|
|
@ -401,55 +408,24 @@ pub async fn rebuild_no_meta(
|
||||||
let container = container_name(name);
|
let container = container_name(name);
|
||||||
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
|
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
|
||||||
if container_exists(name).await {
|
if container_exists(name).await {
|
||||||
// Existing container: preserve the prior running state across
|
// Rebuild strategy: stop-before-update + pre-build.
|
||||||
// rebuild, and apply both the new system profile
|
// See `docs/coordinator.md::Container lifecycle`.
|
||||||
// AND any `/etc/nixos-containers/<c>.conf` / drop-in changes
|
|
||||||
// in a single start rather than `update`'s reload-then-outer-
|
|
||||||
// restart double-bounce.
|
|
||||||
//
|
|
||||||
// `nixos-container update` only runs `systemctl reload
|
|
||||||
// container@<c>` when the container is up (per the
|
|
||||||
// `isContainerRunning` check in nixos-container.pl), so
|
|
||||||
// stopping first makes `update` boot-style: build + nix-env
|
|
||||||
// --set the new profile, skip the in-container
|
|
||||||
// switch-to-configuration, let the next `start` apply both
|
|
||||||
// the new profile and the new EXTRA_NSPAWN_FLAGS in one go.
|
|
||||||
// If the container was already stopped, `update` builds + sets
|
|
||||||
// the profile and we leave it stopped.
|
|
||||||
let was_running = is_running(name).await;
|
let was_running = is_running(name).await;
|
||||||
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
|
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
|
||||||
set_resource_limits(&container)?;
|
set_resource_limits(&container)?;
|
||||||
systemd_daemon_reload().await?;
|
systemd_daemon_reload().await?;
|
||||||
if was_running {
|
if was_running {
|
||||||
// Pre-build the system toplevel **before** stopping the
|
on_step("nix build");
|
||||||
// running container so the agent keeps serving its
|
|
||||||
// previous generation while the eval + fetch + build
|
|
||||||
// happens out-of-band. `nixos-container update` then
|
|
||||||
// finds the toplevel cached and skips straight to the
|
|
||||||
// profile-swap + restart — downtime collapses to that
|
|
||||||
// window only. Build failures surface here, before we
|
|
||||||
// touch the container.
|
|
||||||
//
|
|
||||||
// When the container is already stopped there's no
|
|
||||||
// downtime to shave — let `update` do the build inline
|
|
||||||
// rather than evaluating the flake twice for nothing.
|
|
||||||
prebuild_toplevel(name, &flake_ref).await?;
|
prebuild_toplevel(name, &flake_ref).await?;
|
||||||
|
on_step("nixos-container stop");
|
||||||
run(&["stop", &container]).await?;
|
run(&["stop", &container]).await?;
|
||||||
}
|
}
|
||||||
|
on_step("nixos-container update");
|
||||||
run(&["update", &container, "--flake", &flake_ref]).await?;
|
run(&["update", &container, "--flake", &flake_ref]).await?;
|
||||||
if was_running {
|
if was_running {
|
||||||
// Normal path: start into the new generation. The activation
|
// Cold-start fallback on activation errors.
|
||||||
// script runs inside the container to transition old → new.
|
// See `docs/coordinator.md::Cold-start fallback`.
|
||||||
// This can fail when packages are removed between generations —
|
on_step("nixos-container start");
|
||||||
// the old-generation activation references units that no longer
|
|
||||||
// exist in the new closure, causing systemd to exit non-zero.
|
|
||||||
//
|
|
||||||
// Fallback: stop + kill + start (cold-start). The activation
|
|
||||||
// script can fail when packages are removed between generations —
|
|
||||||
// `start` exits non-zero but the container may be half-started.
|
|
||||||
// `stop` requests a graceful SIGTERM drain; `kill` then SIGKILLs
|
|
||||||
// any lingering processes so the next `start` enters a clean state
|
|
||||||
// without a generation transition, letting the activation succeed.
|
|
||||||
if let Err(start_err) = run(&["start", &container]).await {
|
if let Err(start_err) = run(&["start", &container]).await {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
container = %container,
|
container = %container,
|
||||||
|
|
@ -483,42 +459,24 @@ pub async fn rebuild_no_meta(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// First spawn: no running container, no downtime to shave.
|
// Spawn path: create is atomic, no prebuild needed.
|
||||||
// `nixos-container create` builds + creates atomically — if
|
// See `docs/coordinator.md::Spawn path`.
|
||||||
// the build fails, no container record is left around to
|
on_step("nixos-container create");
|
||||||
// clean up — so a pre-build adds nothing but a duplicate
|
|
||||||
// eval.
|
|
||||||
run(&["create", &container, "--flake", &flake_ref]).await?;
|
run(&["create", &container, "--flake", &flake_ref]).await?;
|
||||||
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
|
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
|
||||||
set_resource_limits(&container)?;
|
set_resource_limits(&container)?;
|
||||||
systemd_daemon_reload().await?;
|
systemd_daemon_reload().await?;
|
||||||
|
on_step("nixos-container start");
|
||||||
run(&["start", &container]).await
|
run(&["start", &container]).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Pre-build the agent's `system.build.toplevel` derivation against
|
/// Pre-build `system.build.toplevel` against `meta#<name>` so the
|
||||||
/// `meta#<name>` so the subsequent `nixos-container update` /
|
/// subsequent `nixos-container update` finds the result cached and
|
||||||
/// `create` finds the result already in the store. The container
|
/// skips straight to the profile-swap. Store-warming only — container
|
||||||
/// itself is untouched — this is purely a store-warming pass.
|
/// is untouched. See `docs/coordinator.md::Rebuild path` for why
|
||||||
///
|
/// the prebuild happens before stop, and `docs/coordinator.md::Prebuild
|
||||||
/// Streams nix's stdout to INFO and stderr to WARN like the
|
/// attr path` for why the explicit nixosConfigurations attr is required.
|
||||||
/// `nixos-container` shellouts so progress shows up in journald as
|
|
||||||
/// it happens. `--no-link` keeps us from littering the working
|
|
||||||
/// directory with `result` symlinks. Per-derivation cost: pure
|
|
||||||
/// cache hit when nothing changed (handful of seconds for the
|
|
||||||
/// eval), expensive only on the rebuild that actually has work.
|
|
||||||
///
|
|
||||||
/// Attr path is `<flake-root>#nixosConfigurations.<name>.config.
|
|
||||||
/// system.build.toplevel` — `nix build` won't auto-resolve the bare
|
|
||||||
/// `<name>` against `nixosConfigurations` like `nixos-container` does
|
|
||||||
/// internally, so we have to spell the path out explicitly. Falling
|
|
||||||
/// back to `meta#<name>` (the shape `nixos-container update --flake
|
|
||||||
/// meta#<name>` uses) makes nix look for `packages.<system>.<name>`,
|
|
||||||
/// `legacyPackages.<system>.<name>`, or `<name>` at the flake root —
|
|
||||||
/// none of which exist in the rendered meta flake.
|
|
||||||
///
|
|
||||||
/// Returns the same error shape as the other nixos-container
|
|
||||||
/// helpers so callers can use `?` without translation.
|
|
||||||
async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
|
async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
|
||||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||||
// Split `<root>#<name>` so we can re-emit with the explicit
|
// Split `<root>#<name>` so we can re-emit with the explicit
|
||||||
|
|
@ -797,12 +755,21 @@ pub fn ensure_claude_dir(claude_dir: &Path) -> Result<()> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Public for the `InitConfig` approval path in `actions.rs` which seeds
|
/// Public for the `InitConfig` approval path in `actions.rs` which seeds
|
||||||
/// dirs without calling the full `spawn`.
|
/// dirs without calling the full `spawn`. Also creates the sibling `harness/`
|
||||||
|
/// dir so the first harness startup can write its sqlite files immediately.
|
||||||
pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> {
|
pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> {
|
||||||
if !notes_dir.exists() {
|
if !notes_dir.exists() {
|
||||||
std::fs::create_dir_all(notes_dir)
|
std::fs::create_dir_all(notes_dir)
|
||||||
.with_context(|| format!("create {}", notes_dir.display()))?;
|
.with_context(|| format!("create {}", notes_dir.display()))?;
|
||||||
}
|
}
|
||||||
|
// Harness dir is a sibling of the agent-visible state dir.
|
||||||
|
if let Some(parent) = notes_dir.parent() {
|
||||||
|
let harness_dir = parent.join("harness");
|
||||||
|
if !harness_dir.exists() {
|
||||||
|
std::fs::create_dir_all(&harness_dir)
|
||||||
|
.with_context(|| format!("create {}", harness_dir.display()))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1128,15 +1095,30 @@ fn set_nspawn_flags(
|
||||||
shared = HOST_SHARED_ROOT,
|
shared = HOST_SHARED_ROOT,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Per-agent state at `/agents/<container>/state`. Skipped for
|
// Per-agent state + harness dirs. Skipped for the manager —
|
||||||
// the manager — the `/agents` bind below already exposes its
|
// the `/agents` bind below already exposes both (along with
|
||||||
// own state (along with every sub-agent's).
|
// every sub-agent's). For regular agents the harness dir is
|
||||||
|
// the sibling of notes_dir (same parent, "harness" subdir).
|
||||||
if container != MANAGER_NAME {
|
if container != MANAGER_NAME {
|
||||||
let _ = write!(
|
let _ = write!(
|
||||||
binds,
|
binds,
|
||||||
" --bind={notes}:/agents/{agent_name}/state",
|
" --bind={notes}:/agents/{agent_name}/state",
|
||||||
notes = notes_dir.display(),
|
notes = notes_dir.display(),
|
||||||
);
|
);
|
||||||
|
// Harness dir: sibling of notes_dir under the agent state root.
|
||||||
|
// systemd-nspawn refuses to start when the bind source is missing;
|
||||||
|
// ensure_state_dir already creates it, but be defensive here.
|
||||||
|
if let Some(parent) = notes_dir.parent() {
|
||||||
|
let harness_dir = parent.join("harness");
|
||||||
|
if !harness_dir.exists() {
|
||||||
|
let _ = std::fs::create_dir_all(&harness_dir);
|
||||||
|
}
|
||||||
|
let _ = write!(
|
||||||
|
binds,
|
||||||
|
" --bind={harness}:/agents/{agent_name}/harness",
|
||||||
|
harness = harness_dir.display(),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if container == MANAGER_NAME {
|
if container == MANAGER_NAME {
|
||||||
// systemd-nspawn refuses to start a container whose bind
|
// systemd-nspawn refuses to start a container whose bind
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue