1468 lines
60 KiB
Rust
1468 lines
60 KiB
Rust
//! `nixos-container` lifecycle + per-agent config flake generation.
|
|
|
|
use std::path::Path;
|
|
|
|
use anyhow::{Context, Result, bail};
|
|
use tokio::process::Command;
|
|
|
|
/// Sub-agent container prefix. `nixos-container` caps the total container name
|
|
/// at 11 chars (it gets encoded into network interface names), so the agent
|
|
/// name itself can be at most `MAX_AGENT_NAME` chars.
|
|
pub const AGENT_PREFIX: &str = "h-";
|
|
pub const MAX_AGENT_NAME: usize = 9;
|
|
/// Container name of the manager. Lives in the same path scheme as sub-agents
|
|
/// (`/var/lib/hyperhive/agents/hm1nd/`, `/var/lib/hyperhive/applied/hm1nd/`),
|
|
/// but its container has no `h-` prefix and extends a different
|
|
/// nixosConfiguration (`manager`, not `agent-base`).
|
|
pub const MANAGER_NAME: &str = "hm1nd";
|
|
|
|
/// Mount point of the per-agent runtime directory inside the container.
|
|
pub const CONTAINER_RUNTIME_MOUNT: &str = "/run/hive";
|
|
|
|
/// Where the per-agent Claude credentials dir mounts inside the
|
|
/// container. The harness service runs as a non-root unix user
|
|
/// whose home is `/home/<agent>/`, so the mount path varies per
|
|
/// agent — `container_claude_mount(name)` returns
|
|
/// `/home/<name>/.claude` for sub-agents and `/home/hm1nd/.claude`
|
|
/// for the manager. `claude` inside the container reads
|
|
/// `$HOME/.claude` and the service environment sets `HOME` to the
|
|
/// same path, so the OAuth session survives container restarts.
|
|
#[must_use]
|
|
pub fn container_claude_mount(name: &str) -> String {
|
|
format!("/home/{name}/.claude")
|
|
}
|
|
|
|
/// Mount point of the shared directory accessible to all agents.
|
|
/// All agents can read/write here; agents should only put things they're
|
|
/// willing to lose (other agents may delete them).
|
|
pub const CONTAINER_SHARED_MOUNT: &str = "/shared";
|
|
|
|
const GIT_NAME: &str = "c0re";
|
|
const GIT_EMAIL: &str = "c0re@hyperhive";
|
|
|
|
/// Sub-agent web UI port range. Deterministic from the agent's name (FNV-1a
|
|
/// hash mod range size), so the dashboard can compute the same port without
|
|
/// asking hive-c0re.
|
|
const WEB_PORT_BASE: u16 = 8100;
|
|
const WEB_PORT_RANGE: u16 = 900;
|
|
|
|
/// Default resource caps applied to every managed container via a systemd
|
|
/// drop-in under `/run/systemd/system/container@<NAME>.service.d/`.
|
|
const DEFAULT_MEMORY_MAX: &str = "2G";
|
|
const DEFAULT_CPU_QUOTA: &str = "50%";
|
|
|
|
/// Per-agent web UI port — `WEB_PORT_BASE + FNV-1a(name) %
|
|
/// WEB_PORT_RANGE` for every agent including the manager. The port
|
|
/// allocation rule reads the same for every name; collisions are
|
|
/// possible (birthday paradox at ~30 agents) and the operator
|
|
/// resolves them by renaming an agent (different hash → different
|
|
/// port). Stable across hosts, restarts, and dashboard renders —
|
|
/// no state-file dance.
|
|
#[must_use]
|
|
pub fn agent_web_port(name: &str) -> u16 {
|
|
let mut hash: u32 = 2_166_136_261;
|
|
for b in name.bytes() {
|
|
hash ^= u32::from(b);
|
|
hash = hash.wrapping_mul(16_777_619);
|
|
}
|
|
// Modulo of a u32 by a u16's value is guaranteed < u16::MAX, so try_from never fails.
|
|
WEB_PORT_BASE + u16::try_from(hash % u32::from(WEB_PORT_RANGE)).unwrap_or(0)
|
|
}
|
|
|
|
#[must_use]
|
|
pub fn container_name(name: &str) -> String {
|
|
if name == MANAGER_NAME {
|
|
MANAGER_NAME.to_owned()
|
|
} else {
|
|
format!("{AGENT_PREFIX}{name}")
|
|
}
|
|
}
|
|
|
|
#[must_use]
|
|
pub fn is_manager(name: &str) -> bool {
|
|
name == MANAGER_NAME
|
|
}
|
|
|
|
/// Read the agent user's `(uid, gid)` from the container's nixos-managed
|
|
/// `/etc/passwd`. Returns `None` when the container hasn't been built
|
|
/// yet, the passwd file is unparseable, or the agent user is missing
|
|
/// (e.g. legacy container that still runs as root).
|
|
///
|
|
/// Used by `forge` + `matrix` after writing per-agent state files so
|
|
/// the bind-mounted host file ends up readable by the agent user
|
|
/// without waiting for the next container activation to run the chown
|
|
/// fixup.
|
|
///
|
|
/// Notes:
|
|
/// - Reads the *container-local* passwd at
|
|
/// `/var/lib/nixos-containers/<container>/etc/passwd`, not the host's.
|
|
/// The container's user-namespace shares uids with the host (no
|
|
/// `PrivateUsers`), so the uid is directly usable in host-side
|
|
/// `chown(2)`.
|
|
/// - Best-effort: caller treats `None` as "skip the chown".
|
|
#[must_use]
|
|
pub fn agent_uid_gid(agent_name: &str) -> Option<(u32, u32)> {
|
|
let container = container_name(agent_name);
|
|
let passwd_path = format!("/var/lib/nixos-containers/{container}/etc/passwd");
|
|
let content = std::fs::read_to_string(&passwd_path).ok()?;
|
|
for line in content.lines() {
|
|
let mut parts = line.split(':');
|
|
let user = parts.next()?;
|
|
if user != agent_name {
|
|
continue;
|
|
}
|
|
let _ = parts.next()?; // x (password placeholder)
|
|
let uid: u32 = parts.next()?.parse().ok()?;
|
|
let gid: u32 = parts.next()?.parse().ok()?;
|
|
return Some((uid, gid));
|
|
}
|
|
None
|
|
}
|
|
|
|
/// Best-effort `chown(path, agent_uid, agent_gid)`. Resolves the agent's
|
|
/// uid/gid via [`agent_uid_gid`] and shells out to `std::os::unix::fs::chown`.
|
|
/// Silently no-ops when the container isn't built yet (`None` from
|
|
/// [`agent_uid_gid`]) and logs at debug on chown syscall failure — the
|
|
/// activation script in `harness-base.nix` is the steady-state safety
|
|
/// net. Used by per-agent state writers in `forge` + `matrix` so the
|
|
/// agent can read the file without waiting for the next container
|
|
/// rebuild.
|
|
pub fn chown_to_agent(name: &str, path: &Path, subsystem: &str) {
|
|
let Some((uid, gid)) = agent_uid_gid(name) else {
|
|
return;
|
|
};
|
|
if let Err(e) = std::os::unix::fs::chown(path, Some(uid), Some(gid)) {
|
|
tracing::debug!(%name, %subsystem, path = %path.display(), error = %e, "chown to agent failed");
|
|
}
|
|
}
|
|
|
|
fn validate(name: &str) -> Result<()> {
|
|
if name.is_empty() {
|
|
bail!("agent name must not be empty");
|
|
}
|
|
if is_manager(name) {
|
|
return Ok(());
|
|
}
|
|
if name.len() > MAX_AGENT_NAME {
|
|
bail!(
|
|
"agent name '{name}' is too long ({} chars); max {MAX_AGENT_NAME}",
|
|
name.len()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// First name (≠ `self_name`) currently running whose hashed port
|
|
/// matches this agent's. The harness inside the colliding container
|
|
/// would otherwise loop on `AddrInUse` forever; we surface the
|
|
/// conflict here so spawn / rebuild fails loudly with an actionable
|
|
/// message instead.
|
|
async fn port_collision(self_name: &str) -> Option<String> {
|
|
let port = agent_web_port(self_name);
|
|
let raw = list().await.unwrap_or_default();
|
|
for c in raw {
|
|
let other = if c == MANAGER_NAME {
|
|
MANAGER_NAME.to_owned()
|
|
} else if let Some(n) = c.strip_prefix(AGENT_PREFIX) {
|
|
n.to_owned()
|
|
} else {
|
|
continue;
|
|
};
|
|
if other == self_name {
|
|
continue;
|
|
}
|
|
if agent_web_port(&other) == port && is_running(&other).await {
|
|
return Some(other);
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub async fn spawn(
|
|
name: &str,
|
|
hyperhive_flake: &str,
|
|
agent_dir: &Path,
|
|
proposed_dir: &Path,
|
|
applied_dir: &Path,
|
|
claude_dir: &Path,
|
|
notes_dir: &Path,
|
|
dashboard_port: u16,
|
|
operator_pronouns: &str,
|
|
context_window_tokens: &std::collections::HashMap<String, u64>,
|
|
) -> Result<()> {
|
|
validate(name)?;
|
|
if let Some(other) = port_collision(name).await {
|
|
bail!(
|
|
"port {} is already taken by '{other}' — rename one of them and retry",
|
|
agent_web_port(name)
|
|
);
|
|
}
|
|
setup_proposed(proposed_dir, name).await?;
|
|
setup_applied(applied_dir, Some(proposed_dir), name).await?;
|
|
ensure_claude_dir(claude_dir)?;
|
|
ensure_state_dir(notes_dir)?;
|
|
// Meta flake gets the new agent's input + nixosConfiguration
|
|
// before `nixos-container create` so the `--flake meta#<name>`
|
|
// ref resolves.
|
|
let agents = agents_after_spawn(name).await?;
|
|
crate::meta::sync_agents(
|
|
hyperhive_flake,
|
|
dashboard_port,
|
|
operator_pronouns,
|
|
context_window_tokens,
|
|
&agents,
|
|
)
|
|
.await?;
|
|
let container = container_name(name);
|
|
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
|
|
run(&["create", &container, "--flake", &flake_ref]).await?;
|
|
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
|
|
set_resource_limits(&container)?;
|
|
systemd_daemon_reload().await?;
|
|
run(&["start", &container]).await
|
|
}
|
|
|
|
/// Build the `AgentSpec` list for the meta flake from `nixos-container
|
|
/// list` + a hypothetical extra name not yet in the list (for spawn
|
|
/// where the new agent's container doesn't exist yet). Pass empty
|
|
/// `name_to_add` from rebuild paths where the agent is already in the
|
|
/// container list.
|
|
async fn agents_for_meta(name_to_add: Option<&str>) -> Result<Vec<crate::meta::AgentSpec>> {
|
|
let containers = list().await.unwrap_or_default();
|
|
let mut out: Vec<crate::meta::AgentSpec> = containers
|
|
.into_iter()
|
|
.filter_map(|c| {
|
|
let (name, is_manager) = if c == MANAGER_NAME {
|
|
(MANAGER_NAME.to_owned(), true)
|
|
} else if let Some(n) = c.strip_prefix(AGENT_PREFIX) {
|
|
(n.to_owned(), false)
|
|
} else {
|
|
return None;
|
|
};
|
|
Some(crate::meta::AgentSpec {
|
|
port: agent_web_port(&name),
|
|
name,
|
|
is_manager,
|
|
})
|
|
})
|
|
.collect();
|
|
if let Some(extra) = name_to_add
|
|
&& !out.iter().any(|a| a.name == extra)
|
|
{
|
|
out.push(crate::meta::AgentSpec {
|
|
name: extra.to_owned(),
|
|
is_manager: is_manager(extra),
|
|
port: agent_web_port(extra),
|
|
});
|
|
}
|
|
out.sort_by(|a, b| a.name.cmp(&b.name));
|
|
Ok(out)
|
|
}
|
|
|
|
async fn agents_after_spawn(name: &str) -> Result<Vec<crate::meta::AgentSpec>> {
|
|
agents_for_meta(Some(name)).await
|
|
}
|
|
|
|
/// Like `agents_for_meta_listing` but with an extra agent added (for a
|
|
/// container that doesn't exist yet). Used by the first-spawn path in
|
|
/// `actions::run_apply_commit` to register the new agent in meta before
|
|
/// `prepare_deploy` tries to update its input lock.
|
|
pub async fn agents_for_meta_listing_with(extra: &str) -> Result<Vec<crate::meta::AgentSpec>> {
|
|
agents_for_meta(Some(extra)).await
|
|
}
|
|
|
|
/// Public enumeration of currently-existing agents (whatever
|
|
/// `nixos-container list` says), sorted, no extras. For callers
|
|
/// outside this module that need to reseed meta after lifecycle
|
|
/// changes — destroy, startup reconciliation, etc.
|
|
pub async fn agents_for_meta_listing() -> Result<Vec<crate::meta::AgentSpec>> {
|
|
agents_for_meta(None).await
|
|
}
|
|
|
|
/// True when the named container already exists (appears in
|
|
/// `nixos-container list`). Used by the apply-commit path to decide
|
|
/// between first-spawn (`nixos-container create`) and normal rebuild
|
|
/// (`nixos-container update`).
|
|
pub async fn container_exists(name: &str) -> bool {
|
|
let container = container_name(name);
|
|
list()
|
|
.await
|
|
.unwrap_or_default()
|
|
.iter()
|
|
.any(|c| c == &container)
|
|
}
|
|
|
|
pub async fn kill(name: &str) -> Result<()> {
|
|
validate(name)?;
|
|
let container = container_name(name);
|
|
run(&["stop", &container]).await
|
|
}
|
|
|
|
pub async fn start(name: &str) -> Result<()> {
|
|
validate(name)?;
|
|
let container = container_name(name);
|
|
run(&["start", &container]).await
|
|
}
|
|
|
|
/// Stop + start without regenerating any config. For "kick the container"
|
|
/// without touching the flake or nspawn flags.
|
|
pub async fn restart(name: &str) -> Result<()> {
|
|
kill(name).await?;
|
|
start(name).await
|
|
}
|
|
|
|
/// True when the container's systemd unit is active. Used by the dashboard
|
|
/// to gate stop/restart buttons.
|
|
pub async fn is_running(name: &str) -> bool {
|
|
let container = container_name(name);
|
|
let unit = format!("container@{container}.service");
|
|
Command::new("systemctl")
|
|
.args(["is-active", "--quiet", &unit])
|
|
.status()
|
|
.await
|
|
.map(|s| s.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// Fully tear down a sub-agent's container: stop + remove via `nixos-container
|
|
/// destroy`, then clean our own systemd drop-in. Leaves it to the caller to
|
|
/// wipe `/var/lib/hyperhive/...` state and the per-agent runtime dir.
|
|
pub async fn destroy(name: &str) -> Result<()> {
|
|
validate(name)?;
|
|
let container = container_name(name);
|
|
// nixos-container destroy handles stop + removal of /var/lib/nixos-containers/<C>
|
|
// and /etc/nixos-containers/<C>.conf. Tolerate "no such container".
|
|
if let Err(e) = run(&["destroy", &container]).await {
|
|
tracing::warn!(error = ?e, "nixos-container destroy returned an error; continuing cleanup");
|
|
}
|
|
let dropin_dir = format!("/run/systemd/system/container@{container}.service.d");
|
|
if std::path::Path::new(&dropin_dir).exists() {
|
|
std::fs::remove_dir_all(&dropin_dir).with_context(|| format!("remove {dropin_dir}"))?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub async fn rebuild(
|
|
name: &str,
|
|
hyperhive_flake: &str,
|
|
agent_dir: &Path,
|
|
applied_dir: &Path,
|
|
claude_dir: &Path,
|
|
notes_dir: &Path,
|
|
dashboard_port: u16,
|
|
operator_pronouns: &str,
|
|
context_window_tokens: &std::collections::HashMap<String, u64>,
|
|
) -> Result<()> {
|
|
// Sync the meta flake (idempotent — no-op when the rendered
|
|
// flake matches disk) so a manual rebuild from the dashboard
|
|
// can also recover from a divergent meta repo (e.g. an agent
|
|
// got added directly via `nixos-container create` outside
|
|
// hive-c0re).
|
|
let agents = agents_for_meta(None).await?;
|
|
crate::meta::sync_agents(
|
|
hyperhive_flake,
|
|
dashboard_port,
|
|
operator_pronouns,
|
|
context_window_tokens,
|
|
&agents,
|
|
)
|
|
.await?;
|
|
// Then bump just this agent's input — picks up whatever
|
|
// `applied/<n>/main` currently points at (deployed/<latest>).
|
|
// Commits the lock if it changed.
|
|
crate::meta::lock_update_for_rebuild(name).await?;
|
|
rebuild_no_meta(name, agent_dir, applied_dir, claude_dir, notes_dir).await
|
|
}
|
|
|
|
/// Container-level rebuild without touching the meta repo. Callers
|
|
/// that own the meta side themselves (`actions::run_apply_commit`
|
|
/// drives meta through the two-phase prepare/finalize/abort flow)
|
|
/// use this directly. Public `rebuild` wraps it with idempotent meta
|
|
/// sync + lock-bump-and-commit.
|
|
pub async fn rebuild_no_meta(
|
|
name: &str,
|
|
agent_dir: &Path,
|
|
applied_dir: &Path,
|
|
claude_dir: &Path,
|
|
notes_dir: &Path,
|
|
) -> Result<()> {
|
|
validate(name)?;
|
|
if let Some(other) = port_collision(name).await {
|
|
bail!(
|
|
"port {} is already taken by '{other}' — rename one of them and retry",
|
|
agent_web_port(name)
|
|
);
|
|
}
|
|
setup_applied(applied_dir, None, name).await?;
|
|
ensure_claude_dir(claude_dir)?;
|
|
ensure_state_dir(notes_dir)?;
|
|
let container = container_name(name);
|
|
let flake_ref = format!("{}#{name}", crate::meta::meta_dir().display());
|
|
if container_exists(name).await {
|
|
// Existing container: preserve the prior running state across
|
|
// rebuild, and apply both the new system profile
|
|
// AND any `/etc/nixos-containers/<c>.conf` / drop-in changes
|
|
// in a single start rather than `update`'s reload-then-outer-
|
|
// restart double-bounce.
|
|
//
|
|
// `nixos-container update` only runs `systemctl reload
|
|
// container@<c>` when the container is up (per the
|
|
// `isContainerRunning` check in nixos-container.pl), so
|
|
// stopping first makes `update` boot-style: build + nix-env
|
|
// --set the new profile, skip the in-container
|
|
// switch-to-configuration, let the next `start` apply both
|
|
// the new profile and the new EXTRA_NSPAWN_FLAGS in one go.
|
|
// If the container was already stopped, `update` builds + sets
|
|
// the profile and we leave it stopped.
|
|
let was_running = is_running(name).await;
|
|
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
|
|
set_resource_limits(&container)?;
|
|
systemd_daemon_reload().await?;
|
|
if was_running {
|
|
// Pre-build the system toplevel **before** stopping the
|
|
// running container so the agent keeps serving its
|
|
// previous generation while the eval + fetch + build
|
|
// happens out-of-band. `nixos-container update` then
|
|
// finds the toplevel cached and skips straight to the
|
|
// profile-swap + restart — downtime collapses to that
|
|
// window only. Build failures surface here, before we
|
|
// touch the container.
|
|
//
|
|
// When the container is already stopped there's no
|
|
// downtime to shave — let `update` do the build inline
|
|
// rather than evaluating the flake twice for nothing.
|
|
prebuild_toplevel(name, &flake_ref).await?;
|
|
run(&["stop", &container]).await?;
|
|
}
|
|
run(&["update", &container, "--flake", &flake_ref]).await?;
|
|
if was_running {
|
|
// Normal path: start into the new generation. The activation
|
|
// script runs inside the container to transition old → new.
|
|
// This can fail when packages are removed between generations —
|
|
// the old-generation activation references units that no longer
|
|
// exist in the new closure, causing systemd to exit non-zero.
|
|
//
|
|
// Fallback: stop + kill + start (cold-start). The activation
|
|
// script can fail when packages are removed between generations —
|
|
// `start` exits non-zero but the container may be half-started.
|
|
// `stop` requests a graceful SIGTERM drain; `kill` then SIGKILLs
|
|
// any lingering processes so the next `start` enters a clean state
|
|
// without a generation transition, letting the activation succeed.
|
|
if let Err(start_err) = run(&["start", &container]).await {
|
|
tracing::warn!(
|
|
container = %container,
|
|
error = %start_err,
|
|
"start after rebuild failed (possible activation error); \
|
|
retrying via stop + kill + start"
|
|
);
|
|
run(&["stop", &container]).await.unwrap_or_else(|e| {
|
|
tracing::warn!(
|
|
container = %container,
|
|
error = %e,
|
|
"stop before cold-start retry failed (ignored)"
|
|
);
|
|
});
|
|
run(&["kill", &container]).await.unwrap_or_else(|e| {
|
|
tracing::warn!(
|
|
container = %container,
|
|
error = %e,
|
|
"kill before cold-start retry failed (ignored)"
|
|
);
|
|
});
|
|
run(&["start", &container]).await
|
|
.map_err(|e| anyhow::anyhow!(
|
|
"cold-start fallback also failed: {e:#} \
|
|
(original start error: {start_err:#})"
|
|
))
|
|
} else {
|
|
Ok(())
|
|
}
|
|
} else {
|
|
Ok(())
|
|
}
|
|
} else {
|
|
// First spawn: no running container, no downtime to shave.
|
|
// `nixos-container create` builds + creates atomically — if
|
|
// the build fails, no container record is left around to
|
|
// clean up — so a pre-build adds nothing but a duplicate
|
|
// eval.
|
|
run(&["create", &container, "--flake", &flake_ref]).await?;
|
|
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir)?;
|
|
set_resource_limits(&container)?;
|
|
systemd_daemon_reload().await?;
|
|
run(&["start", &container]).await
|
|
}
|
|
}
|
|
|
|
/// Pre-build the agent's `system.build.toplevel` derivation against
|
|
/// `meta#<name>` so the subsequent `nixos-container update` /
|
|
/// `create` finds the result already in the store. The container
|
|
/// itself is untouched — this is purely a store-warming pass.
|
|
///
|
|
/// Streams nix's stdout to INFO and stderr to WARN like the
|
|
/// `nixos-container` shellouts so progress shows up in journald as
|
|
/// it happens. `--no-link` keeps us from littering the working
|
|
/// directory with `result` symlinks. Per-derivation cost: pure
|
|
/// cache hit when nothing changed (handful of seconds for the
|
|
/// eval), expensive only on the rebuild that actually has work.
|
|
///
|
|
/// Attr path is `<flake-root>#nixosConfigurations.<name>.config.
|
|
/// system.build.toplevel` — `nix build` won't auto-resolve the bare
|
|
/// `<name>` against `nixosConfigurations` like `nixos-container` does
|
|
/// internally, so we have to spell the path out explicitly. Falling
|
|
/// back to `meta#<name>` (the shape `nixos-container update --flake
|
|
/// meta#<name>` uses) makes nix look for `packages.<system>.<name>`,
|
|
/// `legacyPackages.<system>.<name>`, or `<name>` at the flake root —
|
|
/// none of which exist in the rendered meta flake.
|
|
///
|
|
/// Returns the same error shape as the other nixos-container
|
|
/// helpers so callers can use `?` without translation.
|
|
async fn prebuild_toplevel(name: &str, flake_ref: &str) -> Result<()> {
|
|
use tokio::io::{AsyncBufReadExt, BufReader};
|
|
// Split `<root>#<name>` so we can re-emit with the explicit
|
|
// `nixosConfigurations.<name>` segment. The flake_ref shape is
|
|
// constructed by `rebuild_no_meta` and always contains exactly one
|
|
// `#`; `split_once` returning None here would be a programmer
|
|
// error we'd want to surface loudly rather than paper over.
|
|
let (flake_root, fragment) = flake_ref
|
|
.split_once('#')
|
|
.with_context(|| format!("flake_ref {flake_ref:?} missing '#<name>' fragment"))?;
|
|
// Sanity-check the fragment matches the agent name we were
|
|
// passed — guards against future calls that pass a divergent
|
|
// pair (no current callsite does, but the pair is redundant
|
|
// and worth checking once).
|
|
if fragment != name {
|
|
anyhow::bail!(
|
|
"prebuild_toplevel: flake_ref fragment '{fragment}' ≠ agent name '{name}'"
|
|
);
|
|
}
|
|
let attr = format!("{flake_root}#nixosConfigurations.{name}.config.system.build.toplevel");
|
|
let args = vec![
|
|
"--extra-experimental-features",
|
|
"nix-command flakes",
|
|
"build",
|
|
"--no-link",
|
|
"--print-out-paths",
|
|
&attr,
|
|
];
|
|
let cmdline = format!("nix {}", args.join(" "));
|
|
tracing::info!(%name, %cmdline, "prebuild: warming system toplevel");
|
|
|
|
// Open a build_logs row for this attempt (best-effort — None when
|
|
// the global handle hasn't been installed, e.g. early startup
|
|
// or standalone tests). Lines pumped from stdout/stderr append
|
|
// into the row; `finish` lands the terminal status before we bail.
|
|
let logs = crate::build_logs::global();
|
|
let log_id = logs.as_ref().and_then(|h| {
|
|
h.start(name, "prebuild", &cmdline)
|
|
.map_err(|e| {
|
|
tracing::warn!(error = ?e, "build_logs: start failed (prebuild log dropped)");
|
|
})
|
|
.ok()
|
|
});
|
|
|
|
let mut child = Command::new("nix")
|
|
.args(&args)
|
|
.stdout(std::process::Stdio::piped())
|
|
.stderr(std::process::Stdio::piped())
|
|
.spawn()
|
|
.with_context(|| format!("spawn {cmdline}"))?;
|
|
|
|
let stdout = child.stdout.take().expect("piped stdout");
|
|
let stderr = child.stderr.take().expect("piped stderr");
|
|
|
|
let stdout_cmdline = cmdline.clone();
|
|
let stdout_logs = logs.clone();
|
|
let pump_stdout = tokio::spawn(async move {
|
|
let mut lines = BufReader::new(stdout).lines();
|
|
while let Ok(Some(line)) = lines.next_line().await {
|
|
tracing::info!(target: "nix-prebuild", cmdline = %stdout_cmdline, "{line}");
|
|
if let (Some(h), Some(id)) = (&stdout_logs, log_id) {
|
|
h.append_stdout(id, &line);
|
|
}
|
|
}
|
|
});
|
|
|
|
let stderr_cmdline = cmdline.clone();
|
|
let stderr_logs = logs.clone();
|
|
let pump_stderr = tokio::spawn(async move {
|
|
let mut lines = BufReader::new(stderr).lines();
|
|
while let Ok(Some(line)) = lines.next_line().await {
|
|
tracing::warn!(target: "nix-prebuild", cmdline = %stderr_cmdline, "{line}");
|
|
if let (Some(h), Some(id)) = (&stderr_logs, log_id) {
|
|
h.append_stderr(id, &line);
|
|
}
|
|
}
|
|
});
|
|
|
|
let status = child
|
|
.wait()
|
|
.await
|
|
.with_context(|| format!("wait {cmdline}"))?;
|
|
let _ = pump_stdout.await;
|
|
let _ = pump_stderr.await;
|
|
|
|
let ok = status.success();
|
|
if let (Some(h), Some(id)) = (&logs, log_id) {
|
|
h.finish(
|
|
id,
|
|
if ok {
|
|
crate::build_logs::BuildStatus::Ok
|
|
} else {
|
|
crate::build_logs::BuildStatus::Fail
|
|
},
|
|
);
|
|
}
|
|
if !ok {
|
|
match log_id {
|
|
Some(id) => bail!("prebuild {cmdline} failed ({status}); see build log #{id}"),
|
|
None => bail!("prebuild {cmdline} failed ({status})"),
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub async fn list() -> Result<Vec<String>> {
|
|
let out = Command::new("nixos-container")
|
|
.arg("list")
|
|
.output()
|
|
.await
|
|
.context("invoke nixos-container list")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"nixos-container list exited with status {}: {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok(String::from_utf8_lossy(&out.stdout)
|
|
.lines()
|
|
.map(str::trim)
|
|
.filter(|line| line.starts_with(AGENT_PREFIX) || *line == MANAGER_NAME)
|
|
.map(str::to_owned)
|
|
.collect())
|
|
}
|
|
|
|
/// Initialize the manager-editable proposed repo. Seeds two tracked
|
|
/// files: `agent.nix` (the module the manager edits) and `flake.nix`
|
|
/// (the boilerplate that lets the meta flake import this repo as an
|
|
/// input — meta locks at a specific sha and reads
|
|
/// `nixosModules.default`, so `flake.nix` must be in the commit). The
|
|
/// manager shouldn't edit `flake.nix` (the prompt says so) but it's
|
|
/// visible so they can introspect.
|
|
///
|
|
/// Touched by hive-c0re only on first spawn — never again — so the
|
|
/// manager can't be surprised by hive-c0re commits or working-tree
|
|
/// resets.
|
|
pub async fn setup_proposed(proposed_dir: &Path, name: &str) -> Result<()> {
|
|
let fresh = !proposed_dir.join(".git").exists();
|
|
if fresh {
|
|
std::fs::create_dir_all(proposed_dir)
|
|
.with_context(|| format!("create {}", proposed_dir.display()))?;
|
|
let agent_path = proposed_dir.join("agent.nix");
|
|
if !agent_path.exists() {
|
|
std::fs::write(&agent_path, initial_agent_nix(name))
|
|
.with_context(|| format!("write {}", agent_path.display()))?;
|
|
}
|
|
let flake_path = proposed_dir.join("flake.nix");
|
|
if !flake_path.exists() {
|
|
std::fs::write(&flake_path, initial_flake_nix())
|
|
.with_context(|| format!("write {}", flake_path.display()))?;
|
|
}
|
|
git(proposed_dir, &["init", "--initial-branch=main"]).await?;
|
|
git(proposed_dir, &["add", "agent.nix", "flake.nix"]).await?;
|
|
git_commit(proposed_dir, "hive-c0re init").await?;
|
|
}
|
|
// Idempotently wire the `applied` remote — purely for the
|
|
// manager's ergonomics. The URL is the path inside the manager
|
|
// container (`/applied/<n>/.git`), where the RO bind in
|
|
// `set_nspawn_flags` makes it real. hive-c0re itself never
|
|
// dereferences this remote; the host-side fetch in
|
|
// `request_apply_commit` uses absolute host paths.
|
|
ensure_applied_remote(proposed_dir, name).await
|
|
}
|
|
|
|
async fn ensure_applied_remote(proposed_dir: &Path, name: &str) -> Result<()> {
|
|
let want = format!("/applied/{name}/.git");
|
|
let existing = git_command()
|
|
.current_dir(proposed_dir)
|
|
.args(["remote", "get-url", "applied"])
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("git remote get-url applied in {}", proposed_dir.display()))?;
|
|
if existing.status.success() {
|
|
let current = String::from_utf8_lossy(&existing.stdout).trim().to_owned();
|
|
if current == want {
|
|
return Ok(());
|
|
}
|
|
// URL drifted (path scheme changed, etc.) — re-point it.
|
|
return git(proposed_dir, &["remote", "set-url", "applied", &want]).await;
|
|
}
|
|
git(proposed_dir, &["remote", "add", "applied", &want]).await
|
|
}
|
|
|
|
/// Set up the applied repo. First-spawn only: init the repo, pull
|
|
/// proposed's initial commit in via `git fetch`, tag it `deployed/0`.
|
|
/// This is the *only* time hive-c0re reads from `proposed` for an
|
|
/// agent — subsequent proposals are fetched at `request_apply_commit`
|
|
/// time and tagged `proposal/<id>` (see `actions::approve` for the
|
|
/// tag state machine).
|
|
///
|
|
/// `proposed_dir` is `None` on rebuild paths where the repo already
|
|
/// exists — we just verify it's the right shape and bail otherwise.
|
|
/// Unlike the pre-overhaul code path, `flake.nix` is no longer
|
|
/// regenerated at the host level: it's tracked in proposed (seeded by
|
|
/// `setup_proposed`) and rides along on every fetch.
|
|
pub async fn setup_applied(
|
|
applied_dir: &Path,
|
|
proposed_dir: Option<&Path>,
|
|
name: &str,
|
|
) -> Result<()> {
|
|
std::fs::create_dir_all(applied_dir)
|
|
.with_context(|| format!("create {}", applied_dir.display()))?;
|
|
|
|
if !applied_dir.join(".git").exists() {
|
|
let Some(proposed) = proposed_dir else {
|
|
bail!(
|
|
"applied repo at {} is missing its .git directory; \
|
|
cannot rebuild without a proposed source to seed from. \
|
|
destroy --purge and re-spawn this agent.",
|
|
applied_dir.display()
|
|
);
|
|
};
|
|
git(applied_dir, &["init", "--initial-branch=main"]).await?;
|
|
let proposed_str = proposed.display().to_string();
|
|
// Seed the applied repo at the root (template) commit of proposed,
|
|
// not at `main`. This ensures `deployed/0` is the template baseline
|
|
// so the first ApplyCommit diff shows the manager's real changes
|
|
// rather than an empty diff (which happens when the manager has
|
|
// already committed their config and proposed/main == proposal/<id>).
|
|
let root_sha = git_root_commit(proposed).await?;
|
|
git(
|
|
applied_dir,
|
|
// --update-head-ok lets us fetch into refs/heads/main while
|
|
// HEAD still points there. git's default safeguard refuses
|
|
// to avoid index/working-tree desync, but the working tree
|
|
// is empty (we just `init`'d) and we read-tree-reset right
|
|
// after, so the safeguard is moot here.
|
|
&[
|
|
"fetch",
|
|
"--no-tags",
|
|
"--update-head-ok",
|
|
&proposed_str,
|
|
&format!("{root_sha}:refs/heads/main"),
|
|
],
|
|
)
|
|
.await?;
|
|
git_read_tree_reset(applied_dir, "refs/heads/main").await?;
|
|
git_tag(applied_dir, "deployed/0", "refs/heads/main").await?;
|
|
} else if git_rev_parse(applied_dir, "refs/tags/deployed/0")
|
|
.await
|
|
.is_err()
|
|
{
|
|
// Pre-overhaul applied repo — no deployed/* tag scheme,
|
|
// flake.nix may be untracked, agent.nix possibly authored by
|
|
// hive-c0re directly. The startup auto-migration fixes this
|
|
// in place; if it didn't run (or got skipped), surface a
|
|
// clear error.
|
|
bail!(
|
|
"applied repo at {} predates the meta-flake layout. \
|
|
Restart hive-c0re to let the auto-migration run, or \
|
|
destroy --purge {name} and re-spawn.",
|
|
applied_dir.display()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Create the per-agent Claude credentials dir if missing. Mode 0700 — only
|
|
/// root inside the container reads/writes it. Idempotent: existing dirs are
|
|
/// left untouched (an agent's OAuth tokens survive `destroy`/recreate).
|
|
/// Public for the `InitConfig` approval path in `actions.rs` which seeds
|
|
/// dirs without calling the full `spawn`.
|
|
pub fn ensure_claude_dir(claude_dir: &Path) -> Result<()> {
|
|
if !claude_dir.exists() {
|
|
std::fs::create_dir_all(claude_dir)
|
|
.with_context(|| format!("create {}", claude_dir.display()))?;
|
|
#[cfg(unix)]
|
|
{
|
|
use std::os::unix::fs::PermissionsExt;
|
|
std::fs::set_permissions(claude_dir, std::fs::Permissions::from_mode(0o700))
|
|
.with_context(|| format!("chmod {}", claude_dir.display()))?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Public for the `InitConfig` approval path in `actions.rs` which seeds
|
|
/// dirs without calling the full `spawn`.
|
|
pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> {
|
|
if !notes_dir.exists() {
|
|
std::fs::create_dir_all(notes_dir)
|
|
.with_context(|| format!("create {}", notes_dir.display()))?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn initial_agent_nix(name: &str) -> String {
|
|
format!(
|
|
"{{ config, pkgs, lib, ... }}:\n{{\n # Per-agent overrides for {name}. This is a regular NixOS module\n # — add packages, services, modules, imports as needed.\n #\n # imports = [ ./extra-module.nix ];\n # environment.systemPackages = with pkgs; [ ];\n}}\n",
|
|
)
|
|
}
|
|
|
|
/// Module-only flake exposed by every agent's repo. Consumed by the
|
|
/// hive-c0re-owned meta flake at `/var/lib/hyperhive/meta/` as a flake
|
|
/// input. The wrapper is intentionally permissive:
|
|
///
|
|
/// - Manager edits `inputs.* = …` to add other flakes (e.g. an MCP
|
|
/// server's own flake) — the lock for those lands in the agent's
|
|
/// own `flake.lock` and rolls up into meta's lock transitively.
|
|
/// - The outputs block forwards every input (minus `self`) into
|
|
/// `agent.nix` as the `flakeInputs` module argument, so the
|
|
/// manager just references `flakeInputs.<name>.packages.${pkgs.system}.default`
|
|
/// without further plumbing.
|
|
///
|
|
/// Identity injection (`HIVE_PORT` / `HIVE_LABEL` / dashboard port /
|
|
/// git committer) still lives in the meta flake's wrapper.
|
|
pub fn initial_flake_nix() -> &'static str {
|
|
"{\n description = \"hyperhive agent\";\n inputs = { };\n outputs =\n { self, ... }@inputs:\n {\n nixosModules.default = {\n imports = [ ./agent.nix ];\n _module.args.flakeInputs = builtins.removeAttrs inputs [ \"self\" ];\n };\n };\n}\n"
|
|
}
|
|
|
|
/// Return the SHA of the root (oldest, no-parent) commit in a repo.
|
|
/// Used to seed the applied repo at the template baseline rather than at
|
|
/// `main`, so the first `ApplyCommit` diff shows the manager's real changes.
|
|
async fn git_root_commit(dir: &Path) -> Result<String> {
|
|
let out = git_command()
|
|
.current_dir(dir)
|
|
.args(["rev-list", "--max-parents=0", "HEAD"])
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("git rev-list --max-parents=0 HEAD in {}", dir.display()))?;
|
|
if !out.status.success() {
|
|
anyhow::bail!(
|
|
"git rev-list --max-parents=0 failed: {}",
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
|
|
}
|
|
|
|
async fn git_commit(dir: &Path, message: &str) -> Result<()> {
|
|
git(
|
|
dir,
|
|
&[
|
|
"-c",
|
|
&format!("user.name={GIT_NAME}"),
|
|
"-c",
|
|
&format!("user.email={GIT_EMAIL}"),
|
|
"commit",
|
|
"-m",
|
|
message,
|
|
],
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// Spawn `git` honoring the `HYPERHIVE_GIT` env var (absolute path baked in
|
|
/// by the NixOS module), falling back to bare `git` (PATH lookup) otherwise.
|
|
#[must_use]
|
|
pub fn git_command() -> Command {
|
|
let exe = std::env::var("HYPERHIVE_GIT").unwrap_or_else(|_| "git".into());
|
|
Command::new(exe)
|
|
}
|
|
|
|
pub async fn git(dir: &Path, args: &[&str]) -> Result<()> {
|
|
let out = git_command()
|
|
.current_dir(dir)
|
|
.args(args)
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"git {} failed ({}): {}",
|
|
args.join(" "),
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Fetch the commit `sha` from the `src` git repo into `dst` and pin
|
|
/// it as `refs/tags/<tag>`. Used at `request_apply_commit` time so
|
|
/// hive-c0re captures an immutable handle on the manager's commit;
|
|
/// subsequent amendments / force-pushes in `src` no longer affect
|
|
/// what gets built. Returns the resolved full sha.
|
|
///
|
|
/// `sha` must be a commit sha (short or full) — the caller
|
|
/// (`submit_apply_commit`) shape-checks it first. We resolve it
|
|
/// LOCALLY against `src` rather than asking the remote to resolve
|
|
/// it: `git fetch <remote> <sha>:<dst>` treats the left side as a
|
|
/// remote *ref name*, and a bare sha is not one ("couldn't find
|
|
/// remote ref ..."). Fetching by sha would need a full 40-hex sha
|
|
/// plus `uploadpack.allow*SHA1InWant` on the remote, which the
|
|
/// proposed repos don't set. hive-c0re has direct read access to
|
|
/// `src`, so a local `rev-parse` + a branch-glob fetch sidesteps
|
|
/// the whole sha-want negotiation.
|
|
pub async fn git_fetch_to_tag(dst: &Path, src: &Path, sha: &str, tag: &str) -> Result<String> {
|
|
let src_str = src.display().to_string();
|
|
// Resolve the (short-or-full) sha to a full sha against the
|
|
// source repo. The `^{commit}` peel + non-zero exit on a missing
|
|
// object means a typo'd / stale sha fails loudly right here.
|
|
let full = git_rev_parse(src, &format!("{sha}^{{commit}}"))
|
|
.await
|
|
.with_context(|| format!("commit '{sha}' not found in proposed repo {src_str}"))?;
|
|
// Bring src's objects into dst. Fetching every head pulls the
|
|
// wanted commit's history (always reachable from a branch in the
|
|
// manager's flow) into dst's object db without sha-want.
|
|
git(
|
|
dst,
|
|
&[
|
|
"fetch",
|
|
"--no-tags",
|
|
&src_str,
|
|
"+refs/heads/*:refs/remotes/proposal-src/*",
|
|
],
|
|
)
|
|
.await?;
|
|
// Pin the exact commit as the proposal tag. The objects are now
|
|
// local so this resolves without touching the remote.
|
|
git(dst, &["tag", tag, &full]).await.with_context(|| {
|
|
format!("tag {tag} at {full}: commit not reachable from any branch in proposed repo")
|
|
})?;
|
|
Ok(full)
|
|
}
|
|
|
|
/// Resolve `refname` (a tag, branch, or sha) in `dir` to its full sha.
|
|
pub async fn git_rev_parse(dir: &Path, refname: &str) -> Result<String> {
|
|
let out = git_command()
|
|
.current_dir(dir)
|
|
.args(["rev-parse", refname])
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("git rev-parse {refname} in {}", dir.display()))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"git rev-parse {refname} failed ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
|
|
}
|
|
|
|
/// Plant a lightweight tag at `target`. Errors if the tag already
|
|
/// exists — we want loud failures on id reuse, not silent
|
|
/// overwrites.
|
|
pub async fn git_tag(dir: &Path, name: &str, target: &str) -> Result<()> {
|
|
git(dir, &["tag", name, target]).await
|
|
}
|
|
|
|
/// Plant an annotated tag with `body` as the message. Used for
|
|
/// `failed/<id>` (body = build error) and `denied/<id>` (body =
|
|
/// operator note). Multi-line bodies handled via stdin so we don't
|
|
/// have to escape anything.
|
|
pub async fn git_tag_annotated(dir: &Path, name: &str, target: &str, body: &str) -> Result<()> {
|
|
use tokio::io::AsyncWriteExt;
|
|
// Annotated tags are git objects, so they need a tagger identity
|
|
// (same constraint as a commit). Pass the hive-c0re identity
|
|
// inline rather than relying on a global git config — applied
|
|
// repos are hive-c0re-owned and the host's user might not have
|
|
// user.email set.
|
|
let mut child = git_command()
|
|
.current_dir(dir)
|
|
.args([
|
|
"-c",
|
|
&format!("user.name={GIT_NAME}"),
|
|
"-c",
|
|
&format!("user.email={GIT_EMAIL}"),
|
|
"tag",
|
|
"-a",
|
|
name,
|
|
target,
|
|
"-F",
|
|
"-",
|
|
])
|
|
.stdin(std::process::Stdio::piped())
|
|
.stdout(std::process::Stdio::piped())
|
|
.stderr(std::process::Stdio::piped())
|
|
.spawn()
|
|
.with_context(|| format!("spawn git tag -a {name} in {}", dir.display()))?;
|
|
if let Some(mut stdin) = child.stdin.take() {
|
|
stdin
|
|
.write_all(body.as_bytes())
|
|
.await
|
|
.context("write tag body to git stdin")?;
|
|
// Drop closes stdin so git can finish reading.
|
|
drop(stdin);
|
|
}
|
|
let out = child.wait_with_output().await.context("wait git tag -a")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"git tag -a {name} failed ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Replace working tree + index with the tree at `target` without
|
|
/// moving HEAD. `applied/main` stays pointing at the last known-good
|
|
/// `deployed/*` while we let `nixos-container update` evaluate the
|
|
/// candidate. On build failure callers reset back to HEAD; on
|
|
/// success they fast-forward main to `target`.
|
|
pub async fn git_read_tree_reset(dir: &Path, target: &str) -> Result<()> {
|
|
git(dir, &["read-tree", "--reset", "-u", target]).await
|
|
}
|
|
|
|
/// Hard-set a ref to `target`. Used to fast-forward `refs/heads/main`
|
|
/// to the just-deployed proposal commit. Uses `update-ref`, not
|
|
/// `branch -f`, so it works regardless of where HEAD currently sits.
|
|
pub async fn git_update_ref(dir: &Path, refname: &str, target: &str) -> Result<()> {
|
|
git(dir, &["update-ref", refname, target]).await
|
|
}
|
|
|
|
/// Write a systemd drop-in for `container@<container>.service` that applies
|
|
/// our default resource caps. Goes under `/run/systemd/system/...` so it's
|
|
/// ephemeral (regenerated on every spawn / rebuild).
|
|
fn set_resource_limits(container: &str) -> Result<()> {
|
|
let dir = format!("/run/systemd/system/container@{container}.service.d");
|
|
std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?;
|
|
let path = format!("{dir}/hyperhive-limits.conf");
|
|
let content =
|
|
format!("[Service]\nMemoryMax={DEFAULT_MEMORY_MAX}\nCPUQuota={DEFAULT_CPU_QUOTA}\n",);
|
|
std::fs::write(&path, content).with_context(|| format!("write {path}"))?;
|
|
tracing::info!(
|
|
%path,
|
|
memory_max = DEFAULT_MEMORY_MAX,
|
|
cpu_quota = DEFAULT_CPU_QUOTA,
|
|
"wrote resource limits drop-in"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
async fn systemd_daemon_reload() -> Result<()> {
|
|
let out = Command::new("systemctl")
|
|
.arg("daemon-reload")
|
|
.output()
|
|
.await
|
|
.context("invoke systemctl daemon-reload")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"systemctl daemon-reload failed ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Idempotently rewrite the lines in `/etc/nixos-containers/<container>.conf`
|
|
/// that hive-c0re owns: `PRIVATE_NETWORK` (forced 0 so the agent's web UI port
|
|
/// is reachable on the host) and `EXTRA_NSPAWN_FLAGS` (the runtime-dir bind).
|
|
/// The start script expands `$EXTRA_NSPAWN_FLAGS` unquoted into the
|
|
/// `systemd-nspawn` command.
|
|
/// Where in the container's filesystem the manager sees its agents tree.
|
|
/// Matches the `/agents` path that pre-Phase-8 hosts declared via
|
|
/// `containers.hm1nd.bindMounts."/agents"`.
|
|
pub const CONTAINER_MANAGER_AGENTS_MOUNT: &str = "/agents";
|
|
|
|
/// Where the manager sees the applied trees of every agent, read-only.
|
|
/// Manager runs `git fetch /applied/<n>/.git refs/tags/*:refs/tags/applied/*`
|
|
/// to learn what hive-c0re deployed (or rejected, or failed to
|
|
/// build); the RO bind makes accidental writes impossible from
|
|
/// inside the container.
|
|
pub const CONTAINER_MANAGER_APPLIED_MOUNT: &str = "/applied";
|
|
|
|
/// The on-host root that gets bind-mounted to `/agents` inside the manager.
|
|
/// Hard-coded to match `AGENT_STATE_ROOT` in coordinator.rs (kept duplicated
|
|
/// here so lifecycle stays usable as a leaf module).
|
|
const HOST_AGENTS_ROOT: &str = "/var/lib/hyperhive/agents";
|
|
|
|
/// On-host applied repo root, mirrored RO into the manager. Matches
|
|
/// `APPLIED_STATE_ROOT` in coordinator.rs.
|
|
const HOST_APPLIED_ROOT: &str = "/var/lib/hyperhive/applied";
|
|
|
|
/// On-host meta repo root, mirrored RO into the manager. Matches
|
|
/// `meta::meta_dir()` but duplicated here so lifecycle stays a leaf.
|
|
const HOST_META_ROOT: &str = "/var/lib/hyperhive/meta";
|
|
|
|
/// Shared directory accessible to all agents. All agents bind-mount this RW.
|
|
const HOST_SHARED_ROOT: &str = "/var/lib/hyperhive/shared";
|
|
|
|
fn set_nspawn_flags(
|
|
container: &str,
|
|
runtime_dir: &Path,
|
|
claude_dir: &Path,
|
|
notes_dir: &Path,
|
|
) -> Result<()> {
|
|
use std::fmt::Write as _;
|
|
|
|
// Ensure /shared directory exists before binding. systemd-nspawn requires the bind source to exist.
|
|
std::fs::create_dir_all(HOST_SHARED_ROOT)
|
|
.with_context(|| format!("create {HOST_SHARED_ROOT}"))?;
|
|
|
|
let path = format!("/etc/nixos-containers/{container}.conf");
|
|
let original = std::fs::read_to_string(&path).with_context(|| format!("read {path}"))?;
|
|
|
|
// Logical agent name (container name minus the sub-agent prefix).
|
|
// For the manager the strip is a no-op — harmless, manager paths
|
|
// below are gated on `container == MANAGER_NAME` anyway.
|
|
let agent_name = container.strip_prefix(AGENT_PREFIX).unwrap_or(container);
|
|
|
|
// Claude credentials land at `/home/<agent>/.claude` so the
|
|
// `claude` CLI (which reads `$HOME/.claude`) finds them. The
|
|
// harness service's environment sets `HOME` to the same path
|
|
// (`agent-base.nix` / `manager.nix`), so no `--setenv` plumbing
|
|
// is needed here — the bind alone is enough.
|
|
let claude_mount = container_claude_mount(agent_name);
|
|
|
|
let mut binds = format!(
|
|
"--bind={runtime}:{CONTAINER_RUNTIME_MOUNT} --bind={claude}:{claude_mount} --bind={shared}:{CONTAINER_SHARED_MOUNT}",
|
|
runtime = runtime_dir.display(),
|
|
claude = claude_dir.display(),
|
|
shared = HOST_SHARED_ROOT,
|
|
);
|
|
|
|
// Per-agent state at `/agents/<container>/state`. Skipped for
|
|
// the manager — the `/agents` bind below already exposes its
|
|
// own state (along with every sub-agent's).
|
|
if container != MANAGER_NAME {
|
|
let _ = write!(
|
|
binds,
|
|
" --bind={notes}:/agents/{agent_name}/state",
|
|
notes = notes_dir.display(),
|
|
);
|
|
}
|
|
if container == MANAGER_NAME {
|
|
// systemd-nspawn refuses to start a container whose bind
|
|
// source doesn't exist. The meta repo is created by the
|
|
// startup migration, but make sure the directory is there
|
|
// before the manager comes up in case set_nspawn_flags fires
|
|
// first (e.g. cold start with no agents).
|
|
std::fs::create_dir_all(HOST_META_ROOT)
|
|
.with_context(|| format!("create {HOST_META_ROOT}"))?;
|
|
// Manager edits sub-agent proposed/ repos and its own. RW so it can
|
|
// git-commit. Sub-agents see only their own /run/hive socket and
|
|
// /root/.claude (no /agents or /applied).
|
|
//
|
|
// /applied is a separate RO mount of the hive-c0re-only applied
|
|
// repos so the manager can `git fetch /applied/<n>/.git
|
|
// refs/tags/*:refs/tags/applied/*` to mirror deployed/failed/
|
|
// denied tags into its proposed clones and diff against
|
|
// what's actually deployed. RO bind makes destructive git
|
|
// plumbing inside the container unable to corrupt applied.
|
|
//
|
|
// /meta is a third RO mount exposing the system-wide deploy
|
|
// flake (`git log /meta --oneline` shows every deploy across
|
|
// every agent; `cat /meta/flake.lock` resolves which sha each
|
|
// agent is pinned at right now).
|
|
let _ = write!(
|
|
binds,
|
|
" --bind={HOST_AGENTS_ROOT}:{CONTAINER_MANAGER_AGENTS_MOUNT}",
|
|
);
|
|
let _ = write!(
|
|
binds,
|
|
" --bind-ro={HOST_APPLIED_ROOT}:{CONTAINER_MANAGER_APPLIED_MOUNT}",
|
|
);
|
|
let _ = write!(
|
|
binds,
|
|
" --bind-ro={HOST_META_ROOT}:{mount}",
|
|
mount = crate::meta::CONTAINER_MANAGER_META_MOUNT,
|
|
);
|
|
} else {
|
|
// Sub-agents get a READ-ONLY view of their own proposed
|
|
// config repo at /agents/<name>/config — agent.nix plus
|
|
// whatever extra files the manager split the config into.
|
|
// Lets an agent inspect exactly what defines it and request
|
|
// precise changes from the manager. RO is load-bearing: the
|
|
// agent must NOT edit its own config — changes only ever flow
|
|
// through the manager's RW proposed repo + the approval
|
|
// queue. The manager already has this dir RW via the /agents
|
|
// tree bind above.
|
|
let config_dir = format!("{HOST_AGENTS_ROOT}/{agent_name}/config");
|
|
// nspawn refuses to start when a bind source is missing.
|
|
// `setup_proposed` seeds this dir before spawn reaches here,
|
|
// but create defensively so a missing repo degrades to an
|
|
// empty RO dir instead of a container that won't boot.
|
|
std::fs::create_dir_all(&config_dir).with_context(|| format!("create {config_dir}"))?;
|
|
let _ = write!(binds, " --bind-ro={config_dir}:/agents/{agent_name}/config");
|
|
|
|
// Per-agent socket subdir. Bind-mounts `/run/hive-agent/<name>/`
|
|
// into the container at the same path so the harness's
|
|
// `HIVE_WEB_SOCKET` bind has a stable location both sides can
|
|
// see. Sub-agents only — the manager's UI is served at `/`
|
|
// via the c0re dashboard upstream, not via `/agent/<name>/`,
|
|
// so it never needs the per-agent socket dir.
|
|
//
|
|
// Bind-mounting the SUBDIR (not the socket file) is mandatory:
|
|
// the harness's `bind_unix` helper unlinks any stale socket
|
|
// before calling `bind(2)`, and a file bind-mount drops its
|
|
// host-side anchor on unlink — the rebind would land in the
|
|
// container's private namespace, invisible to the gateway.
|
|
// Dir bind keeps the same dir inode visible on both sides, so
|
|
// the new `web.sock` shows up on the host the moment the
|
|
// harness binds it.
|
|
//
|
|
// Per-agent dir (rather than a shared `/run/hive-agent/`
|
|
// mount) means the agent's container only sees its own
|
|
// subdir — never siblings'. See `docs/gateway.md::Per-agent
|
|
// unix-socket upstream`.
|
|
//
|
|
// mkdir source defensively: nspawn refuses to start when the
|
|
// bind source is missing, and on a fresh host `/run/hive-agent/`
|
|
// doesn't exist yet.
|
|
let socket_dir = crate::agent_sockets::agent_dir_for(agent_name);
|
|
std::fs::create_dir_all(&socket_dir)
|
|
.with_context(|| format!("create {}", socket_dir.display()))?;
|
|
let _ = write!(
|
|
binds,
|
|
" --bind={socket_dir}:{socket_dir}",
|
|
socket_dir = socket_dir.display(),
|
|
);
|
|
}
|
|
let bind_flag = format!("EXTRA_NSPAWN_FLAGS=\"{binds}\"");
|
|
let mut lines: Vec<String> = original
|
|
.lines()
|
|
.filter(|line| {
|
|
let trimmed = line.trim_start();
|
|
// Strip any network-namespace knobs nixos-container's create
|
|
// might have populated. The start script adds `--network-veth`
|
|
// whenever HOST_ADDRESS / LOCAL_ADDRESS (or their IPv6 cousins)
|
|
// are non-empty — and veth implies a private netns, hiding our
|
|
// web-UI port from the host. Force host netns.
|
|
!trimmed.starts_with("EXTRA_NSPAWN_FLAGS=")
|
|
&& !trimmed.starts_with("PRIVATE_NETWORK=")
|
|
&& !trimmed.starts_with("HOST_ADDRESS=")
|
|
&& !trimmed.starts_with("LOCAL_ADDRESS=")
|
|
&& !trimmed.starts_with("HOST_ADDRESS6=")
|
|
&& !trimmed.starts_with("LOCAL_ADDRESS6=")
|
|
&& !trimmed.starts_with("HOST_BRIDGE=")
|
|
})
|
|
.map(str::to_owned)
|
|
.collect();
|
|
lines.push("PRIVATE_NETWORK=0".to_owned());
|
|
lines.push("HOST_ADDRESS=".to_owned());
|
|
lines.push("LOCAL_ADDRESS=".to_owned());
|
|
lines.push("HOST_ADDRESS6=".to_owned());
|
|
lines.push("LOCAL_ADDRESS6=".to_owned());
|
|
lines.push("HOST_BRIDGE=".to_owned());
|
|
lines.push(bind_flag);
|
|
let mut content = lines.join("\n");
|
|
content.push('\n');
|
|
std::fs::write(&path, content).with_context(|| format!("write {path}"))?;
|
|
tracing::info!(%path, "set PRIVATE_NETWORK=0 + EXTRA_NSPAWN_FLAGS");
|
|
Ok(())
|
|
}
|
|
|
|
/// Spawn `nixos-container <args>` and pipe its stdout + stderr into
|
|
/// `tracing` one line at a time so a long-running command (most
|
|
/// notably `update`, which kicks off a full nix build that can run
|
|
/// for minutes on a stale flake) shows progress in journald as it
|
|
/// happens. The buffered `.output()` we used before only flushed the
|
|
/// summary at exit, which made "slow" and "stuck" look identical to
|
|
/// the operator watching `journalctl -u hive-c0re -f`.
|
|
///
|
|
/// stdout lines log at INFO, stderr at WARN. The same lines are
|
|
/// captured per-attempt into `build_logs.sqlite` so the dashboard
|
|
/// can surface the full stream to the operator; on failure we bail
|
|
/// with a `see build log #<id>` pointer instead of the legacy
|
|
/// 32-line ring-buffer tail that routinely truncated eval errors.
|
|
async fn run(args: &[&str]) -> Result<()> {
|
|
use tokio::io::{AsyncBufReadExt, BufReader};
|
|
let cmdline = args.join(" ");
|
|
|
|
// Convention: `nixos-container <verb> <container> ...` — the
|
|
// verb is `args[0]` (kind) and the container is `args[1]`
|
|
// (h-<name> | hm1nd | hive-matrix | ...) for every long-running
|
|
// case we care about. Strip the `h-` prefix for sub-agents so the
|
|
// build_logs row's `agent` column matches the agent's bare name
|
|
// (`alice` rather than `h-alice`) — that's what the dashboard
|
|
// groups by. Manager + sibling containers pass through as-is.
|
|
let kind = args.first().copied().unwrap_or("nixos-container");
|
|
let agent = args
|
|
.get(1)
|
|
.copied()
|
|
.map(|c| c.strip_prefix(AGENT_PREFIX).unwrap_or(c).to_string())
|
|
.unwrap_or_else(|| "<unknown>".to_string());
|
|
|
|
let logs = crate::build_logs::global();
|
|
let log_id = logs.as_ref().and_then(|h| {
|
|
h.start(&agent, kind, &cmdline)
|
|
.map_err(|e| {
|
|
tracing::warn!(error = ?e, "build_logs: start failed (nixos-container log dropped)");
|
|
})
|
|
.ok()
|
|
});
|
|
|
|
let mut child = Command::new("nixos-container")
|
|
.args(args)
|
|
.stdout(std::process::Stdio::piped())
|
|
.stderr(std::process::Stdio::piped())
|
|
.spawn()
|
|
.with_context(|| format!("invoke nixos-container {cmdline}"))?;
|
|
|
|
let stdout = child.stdout.take().expect("piped stdout");
|
|
let stderr = child.stderr.take().expect("piped stderr");
|
|
|
|
let stdout_cmdline = cmdline.clone();
|
|
let stdout_logs = logs.clone();
|
|
let pump_stdout = tokio::spawn(async move {
|
|
let mut lines = BufReader::new(stdout).lines();
|
|
while let Ok(Some(line)) = lines.next_line().await {
|
|
tracing::info!(target: "nixos-container", cmdline = %stdout_cmdline, "{line}");
|
|
if let (Some(h), Some(id)) = (&stdout_logs, log_id) {
|
|
h.append_stdout(id, &line);
|
|
}
|
|
}
|
|
});
|
|
|
|
let stderr_cmdline = cmdline.clone();
|
|
let stderr_logs = logs.clone();
|
|
let pump_stderr = tokio::spawn(async move {
|
|
let mut lines = BufReader::new(stderr).lines();
|
|
while let Ok(Some(line)) = lines.next_line().await {
|
|
tracing::warn!(target: "nixos-container", cmdline = %stderr_cmdline, "{line}");
|
|
if let (Some(h), Some(id)) = (&stderr_logs, log_id) {
|
|
h.append_stderr(id, &line);
|
|
}
|
|
}
|
|
});
|
|
|
|
let status = child
|
|
.wait()
|
|
.await
|
|
.with_context(|| format!("wait nixos-container {cmdline}"))?;
|
|
let _ = pump_stdout.await;
|
|
let _ = pump_stderr.await;
|
|
|
|
let ok = status.success();
|
|
if let (Some(h), Some(id)) = (&logs, log_id) {
|
|
h.finish(
|
|
id,
|
|
if ok {
|
|
crate::build_logs::BuildStatus::Ok
|
|
} else {
|
|
crate::build_logs::BuildStatus::Fail
|
|
},
|
|
);
|
|
}
|
|
if !ok {
|
|
// `container_journal_tail` is best-effort + only fires on
|
|
// `update`; the captured build log holds the full host-side
|
|
// stderr regardless, so the bail message can stay terse: a
|
|
// pointer to the log id + the journal tail (when available)
|
|
// is enough for the operator to drill in without flooding
|
|
// every notification with the eval-error verbatim.
|
|
let journal = container_journal_tail(args).await;
|
|
match log_id {
|
|
Some(id) => bail!(
|
|
"nixos-container {cmdline} failed ({status}); see build log #{id}{journal}"
|
|
),
|
|
None => bail!("nixos-container {cmdline} failed ({status}){journal}"),
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// On a failed `nixos-container update`, the stderr nixos-container
|
|
/// itself prints is often terse ("failed to reload container") — the
|
|
/// real reason (which unit failed `switch-to-configuration` during
|
|
/// the reload phase) lands in the *container's* own journal, not on
|
|
/// the host. Fetch the tail of it so a failed rebuild self-documents
|
|
/// the failing unit in the error string, no second round-trip.
|
|
///
|
|
/// Scoped to `update`: that's the reload-phase case, and the
|
|
/// container is still up (running the old generation) so
|
|
/// `journalctl -M` works. Best-effort — returns "" for other verbs
|
|
/// or when the journal can't be read (machine gone, journalctl
|
|
/// missing); it never produces an error of its own.
|
|
async fn container_journal_tail(args: &[&str]) -> String {
|
|
if args.first().copied() != Some("update") {
|
|
return String::new();
|
|
}
|
|
let Some(container) = args.get(1) else {
|
|
return String::new();
|
|
};
|
|
let out = Command::new("journalctl")
|
|
.args(["-M", container, "-n", "40", "--no-pager", "--output=short"])
|
|
.output()
|
|
.await;
|
|
match out {
|
|
Ok(o) if !o.stdout.is_empty() => format!(
|
|
"\n--- last 40 journal lines from container '{container}' ---\n{}",
|
|
String::from_utf8_lossy(&o.stdout).trim_end()
|
|
),
|
|
_ => String::new(),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
/// Regression test: `setup_proposed` must seed both agent.nix and flake.nix
|
|
/// in the initial commit. Before commit 5b5a93e flake.nix was missing from
|
|
/// the scaffold, requiring manual creation (seen with the damocles agent).
|
|
#[tokio::test]
|
|
async fn setup_proposed_seeds_flake_nix() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let proposed = dir.path().join("proposed");
|
|
setup_proposed(&proposed, "test-agent")
|
|
.await
|
|
.expect("setup_proposed");
|
|
|
|
// Both files must exist on disk.
|
|
assert!(proposed.join("agent.nix").exists(), "agent.nix missing");
|
|
assert!(proposed.join("flake.nix").exists(), "flake.nix missing");
|
|
|
|
// flake.nix must export nixosModules.default (the meta-flake contract).
|
|
let flake = std::fs::read_to_string(proposed.join("flake.nix")).unwrap();
|
|
assert!(
|
|
flake.contains("nixosModules.default"),
|
|
"flake.nix does not export nixosModules.default"
|
|
);
|
|
|
|
// Both files must be tracked in the initial git commit.
|
|
let out = git_command()
|
|
.current_dir(&proposed)
|
|
.args(["show", "--name-only", "--format=", "HEAD"])
|
|
.output()
|
|
.await
|
|
.expect("git show");
|
|
let tracked = String::from_utf8_lossy(&out.stdout);
|
|
assert!(tracked.contains("agent.nix"), "agent.nix not committed");
|
|
assert!(tracked.contains("flake.nix"), "flake.nix not committed");
|
|
}
|
|
|
|
/// `setup_proposed` is idempotent: calling it on an existing repo is a
|
|
/// no-op (the fresh guard skips all writes).
|
|
#[tokio::test]
|
|
async fn setup_proposed_idempotent() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let proposed = dir.path().join("proposed");
|
|
setup_proposed(&proposed, "test-agent")
|
|
.await
|
|
.expect("first call");
|
|
// Second call must not error even though .git already exists.
|
|
setup_proposed(&proposed, "test-agent")
|
|
.await
|
|
.expect("second call");
|
|
// Still one commit.
|
|
let out = git_command()
|
|
.current_dir(&proposed)
|
|
.args(["rev-list", "--count", "HEAD"])
|
|
.output()
|
|
.await
|
|
.expect("git rev-list");
|
|
let count = String::from_utf8_lossy(&out.stdout).trim().to_owned();
|
|
assert_eq!(
|
|
count, "1",
|
|
"expected exactly one commit after idempotent call"
|
|
);
|
|
}
|
|
}
|