961 lines
38 KiB
Rust
961 lines
38 KiB
Rust
//! Per-container host-side config: the nspawn conf rewrite (bind mounts,
|
|
//! network isolation, forwarded credentials), the systemd resource-limits
|
|
//! drop-in, the agent's copy of its memory cap, and
|
|
//! the `write_dropins` verb that re-applies them.
|
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use anyhow::{Context, Result};
|
|
use hive_priv_sock::BindMount;
|
|
use hive_types::Permission;
|
|
|
|
use crate::coordinator::{AgentPaths, HiveEnv};
|
|
|
|
use super::{
|
|
AGENT_PREFIX, CONTAINER_RUNTIME_MOUNT, container_claude_mount, container_name, validate,
|
|
};
|
|
|
|
/// Re-apply the per-container host-side config: nspawn flags (bind
|
|
/// mounts etc.), the systemd resource-limits drop-in, the agent's copy of
|
|
/// its memory cap, and a daemon reload so all of it
|
|
/// takes effect on the next unit (re)start. Idempotent — the job queue's
|
|
/// `WriteDropin` node, also folded into every `Swap` (rebuild is the
|
|
/// reconcile verb).
|
|
pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> {
|
|
validate(name)?;
|
|
let container = container_name(name);
|
|
set_nspawn_flags(&container, hive, &paths.agent, &paths.claude, &paths.notes).await?;
|
|
if let Some((cpu_quota, memory_max)) = stage_limits(
|
|
name,
|
|
&crate::resource_limits::swarm_limits_path(),
|
|
&limits_dropin_path(&container),
|
|
&paths.agent,
|
|
hive,
|
|
HostMemory::read(),
|
|
)? {
|
|
set_resource_limits(
|
|
&container,
|
|
&cpu_quota,
|
|
&memory_max,
|
|
hive.agent_cpu_weight,
|
|
hive.agent_io_weight,
|
|
)
|
|
.await?;
|
|
}
|
|
systemd_daemon_reload().await
|
|
}
|
|
|
|
/// The limits drop-in hive-priv's `write_resource_limits` writes for
|
|
/// `container`. Must match that path: a mismatch reads as "no drop-in
|
|
/// yet" in [`limits_to_apply`].
|
|
fn limits_dropin_path(container: &str) -> PathBuf {
|
|
PathBuf::from(format!(
|
|
"/run/systemd/system/container@{container}.service.d/hyperhive-limits.conf"
|
|
))
|
|
}
|
|
|
|
/// File in the agent's runtime dir, `/run/hive/` in the container, holding the
|
|
/// `MemoryMax=` its drop-in carries: a byte count, or `infinity` for no cap.
|
|
/// `nix/agent-modules/memory-cap.nix` reads it at boot under this name, and
|
|
/// treats a missing file as a fault.
|
|
const MEMORY_CAP_FILE: &str = "memory-max-bytes";
|
|
|
|
/// The host's physical memory and page size, the two numbers systemd resolves
|
|
/// a `MemoryMax=` percentage against.
|
|
#[derive(Clone, Copy, Debug)]
|
|
struct HostMemory {
|
|
bytes: u64,
|
|
page: u64,
|
|
}
|
|
|
|
impl HostMemory {
|
|
/// `sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE)`, as systemd's
|
|
/// `physical_memory()` computes it. systemd also caps that by the root
|
|
/// cgroup's `memory.max`, which the host's root cgroup does not have.
|
|
fn read() -> Option<Self> {
|
|
// SAFETY: `sysconf(3)` only reads system configuration.
|
|
let (pages, page) = unsafe {
|
|
(
|
|
libc::sysconf(libc::_SC_PHYS_PAGES),
|
|
libc::sysconf(libc::_SC_PAGESIZE),
|
|
)
|
|
};
|
|
let pages = u64::try_from(pages).ok().filter(|&p| p > 0)?;
|
|
let page = u64::try_from(page).ok().filter(|&p| p > 0)?;
|
|
Some(Self {
|
|
bytes: pages.checked_mul(page)?,
|
|
page,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// [`limits_to_apply`], plus handing the memory cap to the agent through
|
|
/// [`MEMORY_CAP_FILE`] in `runtime_dir`. The container cannot read the cap
|
|
/// systemd enforces on it, so this file is how it learns the value it sizes
|
|
/// its heap and its subagents against. It is written on every call, from the
|
|
/// `MemoryMax=` the agent will start under: the new one, or the kept drop-in's.
|
|
fn stage_limits(
|
|
agent_name: &str,
|
|
swarm_path: &Path,
|
|
dropin_path: &Path,
|
|
runtime_dir: &Path,
|
|
hive: &HiveEnv,
|
|
host: Option<HostMemory>,
|
|
) -> Result<Option<(String, String)>> {
|
|
let limits = limits_to_apply(agent_name, swarm_path, dropin_path, hive);
|
|
let memory_max = match &limits {
|
|
Some((_, memory_max)) => Some(memory_max.clone()),
|
|
None => dropin_memory_max(agent_name, dropin_path),
|
|
};
|
|
let cap = memory_max.and_then(|m| memory_cap_value(agent_name, &m, host));
|
|
write_memory_cap(runtime_dir, cap.as_deref())?;
|
|
Ok(limits)
|
|
}
|
|
|
|
/// The `MemoryMax=` the drop-in at `path` applies: its last assignment, as
|
|
/// systemd reads it, or `infinity`, systemd's default, when it has none or its
|
|
/// last one is empty. `None` when the file can't be read.
|
|
fn dropin_memory_max(agent_name: &str, path: &Path) -> Option<String> {
|
|
let body = match std::fs::read_to_string(path) {
|
|
Ok(body) => body,
|
|
Err(e) => {
|
|
tracing::warn!(
|
|
agent = %agent_name,
|
|
path = %path.display(),
|
|
error = ?e,
|
|
"limits drop-in unreadable; the agent gets no memory cap"
|
|
);
|
|
return None;
|
|
}
|
|
};
|
|
let value = body
|
|
.lines()
|
|
.rev()
|
|
.find_map(|line| line.trim().strip_prefix("MemoryMax="))
|
|
.map(str::trim)
|
|
.filter(|v| !v.is_empty())
|
|
.unwrap_or("infinity");
|
|
Some(value.to_owned())
|
|
}
|
|
|
|
/// What [`MEMORY_CAP_FILE`] says for `memory_max`, read the way systemd reads
|
|
/// `MemoryMax=`: a size is its byte count, and a percentage is that share of
|
|
/// `host`'s physical memory, rounded down to a page. A value systemd ignores
|
|
/// (zero, over 100%, more than two decimals, unparseable) leaves the unit
|
|
/// uncapped, so it is `infinity` here. `None` for a percentage when `host` is
|
|
/// unknown.
|
|
fn memory_cap_value(
|
|
agent_name: &str,
|
|
memory_max: &str,
|
|
host: Option<HostMemory>,
|
|
) -> Option<String> {
|
|
let bytes = if let Some(percent) = memory_max.strip_suffix('%') {
|
|
let Some(host) = host else {
|
|
tracing::warn!(
|
|
agent = agent_name,
|
|
memory_max,
|
|
"host physical memory unknown; the agent gets no memory cap"
|
|
);
|
|
return None;
|
|
};
|
|
permyriad(percent).and_then(|p| scale_physical_memory(host, p))
|
|
} else {
|
|
crate::resource_limits::parse_bytes(memory_max)
|
|
};
|
|
Some(
|
|
bytes
|
|
.filter(|&b| b > 0)
|
|
.map_or_else(|| "infinity".to_owned(), |b| b.to_string()),
|
|
)
|
|
}
|
|
|
|
/// `percent` (the part before `%`) in hundredths of a percent, with systemd's
|
|
/// `parse_permyriad` rules: digits, at most two decimals, at most 100%.
|
|
fn permyriad(percent: &str) -> Option<u64> {
|
|
let (int, frac) = percent.split_once('.').unwrap_or((percent, ""));
|
|
if int.is_empty() || !int.bytes().all(|b| b.is_ascii_digit()) {
|
|
return None;
|
|
}
|
|
let hundredths = match frac.as_bytes() {
|
|
[] if !percent.contains('.') => 0,
|
|
[d] if d.is_ascii_digit() => u64::from(d - b'0') * 10,
|
|
[d1, d2] if d1.is_ascii_digit() && d2.is_ascii_digit() => {
|
|
u64::from(d1 - b'0') * 10 + u64::from(d2 - b'0')
|
|
}
|
|
_ => return None,
|
|
};
|
|
let value = int
|
|
.parse::<u64>()
|
|
.ok()?
|
|
.checked_mul(100)?
|
|
.checked_add(hundredths)?;
|
|
(value <= 10_000).then_some(value)
|
|
}
|
|
|
|
/// systemd's `physical_memory_scale(permyriad, 10000)`: whole pages, rounded
|
|
/// down, and the exact total at 100%.
|
|
fn scale_physical_memory(host: HostMemory, permyriad: u64) -> Option<u64> {
|
|
if permyriad == 10_000 {
|
|
return Some(host.bytes);
|
|
}
|
|
let pages = u128::from(host.bytes / host.page) * u128::from(permyriad) / 10_000;
|
|
u64::try_from(pages * u128::from(host.page)).ok()
|
|
}
|
|
|
|
/// Write `cap` into [`MEMORY_CAP_FILE`], or remove the file when the cap is
|
|
/// unknown, which the agent reports at boot.
|
|
fn write_memory_cap(runtime_dir: &Path, cap: Option<&str>) -> Result<()> {
|
|
let path = runtime_dir.join(MEMORY_CAP_FILE);
|
|
let Some(cap) = cap else {
|
|
return match std::fs::remove_file(&path) {
|
|
Err(e) if e.kind() != std::io::ErrorKind::NotFound => {
|
|
Err(e).with_context(|| format!("remove {}", path.display()))
|
|
}
|
|
_ => Ok(()),
|
|
};
|
|
};
|
|
std::fs::create_dir_all(runtime_dir)
|
|
.with_context(|| format!("create agent runtime dir {}", runtime_dir.display()))?;
|
|
let tmp = runtime_dir.join(format!(".{MEMORY_CAP_FILE}.tmp"));
|
|
std::fs::write(&tmp, format!("{cap}\n")).with_context(|| format!("write {}", tmp.display()))?;
|
|
std::fs::rename(&tmp, &path).with_context(|| format!("rename onto {}", path.display()))
|
|
}
|
|
|
|
/// The `(CPUQuota, MemoryMax)` to write into `agent_name`'s limits
|
|
/// drop-in, or `None` to leave the drop-in at `dropin_path` as it is.
|
|
///
|
|
/// When the swarm caps at `swarm_path` can't be read, the error is logged and
|
|
/// the agent still starts. The last-applied drop-in is kept, because the hive
|
|
/// defaults can be looser than the swarm's cap.
|
|
/// With no drop-in yet, the agent gets the hive defaults: without one it
|
|
/// would run uncapped.
|
|
fn limits_to_apply(
|
|
agent_name: &str,
|
|
swarm_path: &Path,
|
|
dropin_path: &Path,
|
|
hive: &HiveEnv,
|
|
) -> Option<(String, String)> {
|
|
let hive_cpu = &hive.agent_cpu_quota;
|
|
let hive_mem = &hive.agent_memory_max;
|
|
let e = match crate::resource_limits::effective(swarm_path, agent_name, hive_cpu, hive_mem) {
|
|
Ok(limits) => return Some(limits),
|
|
Err(e) => e,
|
|
};
|
|
if dropin_path.exists() {
|
|
tracing::error!(
|
|
agent = %agent_name,
|
|
path = %swarm_path.display(),
|
|
error = ?e,
|
|
"resource limits unreadable — keeping the last-applied limits drop-in"
|
|
);
|
|
None
|
|
} else {
|
|
tracing::error!(
|
|
agent = %agent_name,
|
|
path = %swarm_path.display(),
|
|
error = ?e,
|
|
"resource limits unreadable and no limits drop-in yet — applying hive defaults"
|
|
);
|
|
Some((hive_cpu.clone(), hive_mem.clone()))
|
|
}
|
|
}
|
|
|
|
/// Write a systemd drop-in for `container@<container>.service` that applies
|
|
/// the agent's effective resource caps — the swarm's cap from
|
|
/// `meta/swarm-resource-limits.json` where set, the hive-wide defaults
|
|
/// otherwise. Goes under `/run/systemd/system/...` so it's ephemeral
|
|
/// (regenerated on every spawn / rebuild, except as [`limits_to_apply`] says).
|
|
///
|
|
/// The weights are hive-wide (`services.hyperhive.agentCpuWeight` /
|
|
/// `agentIoWeight`) — unlike the caps they have no swarm cap, so they come
|
|
/// straight off [`HiveEnv`].
|
|
/// `None` (the nix option set to `null`) means the weight line is left out
|
|
/// and the container keeps the kernel default.
|
|
async fn set_resource_limits(
|
|
container: &str,
|
|
cpu_quota: &str,
|
|
memory_max: &str,
|
|
cpu_weight: Option<u32>,
|
|
io_weight: Option<u32>,
|
|
) -> Result<()> {
|
|
crate::priv_client::write_resource_limits(
|
|
container, memory_max, cpu_quota, cpu_weight, io_weight,
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn systemd_daemon_reload() -> Result<()> {
|
|
crate::priv_client::daemon_reload().await
|
|
}
|
|
|
|
/// Append the bind flag for `child`'s `state` dir into `binds`. See
|
|
/// docs/agent-lifecycle/persistence.md ("Cross-agent access to state") for what
|
|
/// the holder may touch and why. Creates the missing host-side directory so
|
|
/// nspawn doesn't refuse to start; a missing dir is non-fatal.
|
|
///
|
|
/// **One dir, one answer.** `state` is **read-write**: the holder reads and
|
|
/// writes another agent's notes to recover it, which is the one case that
|
|
/// needs to work while that agent is down. The agent's `config` is not mounted
|
|
/// at all: a config change is a PR against its repo on the forge, reviewed and
|
|
/// merged, and the holder reads the config by cloning it from there. Its
|
|
/// `harness` and `work` are absent too: they hold that agent's own runtime
|
|
/// material (`bash-tasks/`, turn-stats and event sqlite dbs) and its, often
|
|
/// very large, worktrees and build trees, none of which the holder has a stated
|
|
/// reason to read, let alone write. hive-c0re itself reaches `harness` directly
|
|
/// on the host (the paused marker), which needs no bind mount into anyone
|
|
/// else's container.
|
|
fn bind_child_agent_dirs(child: &str, binds: &mut Vec<BindMount>) {
|
|
let Ok(child) = hive_types::Ident::parse(child) else {
|
|
tracing::warn!(%child, "skipping child bind: invalid agent name");
|
|
return;
|
|
};
|
|
let host = crate::paths::agent_state_dir(&child).join("state");
|
|
let _ = std::fs::create_dir_all(&host);
|
|
binds.push(BindMount {
|
|
host_path: host.to_string_lossy().into_owned(),
|
|
container_path: format!("/agents/{child}/state"),
|
|
read_only: false,
|
|
});
|
|
}
|
|
|
|
/// Append RW bind flags for `agent_name`'s own `state`, `harness` and `work`
|
|
/// dirs into `binds`. `harness` and `work` are siblings of `notes_dir`;
|
|
/// missing ones are created so nspawn doesn't refuse to start.
|
|
fn bind_own_agent_dirs(agent_name: &str, notes_dir: &Path, binds: &mut Vec<BindMount>) {
|
|
binds.push(BindMount {
|
|
host_path: notes_dir.to_string_lossy().into_owned(),
|
|
container_path: format!("/agents/{agent_name}/state"),
|
|
read_only: false,
|
|
});
|
|
if let Some(state_parent) = notes_dir.parent() {
|
|
for sub in ["harness", crate::paths::WORK_SUBDIR] {
|
|
let dir = state_parent.join(sub);
|
|
if !dir.exists() {
|
|
let _ = std::fs::create_dir_all(&dir);
|
|
}
|
|
binds.push(BindMount {
|
|
host_path: dir.to_string_lossy().into_owned(),
|
|
container_path: format!("/agents/{agent_name}/{sub}"),
|
|
read_only: false,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Whether `agent_name` holds `ManageRootAgent` per the swarm permissions
|
|
/// stored at `path`. An unreadable file denies: the error is logged and the
|
|
/// agent starts without the cross-agent mounts, so one corrupt file never
|
|
/// blocks any agent's start, spawn or swap.
|
|
fn holds_manage_root_agent(agent_name: &str, path: &Path) -> bool {
|
|
crate::permissions::holds(path, agent_name, Permission::ManageRootAgent).unwrap_or_else(|e| {
|
|
tracing::error!(
|
|
agent = %agent_name,
|
|
path = %path.display(),
|
|
error = ?e,
|
|
"swarm permissions unreadable — no ManageRootAgent mounts"
|
|
);
|
|
false
|
|
})
|
|
}
|
|
|
|
/// Idempotently rewrite the lines in `/etc/nixos-containers/<container>.conf`
|
|
/// that hive-c0re owns: `PRIVATE_NETWORK` (always 1), `HOST_ADDRESS` (the
|
|
/// bridge gateway IP) and `EXTRA_NSPAWN_FLAGS` (the runtime-dir bind). What
|
|
/// those network vars mean and why isolation is unconditional:
|
|
/// `docs/networking/network.md` § *What the Rust side does*.
|
|
///
|
|
/// ⚠️ The start script expands `$EXTRA_NSPAWN_FLAGS` unquoted into the
|
|
/// `systemd-nspawn` command.
|
|
#[allow(
|
|
clippy::too_many_lines,
|
|
reason = "one contiguous nspawn-flag assembly block; the length is the flag \
|
|
surface itself, splitting it would just hide the shape"
|
|
)]
|
|
async fn set_nspawn_flags(
|
|
container: &str,
|
|
hive: &HiveEnv,
|
|
runtime_dir: &Path,
|
|
claude_dir: &Path,
|
|
notes_dir: &Path,
|
|
) -> Result<()> {
|
|
// Ensure /knowledge dir exists. It may be empty until forge seeds it;
|
|
// nspawn refuses to start if the bind source is missing entirely.
|
|
std::fs::create_dir_all(crate::knowledge::LOCAL_DIR)
|
|
.with_context(|| format!("create {}", crate::knowledge::LOCAL_DIR))?;
|
|
|
|
// Logical agent name — strip the `h-` prefix.
|
|
// For the manager: `h-ruth` → `ruth`. For sub-agents: `h-iris` → `iris`.
|
|
let agent_name = container.strip_prefix(AGENT_PREFIX).unwrap_or(container);
|
|
|
|
// Claude credentials land at `/home/<agent>/.claude` so the
|
|
// `claude` CLI (which reads `$HOME/.claude`) finds them. The
|
|
// harness service's environment sets `HOME` to the same path
|
|
// (`agent.nix` template), so no `--setenv` plumbing
|
|
// is needed here — the bind alone is enough.
|
|
|
|
// The agent's own identity *at* the swarm secret store, collected under
|
|
// this hive's certificate. Staged here rather than at spawn because this
|
|
// is the one place that writes the container's credential list — see
|
|
// `super::agent_identity` for the whole hop.
|
|
let load_creds = super::agent_identity::stage(agent_name).await;
|
|
|
|
let mut binds = base_binds(runtime_dir, claude_dir, container_claude_mount(agent_name));
|
|
|
|
// Own state, harness and work dirs - same for every agent including the
|
|
// manager. There is no config mount: an agent reads its config by cloning
|
|
// `agent-configs/<name>` from the forge, and changes only ever flow
|
|
// through a config PR there.
|
|
bind_own_agent_dirs(agent_name, notes_dir, &mut binds);
|
|
hive_types::Ident::parse(agent_name)
|
|
.map_err(|e| anyhow::anyhow!("invalid agent name {agent_name:?}: {e}"))?;
|
|
|
|
// `ManageRootAgent` permission: mount *every* agent in the hive as a
|
|
// virtual child. Enables recovery: the holder can read and write another
|
|
// agent's notes even when that agent is down. Also grants RO access
|
|
// to /meta.
|
|
//
|
|
// ⚠️ "every agent" is the definition of this permission, not an
|
|
// accident of how the set is computed. The grant used to hang off a
|
|
// `can_manage_top_level_agents` role and cover `top_level_agents()`
|
|
// — i.e. `parent.is_none()` — which was "everything outside the
|
|
// hierarchy". With that hierarchy removed, every agent is
|
|
// parentless, so that set *was* every agent anyway; the permission
|
|
// now says so out loud instead of deriving it from a field that no
|
|
// longer discriminates.
|
|
//
|
|
// This is the *only* cross-agent mount left. Dropping the topology
|
|
// parent field took with it the unconditional grant every
|
|
// agent used to get over its own direct children — so an agent with
|
|
// no permission now sees its own dirs and nothing else.
|
|
if holds_manage_root_agent(agent_name, &crate::permissions::path()) {
|
|
// Skipping self is a no-op, not a narrowing: `agent_notes_dir` is
|
|
// `agent_state_dir/state`, so binding the holder as its own virtual
|
|
// child reproduced the own-dir mount pushed above, byte for byte.
|
|
for other in crate::topology::all_agents() {
|
|
if other != agent_name {
|
|
bind_child_agent_dirs(&other, &mut binds);
|
|
}
|
|
}
|
|
// systemd-nspawn refuses to start a container whose bind
|
|
// source doesn't exist. The directory is created by the
|
|
// startup migration, but make sure it is there
|
|
// before the role holder comes up in case set_nspawn_flags
|
|
// fires first (e.g. cold start with no agents).
|
|
let meta_root = crate::paths::meta_root();
|
|
std::fs::create_dir_all(&meta_root)
|
|
.with_context(|| format!("create {}", meta_root.display()))?;
|
|
binds.push(BindMount {
|
|
host_path: meta_root.to_string_lossy().into_owned(),
|
|
container_path: crate::meta::CONTAINER_MANAGER_META_MOUNT.to_owned(),
|
|
read_only: true,
|
|
});
|
|
}
|
|
|
|
// Whether the agent's config sets `privateUsers`, as the swarm builder
|
|
// recorded it. Read from the record, never from the agent's nix.
|
|
let private_users = private_users_of(&crate::attic::closure::record(agent_name).await);
|
|
|
|
// Web-socket subdir: bind-mount `/run/hive-agent/<name>/` into the
|
|
// container so the harness can bind `web.sock` there and the host-side
|
|
// gateway sees it. Subdir bind (not socket file) keeps the inode
|
|
// visible after the harness unlinks a stale socket on rebind.
|
|
// Applies to manager and sub-agents alike.
|
|
// hive-priv creates it (`/run` is read-only to hive-c0re) as `0751 root`
|
|
// and leaves an existing one alone; the container's activation hands it
|
|
// to the agent user. Don't chown or chmod it here.
|
|
let socket_dir = crate::agent_sockets::agent_dir_for(agent_name);
|
|
crate::priv_client::ensure_agent_socket_dir(agent_name, private_users).await?;
|
|
binds.push(socket_bind(agent_name, &socket_dir, private_users));
|
|
|
|
// Every container runs in a private network namespace with a veth
|
|
// pair attached to the host bridge — including the manager (all
|
|
// hive-c0re<->agent comms go through bind-mounted UDS, not TCP).
|
|
// There is no non-isolated mode to fall back to, and the settings
|
|
// are process-global, so anything wrong here was already fatal at
|
|
// daemon startup; this call cannot newly fail.
|
|
let isolation = super::network_isolation_from_env()?;
|
|
tracing::info!(
|
|
%agent_name, gateway_ip = %isolation.gateway_ip, bridge = %isolation.bridge,
|
|
"network isolation: PRIVATE_NETWORK=1 (DHCP)"
|
|
);
|
|
|
|
// Delegate the actual conf-file rewrite to hive-priv (runs as root).
|
|
crate::priv_client::write_nspawn_flags(
|
|
container,
|
|
&binds,
|
|
isolation,
|
|
&load_creds,
|
|
super::hive_facts(hive),
|
|
private_users,
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// The bind of the agent's socket dir.
|
|
///
|
|
/// A namespaced agent's binds are mounted `owneridmap`, which maps the owner of
|
|
/// the mount point inside the container to the host owner. nspawn mounts a
|
|
/// fresh tmpfs on `/run` before the binds, so a mount point under `/run` is
|
|
/// created by nspawn owned by root, and the agent user would have no mapping
|
|
/// (EOVERFLOW). A namespaced agent gets the dir at `/agents/<name>/sockets`,
|
|
/// whose mount point hive-priv owns to the agent; the agent module links
|
|
/// `/run/hive-agent/<name>` to it.
|
|
fn socket_bind(agent_name: &str, socket_dir: &Path, private_users: Option<bool>) -> BindMount {
|
|
let host_path = socket_dir.to_string_lossy().into_owned();
|
|
let container_path = if private_users == Some(true) {
|
|
format!("/agents/{agent_name}/sockets")
|
|
} else {
|
|
host_path.clone()
|
|
};
|
|
BindMount {
|
|
host_path,
|
|
container_path,
|
|
read_only: false,
|
|
}
|
|
}
|
|
|
|
/// What hive-priv is told about the agent's user namespace: the record's flag,
|
|
/// or `None` (leave the agent as it is) when there is no record or it cannot be
|
|
/// read, so a swarm outage never turns a namespaced agent off or on.
|
|
fn private_users_of(
|
|
record: &Result<Option<swarm_wire::agent_closures::AgentClosure>>,
|
|
) -> Option<bool> {
|
|
match record {
|
|
Ok(record) => record.as_ref().map(|r| r.private_users),
|
|
Err(e) => {
|
|
tracing::warn!(error = %format!("{e:#}"), "closure record unreadable; the agent's user namespace is left as it is");
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The binds every agent container gets, before its own state, config and
|
|
/// role-specific dirs.
|
|
fn base_binds(runtime_dir: &Path, claude_dir: &Path, claude_mount: String) -> Vec<BindMount> {
|
|
vec![
|
|
BindMount {
|
|
host_path: runtime_dir.to_string_lossy().into_owned(),
|
|
container_path: CONTAINER_RUNTIME_MOUNT.to_owned(),
|
|
read_only: false,
|
|
},
|
|
BindMount {
|
|
host_path: claude_dir.to_string_lossy().into_owned(),
|
|
container_path: claude_mount,
|
|
read_only: false,
|
|
},
|
|
BindMount {
|
|
host_path: crate::knowledge::LOCAL_DIR.to_owned(),
|
|
container_path: crate::knowledge::CONTAINER_MOUNT.to_owned(),
|
|
read_only: true,
|
|
},
|
|
]
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::{
|
|
BindMount, HostMemory, MEMORY_CAP_FILE, base_binds, bind_child_agent_dirs,
|
|
bind_own_agent_dirs, holds_manage_root_agent, limits_to_apply, memory_cap_value,
|
|
stage_limits,
|
|
};
|
|
use crate::coordinator::HiveEnv;
|
|
|
|
fn record_with(private_users: bool) -> swarm_wire::agent_closures::AgentClosure {
|
|
swarm_wire::agent_closures::AgentClosure {
|
|
meta_rev: "a".repeat(40),
|
|
store_path: "/nix/store/x".to_owned(),
|
|
system: "x86_64-linux".to_owned(),
|
|
built_unix: 1,
|
|
cached: true,
|
|
private_users,
|
|
}
|
|
}
|
|
|
|
/// The flag in the record is what hive-priv is told, both ways. No record
|
|
/// and an unreadable one say nothing, so a swarm outage changes no agent's
|
|
/// namespace; a record from before the field reads false.
|
|
/// Only a namespaced agent's socket dir moves off `/run`.
|
|
#[test]
|
|
fn the_socket_dir_leaves_run_only_for_a_namespaced_agent() {
|
|
let dir = std::path::Path::new("/run/hive-agent/iris");
|
|
for private_users in [None, Some(false)] {
|
|
let b = super::socket_bind("iris", dir, private_users);
|
|
assert_eq!(b.container_path, "/run/hive-agent/iris");
|
|
assert_eq!(b.host_path, "/run/hive-agent/iris");
|
|
}
|
|
let b = super::socket_bind("iris", dir, Some(true));
|
|
assert_eq!(b.container_path, "/agents/iris/sockets");
|
|
assert_eq!(b.host_path, "/run/hive-agent/iris");
|
|
assert!(!b.read_only);
|
|
}
|
|
|
|
#[test]
|
|
fn the_records_flag_is_what_the_start_path_passes() {
|
|
assert_eq!(
|
|
super::private_users_of(&Ok(Some(record_with(true)))),
|
|
Some(true)
|
|
);
|
|
assert_eq!(
|
|
super::private_users_of(&Ok(Some(record_with(false)))),
|
|
Some(false)
|
|
);
|
|
assert_eq!(super::private_users_of(&Ok(None)), None);
|
|
assert_eq!(super::private_users_of(&Err(anyhow::anyhow!("down"))), None);
|
|
let old = serde_json::json!({
|
|
"meta_rev": "a", "store_path": "/nix/store/x",
|
|
"system": "s", "built_unix": 1,
|
|
});
|
|
let old = serde_json::from_value(old).expect("decodes");
|
|
assert_eq!(super::private_users_of(&Ok(Some(old))), Some(false));
|
|
}
|
|
|
|
/// No agent container gets a cross-agent scratch dir; `/knowledge` is the
|
|
/// control that the lookup finds a bind that is there.
|
|
#[test]
|
|
fn base_binds_have_no_shared_dir() {
|
|
let binds = base_binds(
|
|
std::path::Path::new("/run/x"),
|
|
std::path::Path::new("/var/lib/x/claude"),
|
|
"/home/a/.claude".to_owned(),
|
|
);
|
|
let targets: Vec<&str> = binds.iter().map(|b| b.container_path.as_str()).collect();
|
|
assert!(targets.contains(&"/knowledge"), "control: {targets:?}");
|
|
assert!(!targets.contains(&"/shared"), "{targets:?}");
|
|
assert!(
|
|
binds.iter().all(|b| !b.host_path.ends_with("/shared")),
|
|
"{binds:?}"
|
|
);
|
|
}
|
|
|
|
fn child_binds() -> Vec<BindMount> {
|
|
let mut binds = Vec::new();
|
|
bind_child_agent_dirs("kiddo", &mut binds);
|
|
binds
|
|
}
|
|
|
|
/// The boundary, asserted as a whole rather than per-dir: a
|
|
/// `ManageRootAgent` holder sees another agent's `state`, and nothing
|
|
/// else. In particular no `config`: it is read by cloning the repo.
|
|
#[test]
|
|
fn holder_sees_only_other_agents_state() {
|
|
let paths: Vec<String> = child_binds()
|
|
.into_iter()
|
|
.map(|b| b.container_path)
|
|
.collect();
|
|
assert_eq!(paths, ["/agents/kiddo/state"]);
|
|
}
|
|
|
|
/// The regression this exists for. `harness` holds the other agent's
|
|
/// own runtime material and was only ever mounted because one loop
|
|
/// treated all three dirs alike — re-adding it to that loop is a
|
|
/// one-word change that nothing else would catch.
|
|
#[test]
|
|
fn a_holder_never_sees_another_agents_harness_dir() {
|
|
for bind in child_binds() {
|
|
assert!(
|
|
!bind.container_path.contains("harness"),
|
|
"another agent's harness must not be bound in: {}",
|
|
bind.container_path
|
|
);
|
|
assert!(
|
|
!bind.host_path.contains("harness"),
|
|
"another agent's harness must not be bound in: {}",
|
|
bind.host_path
|
|
);
|
|
}
|
|
}
|
|
|
|
/// An agent's own `work` dir is bound RW at `/agents/<n>/work` from the
|
|
/// sibling of its state dir, and the host side exists afterwards —
|
|
/// nspawn refuses a missing bind source.
|
|
#[test]
|
|
fn own_binds_include_a_writable_work_dir() {
|
|
let root = tempfile::tempdir().expect("tempdir");
|
|
let notes_dir = root.path().join("state");
|
|
let mut binds = Vec::new();
|
|
bind_own_agent_dirs("iris", ¬es_dir, &mut binds);
|
|
let work_dir = root.path().join("work");
|
|
let work = binds
|
|
.iter()
|
|
.find(|b| b.container_path == "/agents/iris/work")
|
|
.expect("a work bind");
|
|
assert_eq!(work.host_path, work_dir.to_string_lossy());
|
|
assert!(!work.read_only);
|
|
assert!(work_dir.is_dir());
|
|
}
|
|
|
|
/// `work` holds the agent's worktrees and build trees; like `harness` it
|
|
/// is never bound into another agent's container.
|
|
#[test]
|
|
fn a_holder_never_sees_another_agents_work_dir() {
|
|
for bind in child_binds() {
|
|
assert!(
|
|
!bind.container_path.contains("work"),
|
|
"another agent's work dir must not be bound in: {}",
|
|
bind.container_path
|
|
);
|
|
assert!(
|
|
!bind.host_path.ends_with("/work"),
|
|
"another agent's work dir must not be bound in: {}",
|
|
bind.host_path
|
|
);
|
|
}
|
|
}
|
|
|
|
/// An unparseable name yields no mounts at all — the guard must fail
|
|
/// closed, since the alternative is a path built from unvalidated
|
|
/// input.
|
|
#[test]
|
|
fn an_invalid_child_name_binds_nothing() {
|
|
let mut binds = Vec::new();
|
|
bind_child_agent_dirs("../escape", &mut binds);
|
|
assert!(binds.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn corrupt_swarm_permissions_deny_without_failing() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let path = dir.path().join("swarm-agent-permissions.json");
|
|
let truncated = "{\n \"ruth\": {\"permissions\": [\"manage_root_ag";
|
|
std::fs::write(&path, truncated).expect("seed");
|
|
assert!(!holds_manage_root_agent("ruth", &path));
|
|
assert_eq!(std::fs::read(&path).expect("read"), truncated.as_bytes());
|
|
}
|
|
|
|
/// Unset is the default, which gives nobody the recovery mounts, `ruth`
|
|
/// included; only a grant does.
|
|
#[test]
|
|
fn the_swarm_grant_decides_the_recovery_mounts() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let path = dir.path().join("swarm-agent-permissions.json");
|
|
assert!(!holds_manage_root_agent("ruth", &path));
|
|
assert!(!holds_manage_root_agent("alice", &path));
|
|
std::fs::write(
|
|
&path,
|
|
r#"{"ruth": {"permissions": []}, "alice": {"permissions": ["manage_root_agent"]}}"#,
|
|
)
|
|
.expect("seed");
|
|
assert!(!holds_manage_root_agent("ruth", &path));
|
|
assert!(holds_manage_root_agent("alice", &path));
|
|
}
|
|
|
|
const TRUNCATED_LIMITS: &str = "{\n \"sock\": { \"cpu_quota\": \"50%\", \"mem";
|
|
|
|
fn swarm_file(dir: &tempfile::TempDir) -> std::path::PathBuf {
|
|
dir.path().join("swarm-resource-limits.json")
|
|
}
|
|
|
|
#[test]
|
|
fn corrupt_swarm_caps_keep_an_existing_dropin() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let swarm = swarm_file(&dir);
|
|
let dropin = dir.path().join("hyperhive-limits.conf");
|
|
std::fs::write(&swarm, TRUNCATED_LIMITS).expect("seed swarm caps");
|
|
std::fs::write(&dropin, "MemoryMax=1G\n").expect("seed drop-in");
|
|
assert_eq!(
|
|
limits_to_apply("sock", &swarm, &dropin, &HiveEnv::default()),
|
|
None
|
|
);
|
|
assert_eq!(
|
|
std::fs::read(&swarm).expect("read"),
|
|
TRUNCATED_LIMITS.as_bytes()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn corrupt_swarm_caps_without_a_dropin_apply_hive_defaults() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let swarm = swarm_file(&dir);
|
|
std::fs::write(&swarm, TRUNCATED_LIMITS).expect("seed swarm caps");
|
|
let applied = limits_to_apply(
|
|
"sock",
|
|
&swarm,
|
|
&dir.path().join("absent.conf"),
|
|
&HiveEnv::default(),
|
|
);
|
|
assert_eq!(applied, Some(("200%".to_owned(), "4G".to_owned())));
|
|
}
|
|
|
|
#[test]
|
|
fn a_swarm_cap_is_applied_over_the_hive_default() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let swarm = swarm_file(&dir);
|
|
std::fs::write(&swarm, r#"{"sock": {"memory_max": "2G"}}"#).expect("seed swarm caps");
|
|
let applied = limits_to_apply(
|
|
"sock",
|
|
&swarm,
|
|
&dir.path().join("absent.conf"),
|
|
&HiveEnv::default(),
|
|
);
|
|
assert_eq!(applied, Some(("200%".to_owned(), "2G".to_owned())));
|
|
}
|
|
|
|
#[test]
|
|
fn readable_swarm_caps_are_applied_over_an_existing_dropin() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let swarm = swarm_file(&dir);
|
|
let dropin = dir.path().join("hyperhive-limits.conf");
|
|
std::fs::write(&swarm, r#"{"sock": {"memory_max": "1G"}}"#).expect("seed swarm caps");
|
|
std::fs::write(&dropin, "MemoryMax=8G\n").expect("seed drop-in");
|
|
let applied = limits_to_apply("sock", &swarm, &dropin, &HiveEnv::default());
|
|
assert_eq!(applied, Some(("200%".to_owned(), "1G".to_owned())));
|
|
}
|
|
|
|
/// 4 000 003 pages of 4 KiB: odd, so rounding to a page shows.
|
|
const HOST: HostMemory = HostMemory {
|
|
bytes: 4_000_003 * 4096,
|
|
page: 4096,
|
|
};
|
|
|
|
/// Runs [`stage_limits`] against the files in `dir` and returns the
|
|
/// drop-in's `MemoryMax=` and the agent's copy of it, `None` when absent.
|
|
fn stage(dir: &tempfile::TempDir) -> (Option<String>, Option<String>) {
|
|
let runtime = dir.path().join("runtime");
|
|
let applied = stage_limits(
|
|
"sock",
|
|
&swarm_file(dir),
|
|
&dir.path().join("hyperhive-limits.conf"),
|
|
&runtime,
|
|
&HiveEnv::default(),
|
|
Some(HOST),
|
|
)
|
|
.expect("stage limits");
|
|
let copy = std::fs::read_to_string(runtime.join(MEMORY_CAP_FILE)).ok();
|
|
(applied.map(|(_, mem)| mem), copy)
|
|
}
|
|
|
|
#[test]
|
|
fn the_agent_reads_the_cap_its_dropin_carries() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let swarm = swarm_file(&dir);
|
|
|
|
// No swarm cap anywhere: the hive-wide default, 4G.
|
|
assert_eq!(
|
|
stage(&dir),
|
|
(Some("4G".to_owned()), Some("4294967296\n".to_owned()))
|
|
);
|
|
|
|
std::fs::write(&swarm, r#"{"sock": {"memory_max": "16G"}}"#).expect("seed swarm caps");
|
|
assert_eq!(
|
|
stage(&dir),
|
|
(Some("16G".to_owned()), Some("17179869184\n".to_owned()))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn every_kind_of_cap_is_written_for_the_agent() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let swarm = swarm_file(&dir);
|
|
for (memory_max, copy) in [
|
|
("infinity", "infinity\n"),
|
|
("50%", "8192004096\n"),
|
|
("8G", "8589934592\n"),
|
|
] {
|
|
std::fs::write(
|
|
&swarm,
|
|
format!(r#"{{"sock": {{"memory_max": "{memory_max}"}}}}"#),
|
|
)
|
|
.expect("seed swarm caps");
|
|
assert_eq!(
|
|
stage(&dir),
|
|
(Some(memory_max.to_owned()), Some(copy.to_owned())),
|
|
"{memory_max}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_cap_is_read_the_way_systemd_reads_memory_max() {
|
|
for (memory_max, cap) in [
|
|
("4G", "4294967296"),
|
|
("1048576B", "1048576"),
|
|
("3P", "3377699720527872"),
|
|
("infinity", "infinity"),
|
|
("50%", "8192004096"),
|
|
("33.33%", "5460787200"),
|
|
("12.5%", "2048000000"),
|
|
("100%", "16384012288"),
|
|
// systemd ignores these and the unit runs uncapped.
|
|
("0", "infinity"),
|
|
("4g", "infinity"),
|
|
("4GB", "infinity"),
|
|
("0%", "infinity"),
|
|
("150%", "infinity"),
|
|
("12.345%", "infinity"),
|
|
("lots", "infinity"),
|
|
] {
|
|
assert_eq!(
|
|
memory_cap_value("sock", memory_max, Some(HOST)).as_deref(),
|
|
Some(cap),
|
|
"{memory_max}"
|
|
);
|
|
}
|
|
assert_eq!(memory_cap_value("sock", "50%", None), None);
|
|
assert_eq!(
|
|
memory_cap_value("sock", "8G", None).as_deref(),
|
|
Some("8589934592")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn the_host_memory_systemd_would_read_is_found() {
|
|
let host = HostMemory::read().expect("sysconf");
|
|
assert!(
|
|
host.page >= 4096 && host.bytes.is_multiple_of(host.page),
|
|
"{host:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_kept_dropin_hands_the_agent_its_cap() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let swarm = swarm_file(&dir);
|
|
let dropin = dir.path().join("hyperhive-limits.conf");
|
|
let copy = dir.path().join("runtime").join(MEMORY_CAP_FILE);
|
|
std::fs::write(&swarm, r#"{"sock": {"memory_max": "1G"}}"#).expect("seed swarm caps");
|
|
assert_eq!(stage(&dir).1, Some("1073741824\n".to_owned()));
|
|
|
|
// Unreadable caps keep the drop-in; the copy follows the drop-in.
|
|
std::fs::write(&swarm, TRUNCATED_LIMITS).expect("seed swarm caps");
|
|
std::fs::write(
|
|
&dropin,
|
|
"[Unit]\nConditionPathIsDirectory=/rt/sock\n\n[Service]\nRestartSec=5\nMemoryMax=2G\nCPUQuota=200%\n",
|
|
)
|
|
.expect("seed drop-in");
|
|
assert_eq!(stage(&dir), (None, Some("2147483648\n".to_owned())));
|
|
|
|
// A kept drop-in with no copy beside it, as after an upgrade.
|
|
std::fs::remove_file(©).expect("remove copy");
|
|
assert_eq!(stage(&dir), (None, Some("2147483648\n".to_owned())));
|
|
|
|
// The last assignment is the one systemd applies.
|
|
std::fs::write(&dropin, "[Service]\nMemoryMax=2G\nMemoryMax=3G\n").expect("seed drop-in");
|
|
assert_eq!(stage(&dir), (None, Some("3221225472\n".to_owned())));
|
|
|
|
// No `MemoryMax=` is systemd's default.
|
|
std::fs::write(&dropin, "[Service]\nCPUQuota=200%\n").expect("seed drop-in");
|
|
assert_eq!(stage(&dir), (None, Some("infinity\n".to_owned())));
|
|
}
|
|
|
|
#[test]
|
|
fn an_unknown_cap_leaves_no_copy() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let swarm = swarm_file(&dir);
|
|
let dropin = dir.path().join("hyperhive-limits.conf");
|
|
assert_eq!(stage(&dir).1, Some("4294967296\n".to_owned()));
|
|
|
|
// Unreadable caps and an unreadable drop-in: nothing says what the
|
|
// cgroup enforces.
|
|
std::fs::write(&swarm, TRUNCATED_LIMITS).expect("seed swarm caps");
|
|
std::fs::create_dir(&dropin).expect("seed unreadable drop-in");
|
|
assert_eq!(stage(&dir), (None, None));
|
|
|
|
// A percentage with no host memory to resolve it against.
|
|
std::fs::write(&swarm, r#"{"sock": {"memory_max": "50%"}}"#).expect("seed swarm caps");
|
|
let runtime = dir.path().join("runtime");
|
|
let applied = stage_limits("sock", &swarm, &dropin, &runtime, &HiveEnv::default(), None)
|
|
.expect("stage limits");
|
|
assert_eq!(applied.map(|(_, mem)| mem), Some("50%".to_owned()));
|
|
assert!(!runtime.join(MEMORY_CAP_FILE).exists());
|
|
}
|
|
}
|