agents: persistent /agents/<n>/work dir outside state/
Agents keep git worktrees and cargo target dirs under /home/<agent>, which lives in the container rootfs and is deleted on destroy. They can't move them under state/, because a claude started there walks up into state/CLAUDE.md and takes on the agent's identity. Add a per-agent work/ dir next to state/ on the host (/var/lib/hyperhive/agents/<n>/work), bind-mounted RW at /agents/<n>/work. hive-c0re creates it in ensure_state_dir and in set_nspawn_flags (the rebuild path reaches only the latter), the agent's activation chowns it to the agent user like state/ and harness/, and measure_agent_disk counts it. destroy never touches the agent root, so work/ survives it; purge removes the whole root, so work/ goes with it. Closes #4977
This commit is contained in:
parent
962c1548eb
commit
89a5d65119
8 changed files with 123 additions and 36 deletions
|
|
@ -314,6 +314,15 @@ Under `/var/lib/hyperhive/agents/<name>/`:
|
||||||
- `hyperhive-events.sqlite` — turn-loop event log.
|
- `hyperhive-events.sqlite` — turn-loop event log.
|
||||||
- `hyperhive-turn-stats.sqlite` — per-turn timing stats.
|
- `hyperhive-turn-stats.sqlite` — per-turn timing stats.
|
||||||
- `hyperhive-model` — single-line model name override file.
|
- `hyperhive-model` — single-line model name override file.
|
||||||
|
- `work/` — the agent's working space for git worktrees, build dirs
|
||||||
|
and other large working data. Bind-mounted RW to
|
||||||
|
`/agents/<name>/work` inside the container and owned by the agent
|
||||||
|
user. It sits outside `state/` so a claude or subagent started under
|
||||||
|
`work/` doesn't walk up into `state/CLAUDE.md` and load the agent's
|
||||||
|
own identity. Survives `destroy`/recreate, gone on `--purge`. Counted
|
||||||
|
in the per-agent disk metric (`measure_agent_disk`) together with
|
||||||
|
`state/` and the container rootfs. The dashboard doesn't serve it:
|
||||||
|
the state-file endpoint reads `state/` only.
|
||||||
|
|
||||||
### Cross-agent access to state
|
### Cross-agent access to state
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -193,6 +193,17 @@ don't clutter what claude sees as "my notes dir" (see
|
||||||
[`docs/agent-lifecycle/persistence.md`](../agent-lifecycle/persistence.md)). Survives `destroy`/recreate
|
[`docs/agent-lifecycle/persistence.md`](../agent-lifecycle/persistence.md)). Survives `destroy`/recreate
|
||||||
alongside the claude dir.
|
alongside the claude dir.
|
||||||
|
|
||||||
|
### Persistent work dir per agent
|
||||||
|
|
||||||
|
`/var/lib/hyperhive/agents/<name>/work/` bind-mounts to
|
||||||
|
`/agents/<name>/work` (RW, owned by the agent user). It's the place for
|
||||||
|
git worktrees and build dirs: a claude started under `state/` walks up
|
||||||
|
into `state/CLAUDE.md` and takes on the agent's identity, and one
|
||||||
|
started under `work/` doesn't. `/home/<name>` lives in the container
|
||||||
|
rootfs, which `destroy` deletes; `work/` survives `destroy`/recreate
|
||||||
|
and goes on `--purge`. The per-agent disk metric counts it; the
|
||||||
|
dashboard doesn't serve it.
|
||||||
|
|
||||||
## Networking & ports
|
## Networking & ports
|
||||||
|
|
||||||
### Web UI ports collide on hash
|
### Web UI ports collide on hash
|
||||||
|
|
|
||||||
|
|
@ -1130,6 +1130,14 @@ impl Coordinator {
|
||||||
crate::paths::agent_state_dir(name).join("state")
|
crate::paths::agent_state_dir(name).join("state")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Per-agent working dir for worktrees and build dirs. Bind-mounted RW
|
||||||
|
/// into the agent container at `/agents/{name}/work`. Survives
|
||||||
|
/// destroy/recreate. A sibling of `state/` rather than inside it, so a
|
||||||
|
/// claude started under `work/` never walks up into `state/CLAUDE.md`.
|
||||||
|
pub fn agent_work_dir(name: &hive_types::Ident) -> PathBuf {
|
||||||
|
crate::paths::agent_state_dir(name).join("work")
|
||||||
|
}
|
||||||
|
|
||||||
/// Per-agent harness-internal state dir. Bind-mounted RW into the
|
/// Per-agent harness-internal state dir. Bind-mounted RW into the
|
||||||
/// agent container at `/agents/{name}/harness`. Holds sqlite dbs
|
/// agent container at `/agents/{name}/harness`. Holds sqlite dbs
|
||||||
/// and config files owned by the harness (`hyperhive-events.sqlite`,
|
/// and config files owned by the harness (`hyperhive-events.sqlite`,
|
||||||
|
|
|
||||||
|
|
@ -188,6 +188,30 @@ fn bind_child_agent_dirs(child: &str, binds: &mut Vec<BindMount>) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Append RW bind flags for `agent_name`'s own `state`, `harness` and `work`
|
||||||
|
/// dirs into `binds`. `harness` and `work` are siblings of `notes_dir`;
|
||||||
|
/// missing ones are created so nspawn doesn't refuse to start.
|
||||||
|
fn bind_own_agent_dirs(agent_name: &str, notes_dir: &Path, binds: &mut Vec<BindMount>) {
|
||||||
|
binds.push(BindMount {
|
||||||
|
host_path: notes_dir.to_string_lossy().into_owned(),
|
||||||
|
container_path: format!("/agents/{agent_name}/state"),
|
||||||
|
read_only: false,
|
||||||
|
});
|
||||||
|
if let Some(state_parent) = notes_dir.parent() {
|
||||||
|
for sub in ["harness", "work"] {
|
||||||
|
let dir = state_parent.join(sub);
|
||||||
|
if !dir.exists() {
|
||||||
|
let _ = std::fs::create_dir_all(&dir);
|
||||||
|
}
|
||||||
|
binds.push(BindMount {
|
||||||
|
host_path: dir.to_string_lossy().into_owned(),
|
||||||
|
container_path: format!("/agents/{agent_name}/{sub}"),
|
||||||
|
read_only: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Whether `agent_name` holds `ManageRootAgent` per the capabilities file
|
/// Whether `agent_name` holds `ManageRootAgent` per the capabilities file
|
||||||
/// at `caps_path`. An unreadable file denies: the error is logged and the
|
/// at `caps_path`. An unreadable file denies: the error is logged and the
|
||||||
/// agent starts without the cross-agent mounts, so one corrupt file never
|
/// agent starts without the cross-agent mounts, so one corrupt file never
|
||||||
|
|
@ -362,25 +386,10 @@ async fn set_nspawn_flags(
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
// Own state, harness, and config dirs — same for every agent including
|
// Own state, harness, work, and config dirs — same for every agent
|
||||||
// the manager. Config is RO: an agent must not edit its own config; changes
|
// including the manager. Config is RO: an agent must not edit its own
|
||||||
// only ever flow through a config PR on the forge.
|
// config; changes only ever flow through a config PR on the forge.
|
||||||
binds.push(BindMount {
|
bind_own_agent_dirs(agent_name, notes_dir, &mut binds);
|
||||||
host_path: notes_dir.to_string_lossy().into_owned(),
|
|
||||||
container_path: format!("/agents/{agent_name}/state"),
|
|
||||||
read_only: false,
|
|
||||||
});
|
|
||||||
if let Some(state_parent) = notes_dir.parent() {
|
|
||||||
let harness_dir = state_parent.join("harness");
|
|
||||||
if !harness_dir.exists() {
|
|
||||||
let _ = std::fs::create_dir_all(&harness_dir);
|
|
||||||
}
|
|
||||||
binds.push(BindMount {
|
|
||||||
host_path: harness_dir.to_string_lossy().into_owned(),
|
|
||||||
container_path: format!("/agents/{agent_name}/harness"),
|
|
||||||
read_only: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
hive_types::Ident::parse(agent_name)
|
hive_types::Ident::parse(agent_name)
|
||||||
.map_err(|e| anyhow::anyhow!("invalid agent name {agent_name:?}: {e}"))?;
|
.map_err(|e| anyhow::anyhow!("invalid agent name {agent_name:?}: {e}"))?;
|
||||||
let own_config = config_bind_source(agent_name);
|
let own_config = config_bind_source(agent_name);
|
||||||
|
|
@ -476,7 +485,7 @@ async fn set_nspawn_flags(
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
BindMount, QUEUE_CLIENT_ID_CREDENTIAL, QUEUE_SECRET_CREDENTIAL, bind_child_agent_dirs,
|
BindMount, QUEUE_CLIENT_ID_CREDENTIAL, QUEUE_SECRET_CREDENTIAL, bind_child_agent_dirs,
|
||||||
holds_manage_root_agent, limits_to_apply, queue_agent_credentials,
|
bind_own_agent_dirs, holds_manage_root_agent, limits_to_apply, queue_agent_credentials,
|
||||||
};
|
};
|
||||||
use crate::coordinator::HiveEnv;
|
use crate::coordinator::HiveEnv;
|
||||||
|
|
||||||
|
|
@ -539,6 +548,25 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// An agent's own `work` dir is bound RW at `/agents/<n>/work` from the
|
||||||
|
/// sibling of its state dir, and the host side exists afterwards —
|
||||||
|
/// nspawn refuses a missing bind source.
|
||||||
|
#[test]
|
||||||
|
fn own_binds_include_a_writable_work_dir() {
|
||||||
|
let root = tempfile::tempdir().expect("tempdir");
|
||||||
|
let notes_dir = root.path().join("state");
|
||||||
|
let mut binds = Vec::new();
|
||||||
|
bind_own_agent_dirs("iris", ¬es_dir, &mut binds);
|
||||||
|
let work_dir = root.path().join("work");
|
||||||
|
let work = binds
|
||||||
|
.iter()
|
||||||
|
.find(|b| b.container_path == "/agents/iris/work")
|
||||||
|
.expect("a work bind");
|
||||||
|
assert_eq!(work.host_path, work_dir.to_string_lossy());
|
||||||
|
assert!(!work.read_only);
|
||||||
|
assert!(work_dir.is_dir());
|
||||||
|
}
|
||||||
|
|
||||||
/// An unparseable name yields no mounts at all — the guard must fail
|
/// An unparseable name yields no mounts at all — the guard must fail
|
||||||
/// closed, since the alternative is a path built from unvalidated
|
/// closed, since the alternative is a path built from unvalidated
|
||||||
/// input.
|
/// input.
|
||||||
|
|
|
||||||
|
|
@ -210,18 +210,20 @@ pub fn ensure_claude_dir(claude_dir: &Path) -> Result<()> {
|
||||||
|
|
||||||
/// Create the per-agent state dir if missing. Also creates the sibling
|
/// Create the per-agent state dir if missing. Also creates the sibling
|
||||||
/// `harness/` dir so the first harness startup can write its sqlite files
|
/// `harness/` dir so the first harness startup can write its sqlite files
|
||||||
/// immediately.
|
/// immediately, and the sibling `work/` dir.
|
||||||
pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> {
|
pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> {
|
||||||
if !notes_dir.exists() {
|
if !notes_dir.exists() {
|
||||||
std::fs::create_dir_all(notes_dir)
|
std::fs::create_dir_all(notes_dir)
|
||||||
.with_context(|| format!("create {}", notes_dir.display()))?;
|
.with_context(|| format!("create {}", notes_dir.display()))?;
|
||||||
}
|
}
|
||||||
// Harness dir is a sibling of the agent-visible state dir.
|
// Harness and work dirs are siblings of the agent-visible state dir.
|
||||||
if let Some(parent) = notes_dir.parent() {
|
if let Some(parent) = notes_dir.parent() {
|
||||||
let harness_dir = parent.join("harness");
|
for sibling in ["harness", "work"] {
|
||||||
if !harness_dir.exists() {
|
let dir = parent.join(sibling);
|
||||||
std::fs::create_dir_all(&harness_dir)
|
if !dir.exists() {
|
||||||
.with_context(|| format!("create {}", harness_dir.display()))?;
|
std::fs::create_dir_all(&dir)
|
||||||
|
.with_context(|| format!("create {}", dir.display()))?;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
@ -230,7 +232,7 @@ pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> {
|
||||||
/// Ensure agent `name`'s persistent state root
|
/// Ensure agent `name`'s persistent state root
|
||||||
/// (`/var/lib/hyperhive/agents/<name>`) is a btrfs subvolume — when the host
|
/// (`/var/lib/hyperhive/agents/<name>`) is a btrfs subvolume — when the host
|
||||||
/// filesystem supports it — BEFORE the per-agent subdirs (`state/`, `claude/`,
|
/// filesystem supports it — BEFORE the per-agent subdirs (`state/`, `claude/`,
|
||||||
/// `harness/`) are created by `ensure_state_dir` / `ensure_claude_dir`.
|
/// `harness/`, `work/`) are created by `ensure_state_dir` / `ensure_claude_dir`.
|
||||||
///
|
///
|
||||||
/// Progressive enhancement: if the root already exists
|
/// Progressive enhancement: if the root already exists
|
||||||
/// (any agent provisioned before this landed, plain dir or subvol) it's left
|
/// (any agent provisioned before this landed, plain dir or subvol) it's left
|
||||||
|
|
|
||||||
|
|
@ -139,11 +139,20 @@ async fn du_bytes(path: &std::path::Path) -> Option<u64> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Total per-agent disk: the agent's host-side state dir
|
/// Host paths summed into an agent's disk usage: the agent's host-side
|
||||||
/// (`/var/lib/hyperhive/agents/<name>/state`) plus the container's writable
|
/// state and work dirs (`/var/lib/hyperhive/agents/<name>/{state,work}`) plus
|
||||||
/// rootfs (`/var/lib/nixos-containers/h-<name>`). Each measured with `du -sxb`
|
/// the container's writable rootfs (`/var/lib/nixos-containers/h-<name>`).
|
||||||
/// so the shared nix store and other bind mounts are excluded. A path that
|
fn agent_disk_paths(name: &hive_types::Ident) -> [PathBuf; 3] {
|
||||||
/// doesn't exist contributes 0.
|
[
|
||||||
|
Coordinator::agent_notes_dir(name),
|
||||||
|
Coordinator::agent_work_dir(name),
|
||||||
|
PathBuf::from(format!("{NIXOS_CONTAINERS_ROOT}/h-{name}")),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Total per-agent disk over [`agent_disk_paths`]. Each path is measured
|
||||||
|
/// with `du -sxb` so the shared nix store and other bind mounts are
|
||||||
|
/// excluded. A path that doesn't exist contributes 0.
|
||||||
///
|
///
|
||||||
/// The state dir is resolved via [`Coordinator::agent_notes_dir`] — the same
|
/// The state dir is resolved via [`Coordinator::agent_notes_dir`] — the same
|
||||||
/// host path the dashboard reads agent state from. Hardcoding the
|
/// host path the dashboard reads agent state from. Hardcoding the
|
||||||
|
|
@ -153,10 +162,8 @@ async fn du_bytes(path: &std::path::Path) -> Option<u64> {
|
||||||
/// (almost everything is bind-mounted), that surfaced as all agents reporting
|
/// (almost everything is bind-mounted), that surfaced as all agents reporting
|
||||||
/// 0 disk.
|
/// 0 disk.
|
||||||
async fn measure_agent_disk(name: &hive_types::Ident) -> u64 {
|
async fn measure_agent_disk(name: &hive_types::Ident) -> u64 {
|
||||||
let state_dir = Coordinator::agent_notes_dir(name);
|
|
||||||
let rootfs = PathBuf::from(format!("{NIXOS_CONTAINERS_ROOT}/h-{name}"));
|
|
||||||
let mut total = 0u64;
|
let mut total = 0u64;
|
||||||
for path in [state_dir, rootfs] {
|
for path in agent_disk_paths(name) {
|
||||||
if let Some(bytes) = du_bytes(&path).await {
|
if let Some(bytes) = du_bytes(&path).await {
|
||||||
total = total.saturating_add(bytes);
|
total = total.saturating_add(bytes);
|
||||||
}
|
}
|
||||||
|
|
@ -356,6 +363,19 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn disk_paths_cover_state_work_and_rootfs() {
|
||||||
|
let name = hive_types::Ident::parse("iris").expect("valid ident");
|
||||||
|
assert_eq!(
|
||||||
|
agent_disk_paths(&name),
|
||||||
|
[
|
||||||
|
Coordinator::agent_notes_dir(&name),
|
||||||
|
Coordinator::agent_work_dir(&name),
|
||||||
|
PathBuf::from(format!("{NIXOS_CONTAINERS_ROOT}/h-iris")),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn du_bytes_missing_path_is_none() {
|
async fn du_bytes_missing_path_is_none() {
|
||||||
let dir = tempfile::tempdir().expect("tempdir");
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
|
|
||||||
|
|
@ -195,7 +195,7 @@ in
|
||||||
: > "$marker"
|
: > "$marker"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
# Scope state + harness chowns to THIS container's own dirs only.
|
# Scope state + harness + work chowns to THIS container's own dirs only.
|
||||||
# The glob `/agents/*/state` also matches other agents' state dirs
|
# The glob `/agents/*/state` also matches other agents' state dirs
|
||||||
# bind-mounted into a `ManageRootAgent` holder's container, which
|
# bind-mounted into a `ManageRootAgent` holder's container, which
|
||||||
# would clobber the ownership those dirs' own activation scripts set
|
# would clobber the ownership those dirs' own activation scripts set
|
||||||
|
|
@ -209,6 +209,9 @@ in
|
||||||
if [ -d "/agents/$userName/harness" ]; then
|
if [ -d "/agents/$userName/harness" ]; then
|
||||||
chown -hR "$userName:$userName" "/agents/$userName/harness" 2>/dev/null || true
|
chown -hR "$userName:$userName" "/agents/$userName/harness" 2>/dev/null || true
|
||||||
fi
|
fi
|
||||||
|
if [ -d "/agents/$userName/work" ]; then
|
||||||
|
chown -hR "$userName:$userName" "/agents/$userName/work" 2>/dev/null || true
|
||||||
|
fi
|
||||||
# The socket dir is bind-mounted from the host, where hive-priv creates
|
# The socket dir is bind-mounted from the host, where hive-priv creates
|
||||||
# it `0751 root`. The harness binds its sockets here as this user; the
|
# it `0751 root`. The harness binds its sockets here as this user; the
|
||||||
# 0751 is what lets hive-c0re and nginx dial them without listing
|
# 0751 is what lets hive-c0re and nginx dial them without listing
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,12 @@ let
|
||||||
(agentWith { services.hyperhive.agent.user.name = "iris"; })
|
(agentWith { services.hyperhive.agent.user.name = "iris"; })
|
||||||
.system.activationScripts.hive-agent-user-migrate.text;
|
.system.activationScripts.hive-agent-user-migrate.text;
|
||||||
cases = [
|
cases = [
|
||||||
|
{
|
||||||
|
# hive-c0re creates the host work dir as hive-core; this activation is
|
||||||
|
# the only thing that lets the agent write to it.
|
||||||
|
name = "the agent's activation hands its own work dir to the agent user";
|
||||||
|
ok = lib.hasInfix ''chown -hR "$userName:$userName" "/agents/$userName/work"'' migrate;
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# hive-priv creates the host socket dir `0751 root` and never chowns it,
|
# hive-priv creates the host socket dir `0751 root` and never chowns it,
|
||||||
# so this activation is the only thing that lets the harness bind there.
|
# so this activation is the only thing that lets the harness bind there.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue