diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md index f6e104b1..58ffff89 100644 --- a/docs/agent-lifecycle/persistence.md +++ b/docs/agent-lifecycle/persistence.md @@ -314,6 +314,15 @@ Under `/var/lib/hyperhive/agents//`: - `hyperhive-events.sqlite` — turn-loop event log. - `hyperhive-turn-stats.sqlite` — per-turn timing stats. - `hyperhive-model` — single-line model name override file. +- `work/` — the agent's working space for git worktrees, build dirs + and other large working data. Bind-mounted RW to + `/agents//work` inside the container and owned by the agent + user. It sits outside `state/` so a claude or subagent started under + `work/` doesn't walk up into `state/CLAUDE.md` and load the agent's + own identity. Survives `destroy`/recreate, gone on `--purge`. Counted + in the per-agent disk metric (`measure_agent_disk`) together with + `state/` and the container rootfs. The dashboard doesn't serve it: + the state-file endpoint reads `state/` only. ### Cross-agent access to state diff --git a/docs/process/gotchas.md b/docs/process/gotchas.md index c3e4c7ae..1e5bf753 100644 --- a/docs/process/gotchas.md +++ b/docs/process/gotchas.md @@ -193,6 +193,17 @@ don't clutter what claude sees as "my notes dir" (see [`docs/agent-lifecycle/persistence.md`](../agent-lifecycle/persistence.md)). Survives `destroy`/recreate alongside the claude dir. +### Persistent work dir per agent + +`/var/lib/hyperhive/agents//work/` bind-mounts to +`/agents//work` (RW, owned by the agent user). It's the place for +git worktrees and build dirs: a claude started under `state/` walks up +into `state/CLAUDE.md` and takes on the agent's identity, and one +started under `work/` doesn't. `/home/` lives in the container +rootfs, which `destroy` deletes; `work/` survives `destroy`/recreate +and goes on `--purge`. The per-agent disk metric counts it; the +dashboard doesn't serve it. + ## Networking & ports ### Web UI ports collide on hash diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index d06f5969..b56e4722 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -1130,6 +1130,14 @@ impl Coordinator { crate::paths::agent_state_dir(name).join("state") } + /// Per-agent working dir for worktrees and build dirs. Bind-mounted RW + /// into the agent container at `/agents/{name}/work`. Survives + /// destroy/recreate. A sibling of `state/` rather than inside it, so a + /// claude started under `work/` never walks up into `state/CLAUDE.md`. + pub fn agent_work_dir(name: &hive_types::Ident) -> PathBuf { + crate::paths::agent_state_dir(name).join("work") + } + /// Per-agent harness-internal state dir. Bind-mounted RW into the /// agent container at `/agents/{name}/harness`. Holds sqlite dbs /// and config files owned by the harness (`hyperhive-events.sqlite`, diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index e12df5d2..38f92fcc 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -188,6 +188,30 @@ fn bind_child_agent_dirs(child: &str, binds: &mut Vec) { } } +/// Append RW bind flags for `agent_name`'s own `state`, `harness` and `work` +/// dirs into `binds`. `harness` and `work` are siblings of `notes_dir`; +/// missing ones are created so nspawn doesn't refuse to start. +fn bind_own_agent_dirs(agent_name: &str, notes_dir: &Path, binds: &mut Vec) { + binds.push(BindMount { + host_path: notes_dir.to_string_lossy().into_owned(), + container_path: format!("/agents/{agent_name}/state"), + read_only: false, + }); + if let Some(state_parent) = notes_dir.parent() { + for sub in ["harness", "work"] { + let dir = state_parent.join(sub); + if !dir.exists() { + let _ = std::fs::create_dir_all(&dir); + } + binds.push(BindMount { + host_path: dir.to_string_lossy().into_owned(), + container_path: format!("/agents/{agent_name}/{sub}"), + read_only: false, + }); + } + } +} + /// Whether `agent_name` holds `ManageRootAgent` per the capabilities file /// at `caps_path`. An unreadable file denies: the error is logged and the /// agent starts without the cross-agent mounts, so one corrupt file never @@ -362,25 +386,10 @@ async fn set_nspawn_flags( }, ]; - // Own state, harness, and config dirs — same for every agent including - // the manager. Config is RO: an agent must not edit its own config; changes - // only ever flow through a config PR on the forge. - binds.push(BindMount { - host_path: notes_dir.to_string_lossy().into_owned(), - container_path: format!("/agents/{agent_name}/state"), - read_only: false, - }); - if let Some(state_parent) = notes_dir.parent() { - let harness_dir = state_parent.join("harness"); - if !harness_dir.exists() { - let _ = std::fs::create_dir_all(&harness_dir); - } - binds.push(BindMount { - host_path: harness_dir.to_string_lossy().into_owned(), - container_path: format!("/agents/{agent_name}/harness"), - read_only: false, - }); - } + // Own state, harness, work, and config dirs — same for every agent + // including the manager. Config is RO: an agent must not edit its own + // config; changes only ever flow through a config PR on the forge. + bind_own_agent_dirs(agent_name, notes_dir, &mut binds); hive_types::Ident::parse(agent_name) .map_err(|e| anyhow::anyhow!("invalid agent name {agent_name:?}: {e}"))?; let own_config = config_bind_source(agent_name); @@ -476,7 +485,7 @@ async fn set_nspawn_flags( mod tests { use super::{ BindMount, QUEUE_CLIENT_ID_CREDENTIAL, QUEUE_SECRET_CREDENTIAL, bind_child_agent_dirs, - holds_manage_root_agent, limits_to_apply, queue_agent_credentials, + bind_own_agent_dirs, holds_manage_root_agent, limits_to_apply, queue_agent_credentials, }; use crate::coordinator::HiveEnv; @@ -539,6 +548,25 @@ mod tests { } } + /// An agent's own `work` dir is bound RW at `/agents//work` from the + /// sibling of its state dir, and the host side exists afterwards — + /// nspawn refuses a missing bind source. + #[test] + fn own_binds_include_a_writable_work_dir() { + let root = tempfile::tempdir().expect("tempdir"); + let notes_dir = root.path().join("state"); + let mut binds = Vec::new(); + bind_own_agent_dirs("iris", ¬es_dir, &mut binds); + let work_dir = root.path().join("work"); + let work = binds + .iter() + .find(|b| b.container_path == "/agents/iris/work") + .expect("a work bind"); + assert_eq!(work.host_path, work_dir.to_string_lossy()); + assert!(!work.read_only); + assert!(work_dir.is_dir()); + } + /// An unparseable name yields no mounts at all — the guard must fail /// closed, since the alternative is a path built from unvalidated /// input. diff --git a/hive-c0re/src/lifecycle/setup.rs b/hive-c0re/src/lifecycle/setup.rs index 033af761..3367f3ba 100644 --- a/hive-c0re/src/lifecycle/setup.rs +++ b/hive-c0re/src/lifecycle/setup.rs @@ -210,18 +210,20 @@ pub fn ensure_claude_dir(claude_dir: &Path) -> Result<()> { /// Create the per-agent state dir if missing. Also creates the sibling /// `harness/` dir so the first harness startup can write its sqlite files -/// immediately. +/// immediately, and the sibling `work/` dir. pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> { if !notes_dir.exists() { std::fs::create_dir_all(notes_dir) .with_context(|| format!("create {}", notes_dir.display()))?; } - // Harness dir is a sibling of the agent-visible state dir. + // Harness and work dirs are siblings of the agent-visible state dir. if let Some(parent) = notes_dir.parent() { - let harness_dir = parent.join("harness"); - if !harness_dir.exists() { - std::fs::create_dir_all(&harness_dir) - .with_context(|| format!("create {}", harness_dir.display()))?; + for sibling in ["harness", "work"] { + let dir = parent.join(sibling); + if !dir.exists() { + std::fs::create_dir_all(&dir) + .with_context(|| format!("create {}", dir.display()))?; + } } } Ok(()) @@ -230,7 +232,7 @@ pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> { /// Ensure agent `name`'s persistent state root /// (`/var/lib/hyperhive/agents/`) is a btrfs subvolume — when the host /// filesystem supports it — BEFORE the per-agent subdirs (`state/`, `claude/`, -/// `harness/`) are created by `ensure_state_dir` / `ensure_claude_dir`. +/// `harness/`, `work/`) are created by `ensure_state_dir` / `ensure_claude_dir`. /// /// Progressive enhancement: if the root already exists /// (any agent provisioned before this landed, plain dir or subvol) it's left diff --git a/hive-c0re/src/stats/container_stats.rs b/hive-c0re/src/stats/container_stats.rs index 26f677d1..1ba5c0e9 100644 --- a/hive-c0re/src/stats/container_stats.rs +++ b/hive-c0re/src/stats/container_stats.rs @@ -139,11 +139,20 @@ async fn du_bytes(path: &std::path::Path) -> Option { } } -/// Total per-agent disk: the agent's host-side state dir -/// (`/var/lib/hyperhive/agents//state`) plus the container's writable -/// rootfs (`/var/lib/nixos-containers/h-`). Each measured with `du -sxb` -/// so the shared nix store and other bind mounts are excluded. A path that -/// doesn't exist contributes 0. +/// Host paths summed into an agent's disk usage: the agent's host-side +/// state and work dirs (`/var/lib/hyperhive/agents//{state,work}`) plus +/// the container's writable rootfs (`/var/lib/nixos-containers/h-`). +fn agent_disk_paths(name: &hive_types::Ident) -> [PathBuf; 3] { + [ + Coordinator::agent_notes_dir(name), + Coordinator::agent_work_dir(name), + PathBuf::from(format!("{NIXOS_CONTAINERS_ROOT}/h-{name}")), + ] +} + +/// Total per-agent disk over [`agent_disk_paths`]. Each path is measured +/// with `du -sxb` so the shared nix store and other bind mounts are +/// excluded. A path that doesn't exist contributes 0. /// /// The state dir is resolved via [`Coordinator::agent_notes_dir`] — the same /// host path the dashboard reads agent state from. Hardcoding the @@ -153,10 +162,8 @@ async fn du_bytes(path: &std::path::Path) -> Option { /// (almost everything is bind-mounted), that surfaced as all agents reporting /// 0 disk. async fn measure_agent_disk(name: &hive_types::Ident) -> u64 { - let state_dir = Coordinator::agent_notes_dir(name); - let rootfs = PathBuf::from(format!("{NIXOS_CONTAINERS_ROOT}/h-{name}")); let mut total = 0u64; - for path in [state_dir, rootfs] { + for path in agent_disk_paths(name) { if let Some(bytes) = du_bytes(&path).await { total = total.saturating_add(bytes); } @@ -356,6 +363,19 @@ mod tests { ); } + #[test] + fn disk_paths_cover_state_work_and_rootfs() { + let name = hive_types::Ident::parse("iris").expect("valid ident"); + assert_eq!( + agent_disk_paths(&name), + [ + Coordinator::agent_notes_dir(&name), + Coordinator::agent_work_dir(&name), + PathBuf::from(format!("{NIXOS_CONTAINERS_ROOT}/h-iris")), + ] + ); + } + #[tokio::test] async fn du_bytes_missing_path_is_none() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/nix/agent-modules/user.nix b/nix/agent-modules/user.nix index be8bb6e7..f521301f 100644 --- a/nix/agent-modules/user.nix +++ b/nix/agent-modules/user.nix @@ -195,7 +195,7 @@ in : > "$marker" fi fi - # Scope state + harness chowns to THIS container's own dirs only. + # Scope state + harness + work chowns to THIS container's own dirs only. # The glob `/agents/*/state` also matches other agents' state dirs # bind-mounted into a `ManageRootAgent` holder's container, which # would clobber the ownership those dirs' own activation scripts set @@ -209,6 +209,9 @@ in if [ -d "/agents/$userName/harness" ]; then chown -hR "$userName:$userName" "/agents/$userName/harness" 2>/dev/null || true fi + if [ -d "/agents/$userName/work" ]; then + chown -hR "$userName:$userName" "/agents/$userName/work" 2>/dev/null || true + fi # The socket dir is bind-mounted from the host, where hive-priv creates # it `0751 root`. The harness binds its sockets here as this user; the # 0751 is what lets hive-c0re and nginx dial them without listing diff --git a/nix/module-eval/agent-user.nix b/nix/module-eval/agent-user.nix index 4f3de42d..2be03f84 100644 --- a/nix/module-eval/agent-user.nix +++ b/nix/module-eval/agent-user.nix @@ -26,6 +26,12 @@ let (agentWith { services.hyperhive.agent.user.name = "iris"; }) .system.activationScripts.hive-agent-user-migrate.text; cases = [ + { + # hive-c0re creates the host work dir as hive-core; this activation is + # the only thing that lets the agent write to it. + name = "the agent's activation hands its own work dir to the agent user"; + ok = lib.hasInfix ''chown -hR "$userName:$userName" "/agents/$userName/work"'' migrate; + } { # hive-priv creates the host socket dir `0751 root` and never chowns it, # so this activation is the only thing that lets the harness bind there.