From 1f602d5fda5cd9a930893bec36d549ecda6f5c7d Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 19 Jun 2026 12:28:17 +0200 Subject: [PATCH] hive-c0re: back agent state dirs with btrfs subvolumes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Progressive enhancement: a brand-new agent's state root under /var/lib/hyperhive/agents is created as a btrfs subvolume when the host filesystem is btrfs, otherwise it falls back to a plain directory. No existing agent is auto-migrated — the new path only fires when the root does not yet exist, so plain-dir agents are left untouched until an explicit opt-in upgrade. Two new privileged ops (subvolume create/delete are root-only): EnsureAgentSubvolume statfs-gates on btrfs, creates the subvolume, and chowns it to the hive-core user so the normal state/claude/harness mkdirs succeed inside it; DeleteAgentSubvolume btrfs-subvolume-deletes the root iff it is actually a subvolume. hive-c0re calls Ensure before the per-agent dirs are created (spawn/rebuild/InitConfig) and Delete on the purge path only — destroy keeps the subvolume for revival, matching plain-dir semantics. btrfs-progs added to the hive-priv unit PATH. Per-subvolume usage accounting + optional quota is a separate follow-up. --- Cargo.lock | 1 + docs/persistence.md | 27 ++++++++ hive-c0re/src/actions.rs | 8 +++ hive-c0re/src/lifecycle.rs | 24 +++++++ hive-c0re/src/priv_client.rs | 21 ++++++ hive-priv/Cargo.toml | 1 + hive-priv/src/main.rs | 122 +++++++++++++++++++++++++++++++++++ hive-sh4re/src/priv_proto.rs | 37 +++++++++++ nix/modules/hive-c0re.nix | 1 + 9 files changed, 242 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f7e5ca18..f6f6cfd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1406,6 +1406,7 @@ version = "0.1.0" dependencies = [ "anyhow", "hive-sh4re", + "libc", "serde_json", "tokio", "tracing", diff --git a/docs/persistence.md b/docs/persistence.md index d1f297b2..233fedfa 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -288,6 +288,33 @@ step would re-fire). The manager is non-destroyable from both paths (declarative container; would fight with the host's NixOS config). +### btrfs subvolumes for `/var/lib/hyperhive/agents/` + +On a btrfs host, a brand-new agent's state root is created as a +**btrfs subvolume** instead of a plain directory (progressive +enhancement — see the #1762 lane). This is a no-op fallback on +non-btrfs hosts and for any agent whose root already exists, so +nothing is auto-migrated: existing agents keep their plain dirs +until an explicit opt-in upgrade. + +- **Creation:** `lifecycle::ensure_agent_state_subvolume` runs before + the per-agent subdirs are created (spawn / rebuild / InitConfig). + It skips the work when the root already exists; otherwise it asks + hive-priv (`EnsureAgentSubvolume`) to `btrfs subvolume create` the + root when the FS is btrfs (`statfs` magic gate) and chown it to the + `hive-core` user so the normal `state/` `claude/` `harness/` mkdirs + succeed inside it. +- **DESTR0Y keeps the subvolume** exactly like a plain dir — revival + reuses it untouched. +- **PURG3 deletes it correctly:** a subvolume root can't be removed + with `rmdir`/`remove_dir_all`, so purge first calls hive-priv + (`DeleteAgentSubvolume`) which `btrfs subvolume delete`s it iff it's + actually a subvolume, then the normal `remove_dir_all` sweep covers + plain-dir agents + the applied dir. + +Per-subvolume disk-usage accounting and optional quotas are a +follow-up (the qgroup work), not part of the base migration. + ## Run-time dirs `/run/hyperhive/` is tmpfs-backed (systemd `RuntimeDirectory=`) but diff --git a/hive-c0re/src/actions.rs b/hive-c0re/src/actions.rs index d3c98ef7..84c782fd 100644 --- a/hive-c0re/src/actions.rs +++ b/hive-c0re/src/actions.rs @@ -315,6 +315,7 @@ async fn run_approval_init_config( ) -> Result<()> { let result: Result<()> = async { lifecycle::setup_proposed(&proposed_dir, &approval.agent).await?; + lifecycle::ensure_agent_state_subvolume(&approval.agent).await?; lifecycle::ensure_claude_dir(&claude_dir)?; lifecycle::ensure_state_dir(¬es_dir)?; Ok(()) @@ -659,6 +660,13 @@ pub async fn destroy(coord: &Arc, name: &str, purge: bool) -> Resul let _ = std::fs::remove_dir_all(&runtime); } if purge { + // The state root may be a btrfs subvolume (#1762): a subvolume root + // can't be removed with rmdir/`remove_dir_all`, so delete it via + // hive-priv (root) first. No-op for plain-dir agents — the loop below + // then handles the plain-dir state root plus the applied dir. + if let Err(e) = crate::priv_client::delete_agent_subvolume(name).await { + tracing::warn!(error = ?e, %name, "purge: delete state subvolume failed"); + } for dir in [ Coordinator::agent_state_root(name), Coordinator::agent_applied_dir(name), diff --git a/hive-c0re/src/lifecycle.rs b/hive-c0re/src/lifecycle.rs index 9185ea55..b0c03169 100644 --- a/hive-c0re/src/lifecycle.rs +++ b/hive-c0re/src/lifecycle.rs @@ -269,6 +269,7 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> } setup_proposed(&paths.proposed_dir, name).await?; setup_applied(&paths.applied_dir, Some(&paths.proposed_dir), name).await?; + ensure_agent_state_subvolume(name).await?; ensure_claude_dir(&paths.claude_dir)?; ensure_state_dir(&paths.notes_dir)?; // Meta flake gets the new agent's input + nixosConfiguration @@ -459,6 +460,7 @@ pub async fn rebuild_no_meta( ); } setup_applied(&paths.applied_dir, None, name).await?; + ensure_agent_state_subvolume(name).await?; ensure_claude_dir(&paths.claude_dir)?; ensure_state_dir(&paths.notes_dir)?; let container = container_name(name); @@ -871,6 +873,28 @@ pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> { Ok(()) } +/// Ensure agent `name`'s persistent state root +/// (`/var/lib/hyperhive/agents/`) is a btrfs subvolume — when the host +/// filesystem supports it — BEFORE the per-agent subdirs (`state/`, `claude/`, +/// `harness/`) are created by `ensure_state_dir` / `ensure_claude_dir`. +/// +/// Progressive enhancement (the #1762 model): if the root already exists +/// (any agent provisioned before this landed, plain dir or subvol) it's left +/// exactly as-is — no auto-migration — and the priv round-trip is skipped. On +/// a non-btrfs host the priv op no-ops and the root is later created as a +/// plain dir by `ensure_*_dir`, identical to the old behaviour. Only a +/// brand-new agent on a btrfs host gets a real subvolume. Subvolume creation +/// is privileged, so it's delegated to hive-priv. +pub async fn ensure_agent_state_subvolume(name: &str) -> Result<()> { + let root = Path::new(HOST_AGENTS_ROOT).join(name); + if root.exists() { + return Ok(()); + } + crate::priv_client::ensure_agent_subvolume(name) + .await + .with_context(|| format!("ensure btrfs subvolume for agent {name}")) +} + fn initial_agent_nix(name: &str) -> String { format!( "{{ config, pkgs, lib, ... }}:\n{{\n # Per-agent overrides for {name}. This is a regular NixOS module\n # — add packages, services, modules, imports as needed.\n #\n # imports = [ ./extra-module.nix ];\n # environment.systemPackages = with pkgs; [ ];\n}}\n", diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index b6a25b5f..dd4cc41d 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -305,6 +305,27 @@ pub async fn control_infra_container(container: &str, action: InfraAction) -> Re .await?) } +/// Ensure the agent's persistent state root is a btrfs subvolume when the +/// host FS supports it (via hive-priv, which runs as root). Idempotent and +/// progressive: a no-op when the root already exists or the FS isn't btrfs. +/// Safe to call on every provision. +pub async fn ensure_agent_subvolume(agent_name: &str) -> Result<()> { + ok(call(&PrivRequest::EnsureAgentSubvolume { + agent_name: agent_name.to_owned(), + }) + .await?) +} + +/// Delete the agent's state root iff it is a btrfs subvolume (purge path +/// only). No-op for plain dirs / missing paths — hive-c0re's own +/// `remove_dir_all` handles those. +pub async fn delete_agent_subvolume(agent_name: &str) -> Result<()> { + ok(call(&PrivRequest::DeleteAgentSubvolume { + agent_name: agent_name.to_owned(), + }) + .await?) +} + fn check(resp: PrivResponse) -> Result<(String, String)> { if resp.ok { Ok((resp.stdout, resp.stderr)) diff --git a/hive-priv/Cargo.toml b/hive-priv/Cargo.toml index eccd4f69..39a369fc 100644 --- a/hive-priv/Cargo.toml +++ b/hive-priv/Cargo.toml @@ -9,6 +9,7 @@ workspace = true [dependencies] anyhow.workspace = true hive-sh4re.workspace = true +libc.workspace = true serde_json.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 47e9cce1..ee5b6ab7 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -269,6 +269,16 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String, ref container, action, } => control_infra_container(container, action).await, + + PrivRequest::EnsureAgentSubvolume { ref agent_name } => { + validate_agent_name(agent_name)?; + ensure_agent_subvolume(agent_name).await + } + + PrivRequest::DeleteAgentSubvolume { ref agent_name } => { + validate_agent_name(agent_name)?; + delete_agent_subvolume(agent_name).await + } } } @@ -484,6 +494,118 @@ fn write_agent_state_file( Ok((String::new(), String::new())) } +/// btrfs superblock magic, as reported by `statfs(2)`'s `f_type`. +const BTRFS_SUPER_MAGIC: i64 = 0x9123_683E; + +/// Inode number of a btrfs subvolume root (`BTRFS_FIRST_FREE_OBJECTID`). +/// Every subvolume's top directory has this inode; plain directories do +/// not, so `statfs == btrfs && st_ino == 256` reliably identifies a +/// subvolume root. +const BTRFS_SUBVOL_ROOT_INO: u64 = 256; + +/// Whether `path` lives on a btrfs filesystem (via `statfs(2)`). +fn is_on_btrfs(path: &Path) -> Result { + use std::os::unix::ffi::OsStrExt as _; + let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()) + .with_context(|| format!("path {} has an interior null byte", path.display()))?; + // SAFETY: `c_path` is a valid NUL-terminated C string that outlives the + // call; `statfs` only writes into the zero-initialised `buf`. + let mut buf: libc::statfs = unsafe { std::mem::zeroed() }; + let rc = unsafe { libc::statfs(c_path.as_ptr(), &raw mut buf) }; + if rc != 0 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("statfs {}", path.display())); + } + Ok(buf.f_type == BTRFS_SUPER_MAGIC) +} + +/// Whether `path` is the root of a btrfs subvolume (on btrfs and inode 256). +fn is_btrfs_subvolume(path: &Path) -> bool { + use std::os::unix::fs::MetadataExt as _; + let on_btrfs = is_on_btrfs(path).unwrap_or(false); + let ino_match = std::fs::metadata(path).is_ok_and(|m| m.ino() == BTRFS_SUBVOL_ROOT_INO); + on_btrfs && ino_match +} + +/// `EnsureAgentSubvolume` — make the agent's state root a btrfs subvolume +/// when the FS supports it. Idempotent + progressive: no-op when the root +/// already exists or the FS isn't btrfs. See the wire doc on the variant. +async fn ensure_agent_subvolume(agent_name: &str) -> Result<(String, String)> { + use std::os::unix::fs::MetadataExt as _; + + let root = PathBuf::from(AGENT_STATE_ROOT); + let agent_root = root.join(agent_name); + + // Progressive: existing agents (plain dir OR already a subvol) are left + // untouched — never auto-migrated. + if agent_root.exists() { + return Ok((String::new(), String::new())); + } + + // Only btrfs supports subvolumes; on anything else hive-c0re's normal + // `create_dir_all` makes a plain dir (the pre-subvolume behaviour). The + // parent must exist for both statfs and `btrfs subvolume create`. + std::fs::create_dir_all(&root) + .with_context(|| format!("create agents root {}", root.display()))?; + if !is_on_btrfs(&root)? { + return Ok((String::new(), String::new())); + } + + let out = Command::new("btrfs") + .args(["subvolume", "create"]) + .arg(&agent_root) + .output() + .await + .with_context(|| format!("spawn btrfs subvolume create {}", agent_root.display()))?; + if !out.status.success() { + bail!( + "btrfs subvolume create {} failed: {}", + agent_root.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + + // The subvol root is created root-owned; hive-c0re (the `hive-core` user) + // must be able to mkdir state/ claude/ harness/ inside it, exactly as it + // would in a plain dir. Match the owner of AGENT_STATE_ROOT (hive-core). + if let Ok(meta) = std::fs::metadata(&root) + && let Err(e) = std::os::unix::fs::chown(&agent_root, Some(meta.uid()), Some(meta.gid())) + { + tracing::warn!( + agent = %agent_name, + error = %e, + "ensure_agent_subvolume: chown subvol to agents-root owner failed" + ); + } + tracing::info!(agent = %agent_name, path = %agent_root.display(), "created agent state subvolume"); + Ok((String::new(), String::new())) +} + +/// `DeleteAgentSubvolume` — delete the agent's state root iff it's a btrfs +/// subvolume (purge path only). No-op for plain dirs / missing paths; +/// hive-c0re's own `remove_dir_all` covers those. See the wire doc. +async fn delete_agent_subvolume(agent_name: &str) -> Result<(String, String)> { + let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); + if !is_btrfs_subvolume(&agent_root) { + return Ok((String::new(), String::new())); + } + let out = Command::new("btrfs") + .args(["subvolume", "delete"]) + .arg(&agent_root) + .output() + .await + .with_context(|| format!("spawn btrfs subvolume delete {}", agent_root.display()))?; + if !out.status.success() { + bail!( + "btrfs subvolume delete {} failed: {}", + agent_root.display(), + String::from_utf8_lossy(&out.stderr).trim() + ); + } + tracing::info!(agent = %agent_name, path = %agent_root.display(), "deleted agent state subvolume"); + Ok((String::new(), String::new())) +} + /// Validate a single argument destined for `forgejo admin`. Rejects /// null bytes and newlines (which could corrupt the subprocess args list /// or log output). Shell metacharacters are harmless since the command diff --git a/hive-sh4re/src/priv_proto.rs b/hive-sh4re/src/priv_proto.rs index 1b4a3e38..42520ff6 100644 --- a/hive-sh4re/src/priv_proto.rs +++ b/hive-sh4re/src/priv_proto.rs @@ -336,6 +336,43 @@ pub enum PrivRequest { container: String, action: InfraAction, }, + + // --- Agent state subvolumes (btrfs) --- + /// Ensure the agent's persistent state root + /// (`/`) is a btrfs subvolume — IF the + /// underlying filesystem is btrfs and the root doesn't already exist. + /// + /// hive-priv derives the path from `agent_name` (never passed over the + /// wire), validates the name, then: + /// - path already exists (dir or subvol) → no-op (progressive: existing + /// agents are left exactly as they are, never auto-migrated); + /// - parent FS is not btrfs → no-op (hive-c0re's normal `create_dir_all` + /// makes a plain directory, the pre-subvolume behaviour); + /// - else → `btrfs subvolume create ` and chown it to the owner of + /// `AGENT_STATE_ROOT` (the `hive-core` user) so hive-c0re can create the + /// `state/` / `claude/` / `harness/` subdirs inside it as before. + /// + /// Idempotent and safe to call on every provision. Requires root: btrfs + /// subvolume creation is privileged. + EnsureAgentSubvolume { + /// Logical agent name (validated by `validate_agent_name`). + agent_name: String, + }, + + /// Delete the agent's persistent state root if — and only if — it is a + /// btrfs subvolume. Called by hive-c0re on the **purge** path only + /// (never on a plain destroy, which keeps state for revival). + /// + /// A subvolume root cannot be removed with `rmdir`/`remove_dir_all`, so + /// this routes through hive-priv to run `btrfs subvolume delete`. If the + /// path is a plain directory (pre-subvolume agent) or doesn't exist, it's + /// a no-op — hive-c0re's own `remove_dir_all` handles the plain-dir case. + /// hive-priv derives + validates the path the same way as + /// [`PrivRequest::EnsureAgentSubvolume`]. Requires root. + DeleteAgentSubvolume { + /// Logical agent name (validated by `validate_agent_name`). + agent_name: String, + }, } /// Response from the privileged helper. diff --git a/nix/modules/hive-c0re.nix b/nix/modules/hive-c0re.nix index 5dd3000c..1a364204 100644 --- a/nix/modules/hive-c0re.nix +++ b/nix/modules/hive-c0re.nix @@ -930,6 +930,7 @@ in pkgs.nix # nix, nix-env, nix-instantiate — create + update pkgs.util-linux # umount (nsenter is hardcoded in the script) pkgs.e2fsprogs # chattr + pkgs.btrfs-progs # btrfs subvolume create/delete — Ensure/DeleteAgentSubvolume ]; environment = { # `nixos-container update/create` runs `nix`, which writes its