diff --git a/Cargo.lock b/Cargo.lock index f6f6cfd7..f7e5ca18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1406,7 +1406,6 @@ version = "0.1.0" dependencies = [ "anyhow", "hive-sh4re", - "libc", "serde_json", "tokio", "tracing", diff --git a/docs/persistence.md b/docs/persistence.md index 233fedfa..d1f297b2 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -288,33 +288,6 @@ step would re-fire). The manager is non-destroyable from both paths (declarative container; would fight with the host's NixOS config). -### btrfs subvolumes for `/var/lib/hyperhive/agents/` - -On a btrfs host, a brand-new agent's state root is created as a -**btrfs subvolume** instead of a plain directory (progressive -enhancement — see the #1762 lane). This is a no-op fallback on -non-btrfs hosts and for any agent whose root already exists, so -nothing is auto-migrated: existing agents keep their plain dirs -until an explicit opt-in upgrade. - -- **Creation:** `lifecycle::ensure_agent_state_subvolume` runs before - the per-agent subdirs are created (spawn / rebuild / InitConfig). - It skips the work when the root already exists; otherwise it asks - hive-priv (`EnsureAgentSubvolume`) to `btrfs subvolume create` the - root when the FS is btrfs (`statfs` magic gate) and chown it to the - `hive-core` user so the normal `state/` `claude/` `harness/` mkdirs - succeed inside it. -- **DESTR0Y keeps the subvolume** exactly like a plain dir — revival - reuses it untouched. -- **PURG3 deletes it correctly:** a subvolume root can't be removed - with `rmdir`/`remove_dir_all`, so purge first calls hive-priv - (`DeleteAgentSubvolume`) which `btrfs subvolume delete`s it iff it's - actually a subvolume, then the normal `remove_dir_all` sweep covers - plain-dir agents + the applied dir. - -Per-subvolume disk-usage accounting and optional quotas are a -follow-up (the qgroup work), not part of the base migration. - ## Run-time dirs `/run/hyperhive/` is tmpfs-backed (systemd `RuntimeDirectory=`) but diff --git a/hive-c0re/src/actions.rs b/hive-c0re/src/actions.rs index 61c06b7e..d3c98ef7 100644 --- a/hive-c0re/src/actions.rs +++ b/hive-c0re/src/actions.rs @@ -315,7 +315,6 @@ async fn run_approval_init_config( ) -> Result<()> { let result: Result<()> = async { lifecycle::setup_proposed(&proposed_dir, &approval.agent).await?; - lifecycle::ensure_agent_state_subvolume(&approval.agent).await?; lifecycle::ensure_claude_dir(&claude_dir)?; lifecycle::ensure_state_dir(¬es_dir)?; Ok(()) @@ -660,13 +659,6 @@ pub async fn destroy(coord: &Arc, name: &str, purge: bool) -> Resul let _ = std::fs::remove_dir_all(&runtime); } if purge { - // The state root may be a btrfs subvolume: a subvolume root - // can't be removed with rmdir/`remove_dir_all`, so delete it via - // hive-priv (root) first. No-op for plain-dir agents — the loop below - // then handles the plain-dir state root plus the applied dir. - if let Err(e) = crate::priv_client::delete_agent_subvolume(name).await { - tracing::warn!(error = ?e, %name, "purge: delete state subvolume failed"); - } for dir in [ Coordinator::agent_state_root(name), Coordinator::agent_applied_dir(name), diff --git a/hive-c0re/src/lifecycle.rs b/hive-c0re/src/lifecycle.rs index f46c50fd..9185ea55 100644 --- a/hive-c0re/src/lifecycle.rs +++ b/hive-c0re/src/lifecycle.rs @@ -269,7 +269,6 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()> } setup_proposed(&paths.proposed_dir, name).await?; setup_applied(&paths.applied_dir, Some(&paths.proposed_dir), name).await?; - ensure_agent_state_subvolume(name).await?; ensure_claude_dir(&paths.claude_dir)?; ensure_state_dir(&paths.notes_dir)?; // Meta flake gets the new agent's input + nixosConfiguration @@ -460,7 +459,6 @@ pub async fn rebuild_no_meta( ); } setup_applied(&paths.applied_dir, None, name).await?; - ensure_agent_state_subvolume(name).await?; ensure_claude_dir(&paths.claude_dir)?; ensure_state_dir(&paths.notes_dir)?; let container = container_name(name); @@ -873,28 +871,6 @@ pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> { Ok(()) } -/// Ensure agent `name`'s persistent state root -/// (`/var/lib/hyperhive/agents/`) is a btrfs subvolume — when the host -/// filesystem supports it — BEFORE the per-agent subdirs (`state/`, `claude/`, -/// `harness/`) are created by `ensure_state_dir` / `ensure_claude_dir`. -/// -/// Progressive enhancement: if the root already exists -/// (any agent provisioned before this landed, plain dir or subvol) it's left -/// exactly as-is — no auto-migration — and the priv round-trip is skipped. On -/// a non-btrfs host the priv op no-ops and the root is later created as a -/// plain dir by `ensure_*_dir`, identical to the old behaviour. Only a -/// brand-new agent on a btrfs host gets a real subvolume. Subvolume creation -/// is privileged, so it's delegated to hive-priv. -pub async fn ensure_agent_state_subvolume(name: &str) -> Result<()> { - let root = Path::new(HOST_AGENTS_ROOT).join(name); - if root.exists() { - return Ok(()); - } - crate::priv_client::ensure_agent_subvolume(name) - .await - .with_context(|| format!("ensure btrfs subvolume for agent {name}")) -} - fn initial_agent_nix(name: &str) -> String { format!( "{{ config, pkgs, lib, ... }}:\n{{\n # Per-agent overrides for {name}. This is a regular NixOS module\n # — add packages, services, modules, imports as needed.\n #\n # imports = [ ./extra-module.nix ];\n # environment.systemPackages = with pkgs; [ ];\n}}\n", diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index dd4cc41d..b6a25b5f 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -305,27 +305,6 @@ pub async fn control_infra_container(container: &str, action: InfraAction) -> Re .await?) } -/// Ensure the agent's persistent state root is a btrfs subvolume when the -/// host FS supports it (via hive-priv, which runs as root). Idempotent and -/// progressive: a no-op when the root already exists or the FS isn't btrfs. -/// Safe to call on every provision. -pub async fn ensure_agent_subvolume(agent_name: &str) -> Result<()> { - ok(call(&PrivRequest::EnsureAgentSubvolume { - agent_name: agent_name.to_owned(), - }) - .await?) -} - -/// Delete the agent's state root iff it is a btrfs subvolume (purge path -/// only). No-op for plain dirs / missing paths — hive-c0re's own -/// `remove_dir_all` handles those. -pub async fn delete_agent_subvolume(agent_name: &str) -> Result<()> { - ok(call(&PrivRequest::DeleteAgentSubvolume { - agent_name: agent_name.to_owned(), - }) - .await?) -} - fn check(resp: PrivResponse) -> Result<(String, String)> { if resp.ok { Ok((resp.stdout, resp.stderr)) diff --git a/hive-priv/Cargo.toml b/hive-priv/Cargo.toml index 39a369fc..eccd4f69 100644 --- a/hive-priv/Cargo.toml +++ b/hive-priv/Cargo.toml @@ -9,7 +9,6 @@ workspace = true [dependencies] anyhow.workspace = true hive-sh4re.workspace = true -libc.workspace = true serde_json.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 7bffe4d1..47e9cce1 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -269,16 +269,6 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String, ref container, action, } => control_infra_container(container, action).await, - - PrivRequest::EnsureAgentSubvolume { ref agent_name } => { - validate_agent_name(agent_name)?; - ensure_agent_subvolume(agent_name).await - } - - PrivRequest::DeleteAgentSubvolume { ref agent_name } => { - validate_agent_name(agent_name)?; - delete_agent_subvolume(agent_name).await - } } } @@ -494,133 +484,6 @@ fn write_agent_state_file( Ok((String::new(), String::new())) } -/// btrfs superblock magic, as reported by `statfs(2)`'s `f_type`. -const BTRFS_SUPER_MAGIC: i64 = 0x9123_683E; - -/// Inode number of a btrfs subvolume root (`BTRFS_FIRST_FREE_OBJECTID`). -/// Every subvolume's top directory has this inode; plain directories do -/// not, so `statfs == btrfs && st_ino == 256` reliably identifies a -/// subvolume root. -const BTRFS_SUBVOL_ROOT_INO: u64 = 256; - -/// Whether `path` lives on a btrfs filesystem (via `statfs(2)`). -fn is_on_btrfs(path: &Path) -> Result { - use std::os::unix::ffi::OsStrExt as _; - let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()) - .with_context(|| format!("path {} has an interior null byte", path.display()))?; - // SAFETY: `c_path` is a valid NUL-terminated C string that outlives the - // call; `statfs` only writes into the zero-initialised `buf`. - let mut buf: libc::statfs = unsafe { std::mem::zeroed() }; - let rc = unsafe { libc::statfs(c_path.as_ptr(), &raw mut buf) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()) - .with_context(|| format!("statfs {}", path.display())); - } - Ok(buf.f_type == BTRFS_SUPER_MAGIC) -} - -/// Whether `path` is the root of a btrfs subvolume (on btrfs and inode 256). -fn is_btrfs_subvolume(path: &Path) -> bool { - use std::os::unix::fs::MetadataExt as _; - let on_btrfs = is_on_btrfs(path).unwrap_or(false); - let ino_match = std::fs::metadata(path).is_ok_and(|m| m.ino() == BTRFS_SUBVOL_ROOT_INO); - on_btrfs && ino_match -} - -/// `EnsureAgentSubvolume` — make the agent's state root a btrfs subvolume -/// when the FS supports it. Idempotent + progressive: no-op when the root -/// already exists or the FS isn't btrfs. See the wire doc on the variant. -async fn ensure_agent_subvolume(agent_name: &str) -> Result<(String, String)> { - use std::os::unix::fs::MetadataExt as _; - - let root = PathBuf::from(AGENT_STATE_ROOT); - let agent_root = root.join(agent_name); - - // Progressive: existing agents (plain dir OR already a subvol) are left - // untouched — never auto-migrated. - if agent_root.exists() { - return Ok((String::new(), String::new())); - } - - // Only btrfs supports subvolumes; on anything else hive-c0re's normal - // `create_dir_all` makes a plain dir (the pre-subvolume behaviour). The - // parent must exist for both statfs and `btrfs subvolume create`. - std::fs::create_dir_all(&root) - .with_context(|| format!("create agents root {}", root.display()))?; - if !is_on_btrfs(&root)? { - return Ok((String::new(), String::new())); - } - - let out = Command::new("btrfs") - .args(["subvolume", "create"]) - .arg(&agent_root) - .output() - .await - .with_context(|| format!("spawn btrfs subvolume create {}", agent_root.display()))?; - if !out.status.success() { - bail!( - "btrfs subvolume create {} failed: {}", - agent_root.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - - // The subvol root is created root-owned, but hive-c0re (the `hive-core` - // user) must be able to mkdir state/ claude/ harness/ inside it — exactly - // as it would in a plain dir. Chown it to AGENT_STATE_ROOT's owner - // (hive-core). This MUST succeed: a root-owned subvol would make the - // downstream dir creation fail with a confusing permission error, and the - // c0re-side exists-check would then skip re-running this op on retry, - // wedging the agent. So on any failure roll the subvol back and bail — the - // create path surfaces a clear error and a retry starts clean. - let chown_result = std::fs::metadata(&root) - .with_context(|| format!("stat agents root {} for ownership", root.display())) - .and_then(|meta| { - std::os::unix::fs::chown(&agent_root, Some(meta.uid()), Some(meta.gid())).with_context( - || format!("chown subvol {} to agents-root owner", agent_root.display()), - ) - }); - if let Err(e) = chown_result { - // Best-effort rollback so we never leave a root-owned subvol behind. - let _ = Command::new("btrfs") - .args(["subvolume", "delete"]) - .arg(&agent_root) - .output() - .await; - return Err(e.context(format!( - "rolled back subvolume {} after chown failed", - agent_root.display() - ))); - } - tracing::info!(agent = %agent_name, path = %agent_root.display(), "created agent state subvolume"); - Ok((String::new(), String::new())) -} - -/// `DeleteAgentSubvolume` — delete the agent's state root iff it's a btrfs -/// subvolume (purge path only). No-op for plain dirs / missing paths; -/// hive-c0re's own `remove_dir_all` covers those. See the wire doc. -async fn delete_agent_subvolume(agent_name: &str) -> Result<(String, String)> { - let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name); - if !is_btrfs_subvolume(&agent_root) { - return Ok((String::new(), String::new())); - } - let out = Command::new("btrfs") - .args(["subvolume", "delete"]) - .arg(&agent_root) - .output() - .await - .with_context(|| format!("spawn btrfs subvolume delete {}", agent_root.display()))?; - if !out.status.success() { - bail!( - "btrfs subvolume delete {} failed: {}", - agent_root.display(), - String::from_utf8_lossy(&out.stderr).trim() - ); - } - tracing::info!(agent = %agent_name, path = %agent_root.display(), "deleted agent state subvolume"); - Ok((String::new(), String::new())) -} - /// Validate a single argument destined for `forgejo admin`. Rejects /// null bytes and newlines (which could corrupt the subprocess args list /// or log output). Shell metacharacters are harmless since the command diff --git a/hive-sh4re/src/priv_proto.rs b/hive-sh4re/src/priv_proto.rs index 42520ff6..1b4a3e38 100644 --- a/hive-sh4re/src/priv_proto.rs +++ b/hive-sh4re/src/priv_proto.rs @@ -336,43 +336,6 @@ pub enum PrivRequest { container: String, action: InfraAction, }, - - // --- Agent state subvolumes (btrfs) --- - /// Ensure the agent's persistent state root - /// (`/`) is a btrfs subvolume — IF the - /// underlying filesystem is btrfs and the root doesn't already exist. - /// - /// hive-priv derives the path from `agent_name` (never passed over the - /// wire), validates the name, then: - /// - path already exists (dir or subvol) → no-op (progressive: existing - /// agents are left exactly as they are, never auto-migrated); - /// - parent FS is not btrfs → no-op (hive-c0re's normal `create_dir_all` - /// makes a plain directory, the pre-subvolume behaviour); - /// - else → `btrfs subvolume create ` and chown it to the owner of - /// `AGENT_STATE_ROOT` (the `hive-core` user) so hive-c0re can create the - /// `state/` / `claude/` / `harness/` subdirs inside it as before. - /// - /// Idempotent and safe to call on every provision. Requires root: btrfs - /// subvolume creation is privileged. - EnsureAgentSubvolume { - /// Logical agent name (validated by `validate_agent_name`). - agent_name: String, - }, - - /// Delete the agent's persistent state root if — and only if — it is a - /// btrfs subvolume. Called by hive-c0re on the **purge** path only - /// (never on a plain destroy, which keeps state for revival). - /// - /// A subvolume root cannot be removed with `rmdir`/`remove_dir_all`, so - /// this routes through hive-priv to run `btrfs subvolume delete`. If the - /// path is a plain directory (pre-subvolume agent) or doesn't exist, it's - /// a no-op — hive-c0re's own `remove_dir_all` handles the plain-dir case. - /// hive-priv derives + validates the path the same way as - /// [`PrivRequest::EnsureAgentSubvolume`]. Requires root. - DeleteAgentSubvolume { - /// Logical agent name (validated by `validate_agent_name`). - agent_name: String, - }, } /// Response from the privileged helper. diff --git a/nix/modules/hive-c0re.nix b/nix/modules/hive-c0re.nix index 1a364204..5dd3000c 100644 --- a/nix/modules/hive-c0re.nix +++ b/nix/modules/hive-c0re.nix @@ -930,7 +930,6 @@ in pkgs.nix # nix, nix-env, nix-instantiate — create + update pkgs.util-linux # umount (nsenter is hardcoded in the script) pkgs.e2fsprogs # chattr - pkgs.btrfs-progs # btrfs subvolume create/delete — Ensure/DeleteAgentSubvolume ]; environment = { # `nixos-container update/create` runs `nix`, which writes its