hive-c0re: back agent state dirs with btrfs subvolumes
Progressive enhancement: a brand-new agent's state root under /var/lib/hyperhive/agents is created as a btrfs subvolume when the host filesystem is btrfs, otherwise it falls back to a plain directory. No existing agent is auto-migrated — the new path only fires when the root does not yet exist, so plain-dir agents are left untouched until an explicit opt-in upgrade. Two new privileged ops (subvolume create/delete are root-only): EnsureAgentSubvolume statfs-gates on btrfs, creates the subvolume, and chowns it to the hive-core user so the normal state/claude/harness mkdirs succeed inside it; DeleteAgentSubvolume btrfs-subvolume-deletes the root iff it is actually a subvolume. hive-c0re calls Ensure before the per-agent dirs are created (spawn/rebuild/InitConfig) and Delete on the purge path only — destroy keeps the subvolume for revival, matching plain-dir semantics. btrfs-progs added to the hive-priv unit PATH. Per-subvolume usage accounting + optional quota is a separate follow-up.
This commit is contained in:
parent
309cdc7546
commit
1f602d5fda
9 changed files with 242 additions and 0 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -1406,6 +1406,7 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"anyhow",
|
||||
"hive-sh4re",
|
||||
"libc",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tracing",
|
||||
|
|
|
|||
|
|
@ -288,6 +288,33 @@ step would re-fire).
|
|||
The manager is non-destroyable from both paths (declarative
|
||||
container; would fight with the host's NixOS config).
|
||||
|
||||
### btrfs subvolumes for `/var/lib/hyperhive/agents/<name>`
|
||||
|
||||
On a btrfs host, a brand-new agent's state root is created as a
|
||||
**btrfs subvolume** instead of a plain directory (progressive
|
||||
enhancement — see the #1762 lane). This is a no-op fallback on
|
||||
non-btrfs hosts and for any agent whose root already exists, so
|
||||
nothing is auto-migrated: existing agents keep their plain dirs
|
||||
until an explicit opt-in upgrade.
|
||||
|
||||
- **Creation:** `lifecycle::ensure_agent_state_subvolume` runs before
|
||||
the per-agent subdirs are created (spawn / rebuild / InitConfig).
|
||||
It skips the work when the root already exists; otherwise it asks
|
||||
hive-priv (`EnsureAgentSubvolume`) to `btrfs subvolume create` the
|
||||
root when the FS is btrfs (`statfs` magic gate) and chown it to the
|
||||
`hive-core` user so the normal `state/` `claude/` `harness/` mkdirs
|
||||
succeed inside it.
|
||||
- **DESTR0Y keeps the subvolume** exactly like a plain dir — revival
|
||||
reuses it untouched.
|
||||
- **PURG3 deletes it correctly:** a subvolume root can't be removed
|
||||
with `rmdir`/`remove_dir_all`, so purge first calls hive-priv
|
||||
(`DeleteAgentSubvolume`) which `btrfs subvolume delete`s it iff it's
|
||||
actually a subvolume, then the normal `remove_dir_all` sweep covers
|
||||
plain-dir agents + the applied dir.
|
||||
|
||||
Per-subvolume disk-usage accounting and optional quotas are a
|
||||
follow-up (the qgroup work), not part of the base migration.
|
||||
|
||||
## Run-time dirs
|
||||
|
||||
`/run/hyperhive/` is tmpfs-backed (systemd `RuntimeDirectory=`) but
|
||||
|
|
|
|||
|
|
@ -315,6 +315,7 @@ async fn run_approval_init_config(
|
|||
) -> Result<()> {
|
||||
let result: Result<()> = async {
|
||||
lifecycle::setup_proposed(&proposed_dir, &approval.agent).await?;
|
||||
lifecycle::ensure_agent_state_subvolume(&approval.agent).await?;
|
||||
lifecycle::ensure_claude_dir(&claude_dir)?;
|
||||
lifecycle::ensure_state_dir(¬es_dir)?;
|
||||
Ok(())
|
||||
|
|
@ -659,6 +660,13 @@ pub async fn destroy(coord: &Arc<Coordinator>, name: &str, purge: bool) -> Resul
|
|||
let _ = std::fs::remove_dir_all(&runtime);
|
||||
}
|
||||
if purge {
|
||||
// The state root may be a btrfs subvolume (#1762): a subvolume root
|
||||
// can't be removed with rmdir/`remove_dir_all`, so delete it via
|
||||
// hive-priv (root) first. No-op for plain-dir agents — the loop below
|
||||
// then handles the plain-dir state root plus the applied dir.
|
||||
if let Err(e) = crate::priv_client::delete_agent_subvolume(name).await {
|
||||
tracing::warn!(error = ?e, %name, "purge: delete state subvolume failed");
|
||||
}
|
||||
for dir in [
|
||||
Coordinator::agent_state_root(name),
|
||||
Coordinator::agent_applied_dir(name),
|
||||
|
|
|
|||
|
|
@ -269,6 +269,7 @@ pub async fn spawn(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Result<()>
|
|||
}
|
||||
setup_proposed(&paths.proposed_dir, name).await?;
|
||||
setup_applied(&paths.applied_dir, Some(&paths.proposed_dir), name).await?;
|
||||
ensure_agent_state_subvolume(name).await?;
|
||||
ensure_claude_dir(&paths.claude_dir)?;
|
||||
ensure_state_dir(&paths.notes_dir)?;
|
||||
// Meta flake gets the new agent's input + nixosConfiguration
|
||||
|
|
@ -459,6 +460,7 @@ pub async fn rebuild_no_meta(
|
|||
);
|
||||
}
|
||||
setup_applied(&paths.applied_dir, None, name).await?;
|
||||
ensure_agent_state_subvolume(name).await?;
|
||||
ensure_claude_dir(&paths.claude_dir)?;
|
||||
ensure_state_dir(&paths.notes_dir)?;
|
||||
let container = container_name(name);
|
||||
|
|
@ -871,6 +873,28 @@ pub fn ensure_state_dir(notes_dir: &Path) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensure agent `name`'s persistent state root
|
||||
/// (`/var/lib/hyperhive/agents/<name>`) is a btrfs subvolume — when the host
|
||||
/// filesystem supports it — BEFORE the per-agent subdirs (`state/`, `claude/`,
|
||||
/// `harness/`) are created by `ensure_state_dir` / `ensure_claude_dir`.
|
||||
///
|
||||
/// Progressive enhancement (the #1762 model): if the root already exists
|
||||
/// (any agent provisioned before this landed, plain dir or subvol) it's left
|
||||
/// exactly as-is — no auto-migration — and the priv round-trip is skipped. On
|
||||
/// a non-btrfs host the priv op no-ops and the root is later created as a
|
||||
/// plain dir by `ensure_*_dir`, identical to the old behaviour. Only a
|
||||
/// brand-new agent on a btrfs host gets a real subvolume. Subvolume creation
|
||||
/// is privileged, so it's delegated to hive-priv.
|
||||
pub async fn ensure_agent_state_subvolume(name: &str) -> Result<()> {
|
||||
let root = Path::new(HOST_AGENTS_ROOT).join(name);
|
||||
if root.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
crate::priv_client::ensure_agent_subvolume(name)
|
||||
.await
|
||||
.with_context(|| format!("ensure btrfs subvolume for agent {name}"))
|
||||
}
|
||||
|
||||
fn initial_agent_nix(name: &str) -> String {
|
||||
format!(
|
||||
"{{ config, pkgs, lib, ... }}:\n{{\n # Per-agent overrides for {name}. This is a regular NixOS module\n # — add packages, services, modules, imports as needed.\n #\n # imports = [ ./extra-module.nix ];\n # environment.systemPackages = with pkgs; [ ];\n}}\n",
|
||||
|
|
|
|||
|
|
@ -305,6 +305,27 @@ pub async fn control_infra_container(container: &str, action: InfraAction) -> Re
|
|||
.await?)
|
||||
}
|
||||
|
||||
/// Ensure the agent's persistent state root is a btrfs subvolume when the
|
||||
/// host FS supports it (via hive-priv, which runs as root). Idempotent and
|
||||
/// progressive: a no-op when the root already exists or the FS isn't btrfs.
|
||||
/// Safe to call on every provision.
|
||||
pub async fn ensure_agent_subvolume(agent_name: &str) -> Result<()> {
|
||||
ok(call(&PrivRequest::EnsureAgentSubvolume {
|
||||
agent_name: agent_name.to_owned(),
|
||||
})
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// Delete the agent's state root iff it is a btrfs subvolume (purge path
|
||||
/// only). No-op for plain dirs / missing paths — hive-c0re's own
|
||||
/// `remove_dir_all` handles those.
|
||||
pub async fn delete_agent_subvolume(agent_name: &str) -> Result<()> {
|
||||
ok(call(&PrivRequest::DeleteAgentSubvolume {
|
||||
agent_name: agent_name.to_owned(),
|
||||
})
|
||||
.await?)
|
||||
}
|
||||
|
||||
fn check(resp: PrivResponse) -> Result<(String, String)> {
|
||||
if resp.ok {
|
||||
Ok((resp.stdout, resp.stderr))
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ workspace = true
|
|||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
hive-sh4re.workspace = true
|
||||
libc.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
|
|
|
|||
|
|
@ -269,6 +269,16 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String,
|
|||
ref container,
|
||||
action,
|
||||
} => control_infra_container(container, action).await,
|
||||
|
||||
PrivRequest::EnsureAgentSubvolume { ref agent_name } => {
|
||||
validate_agent_name(agent_name)?;
|
||||
ensure_agent_subvolume(agent_name).await
|
||||
}
|
||||
|
||||
PrivRequest::DeleteAgentSubvolume { ref agent_name } => {
|
||||
validate_agent_name(agent_name)?;
|
||||
delete_agent_subvolume(agent_name).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -484,6 +494,118 @@ fn write_agent_state_file(
|
|||
Ok((String::new(), String::new()))
|
||||
}
|
||||
|
||||
/// btrfs superblock magic, as reported by `statfs(2)`'s `f_type`.
|
||||
const BTRFS_SUPER_MAGIC: i64 = 0x9123_683E;
|
||||
|
||||
/// Inode number of a btrfs subvolume root (`BTRFS_FIRST_FREE_OBJECTID`).
|
||||
/// Every subvolume's top directory has this inode; plain directories do
|
||||
/// not, so `statfs == btrfs && st_ino == 256` reliably identifies a
|
||||
/// subvolume root.
|
||||
const BTRFS_SUBVOL_ROOT_INO: u64 = 256;
|
||||
|
||||
/// Whether `path` lives on a btrfs filesystem (via `statfs(2)`).
|
||||
fn is_on_btrfs(path: &Path) -> Result<bool> {
|
||||
use std::os::unix::ffi::OsStrExt as _;
|
||||
let c_path = std::ffi::CString::new(path.as_os_str().as_bytes())
|
||||
.with_context(|| format!("path {} has an interior null byte", path.display()))?;
|
||||
// SAFETY: `c_path` is a valid NUL-terminated C string that outlives the
|
||||
// call; `statfs` only writes into the zero-initialised `buf`.
|
||||
let mut buf: libc::statfs = unsafe { std::mem::zeroed() };
|
||||
let rc = unsafe { libc::statfs(c_path.as_ptr(), &raw mut buf) };
|
||||
if rc != 0 {
|
||||
return Err(std::io::Error::last_os_error())
|
||||
.with_context(|| format!("statfs {}", path.display()));
|
||||
}
|
||||
Ok(buf.f_type == BTRFS_SUPER_MAGIC)
|
||||
}
|
||||
|
||||
/// Whether `path` is the root of a btrfs subvolume (on btrfs and inode 256).
|
||||
fn is_btrfs_subvolume(path: &Path) -> bool {
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
let on_btrfs = is_on_btrfs(path).unwrap_or(false);
|
||||
let ino_match = std::fs::metadata(path).is_ok_and(|m| m.ino() == BTRFS_SUBVOL_ROOT_INO);
|
||||
on_btrfs && ino_match
|
||||
}
|
||||
|
||||
/// `EnsureAgentSubvolume` — make the agent's state root a btrfs subvolume
|
||||
/// when the FS supports it. Idempotent + progressive: no-op when the root
|
||||
/// already exists or the FS isn't btrfs. See the wire doc on the variant.
|
||||
async fn ensure_agent_subvolume(agent_name: &str) -> Result<(String, String)> {
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
|
||||
let root = PathBuf::from(AGENT_STATE_ROOT);
|
||||
let agent_root = root.join(agent_name);
|
||||
|
||||
// Progressive: existing agents (plain dir OR already a subvol) are left
|
||||
// untouched — never auto-migrated.
|
||||
if agent_root.exists() {
|
||||
return Ok((String::new(), String::new()));
|
||||
}
|
||||
|
||||
// Only btrfs supports subvolumes; on anything else hive-c0re's normal
|
||||
// `create_dir_all` makes a plain dir (the pre-subvolume behaviour). The
|
||||
// parent must exist for both statfs and `btrfs subvolume create`.
|
||||
std::fs::create_dir_all(&root)
|
||||
.with_context(|| format!("create agents root {}", root.display()))?;
|
||||
if !is_on_btrfs(&root)? {
|
||||
return Ok((String::new(), String::new()));
|
||||
}
|
||||
|
||||
let out = Command::new("btrfs")
|
||||
.args(["subvolume", "create"])
|
||||
.arg(&agent_root)
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("spawn btrfs subvolume create {}", agent_root.display()))?;
|
||||
if !out.status.success() {
|
||||
bail!(
|
||||
"btrfs subvolume create {} failed: {}",
|
||||
agent_root.display(),
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
);
|
||||
}
|
||||
|
||||
// The subvol root is created root-owned; hive-c0re (the `hive-core` user)
|
||||
// must be able to mkdir state/ claude/ harness/ inside it, exactly as it
|
||||
// would in a plain dir. Match the owner of AGENT_STATE_ROOT (hive-core).
|
||||
if let Ok(meta) = std::fs::metadata(&root)
|
||||
&& let Err(e) = std::os::unix::fs::chown(&agent_root, Some(meta.uid()), Some(meta.gid()))
|
||||
{
|
||||
tracing::warn!(
|
||||
agent = %agent_name,
|
||||
error = %e,
|
||||
"ensure_agent_subvolume: chown subvol to agents-root owner failed"
|
||||
);
|
||||
}
|
||||
tracing::info!(agent = %agent_name, path = %agent_root.display(), "created agent state subvolume");
|
||||
Ok((String::new(), String::new()))
|
||||
}
|
||||
|
||||
/// `DeleteAgentSubvolume` — delete the agent's state root iff it's a btrfs
|
||||
/// subvolume (purge path only). No-op for plain dirs / missing paths;
|
||||
/// hive-c0re's own `remove_dir_all` covers those. See the wire doc.
|
||||
async fn delete_agent_subvolume(agent_name: &str) -> Result<(String, String)> {
|
||||
let agent_root = PathBuf::from(AGENT_STATE_ROOT).join(agent_name);
|
||||
if !is_btrfs_subvolume(&agent_root) {
|
||||
return Ok((String::new(), String::new()));
|
||||
}
|
||||
let out = Command::new("btrfs")
|
||||
.args(["subvolume", "delete"])
|
||||
.arg(&agent_root)
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("spawn btrfs subvolume delete {}", agent_root.display()))?;
|
||||
if !out.status.success() {
|
||||
bail!(
|
||||
"btrfs subvolume delete {} failed: {}",
|
||||
agent_root.display(),
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
);
|
||||
}
|
||||
tracing::info!(agent = %agent_name, path = %agent_root.display(), "deleted agent state subvolume");
|
||||
Ok((String::new(), String::new()))
|
||||
}
|
||||
|
||||
/// Validate a single argument destined for `forgejo admin`. Rejects
|
||||
/// null bytes and newlines (which could corrupt the subprocess args list
|
||||
/// or log output). Shell metacharacters are harmless since the command
|
||||
|
|
|
|||
|
|
@ -336,6 +336,43 @@ pub enum PrivRequest {
|
|||
container: String,
|
||||
action: InfraAction,
|
||||
},
|
||||
|
||||
// --- Agent state subvolumes (btrfs) ---
|
||||
/// Ensure the agent's persistent state root
|
||||
/// (`<AGENT_STATE_ROOT>/<agent_name>`) is a btrfs subvolume — IF the
|
||||
/// underlying filesystem is btrfs and the root doesn't already exist.
|
||||
///
|
||||
/// hive-priv derives the path from `agent_name` (never passed over the
|
||||
/// wire), validates the name, then:
|
||||
/// - path already exists (dir or subvol) → no-op (progressive: existing
|
||||
/// agents are left exactly as they are, never auto-migrated);
|
||||
/// - parent FS is not btrfs → no-op (hive-c0re's normal `create_dir_all`
|
||||
/// makes a plain directory, the pre-subvolume behaviour);
|
||||
/// - else → `btrfs subvolume create <path>` and chown it to the owner of
|
||||
/// `AGENT_STATE_ROOT` (the `hive-core` user) so hive-c0re can create the
|
||||
/// `state/` / `claude/` / `harness/` subdirs inside it as before.
|
||||
///
|
||||
/// Idempotent and safe to call on every provision. Requires root: btrfs
|
||||
/// subvolume creation is privileged.
|
||||
EnsureAgentSubvolume {
|
||||
/// Logical agent name (validated by `validate_agent_name`).
|
||||
agent_name: String,
|
||||
},
|
||||
|
||||
/// Delete the agent's persistent state root if — and only if — it is a
|
||||
/// btrfs subvolume. Called by hive-c0re on the **purge** path only
|
||||
/// (never on a plain destroy, which keeps state for revival).
|
||||
///
|
||||
/// A subvolume root cannot be removed with `rmdir`/`remove_dir_all`, so
|
||||
/// this routes through hive-priv to run `btrfs subvolume delete`. If the
|
||||
/// path is a plain directory (pre-subvolume agent) or doesn't exist, it's
|
||||
/// a no-op — hive-c0re's own `remove_dir_all` handles the plain-dir case.
|
||||
/// hive-priv derives + validates the path the same way as
|
||||
/// [`PrivRequest::EnsureAgentSubvolume`]. Requires root.
|
||||
DeleteAgentSubvolume {
|
||||
/// Logical agent name (validated by `validate_agent_name`).
|
||||
agent_name: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// Response from the privileged helper.
|
||||
|
|
|
|||
|
|
@ -930,6 +930,7 @@ in
|
|||
pkgs.nix # nix, nix-env, nix-instantiate — create + update
|
||||
pkgs.util-linux # umount (nsenter is hardcoded in the script)
|
||||
pkgs.e2fsprogs # chattr
|
||||
pkgs.btrfs-progs # btrfs subvolume create/delete — Ensure/DeleteAgentSubvolume
|
||||
];
|
||||
environment = {
|
||||
# `nixos-container update/create` runs `nix`, which writes its
|
||||
|
|
|
|||
Loading…
Reference in a new issue