hive-c0re: per-agent btrfs disk usage + optional quota (#1793)
Follow-up to the btrfs-subvolume migration. Operator-opt-in disk accounting + quotas on agent state subvolumes via btrfs qgroups: - three privileged ops (qgroup ops need root): EnsureBtrfsQuota (btrfs quota enable on the agent-state filesystem — statfs-gated, idempotent, no-op off btrfs), ReadSubvolumeUsage (btrfs qgroup show -f --raw for one agent), SetSubvolumeQuota (btrfs qgroup limit, or clear). Reuses the is_on_btrfs helper from the subvolume work. - priv_client wrappers, incl parse_qgroup_usage -> (referenced, exclusive) bytes. - hivectl 'quota' subcommand: enable / show [agent] / limit <agent> <size|none>, with a K/M/G/T size parser + human-readable output. Quota is deliberately NOT auto-enabled: btrfs quota enable triggers a full rescan that is I/O-heavy on a large filesystem, and the operator should choose when to pay that. 'quota show' on a plain-dir agent (no subvolume) reports no qgroup data rather than erroring.
This commit is contained in:
atlas
mara
parent
3a2cdaa37b
commit
9ff55399e5
5 changed files with 373 additions and 0 deletions
|
|
@ -161,6 +161,18 @@ enum Cmd {
|
|||
#[arg(long)]
|
||||
graceful: bool,
|
||||
},
|
||||
/// Per-agent disk accounting + optional quotas via btrfs qgroups.
|
||||
///
|
||||
/// Opt-in: `quota enable` turns on btrfs qgroup accounting for the
|
||||
/// agent-state filesystem (a one-time, I/O-heavy rescan — that's why
|
||||
/// it isn't automatic). Then `quota show` reports per-agent usage and
|
||||
/// `quota limit` caps an agent. No-op on non-btrfs hosts. Operates on
|
||||
/// agent state subvolumes (created by the btrfs-subvolume migration);
|
||||
/// agents still on a plain dir report no qgroup usage.
|
||||
Quota {
|
||||
#[command(subcommand)]
|
||||
cmd: QuotaCmd,
|
||||
},
|
||||
/// Emit the full CLI reference as `CommonMark` to stdout.
|
||||
///
|
||||
/// Hidden tooling command (not part of day-to-day operator admin):
|
||||
|
|
@ -428,6 +440,32 @@ enum WgCmd {
|
|||
Status,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum QuotaCmd {
|
||||
/// Enable btrfs qgroup accounting on the agent-state filesystem. Run
|
||||
/// once before `show` / `limit`. Triggers a full btrfs rescan (I/O
|
||||
/// heavy on a large filesystem), so it's a deliberate opt-in.
|
||||
/// Idempotent; a no-op on non-btrfs hosts.
|
||||
Enable,
|
||||
/// Report per-agent disk usage (referenced + exclusive bytes) from
|
||||
/// btrfs qgroups. With no name, shows every agent that has a state
|
||||
/// subvolume; pass a name to show just that one. Requires `enable`
|
||||
/// first.
|
||||
Show {
|
||||
/// Agent to show (omit for all agents with a state subvolume).
|
||||
name: Option<String>,
|
||||
},
|
||||
/// Set or clear an agent's disk quota (a referenced-usage cap). `size`
|
||||
/// accepts a byte count or a `K`/`M`/`G`/`T` suffix (e.g. `5G`), or
|
||||
/// `none` to clear the limit. Requires `enable` first.
|
||||
Limit {
|
||||
/// Agent whose state subvolume to limit.
|
||||
name: String,
|
||||
/// Size cap (`5G`, `500M`, `1073741824`) or `none` to clear.
|
||||
size: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// Default host admin socket path. Must match `hive-c0re`'s default in
|
||||
/// `main.rs` (`/run/hyperhive/host.sock`) — the daemon binds there and
|
||||
/// `hivectl agents` connects to it.
|
||||
|
|
@ -509,6 +547,11 @@ async fn main() -> Result<()> {
|
|||
Cmd::Start { scope } => start(&socket, scope.to_scope()).await,
|
||||
Cmd::Restart { scope, graceful } => restart(&socket, scope.to_scope(), graceful).await,
|
||||
Cmd::Choom { name, fresh } => choom(&name, fresh),
|
||||
Cmd::Quota { cmd } => match cmd {
|
||||
QuotaCmd::Enable => quota_enable().await,
|
||||
QuotaCmd::Show { name } => quota_show(name.as_deref()).await,
|
||||
QuotaCmd::Limit { name, size } => quota_limit(&name, &size).await,
|
||||
},
|
||||
Cmd::MarkdownDocs => {
|
||||
print!("{}", clap_markdown::help_markdown::<Cli>());
|
||||
Ok(())
|
||||
|
|
@ -641,6 +684,100 @@ fn wg_status() -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// `quota enable` — turn on btrfs qgroup accounting (operator opt-in).
|
||||
async fn quota_enable() -> Result<()> {
|
||||
hive_c0re::priv_client::ensure_btrfs_quota()
|
||||
.await
|
||||
.context("enable btrfs qgroup accounting")?;
|
||||
println!("btrfs qgroup accounting enabled on the agent-state filesystem.");
|
||||
println!("(usage may read 0 until btrfs finishes its background rescan)");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `quota show [name]` — report per-agent disk usage from btrfs qgroups.
|
||||
async fn quota_show(name: Option<&str>) -> Result<()> {
|
||||
let agents: Vec<String> = match name {
|
||||
Some(n) => vec![n.to_owned()],
|
||||
None => Coordinator::kept_state_names(),
|
||||
};
|
||||
if agents.is_empty() {
|
||||
println!("no agents with a state dir found");
|
||||
return Ok(());
|
||||
}
|
||||
for agent in &agents {
|
||||
match hive_c0re::priv_client::read_subvolume_usage(agent).await {
|
||||
Ok((rfer, excl)) => {
|
||||
println!(
|
||||
"{agent:<12} referenced {:>10} exclusive {:>10}",
|
||||
human_bytes(rfer),
|
||||
human_bytes(excl)
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("{e:#}");
|
||||
if msg.contains("not enabled") {
|
||||
bail!("btrfs quota not enabled — run `hivectl quota enable` first");
|
||||
}
|
||||
// A plain-dir agent (no subvolume) has no qgroup; report it
|
||||
// inline and keep going rather than aborting the whole sweep.
|
||||
println!("{agent:<12} (no qgroup data — plain dir or: {msg})");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `quota limit <name> <size>` — set or clear an agent's disk quota.
|
||||
async fn quota_limit(name: &str, size: &str) -> Result<()> {
|
||||
let limit = parse_quota_size(size)?;
|
||||
hive_c0re::priv_client::set_subvolume_quota(name, limit)
|
||||
.await
|
||||
.with_context(|| format!("set quota for {name}"))?;
|
||||
match limit {
|
||||
Some(n) => println!("set {name} quota to {} ({n} bytes)", human_bytes(n)),
|
||||
None => println!("cleared {name} quota"),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Parse a quota size: a byte count, a `K`/`M`/`G`/`T`-suffixed value
|
||||
/// (powers of 1024), or `none` to clear. Rejects overflow + bad input.
|
||||
fn parse_quota_size(s: &str) -> Result<Option<u64>> {
|
||||
let t = s.trim();
|
||||
if t.eq_ignore_ascii_case("none") {
|
||||
return Ok(None);
|
||||
}
|
||||
let (num, mult) = match t.chars().last() {
|
||||
Some('K' | 'k') => (&t[..t.len() - 1], 1024u64),
|
||||
Some('M' | 'm') => (&t[..t.len() - 1], 1024 * 1024),
|
||||
Some('G' | 'g') => (&t[..t.len() - 1], 1024 * 1024 * 1024),
|
||||
Some('T' | 't') => (&t[..t.len() - 1], 1024u64 * 1024 * 1024 * 1024),
|
||||
_ => (t, 1),
|
||||
};
|
||||
let n: u64 = num.trim().parse().with_context(|| {
|
||||
format!("invalid size {s:?} — use a byte count, a K/M/G/T suffix (e.g. 5G), or `none`")
|
||||
})?;
|
||||
n.checked_mul(mult)
|
||||
.map(Some)
|
||||
.with_context(|| format!("size {s:?} overflows u64"))
|
||||
}
|
||||
|
||||
/// Format a byte count as a human-friendly KiB/MiB/GiB/TiB string.
|
||||
#[allow(clippy::cast_precision_loss)] // display-only; precision loss is cosmetic
|
||||
fn human_bytes(n: u64) -> String {
|
||||
const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"];
|
||||
if n < 1024 {
|
||||
return format!("{n} B");
|
||||
}
|
||||
let mut v = n as f64;
|
||||
let mut i = 0;
|
||||
while v >= 1024.0 && i < UNITS.len() - 1 {
|
||||
v /= 1024.0;
|
||||
i += 1;
|
||||
}
|
||||
format!("{v:.1} {}", UNITS[i])
|
||||
}
|
||||
|
||||
/// True when `name` matches an existing hyperhive agent — i.e. it has a
|
||||
/// persistent state dir under `/var/lib/hyperhive/agents/`. We use the
|
||||
/// state dir (not the live container list) so kept-state tombstones
|
||||
|
|
|
|||
|
|
@ -326,6 +326,52 @@ pub async fn delete_agent_subvolume(agent_name: &str) -> Result<()> {
|
|||
.await?)
|
||||
}
|
||||
|
||||
/// Enable btrfs qgroup accounting on the agent-state filesystem (operator
|
||||
/// opt-in; prerequisite for usage reads + quotas). Idempotent; no-op off
|
||||
/// btrfs. Via hive-priv (root).
|
||||
pub async fn ensure_btrfs_quota() -> Result<()> {
|
||||
ok(call(&PrivRequest::EnsureBtrfsQuota).await?)
|
||||
}
|
||||
|
||||
/// Read an agent state subvolume's btrfs qgroup usage, returning
|
||||
/// `(referenced_bytes, exclusive_bytes)`. Errors propagate (e.g. quota not
|
||||
/// enabled) so the caller can surface them. Via hive-priv (root).
|
||||
pub async fn read_subvolume_usage(agent_name: &str) -> Result<(u64, u64)> {
|
||||
let (stdout, _) = check(
|
||||
call(&PrivRequest::ReadSubvolumeUsage {
|
||||
agent_name: agent_name.to_owned(),
|
||||
})
|
||||
.await?,
|
||||
)?;
|
||||
parse_qgroup_usage(&stdout).with_context(|| format!("parse qgroup usage for {agent_name}"))
|
||||
}
|
||||
|
||||
/// Set or clear a btrfs qgroup size limit on an agent's state subvolume.
|
||||
/// `limit_bytes = None` clears it. Requires quota enabled. Via hive-priv.
|
||||
pub async fn set_subvolume_quota(agent_name: &str, limit_bytes: Option<u64>) -> Result<()> {
|
||||
ok(call(&PrivRequest::SetSubvolumeQuota {
|
||||
agent_name: agent_name.to_owned(),
|
||||
limit_bytes,
|
||||
})
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// Parse `(referenced, exclusive)` bytes from `btrfs qgroup show -f --raw`
|
||||
/// output. Skips the header rows and reads the last data row (a qgroup row
|
||||
/// is `<id-with-slash> <rfer> <excl> …`).
|
||||
fn parse_qgroup_usage(out: &str) -> Result<(u64, u64)> {
|
||||
for line in out.lines().rev() {
|
||||
let cols: Vec<&str> = line.split_whitespace().collect();
|
||||
if cols.len() >= 3
|
||||
&& cols[0].contains('/')
|
||||
&& let (Ok(rfer), Ok(excl)) = (cols[1].parse::<u64>(), cols[2].parse::<u64>())
|
||||
{
|
||||
return Ok((rfer, excl));
|
||||
}
|
||||
}
|
||||
bail!("no qgroup data row in `btrfs qgroup show` output: {out:?}")
|
||||
}
|
||||
|
||||
fn check(resp: PrivResponse) -> Result<(String, String)> {
|
||||
if resp.ok {
|
||||
Ok((resp.stdout, resp.stderr))
|
||||
|
|
|
|||
Loading…
Reference in a new issue