feat(#2754): hive-wide CPUWeight= / IOWeight= for agent containers
`CPUQuota=`/`MemoryMax=` are hard caps: they throttle an agent even when
the host is idle, so they are the wrong tool for "be polite under
contention". The cgroup v2 relative shares are, and neither was wired.
Adds `services.hyperhive.{agentCpuWeight,agentIoWeight}` (1..=10000,
default 80) threaded through the existing drop-in path: HiveEnv ->
write_dropins -> WriteResourceLimits -> hyperhive-limits.conf, next to
the caps already there. Hive-wide only, as the operator scoped it on the
issue: no per-agent override, no resource-limits.json field, no
dashboard form.
The default of 80 is below the kernel's 100, so agent containers yield
to everything *not* on this drop-in path -- host services and the infra
containers (hive-ci, hive-forge, hive-gateway, hive-matrix). It does not
rank agents against each other; they all carry the same weight.
`WriteResourceLimits` gains two `#[serde(default)]` fields, and the
writer treats weight 0 as "not configured" and omits the line, so an
older hive-c0re talking to a newer hive-priv still produces the exact
pre-weights drop-in. The body is extracted into `limits_dropin_body` so
that is covered by a test rather than asserted by eye.
This commit is contained in:
parent
80650041d9
commit
e407fa93df
8 changed files with 221 additions and 16 deletions
|
|
@ -85,6 +85,14 @@ pub struct Coordinator {
|
|||
pub agent_cpu_quota: String,
|
||||
/// Per-agent systemd `MemoryMax=` value (e.g. `"4G"`). Same drop-in.
|
||||
pub agent_memory_max: String,
|
||||
/// Per-agent systemd `CPUWeight=` (cgroup v2 `cpu.weight`, 1..=10000).
|
||||
/// Same drop-in. A *relative share* under contention, not a cap — see
|
||||
/// the `agentCpuWeight` NixOS option for the full semantics.
|
||||
pub agent_cpu_weight: u32,
|
||||
/// Per-agent systemd `IOWeight=` (cgroup v2 `io.weight`, 1..=10000).
|
||||
/// Same drop-in, same relative-share semantics as
|
||||
/// [`Self::agent_cpu_weight`].
|
||||
pub agent_io_weight: u32,
|
||||
/// Operator-tunable model→price table backing the hive-wide cost
|
||||
/// estimate on the ST4TS tab. Set via `services.hyperhive.modelPrices`
|
||||
/// and passed to `hive-c0re serve --model-prices <json>`. Models not
|
||||
|
|
@ -210,6 +218,12 @@ pub struct HiveEnv {
|
|||
pub agent_cpu_quota: String,
|
||||
/// Per-agent systemd `MemoryMax=` value (e.g. `"4G"`).
|
||||
pub agent_memory_max: String,
|
||||
/// Per-agent systemd `CPUWeight=` — cgroup v2 `cpu.weight`, 1..=10000.
|
||||
/// Relative share under contention, not a cap; see `agentCpuWeight`.
|
||||
pub agent_cpu_weight: u32,
|
||||
/// Per-agent systemd `IOWeight=` — cgroup v2 `io.weight`, 1..=10000.
|
||||
/// Same semantics as [`Self::agent_cpu_weight`]; see `agentIoWeight`.
|
||||
pub agent_io_weight: u32,
|
||||
}
|
||||
|
||||
impl Default for HiveEnv {
|
||||
|
|
@ -227,6 +241,11 @@ impl Default for HiveEnv {
|
|||
]),
|
||||
agent_cpu_quota: "200%".to_string(),
|
||||
agent_memory_max: "4G".to_string(),
|
||||
// Slightly below the kernel default of 100, so agent
|
||||
// containers yield to host services + the infra containers
|
||||
// (which carry no drop-in and stay at 100) under contention.
|
||||
agent_cpu_weight: 80,
|
||||
agent_io_weight: 80,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -457,6 +476,8 @@ impl Coordinator {
|
|||
context_window_tokens,
|
||||
agent_cpu_quota,
|
||||
agent_memory_max,
|
||||
agent_cpu_weight,
|
||||
agent_io_weight,
|
||||
} = env;
|
||||
let broker = Broker::open(db_path).context("open broker")?;
|
||||
let approvals = Approvals::open(db_path).context("open approvals")?;
|
||||
|
|
@ -501,6 +522,8 @@ impl Coordinator {
|
|||
context_window_tokens,
|
||||
agent_cpu_quota,
|
||||
agent_memory_max,
|
||||
agent_cpu_weight,
|
||||
agent_io_weight,
|
||||
model_prices,
|
||||
agents: Mutex::new(HashMap::new()),
|
||||
transient: Mutex::new(HashMap::new()),
|
||||
|
|
@ -532,6 +555,8 @@ impl Coordinator {
|
|||
context_window_tokens: self.context_window_tokens.clone(),
|
||||
agent_cpu_quota: self.agent_cpu_quota.clone(),
|
||||
agent_memory_max: self.agent_memory_max.clone(),
|
||||
agent_cpu_weight: self.agent_cpu_weight,
|
||||
agent_io_weight: self.agent_io_weight,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,14 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re
|
|||
set_nspawn_flags(&container, &paths.agent, &paths.claude, &paths.notes).await?;
|
||||
let (cpu_quota, memory_max) =
|
||||
crate::resource_limits::effective(name, &hive.agent_cpu_quota, &hive.agent_memory_max);
|
||||
set_resource_limits(&container, &cpu_quota, &memory_max).await?;
|
||||
set_resource_limits(
|
||||
&container,
|
||||
&cpu_quota,
|
||||
&memory_max,
|
||||
hive.agent_cpu_weight,
|
||||
hive.agent_io_weight,
|
||||
)
|
||||
.await?;
|
||||
systemd_daemon_reload().await
|
||||
}
|
||||
|
||||
|
|
@ -34,8 +41,21 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re
|
|||
/// `meta/resource-limits.json` where set, the hive-wide defaults
|
||||
/// otherwise. Goes under `/run/systemd/system/...` so it's ephemeral
|
||||
/// (regenerated on every spawn / rebuild).
|
||||
async fn set_resource_limits(container: &str, cpu_quota: &str, memory_max: &str) -> Result<()> {
|
||||
crate::priv_client::write_resource_limits(container, memory_max, cpu_quota).await
|
||||
///
|
||||
/// The weights are hive-wide (`services.hyperhive.agentCpuWeight` /
|
||||
/// `agentIoWeight`) — unlike the caps they have no per-agent override in
|
||||
/// `meta/resource-limits.json`, so they come straight off [`HiveEnv`].
|
||||
async fn set_resource_limits(
|
||||
container: &str,
|
||||
cpu_quota: &str,
|
||||
memory_max: &str,
|
||||
cpu_weight: u32,
|
||||
io_weight: u32,
|
||||
) -> Result<()> {
|
||||
crate::priv_client::write_resource_limits(
|
||||
container, memory_max, cpu_quota, cpu_weight, io_weight,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn systemd_daemon_reload() -> Result<()> {
|
||||
|
|
|
|||
|
|
@ -110,6 +110,17 @@ enum Cmd {
|
|||
/// container. Set via `services.hyperhive.agentMemoryMax`.
|
||||
#[arg(long)]
|
||||
agent_memory_max: Option<String>,
|
||||
/// Override: systemd `CPUWeight=` applied to every agent container
|
||||
/// via the same drop-in — a cgroup v2 relative share under
|
||||
/// contention (1..=10000), not a cap. Set via
|
||||
/// `services.hyperhive.agentCpuWeight`.
|
||||
#[arg(long)]
|
||||
agent_cpu_weight: Option<u32>,
|
||||
/// Override: systemd `IOWeight=` applied to every agent container,
|
||||
/// the block-IO counterpart of `--agent-cpu-weight`. Set via
|
||||
/// `services.hyperhive.agentIoWeight`.
|
||||
#[arg(long)]
|
||||
agent_io_weight: Option<u32>,
|
||||
/// Override: per-model USD prices (per million tokens) for the
|
||||
/// hive-wide ST4TS cost estimate, as a JSON object mapping a
|
||||
/// model-family short name to `{input, output, cache_read,
|
||||
|
|
@ -147,6 +158,8 @@ async fn main() -> Result<()> {
|
|||
context_window_tokens,
|
||||
agent_cpu_quota,
|
||||
agent_memory_max,
|
||||
agent_cpu_weight,
|
||||
agent_io_weight,
|
||||
model_prices,
|
||||
build_slots,
|
||||
} => {
|
||||
|
|
@ -184,6 +197,12 @@ async fn main() -> Result<()> {
|
|||
if let Some(v) = agent_memory_max {
|
||||
sc.env.agent_memory_max = v;
|
||||
}
|
||||
if let Some(v) = agent_cpu_weight {
|
||||
sc.env.agent_cpu_weight = v;
|
||||
}
|
||||
if let Some(v) = agent_io_weight {
|
||||
sc.env.agent_io_weight = v;
|
||||
}
|
||||
if let Some(v) = model_prices {
|
||||
sc.model_prices =
|
||||
serde_json::from_str(&v).context("--model-prices: invalid JSON")?;
|
||||
|
|
|
|||
|
|
@ -174,11 +174,15 @@ pub async fn write_resource_limits(
|
|||
container: &str,
|
||||
memory_max: &str,
|
||||
cpu_quota: &str,
|
||||
cpu_weight: u32,
|
||||
io_weight: u32,
|
||||
) -> Result<()> {
|
||||
ok(call(&PrivRequest::WriteResourceLimits {
|
||||
container: container.to_owned(),
|
||||
memory_max: memory_max.to_owned(),
|
||||
cpu_quota: cpu_quota.to_owned(),
|
||||
cpu_weight,
|
||||
io_weight,
|
||||
})
|
||||
.await?)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -336,12 +336,22 @@ pub enum PrivRequest {
|
|||
},
|
||||
|
||||
/// Write `/run/systemd/system/container@<container>.service.d/hyperhive-limits.conf`
|
||||
/// with `[Service]\nMemoryMax=<memory_max>\nCPUQuota=<cpu_quota>\n`.
|
||||
/// with `[Service]` carrying `MemoryMax=` / `CPUQuota=` (hard caps) and
|
||||
/// `CPUWeight=` / `IOWeight=` (cgroup v2 relative shares, contention-only).
|
||||
/// Written by `lifecycle::set_resource_limits`.
|
||||
WriteResourceLimits {
|
||||
container: String,
|
||||
memory_max: String,
|
||||
cpu_quota: String,
|
||||
/// cgroup v2 `cpu.weight`, 1..=10000. `#[serde(default)]` so a
|
||||
/// hive-priv built before this field still deserialises new
|
||||
/// requests; 0 is treated as "omit the line" by the writer.
|
||||
#[serde(default)]
|
||||
cpu_weight: u32,
|
||||
/// cgroup v2 `io.weight`, 1..=10000. Same default/omit rule as
|
||||
/// `cpu_weight`.
|
||||
#[serde(default)]
|
||||
io_weight: u32,
|
||||
},
|
||||
|
||||
/// Remove `/run/systemd/system/container@<container>.service.d/` if present.
|
||||
|
|
|
|||
|
|
@ -223,7 +223,9 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String,
|
|||
ref container,
|
||||
ref memory_max,
|
||||
ref cpu_quota,
|
||||
} => write_resource_limits(container, memory_max, cpu_quota),
|
||||
cpu_weight,
|
||||
io_weight,
|
||||
} => write_resource_limits(container, memory_max, cpu_quota, cpu_weight, io_weight),
|
||||
|
||||
PrivRequest::RemoveServiceDropin { ref container } => remove_service_dropin(container),
|
||||
|
||||
|
|
@ -527,10 +529,17 @@ fn chmod_socket_dir(agent_name: &str, mode: u32) -> Result<(String, String)> {
|
|||
Ok((String::new(), String::new()))
|
||||
}
|
||||
|
||||
/// `WriteResourceLimits` — drop a systemd `MemoryMax`/`CPUQuota`
|
||||
/// override into the container service's drop-in dir, together with a
|
||||
/// `WriteResourceLimits` — drop the systemd resource settings into the
|
||||
/// container service's drop-in dir, together with a
|
||||
/// `ConditionPathIsDirectory=` guard on the agent's MCP runtime dir.
|
||||
///
|
||||
/// Two different kinds of setting land in the same file. `MemoryMax=` /
|
||||
/// `CPUQuota=` are hard caps that throttle even on an idle host;
|
||||
/// `CPUWeight=` / `IOWeight=` are cgroup v2 relative shares that only
|
||||
/// decide who yields *under contention*. A zero weight means "not
|
||||
/// configured" and omits the line, so a hive-c0re built before the weights
|
||||
/// existed keeps producing the old two-line drop-in.
|
||||
///
|
||||
/// The condition causes systemd to *skip* (not *fail*) the unit when the
|
||||
/// bind-mount source dir is absent — result is `condition`, which does not
|
||||
/// increment the start-limit counter. This is belt-and-braces on top of
|
||||
|
|
@ -541,6 +550,8 @@ fn write_resource_limits(
|
|||
container: &str,
|
||||
memory_max: &str,
|
||||
cpu_quota: &str,
|
||||
cpu_weight: u32,
|
||||
io_weight: u32,
|
||||
) -> Result<(String, String)> {
|
||||
validate_container_system_name(container)?;
|
||||
// Derive the logical agent name (strip h- prefix) to form the runtime
|
||||
|
|
@ -551,19 +562,48 @@ fn write_resource_limits(
|
|||
let dir = format!("/run/systemd/system/container@{container}.service.d");
|
||||
std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?;
|
||||
let path = format!("{dir}/hyperhive-limits.conf");
|
||||
// [Unit] section: condition checked at start time — skips (not fails)
|
||||
// the unit when the MCP socket dir is absent, avoiding restart loops.
|
||||
// [Service] section: resource caps.
|
||||
let content = format!(
|
||||
let content = limits_dropin_body(&runtime_dir, memory_max, cpu_quota, cpu_weight, io_weight);
|
||||
std::fs::write(&path, content).with_context(|| format!("write {path}"))?;
|
||||
Ok((String::new(), String::new()))
|
||||
}
|
||||
|
||||
/// Render the body of `hyperhive-limits.conf`.
|
||||
///
|
||||
/// `[Unit]`: the condition is checked at start time — it skips (not fails)
|
||||
/// the unit when the MCP socket dir is absent, avoiding restart loops.
|
||||
/// `[Service]`: the hard caps first, then the relative weights. A weight of
|
||||
/// `0` means "not configured" and omits its line entirely, so a request from
|
||||
/// a hive-c0re built before the weights existed reproduces the old
|
||||
/// two-setting drop-in byte for byte.
|
||||
fn limits_dropin_body(
|
||||
runtime_dir: &str,
|
||||
memory_max: &str,
|
||||
cpu_quota: &str,
|
||||
cpu_weight: u32,
|
||||
io_weight: u32,
|
||||
) -> String {
|
||||
// Built as two possibly-empty lines rather than pushed onto the
|
||||
// string: `format!` appended to a `String` trips clippy::pedantic's
|
||||
// `format_push_string`, and a `write!` would need an unwrap.
|
||||
let cpu_weight_line = if cpu_weight > 0 {
|
||||
format!("CPUWeight={cpu_weight}\n")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let io_weight_line = if io_weight > 0 {
|
||||
format!("IOWeight={io_weight}\n")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
format!(
|
||||
"[Unit]\n\
|
||||
ConditionPathIsDirectory={runtime_dir}\n\
|
||||
\n\
|
||||
[Service]\n\
|
||||
MemoryMax={memory_max}\n\
|
||||
CPUQuota={cpu_quota}\n"
|
||||
);
|
||||
std::fs::write(&path, content).with_context(|| format!("write {path}"))?;
|
||||
Ok((String::new(), String::new()))
|
||||
CPUQuota={cpu_quota}\n\
|
||||
{cpu_weight_line}{io_weight_line}"
|
||||
)
|
||||
}
|
||||
|
||||
/// `DaemonReload` — `systemctl daemon-reload` on the host.
|
||||
|
|
@ -2185,11 +2225,53 @@ async fn sync_agent_tmpfiles(agents: &[String]) -> Result<(String, String)> {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
PAUSED_MARKER_FILE, redact_password_line, remove_marker_in, write_state_file_nofollow,
|
||||
PAUSED_MARKER_FILE, limits_dropin_body, redact_password_line, remove_marker_in,
|
||||
write_state_file_nofollow,
|
||||
};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
|
||||
/// A zero weight is "not configured": the drop-in must come out
|
||||
/// byte-identical to the pre-weights two-setting body, so a hive-c0re
|
||||
/// older than this field can't change what lands on disk.
|
||||
#[test]
|
||||
fn zero_weights_reproduce_the_pre_weights_dropin() {
|
||||
assert_eq!(
|
||||
limits_dropin_body("/run/hyperhive/agents/iris", "4G", "200%", 0, 0),
|
||||
"[Unit]\n\
|
||||
ConditionPathIsDirectory=/run/hyperhive/agents/iris\n\
|
||||
\n\
|
||||
[Service]\n\
|
||||
MemoryMax=4G\n\
|
||||
CPUQuota=200%\n"
|
||||
);
|
||||
}
|
||||
|
||||
/// Weights are appended to the `[Service]` section, each omitted
|
||||
/// independently when zero.
|
||||
#[test]
|
||||
fn weights_are_emitted_only_when_set() {
|
||||
let both = limits_dropin_body("/rt/x", "4G", "200%", 80, 80);
|
||||
assert!(
|
||||
both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"),
|
||||
"{both}"
|
||||
);
|
||||
|
||||
let cpu_only = limits_dropin_body("/rt/x", "4G", "200%", 80, 0);
|
||||
assert!(
|
||||
cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"),
|
||||
"{cpu_only}"
|
||||
);
|
||||
assert!(!cpu_only.contains("IOWeight"), "{cpu_only}");
|
||||
|
||||
let io_only = limits_dropin_body("/rt/x", "4G", "200%", 0, 80);
|
||||
assert!(
|
||||
io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"),
|
||||
"{io_only}"
|
||||
);
|
||||
assert!(!io_only.contains("CPUWeight"), "{io_only}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redacts_lines_mentioning_password_case_insensitively() {
|
||||
assert_eq!(
|
||||
|
|
|
|||
|
|
@ -81,6 +81,8 @@ let
|
|||
context_window_tokens = cfg.contextWindowTokens;
|
||||
agent_cpu_quota = cfg.agentCpuQuota;
|
||||
agent_memory_max = cfg.agentMemoryMax;
|
||||
agent_cpu_weight = cfg.agentCpuWeight;
|
||||
agent_io_weight = cfg.agentIoWeight;
|
||||
model_prices = cfg.modelPrices;
|
||||
build_slots = cfg.buildSlots;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -301,6 +301,49 @@
|
|||
'';
|
||||
};
|
||||
|
||||
agentCpuWeight = lib.mkOption {
|
||||
type = lib.types.ints.between 1 10000;
|
||||
default = 80;
|
||||
example = 50;
|
||||
description = ''
|
||||
systemd `CPUWeight=` applied to every agent container via the
|
||||
same drop-in as `agentCpuQuota`. This is the cgroup v2
|
||||
`cpu.weight` relative share, **not** a cap: a low-weight
|
||||
container still gets the whole machine when nothing else wants
|
||||
it, and the weight only decides who yields under contention.
|
||||
That makes it the complement of `agentCpuQuota`, which throttles
|
||||
even on an idle host.
|
||||
|
||||
The kernel default is `100`. The hyperhive default of `80` means
|
||||
agent containers yield slightly to everything that is *not* on
|
||||
this drop-in path — host services and the infrastructure
|
||||
containers (`hive-ci`, `hive-forge`, `hive-gateway`,
|
||||
`hive-matrix`), which stay at `100`. Note this is a hive-wide
|
||||
value, so it does not rank agents against *each other*: they all
|
||||
share one weight.
|
||||
'';
|
||||
};
|
||||
|
||||
agentIoWeight = lib.mkOption {
|
||||
type = lib.types.ints.between 1 10000;
|
||||
default = 80;
|
||||
example = 50;
|
||||
description = ''
|
||||
systemd `IOWeight=` applied to every agent container via the
|
||||
same drop-in as `agentCpuQuota` — the block-IO counterpart of
|
||||
`agentCpuWeight`, with the same relative-share, contention-only
|
||||
semantics.
|
||||
|
||||
Caveat: `IOWeight=` maps to the cgroup v2 `io.weight` knob, which
|
||||
is only honoured when the `io.cost` (blk-iocost) controller is
|
||||
enabled for the backing device, or when the device uses the BFQ
|
||||
scheduler. On a host running `none`/`mq-deadline`/`kyber` without
|
||||
iocost QoS configured, systemd writes the value and the kernel
|
||||
ignores it — harmless, but it will measure as a no-op. Check with
|
||||
`cat /sys/fs/cgroup/io.cost.qos` on the host.
|
||||
'';
|
||||
};
|
||||
|
||||
buildSlots = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 1;
|
||||
|
|
|
|||
Loading…
Reference in a new issue