diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index 3c3ef5bb..a53f6901 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -85,6 +85,14 @@ pub struct Coordinator { pub agent_cpu_quota: String, /// Per-agent systemd `MemoryMax=` value (e.g. `"4G"`). Same drop-in. pub agent_memory_max: String, + /// Per-agent systemd `CPUWeight=` (cgroup v2 `cpu.weight`, 1..=10000). + /// Same drop-in. A *relative share* under contention, not a cap — see + /// the `agentCpuWeight` NixOS option for the full semantics. + pub agent_cpu_weight: u32, + /// Per-agent systemd `IOWeight=` (cgroup v2 `io.weight`, 1..=10000). + /// Same drop-in, same relative-share semantics as + /// [`Self::agent_cpu_weight`]. + pub agent_io_weight: u32, /// Operator-tunable model→price table backing the hive-wide cost /// estimate on the ST4TS tab. Set via `services.hyperhive.modelPrices` /// and passed to `hive-c0re serve --model-prices `. Models not @@ -210,6 +218,12 @@ pub struct HiveEnv { pub agent_cpu_quota: String, /// Per-agent systemd `MemoryMax=` value (e.g. `"4G"`). pub agent_memory_max: String, + /// Per-agent systemd `CPUWeight=` — cgroup v2 `cpu.weight`, 1..=10000. + /// Relative share under contention, not a cap; see `agentCpuWeight`. + pub agent_cpu_weight: u32, + /// Per-agent systemd `IOWeight=` — cgroup v2 `io.weight`, 1..=10000. + /// Same semantics as [`Self::agent_cpu_weight`]; see `agentIoWeight`. + pub agent_io_weight: u32, } impl Default for HiveEnv { @@ -227,6 +241,11 @@ impl Default for HiveEnv { ]), agent_cpu_quota: "200%".to_string(), agent_memory_max: "4G".to_string(), + // Slightly below the kernel default of 100, so agent + // containers yield to host services + the infra containers + // (which carry no drop-in and stay at 100) under contention. + agent_cpu_weight: 80, + agent_io_weight: 80, } } } @@ -457,6 +476,8 @@ impl Coordinator { context_window_tokens, agent_cpu_quota, agent_memory_max, + agent_cpu_weight, + agent_io_weight, } = env; let broker = Broker::open(db_path).context("open broker")?; let approvals = Approvals::open(db_path).context("open approvals")?; @@ -501,6 +522,8 @@ impl Coordinator { context_window_tokens, agent_cpu_quota, agent_memory_max, + agent_cpu_weight, + agent_io_weight, model_prices, agents: Mutex::new(HashMap::new()), transient: Mutex::new(HashMap::new()), @@ -532,6 +555,8 @@ impl Coordinator { context_window_tokens: self.context_window_tokens.clone(), agent_cpu_quota: self.agent_cpu_quota.clone(), agent_memory_max: self.agent_memory_max.clone(), + agent_cpu_weight: self.agent_cpu_weight, + agent_io_weight: self.agent_io_weight, } } diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index 341b27f1..b6795dbe 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -25,7 +25,14 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re set_nspawn_flags(&container, &paths.agent, &paths.claude, &paths.notes).await?; let (cpu_quota, memory_max) = crate::resource_limits::effective(name, &hive.agent_cpu_quota, &hive.agent_memory_max); - set_resource_limits(&container, &cpu_quota, &memory_max).await?; + set_resource_limits( + &container, + &cpu_quota, + &memory_max, + hive.agent_cpu_weight, + hive.agent_io_weight, + ) + .await?; systemd_daemon_reload().await } @@ -34,8 +41,21 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re /// `meta/resource-limits.json` where set, the hive-wide defaults /// otherwise. Goes under `/run/systemd/system/...` so it's ephemeral /// (regenerated on every spawn / rebuild). -async fn set_resource_limits(container: &str, cpu_quota: &str, memory_max: &str) -> Result<()> { - crate::priv_client::write_resource_limits(container, memory_max, cpu_quota).await +/// +/// The weights are hive-wide (`services.hyperhive.agentCpuWeight` / +/// `agentIoWeight`) — unlike the caps they have no per-agent override in +/// `meta/resource-limits.json`, so they come straight off [`HiveEnv`]. +async fn set_resource_limits( + container: &str, + cpu_quota: &str, + memory_max: &str, + cpu_weight: u32, + io_weight: u32, +) -> Result<()> { + crate::priv_client::write_resource_limits( + container, memory_max, cpu_quota, cpu_weight, io_weight, + ) + .await } async fn systemd_daemon_reload() -> Result<()> { diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index d869cd48..351df29b 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -110,6 +110,17 @@ enum Cmd { /// container. Set via `services.hyperhive.agentMemoryMax`. #[arg(long)] agent_memory_max: Option, + /// Override: systemd `CPUWeight=` applied to every agent container + /// via the same drop-in — a cgroup v2 relative share under + /// contention (1..=10000), not a cap. Set via + /// `services.hyperhive.agentCpuWeight`. + #[arg(long)] + agent_cpu_weight: Option, + /// Override: systemd `IOWeight=` applied to every agent container, + /// the block-IO counterpart of `--agent-cpu-weight`. Set via + /// `services.hyperhive.agentIoWeight`. + #[arg(long)] + agent_io_weight: Option, /// Override: per-model USD prices (per million tokens) for the /// hive-wide ST4TS cost estimate, as a JSON object mapping a /// model-family short name to `{input, output, cache_read, @@ -147,6 +158,8 @@ async fn main() -> Result<()> { context_window_tokens, agent_cpu_quota, agent_memory_max, + agent_cpu_weight, + agent_io_weight, model_prices, build_slots, } => { @@ -184,6 +197,12 @@ async fn main() -> Result<()> { if let Some(v) = agent_memory_max { sc.env.agent_memory_max = v; } + if let Some(v) = agent_cpu_weight { + sc.env.agent_cpu_weight = v; + } + if let Some(v) = agent_io_weight { + sc.env.agent_io_weight = v; + } if let Some(v) = model_prices { sc.model_prices = serde_json::from_str(&v).context("--model-prices: invalid JSON")?; diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index 66ab51e7..8059ff56 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -174,11 +174,15 @@ pub async fn write_resource_limits( container: &str, memory_max: &str, cpu_quota: &str, + cpu_weight: u32, + io_weight: u32, ) -> Result<()> { ok(call(&PrivRequest::WriteResourceLimits { container: container.to_owned(), memory_max: memory_max.to_owned(), cpu_quota: cpu_quota.to_owned(), + cpu_weight, + io_weight, }) .await?) } diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index ab9b0ec8..979bc112 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -336,12 +336,22 @@ pub enum PrivRequest { }, /// Write `/run/systemd/system/container@.service.d/hyperhive-limits.conf` - /// with `[Service]\nMemoryMax=\nCPUQuota=\n`. + /// with `[Service]` carrying `MemoryMax=` / `CPUQuota=` (hard caps) and + /// `CPUWeight=` / `IOWeight=` (cgroup v2 relative shares, contention-only). /// Written by `lifecycle::set_resource_limits`. WriteResourceLimits { container: String, memory_max: String, cpu_quota: String, + /// cgroup v2 `cpu.weight`, 1..=10000. `#[serde(default)]` so a + /// hive-priv built before this field still deserialises new + /// requests; 0 is treated as "omit the line" by the writer. + #[serde(default)] + cpu_weight: u32, + /// cgroup v2 `io.weight`, 1..=10000. Same default/omit rule as + /// `cpu_weight`. + #[serde(default)] + io_weight: u32, }, /// Remove `/run/systemd/system/container@.service.d/` if present. diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index f6290057..41e900a2 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -223,7 +223,9 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String, ref container, ref memory_max, ref cpu_quota, - } => write_resource_limits(container, memory_max, cpu_quota), + cpu_weight, + io_weight, + } => write_resource_limits(container, memory_max, cpu_quota, cpu_weight, io_weight), PrivRequest::RemoveServiceDropin { ref container } => remove_service_dropin(container), @@ -527,10 +529,17 @@ fn chmod_socket_dir(agent_name: &str, mode: u32) -> Result<(String, String)> { Ok((String::new(), String::new())) } -/// `WriteResourceLimits` — drop a systemd `MemoryMax`/`CPUQuota` -/// override into the container service's drop-in dir, together with a +/// `WriteResourceLimits` — drop the systemd resource settings into the +/// container service's drop-in dir, together with a /// `ConditionPathIsDirectory=` guard on the agent's MCP runtime dir. /// +/// Two different kinds of setting land in the same file. `MemoryMax=` / +/// `CPUQuota=` are hard caps that throttle even on an idle host; +/// `CPUWeight=` / `IOWeight=` are cgroup v2 relative shares that only +/// decide who yields *under contention*. A zero weight means "not +/// configured" and omits the line, so a hive-c0re built before the weights +/// existed keeps producing the old two-line drop-in. +/// /// The condition causes systemd to *skip* (not *fail*) the unit when the /// bind-mount source dir is absent — result is `condition`, which does not /// increment the start-limit counter. This is belt-and-braces on top of @@ -541,6 +550,8 @@ fn write_resource_limits( container: &str, memory_max: &str, cpu_quota: &str, + cpu_weight: u32, + io_weight: u32, ) -> Result<(String, String)> { validate_container_system_name(container)?; // Derive the logical agent name (strip h- prefix) to form the runtime @@ -551,19 +562,48 @@ fn write_resource_limits( let dir = format!("/run/systemd/system/container@{container}.service.d"); std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?; let path = format!("{dir}/hyperhive-limits.conf"); - // [Unit] section: condition checked at start time — skips (not fails) - // the unit when the MCP socket dir is absent, avoiding restart loops. - // [Service] section: resource caps. - let content = format!( + let content = limits_dropin_body(&runtime_dir, memory_max, cpu_quota, cpu_weight, io_weight); + std::fs::write(&path, content).with_context(|| format!("write {path}"))?; + Ok((String::new(), String::new())) +} + +/// Render the body of `hyperhive-limits.conf`. +/// +/// `[Unit]`: the condition is checked at start time — it skips (not fails) +/// the unit when the MCP socket dir is absent, avoiding restart loops. +/// `[Service]`: the hard caps first, then the relative weights. A weight of +/// `0` means "not configured" and omits its line entirely, so a request from +/// a hive-c0re built before the weights existed reproduces the old +/// two-setting drop-in byte for byte. +fn limits_dropin_body( + runtime_dir: &str, + memory_max: &str, + cpu_quota: &str, + cpu_weight: u32, + io_weight: u32, +) -> String { + // Built as two possibly-empty lines rather than pushed onto the + // string: `format!` appended to a `String` trips clippy::pedantic's + // `format_push_string`, and a `write!` would need an unwrap. + let cpu_weight_line = if cpu_weight > 0 { + format!("CPUWeight={cpu_weight}\n") + } else { + String::new() + }; + let io_weight_line = if io_weight > 0 { + format!("IOWeight={io_weight}\n") + } else { + String::new() + }; + format!( "[Unit]\n\ ConditionPathIsDirectory={runtime_dir}\n\ \n\ [Service]\n\ MemoryMax={memory_max}\n\ - CPUQuota={cpu_quota}\n" - ); - std::fs::write(&path, content).with_context(|| format!("write {path}"))?; - Ok((String::new(), String::new())) + CPUQuota={cpu_quota}\n\ + {cpu_weight_line}{io_weight_line}" + ) } /// `DaemonReload` — `systemctl daemon-reload` on the host. @@ -2185,11 +2225,53 @@ async fn sync_agent_tmpfiles(agents: &[String]) -> Result<(String, String)> { #[cfg(test)] mod tests { use super::{ - PAUSED_MARKER_FILE, redact_password_line, remove_marker_in, write_state_file_nofollow, + PAUSED_MARKER_FILE, limits_dropin_body, redact_password_line, remove_marker_in, + write_state_file_nofollow, }; use std::path::PathBuf; use std::sync::atomic::{AtomicU32, Ordering}; + /// A zero weight is "not configured": the drop-in must come out + /// byte-identical to the pre-weights two-setting body, so a hive-c0re + /// older than this field can't change what lands on disk. + #[test] + fn zero_weights_reproduce_the_pre_weights_dropin() { + assert_eq!( + limits_dropin_body("/run/hyperhive/agents/iris", "4G", "200%", 0, 0), + "[Unit]\n\ + ConditionPathIsDirectory=/run/hyperhive/agents/iris\n\ + \n\ + [Service]\n\ + MemoryMax=4G\n\ + CPUQuota=200%\n" + ); + } + + /// Weights are appended to the `[Service]` section, each omitted + /// independently when zero. + #[test] + fn weights_are_emitted_only_when_set() { + let both = limits_dropin_body("/rt/x", "4G", "200%", 80, 80); + assert!( + both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"), + "{both}" + ); + + let cpu_only = limits_dropin_body("/rt/x", "4G", "200%", 80, 0); + assert!( + cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"), + "{cpu_only}" + ); + assert!(!cpu_only.contains("IOWeight"), "{cpu_only}"); + + let io_only = limits_dropin_body("/rt/x", "4G", "200%", 0, 80); + assert!( + io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"), + "{io_only}" + ); + assert!(!io_only.contains("CPUWeight"), "{io_only}"); + } + #[test] fn redacts_lines_mentioning_password_case_insensitively() { assert_eq!( diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 0e9ceed9..5372ed7a 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -81,6 +81,8 @@ let context_window_tokens = cfg.contextWindowTokens; agent_cpu_quota = cfg.agentCpuQuota; agent_memory_max = cfg.agentMemoryMax; + agent_cpu_weight = cfg.agentCpuWeight; + agent_io_weight = cfg.agentIoWeight; model_prices = cfg.modelPrices; build_slots = cfg.buildSlots; }; diff --git a/nix/host-modules/hive-c0re/options.nix b/nix/host-modules/hive-c0re/options.nix index 9822e2c6..8725e28e 100644 --- a/nix/host-modules/hive-c0re/options.nix +++ b/nix/host-modules/hive-c0re/options.nix @@ -301,6 +301,49 @@ ''; }; + agentCpuWeight = lib.mkOption { + type = lib.types.ints.between 1 10000; + default = 80; + example = 50; + description = '' + systemd `CPUWeight=` applied to every agent container via the + same drop-in as `agentCpuQuota`. This is the cgroup v2 + `cpu.weight` relative share, **not** a cap: a low-weight + container still gets the whole machine when nothing else wants + it, and the weight only decides who yields under contention. + That makes it the complement of `agentCpuQuota`, which throttles + even on an idle host. + + The kernel default is `100`. The hyperhive default of `80` means + agent containers yield slightly to everything that is *not* on + this drop-in path — host services and the infrastructure + containers (`hive-ci`, `hive-forge`, `hive-gateway`, + `hive-matrix`), which stay at `100`. Note this is a hive-wide + value, so it does not rank agents against *each other*: they all + share one weight. + ''; + }; + + agentIoWeight = lib.mkOption { + type = lib.types.ints.between 1 10000; + default = 80; + example = 50; + description = '' + systemd `IOWeight=` applied to every agent container via the + same drop-in as `agentCpuQuota` — the block-IO counterpart of + `agentCpuWeight`, with the same relative-share, contention-only + semantics. + + Caveat: `IOWeight=` maps to the cgroup v2 `io.weight` knob, which + is only honoured when the `io.cost` (blk-iocost) controller is + enabled for the backing device, or when the device uses the BFQ + scheduler. On a host running `none`/`mq-deadline`/`kyber` without + iocost QoS configured, systemd writes the value and the kernel + ignores it — harmless, but it will measure as a no-op. Check with + `cat /sys/fs/cgroup/io.cost.qos` on the host. + ''; + }; + buildSlots = lib.mkOption { type = lib.types.ints.positive; default = 1;