diff --git a/docs/coordinator.md b/docs/coordinator.md index c8df2b12..15d79a46 100644 --- a/docs/coordinator.md +++ b/docs/coordinator.md @@ -423,36 +423,10 @@ regardless. rebuild, so changes take effect on the next lifecycle op without requiring a host rebuild. -The same drop-in carries `CPUWeight=` / `IOWeight=` from -`agentCpuWeight` / `agentIoWeight`. Those are a different kind of -setting: the quota and the memory max are **hard caps** that throttle -an agent even on a completely idle host, while the weights are cgroup -v2 **relative shares** that only decide who yields *under contention*. -A low-weight container still gets the whole machine when nothing else -wants it. - | Option | Default | Description | | ---------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------ | | `services.hyperhive.c0re.agentCpuQuota` | `"200%"` | CPU cap per agent, as a percentage of one core (`"200%"` = 2 cores). Raise if agents hit CPU limits during builds or heavy tool use. | | `services.hyperhive.c0re.agentMemoryMax` | `"4G"` | Memory cap per agent. Raise for agents that run large nix builds or hold big in-memory data. | -| `services.hyperhive.c0re.agentCpuWeight` | `80` | `cpu.weight` share per agent, `1`–`10000` or `null` to omit the setting. Kernel default is `100`, so `80` makes agents yield. | -| `services.hyperhive.c0re.agentIoWeight` | `80` | `io.weight` share per agent, same range and `null` handling. See the caveat below — it is a no-op on many hosts. | - -Two things to know about the weights: - -- They are **hive-wide** — unlike the caps there is no per-agent - override in `meta/resource-limits.json`, so every agent carries the - same value and the weight does *not* rank agents against each other. - What `80` buys is that agents yield to everything **not** on this - drop-in path: host services and the infra containers (`hive-ci`, - `hive-forge`, `hive-gateway`, `hive-matrix`), which stay at the - kernel default of `100`. -- `IOWeight=` is only honoured when the backing device runs the BFQ - scheduler or has blk-iocost QoS enabled. On a host using - `none`/`mq-deadline`/`kyber` without iocost, systemd writes the value - and the kernel ignores it — harmless, but it will measure as nothing. - Check with `cat /sys/fs/cgroup/io.cost.qos`, and set the option to - `null` if you would rather not write a setting nothing reads. For a hive-wide cap across all containers together, set `systemd.slices.machine.serviceConfig.CPUQuota` in your NixOS diff --git a/docs/persistence.md b/docs/persistence.md index 042fd35a..dcad76c0 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -318,9 +318,7 @@ Contents: an absent file, absent agent, or absent field falls back to the hive-wide `services.hyperhive.agentCpuQuota` / `agentMemoryMax`, so an agent can override only its memory and still track the hive - default for CPU. The `CPUWeight=` / `IOWeight=` shares in the same - drop-in have **no** per-agent override — they are hive-wide only and - come straight off `HiveEnv`, so this file has no field for them. + default for CPU. The root agent has the meta dir RO-mounted at `/meta/`. diff --git a/docs/security.md b/docs/security.md index 6a2b5e1b..0baec3f0 100644 --- a/docs/security.md +++ b/docs/security.md @@ -209,7 +209,7 @@ known operations; there is no arbitrary command pass-through: | `ReadContainerJournal` | `journalctl -M -n [filters...]` | | `ReloadGatewayNginx` | `systemctl -M hive-gateway reload/start/reset-failed nginx` | | `WriteNspawnFlags` | write `/etc/nixos-containers/.conf` (bind-mount list + network isolation vars) | -| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=`/`CPUWeight=`/`IOWeight=` systemd drop-in for agent container | +| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=` systemd drop-in for agent container | | `RemoveServiceDropin` | remove `container@.service.d/` drop-in on destroy | | `DaemonReload` | `systemctl daemon-reload` | | `ChownSocketDir` / `ChmodSocketDir` | chown/chmod `/run/hive-agent//` socket directory | diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index c08927df..3c3ef5bb 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -85,16 +85,6 @@ pub struct Coordinator { pub agent_cpu_quota: String, /// Per-agent systemd `MemoryMax=` value (e.g. `"4G"`). Same drop-in. pub agent_memory_max: String, - /// Per-agent systemd `CPUWeight=` (cgroup v2 `cpu.weight`, 1..=10000). - /// Same drop-in. A *relative share* under contention, not a cap — see - /// the `agentCpuWeight` NixOS option for the full semantics. `None` - /// (option set to `null`) omits the setting: kernel default, and the - /// drop-in is byte-identical to the one written before weights existed. - pub agent_cpu_weight: Option, - /// Per-agent systemd `IOWeight=` (cgroup v2 `io.weight`, 1..=10000). - /// Same drop-in, same relative-share and `None` semantics as - /// [`Self::agent_cpu_weight`]. - pub agent_io_weight: Option, /// Operator-tunable model→price table backing the hive-wide cost /// estimate on the ST4TS tab. Set via `services.hyperhive.modelPrices` /// and passed to `hive-c0re serve --model-prices `. Models not @@ -220,13 +210,6 @@ pub struct HiveEnv { pub agent_cpu_quota: String, /// Per-agent systemd `MemoryMax=` value (e.g. `"4G"`). pub agent_memory_max: String, - /// Per-agent systemd `CPUWeight=` — cgroup v2 `cpu.weight`, 1..=10000. - /// Relative share under contention, not a cap; see `agentCpuWeight`. - /// `None` = leave the setting out of the drop-in (kernel default). - pub agent_cpu_weight: Option, - /// Per-agent systemd `IOWeight=` — cgroup v2 `io.weight`, 1..=10000. - /// Same semantics as [`Self::agent_cpu_weight`]; see `agentIoWeight`. - pub agent_io_weight: Option, } impl Default for HiveEnv { @@ -244,11 +227,6 @@ impl Default for HiveEnv { ]), agent_cpu_quota: "200%".to_string(), agent_memory_max: "4G".to_string(), - // Slightly below the kernel default of 100, so agent - // containers yield to host services + the infra containers - // (which carry no drop-in and stay at 100) under contention. - agent_cpu_weight: Some(80), - agent_io_weight: Some(80), } } } @@ -479,8 +457,6 @@ impl Coordinator { context_window_tokens, agent_cpu_quota, agent_memory_max, - agent_cpu_weight, - agent_io_weight, } = env; let broker = Broker::open(db_path).context("open broker")?; let approvals = Approvals::open(db_path).context("open approvals")?; @@ -525,8 +501,6 @@ impl Coordinator { context_window_tokens, agent_cpu_quota, agent_memory_max, - agent_cpu_weight, - agent_io_weight, model_prices, agents: Mutex::new(HashMap::new()), transient: Mutex::new(HashMap::new()), @@ -558,8 +532,6 @@ impl Coordinator { context_window_tokens: self.context_window_tokens.clone(), agent_cpu_quota: self.agent_cpu_quota.clone(), agent_memory_max: self.agent_memory_max.clone(), - agent_cpu_weight: self.agent_cpu_weight, - agent_io_weight: self.agent_io_weight, } } diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index 1574f1a5..341b27f1 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -25,14 +25,7 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re set_nspawn_flags(&container, &paths.agent, &paths.claude, &paths.notes).await?; let (cpu_quota, memory_max) = crate::resource_limits::effective(name, &hive.agent_cpu_quota, &hive.agent_memory_max); - set_resource_limits( - &container, - &cpu_quota, - &memory_max, - hive.agent_cpu_weight, - hive.agent_io_weight, - ) - .await?; + set_resource_limits(&container, &cpu_quota, &memory_max).await?; systemd_daemon_reload().await } @@ -41,23 +34,8 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re /// `meta/resource-limits.json` where set, the hive-wide defaults /// otherwise. Goes under `/run/systemd/system/...` so it's ephemeral /// (regenerated on every spawn / rebuild). -/// -/// The weights are hive-wide (`services.hyperhive.agentCpuWeight` / -/// `agentIoWeight`) — unlike the caps they have no per-agent override in -/// `meta/resource-limits.json`, so they come straight off [`HiveEnv`]. -/// `None` (the nix option set to `null`) means the weight line is left out -/// and the container keeps the kernel default. -async fn set_resource_limits( - container: &str, - cpu_quota: &str, - memory_max: &str, - cpu_weight: Option, - io_weight: Option, -) -> Result<()> { - crate::priv_client::write_resource_limits( - container, memory_max, cpu_quota, cpu_weight, io_weight, - ) - .await +async fn set_resource_limits(container: &str, cpu_quota: &str, memory_max: &str) -> Result<()> { + crate::priv_client::write_resource_limits(container, memory_max, cpu_quota).await } async fn systemd_daemon_reload() -> Result<()> { diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index c6264986..d869cd48 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -110,17 +110,6 @@ enum Cmd { /// container. Set via `services.hyperhive.agentMemoryMax`. #[arg(long)] agent_memory_max: Option, - /// Override: systemd `CPUWeight=` applied to every agent container - /// via the same drop-in — a cgroup v2 relative share under - /// contention (1..=10000), not a cap. Set via - /// `services.hyperhive.agentCpuWeight`. - #[arg(long)] - agent_cpu_weight: Option, - /// Override: systemd `IOWeight=` applied to every agent container, - /// the block-IO counterpart of `--agent-cpu-weight`. Set via - /// `services.hyperhive.agentIoWeight`. - #[arg(long)] - agent_io_weight: Option, /// Override: per-model USD prices (per million tokens) for the /// hive-wide ST4TS cost estimate, as a JSON object mapping a /// model-family short name to `{input, output, cache_read, @@ -158,8 +147,6 @@ async fn main() -> Result<()> { context_window_tokens, agent_cpu_quota, agent_memory_max, - agent_cpu_weight, - agent_io_weight, model_prices, build_slots, } => { @@ -197,16 +184,6 @@ async fn main() -> Result<()> { if let Some(v) = agent_memory_max { sc.env.agent_memory_max = v; } - // Passing the flag sets a weight; omitting it keeps whatever the - // config file says (including `null` = don't emit the setting). - // There's deliberately no flag spelling for "clear it" — that's - // what the nix option's `null` is for. - if let Some(v) = agent_cpu_weight { - sc.env.agent_cpu_weight = Some(v); - } - if let Some(v) = agent_io_weight { - sc.env.agent_io_weight = Some(v); - } if let Some(v) = model_prices { sc.model_prices = serde_json::from_str(&v).context("--model-prices: invalid JSON")?; diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index 2ce0020a..66ab51e7 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -174,15 +174,11 @@ pub async fn write_resource_limits( container: &str, memory_max: &str, cpu_quota: &str, - cpu_weight: Option, - io_weight: Option, ) -> Result<()> { ok(call(&PrivRequest::WriteResourceLimits { container: container.to_owned(), memory_max: memory_max.to_owned(), cpu_quota: cpu_quota.to_owned(), - cpu_weight, - io_weight, }) .await?) } diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index fcb71318..ab9b0ec8 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -336,24 +336,12 @@ pub enum PrivRequest { }, /// Write `/run/systemd/system/container@.service.d/hyperhive-limits.conf` - /// with `[Service]` carrying `MemoryMax=` / `CPUQuota=` (hard caps) and - /// `CPUWeight=` / `IOWeight=` (cgroup v2 relative shares, contention-only). + /// with `[Service]\nMemoryMax=\nCPUQuota=\n`. /// Written by `lifecycle::set_resource_limits`. WriteResourceLimits { container: String, memory_max: String, cpu_quota: String, - /// cgroup v2 `cpu.weight`, 1..=10000. `None` means "not - /// configured" — the writer omits the line entirely, leaving the - /// kernel default. `#[serde(default)]` so a request from a - /// hive-c0re built before this field existed deserialises to - /// `None` and reproduces the pre-weights drop-in. - #[serde(default)] - cpu_weight: Option, - /// cgroup v2 `io.weight`, 1..=10000. Same `None` = omit rule as - /// `cpu_weight`. - #[serde(default)] - io_weight: Option, }, /// Remove `/run/systemd/system/container@.service.d/` if present. diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 064b428a..f6290057 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -223,9 +223,7 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String, ref container, ref memory_max, ref cpu_quota, - cpu_weight, - io_weight, - } => write_resource_limits(container, memory_max, cpu_quota, cpu_weight, io_weight), + } => write_resource_limits(container, memory_max, cpu_quota), PrivRequest::RemoveServiceDropin { ref container } => remove_service_dropin(container), @@ -529,17 +527,10 @@ fn chmod_socket_dir(agent_name: &str, mode: u32) -> Result<(String, String)> { Ok((String::new(), String::new())) } -/// `WriteResourceLimits` — drop the systemd resource settings into the -/// container service's drop-in dir, together with a +/// `WriteResourceLimits` — drop a systemd `MemoryMax`/`CPUQuota` +/// override into the container service's drop-in dir, together with a /// `ConditionPathIsDirectory=` guard on the agent's MCP runtime dir. /// -/// Two different kinds of setting land in the same file. `MemoryMax=` / -/// `CPUQuota=` are hard caps that throttle even on an idle host; -/// `CPUWeight=` / `IOWeight=` are cgroup v2 relative shares that only -/// decide who yields *under contention*. A weight of `None` means "not -/// configured" and omits the line, so a hive-c0re built before the weights -/// existed keeps producing the old two-line drop-in. -/// /// The condition causes systemd to *skip* (not *fail*) the unit when the /// bind-mount source dir is absent — result is `condition`, which does not /// increment the start-limit counter. This is belt-and-braces on top of @@ -550,8 +541,6 @@ fn write_resource_limits( container: &str, memory_max: &str, cpu_quota: &str, - cpu_weight: Option, - io_weight: Option, ) -> Result<(String, String)> { validate_container_system_name(container)?; // Derive the logical agent name (strip h- prefix) to form the runtime @@ -562,40 +551,19 @@ fn write_resource_limits( let dir = format!("/run/systemd/system/container@{container}.service.d"); std::fs::create_dir_all(&dir).with_context(|| format!("create {dir}"))?; let path = format!("{dir}/hyperhive-limits.conf"); - let content = limits_dropin_body(&runtime_dir, memory_max, cpu_quota, cpu_weight, io_weight); - std::fs::write(&path, content).with_context(|| format!("write {path}"))?; - Ok((String::new(), String::new())) -} - -/// Render the body of `hyperhive-limits.conf`. -/// -/// `[Unit]`: the condition is checked at start time — it skips (not fails) -/// the unit when the MCP socket dir is absent, avoiding restart loops. -/// `[Service]`: the hard caps first, then the relative weights. A weight of -/// `None` means "not configured" and omits its line entirely, so a request -/// from a hive-c0re built before the weights existed — or one whose nix -/// option is `null` — reproduces the old two-setting drop-in byte for byte. -fn limits_dropin_body( - runtime_dir: &str, - memory_max: &str, - cpu_quota: &str, - cpu_weight: Option, - io_weight: Option, -) -> String { - // Built as two possibly-empty lines rather than pushed onto the - // string: `format!` appended to a `String` trips clippy::pedantic's - // `format_push_string`, and a `write!` would need an unwrap. - let cpu_weight_line = cpu_weight.map_or_else(String::new, |w| format!("CPUWeight={w}\n")); - let io_weight_line = io_weight.map_or_else(String::new, |w| format!("IOWeight={w}\n")); - format!( + // [Unit] section: condition checked at start time — skips (not fails) + // the unit when the MCP socket dir is absent, avoiding restart loops. + // [Service] section: resource caps. + let content = format!( "[Unit]\n\ ConditionPathIsDirectory={runtime_dir}\n\ \n\ [Service]\n\ MemoryMax={memory_max}\n\ - CPUQuota={cpu_quota}\n\ - {cpu_weight_line}{io_weight_line}" - ) + CPUQuota={cpu_quota}\n" + ); + std::fs::write(&path, content).with_context(|| format!("write {path}"))?; + Ok((String::new(), String::new())) } /// `DaemonReload` — `systemctl daemon-reload` on the host. @@ -2217,53 +2185,11 @@ async fn sync_agent_tmpfiles(agents: &[String]) -> Result<(String, String)> { #[cfg(test)] mod tests { use super::{ - PAUSED_MARKER_FILE, limits_dropin_body, redact_password_line, remove_marker_in, - write_state_file_nofollow, + PAUSED_MARKER_FILE, redact_password_line, remove_marker_in, write_state_file_nofollow, }; use std::path::PathBuf; use std::sync::atomic::{AtomicU32, Ordering}; - /// An unset weight is "not configured": the drop-in must come out - /// byte-identical to the pre-weights two-setting body, so a hive-c0re - /// older than this field can't change what lands on disk. - #[test] - fn unset_weights_reproduce_the_pre_weights_dropin() { - assert_eq!( - limits_dropin_body("/run/hyperhive/agents/iris", "4G", "200%", None, None), - "[Unit]\n\ - ConditionPathIsDirectory=/run/hyperhive/agents/iris\n\ - \n\ - [Service]\n\ - MemoryMax=4G\n\ - CPUQuota=200%\n" - ); - } - - /// Weights are appended to the `[Service]` section, each omitted - /// independently when `None`. - #[test] - fn weights_are_emitted_only_when_set() { - let both = limits_dropin_body("/rt/x", "4G", "200%", Some(80), Some(80)); - assert!( - both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"), - "{both}" - ); - - let cpu_only = limits_dropin_body("/rt/x", "4G", "200%", Some(80), None); - assert!( - cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"), - "{cpu_only}" - ); - assert!(!cpu_only.contains("IOWeight"), "{cpu_only}"); - - let io_only = limits_dropin_body("/rt/x", "4G", "200%", None, Some(80)); - assert!( - io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"), - "{io_only}" - ); - assert!(!io_only.contains("CPUWeight"), "{io_only}"); - } - #[test] fn redacts_lines_mentioning_password_case_insensitively() { assert_eq!( diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 5372ed7a..0e9ceed9 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -81,8 +81,6 @@ let context_window_tokens = cfg.contextWindowTokens; agent_cpu_quota = cfg.agentCpuQuota; agent_memory_max = cfg.agentMemoryMax; - agent_cpu_weight = cfg.agentCpuWeight; - agent_io_weight = cfg.agentIoWeight; model_prices = cfg.modelPrices; build_slots = cfg.buildSlots; }; diff --git a/nix/host-modules/hive-c0re/options.nix b/nix/host-modules/hive-c0re/options.nix index eee0237c..9822e2c6 100644 --- a/nix/host-modules/hive-c0re/options.nix +++ b/nix/host-modules/hive-c0re/options.nix @@ -301,54 +301,6 @@ ''; }; - agentCpuWeight = lib.mkOption { - type = lib.types.nullOr (lib.types.ints.between 1 10000); - default = 80; - example = null; - description = '' - systemd `CPUWeight=` applied to every agent container via the - same drop-in as `agentCpuQuota`. This is the cgroup v2 - `cpu.weight` relative share, **not** a cap: a low-weight - container still gets the whole machine when nothing else wants - it, and the weight only decides who yields under contention. - That makes it the complement of `agentCpuQuota`, which throttles - even on an idle host. - - The kernel default is `100`. The hyperhive default of `80` means - agent containers yield slightly to everything that is *not* on - this drop-in path — host services and the infrastructure - containers (`hive-ci`, `hive-forge`, `hive-gateway`, - `hive-matrix`), which stay at `100`. Note this is a hive-wide - value, so it does not rank agents against *each other*: they all - share one weight. - - Set to `null` to leave `CPUWeight=` out of the drop-in entirely — - the container then inherits the kernel default and the generated - unit file is identical to one from before this option existed. - ''; - }; - - agentIoWeight = lib.mkOption { - type = lib.types.nullOr (lib.types.ints.between 1 10000); - default = 80; - example = null; - description = '' - systemd `IOWeight=` applied to every agent container via the - same drop-in as `agentCpuQuota` — the block-IO counterpart of - `agentCpuWeight`, with the same relative-share, contention-only - semantics. - - Caveat: `IOWeight=` maps to the cgroup v2 `io.weight` knob, which - is only honoured when the `io.cost` (blk-iocost) controller is - enabled for the backing device, or when the device uses the BFQ - scheduler. On a host running `none`/`mq-deadline`/`kyber` without - iocost QoS configured, systemd writes the value and the kernel - ignores it — harmless, but it will measure as a no-op. Check with - `cat /sys/fs/cgroup/io.cost.qos` on the host, and set this to - `null` to omit the setting rather than write one nothing reads. - ''; - }; - buildSlots = lib.mkOption { type = lib.types.ints.positive; default = 1;