From 37f6a845a0d7dc3f5b1009613937f12632245442 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 12 Oct 2026 11:26:18 +0200 Subject: [PATCH 1/6] hive-c0re: let a swarm cap replace the hive's answer, not join its chain mara's ruling: "swarm level overrides replace the hive ui limits. otherwise dont change the logic." swarm value present -> it is the value; nothing hive-side is consulted swarm value absent -> exactly today's chain: per-agent field, else hive Per field, so a swarm memory ceiling leaves CPU on the hive's chain. The fallback chain itself is not touched: effective_from_with_swarm calls effective_from as it was and lets the swarm replace its result, and effective_with_swarm delegates to effective, so there stays one place the chain lives. `an_empty_swarm_map_leaves_the_existing_chain_bit_for_bit` asserts the wrapper is a no-op across every shape the chain distinguishes. The swarm's caps live in their own swarm-resource-limits.json, not in the operator's resource-limits.json: different writers, different lifetimes, and a swarm write into the operator's file would make "reset to hive defaults" destroy something the operator never set. It is read and written through the same read_map / write_map, so an unreadable file is an error, and write_dropins resolves both files through limits_to_apply, whose keep-the-last-applied-drop-in policy covers the swarm file too. stage_limits hands limits_to_apply the swarm caps file too, so the agent's memory-max-bytes copy, which it sizes its heap and subagents against, is the MemoryMax= systemd enforces: the swarm cap when there is one. Nothing writes swarm-resource-limits.json yet; the reader comes with the swarm's agent-limits bucket later in this series. --- hive-c0re/src/agent_config/resource_limits.rs | 128 ++++++++++++++++++ hive-c0re/src/container_view.rs | 11 +- hive-c0re/src/lifecycle/host_config.rs | 99 +++++++++++++- hive-c0re/src/server.rs | 3 +- 4 files changed, 232 insertions(+), 9 deletions(-) diff --git a/hive-c0re/src/agent_config/resource_limits.rs b/hive-c0re/src/agent_config/resource_limits.rs index fd68b78ae..880c87da0 100644 --- a/hive-c0re/src/agent_config/resource_limits.rs +++ b/hive-c0re/src/agent_config/resource_limits.rs @@ -187,6 +187,86 @@ fn set_limits_at(path: &Path, name: &str, limits: &AgentLimits) -> std::io::Resu super::write_map(path, ¤t) } +/// Caps the swarm controller declared for this hive's agents. +/// +/// A separate file from the operator's overrides above, not a merge into it: +/// the two have different writers and different lifetimes, and a swarm +/// declaration landing in the file an operator edits would make "reset to +/// hive defaults" destroy something the operator never set. +const SWARM_LIMITS_FILE: &str = "swarm-resource-limits.json"; + +#[must_use] +pub fn swarm_limits_path() -> PathBuf { + crate::paths::meta_root().join(SWARM_LIMITS_FILE) +} + +/// Read the swarm-declared caps. An absent file is the empty map — no swarm +/// opinion, so the hive's own chain decides. A file that exists but can't be +/// read or parsed is an error, for the reason [`read`] gives: reading it as +/// empty would silently drop a cap. +pub fn read_swarm() -> std::io::Result> { + super::read_map(&swarm_limits_path()) +} + +/// Persist the swarm-declared caps atomically. +/// +/// # Errors +/// The meta dir cannot be created, or the file cannot be written. +pub fn write_swarm(map: &BTreeMap) -> std::io::Result<()> { + super::write_map(&swarm_limits_path(), map) +} + +/// [`effective_from`] with the swarm's declaration on top. +/// +/// A swarm cap **replaces** the hive's answer for that field rather than +/// joining the fallback chain as another tier: a tier a hive can out-set is +/// not a cap. Per field, so a swarm memory ceiling leaves CPU on the hive's +/// chain. +/// +/// The chain itself is untouched — [`effective_from`] is called exactly as +/// before and its result is what a missing swarm value falls back to. +#[must_use] +pub fn effective_from_with_swarm( + swarm: &BTreeMap, + limits: &BTreeMap, + name: &str, + hive_cpu_quota: &str, + hive_memory_max: &str, +) -> (String, String) { + let (cpu, mem) = effective_from(limits, name, hive_cpu_quota, hive_memory_max); + match swarm.get(name) { + Some(s) => ( + s.cpu_quota.clone().unwrap_or(cpu), + s.memory_max.clone().unwrap_or(mem), + ), + None => (cpu, mem), + } +} + +/// [`effective_from_with_swarm`] for a caller resolving a single agent, from +/// the operator overrides at `limits_path` and the swarm caps at `swarm_path`. +/// +/// Delegates to [`effective`] rather than re-deriving the hive's answer, so +/// there stays exactly one place the fallback chain lives. Errors as [`read`] +/// does, for either file. +pub fn effective_with_swarm( + limits_path: &Path, + swarm_path: &Path, + name: &str, + hive_cpu_quota: &str, + hive_memory_max: &str, +) -> std::io::Result<(String, String)> { + let (cpu, mem) = effective(limits_path, name, hive_cpu_quota, hive_memory_max)?; + let swarm: BTreeMap = super::read_map(swarm_path)?; + Ok(match swarm.get(name) { + Some(s) => ( + s.cpu_quota.clone().unwrap_or(cpu), + s.memory_max.clone().unwrap_or(mem), + ), + None => (cpu, mem), + }) +} + /// Validate a systemd `CPUQuota=` value. Percentages only, and values /// above 100% are legitimate (one full core is 100%). /// @@ -525,4 +605,52 @@ mod tests { ); } } + + #[test] + fn a_swarm_cap_replaces_the_hive_answer_rather_than_joining_the_chain() { + // The whole point of the ruling: a tier a hive can out-set is not a + // cap. The agent override below is the strongest hive-side value + // there is, and the swarm still wins. + let hive_side = BTreeMap::from([("sock".to_owned(), limits(Some("800%"), Some("16G")))]); + let swarm = BTreeMap::from([("sock".to_owned(), limits(Some("150%"), Some("2G")))]); + assert_eq!( + effective_from_with_swarm(&swarm, &hive_side, "sock", HIVE_CPU, HIVE_MEM), + ("150%".to_owned(), "2G".to_owned()) + ); + } + + #[test] + fn a_swarm_cap_is_per_field_like_every_other_tier() { + // A swarm that caps memory and says nothing about CPU leaves CPU on + // the hive's chain — here the per-agent override, not the hive default. + let hive_side = BTreeMap::from([("sock".to_owned(), limits(Some("800%"), None))]); + let swarm = BTreeMap::from([("sock".to_owned(), limits(None, Some("2G")))]); + assert_eq!( + effective_from_with_swarm(&swarm, &hive_side, "sock", HIVE_CPU, HIVE_MEM), + ("800%".to_owned(), "2G".to_owned()) + ); + } + + #[test] + fn an_empty_swarm_map_leaves_the_existing_chain_bit_for_bit() { + // The control that makes the two tests above mean something: the + // wrapper must be a no-op when the swarm has said nothing, for every + // shape the old chain distinguishes. If this drifts, the swarm tier + // changed behaviour it was explicitly told not to change. + let none = BTreeMap::new(); + let cases = [ + BTreeMap::new(), + BTreeMap::from([("sock".to_owned(), limits(Some("400%"), Some("8G")))]), + BTreeMap::from([("sock".to_owned(), limits(Some("400%"), None))]), + BTreeMap::from([("sock".to_owned(), limits(None, Some("8G")))]), + BTreeMap::from([("other".to_owned(), limits(Some("100%"), Some("1G")))]), + ]; + for hive_side in &cases { + assert_eq!( + effective_from_with_swarm(&none, hive_side, "sock", HIVE_CPU, HIVE_MEM), + effective_from(hive_side, "sock", HIVE_CPU, HIVE_MEM), + "the wrapper changed the answer with no swarm value present" + ); + } + } } diff --git a/hive-c0re/src/container_view.rs b/hive-c0re/src/container_view.rs index 012a9451f..d55686cc8 100644 --- a/hive-c0re/src/container_view.rs +++ b/hive-c0re/src/container_view.rs @@ -171,6 +171,14 @@ pub async fn build_all(hive: &crate::coordinator::HiveEnv) -> anyhow::Result anyhow::Result Re if let Some((cpu_quota, memory_max)) = stage_limits( name, &crate::resource_limits::resource_limits_path(), + &crate::resource_limits::swarm_limits_path(), &limits_dropin_path(&container), &paths.agent, hive, @@ -97,12 +98,13 @@ impl HostMemory { fn stage_limits( agent_name: &str, limits_path: &Path, + swarm_path: &Path, dropin_path: &Path, runtime_dir: &Path, hive: &HiveEnv, host: Option, ) -> Result> { - let limits = limits_to_apply(agent_name, limits_path, dropin_path, hive); + let limits = limits_to_apply(agent_name, limits_path, swarm_path, dropin_path, hive); let memory_max = match &limits { Some((_, memory_max)) => Some(memory_max.clone()), None => dropin_memory_max(agent_name, dropin_path), @@ -224,20 +226,28 @@ fn write_memory_cap(runtime_dir: &Path, cap: Option<&str>) -> Result<()> { /// The `(CPUQuota, MemoryMax)` to write into `agent_name`'s limits /// drop-in, or `None` to leave the drop-in at `dropin_path` as it is. /// -/// When the overrides file at `limits_path` can't be read, the error is -/// logged and the agent still starts. The last-applied drop-in is kept, -/// because the hive defaults can be looser than the agent's own override. +/// When the overrides file at `limits_path` or the swarm caps at +/// `swarm_path` can't be read, the error is logged and the agent still starts. +/// The last-applied drop-in is kept, because the hive defaults can be looser +/// than the agent's own override. /// With no drop-in yet, the agent gets the hive defaults: without one it /// would run uncapped. fn limits_to_apply( agent_name: &str, limits_path: &Path, + swarm_path: &Path, dropin_path: &Path, hive: &HiveEnv, ) -> Option<(String, String)> { let hive_cpu = &hive.agent_cpu_quota; let hive_mem = &hive.agent_memory_max; - let e = match crate::resource_limits::effective(limits_path, agent_name, hive_cpu, hive_mem) { + let e = match crate::resource_limits::effective_with_swarm( + limits_path, + swarm_path, + agent_name, + hive_cpu, + hive_mem, + ) { Ok(limits) => return Some(limits), Err(e) => e, }; @@ -838,6 +848,10 @@ mod tests { const TRUNCATED_LIMITS: &str = "{\n \"sock\": { \"cpu_quota\": \"50%\", \"mem"; + fn absent_swarm(dir: &tempfile::TempDir) -> std::path::PathBuf { + dir.path().join("absent-swarm-resource-limits.json") + } + #[test] fn corrupt_limits_keep_an_existing_dropin() { let dir = tempfile::tempdir().expect("tempdir"); @@ -846,7 +860,13 @@ mod tests { std::fs::write(&limits, TRUNCATED_LIMITS).expect("seed limits"); std::fs::write(&dropin, "MemoryMax=1G\n").expect("seed drop-in"); assert_eq!( - limits_to_apply("sock", &limits, &dropin, &HiveEnv::default()), + limits_to_apply( + "sock", + &limits, + &absent_swarm(&dir), + &dropin, + &HiveEnv::default() + ), None ); assert_eq!( @@ -863,12 +883,47 @@ mod tests { let applied = limits_to_apply( "sock", &limits, + &absent_swarm(&dir), &dir.path().join("absent.conf"), &HiveEnv::default(), ); assert_eq!(applied, Some(("200%".to_owned(), "4G".to_owned()))); } + #[test] + fn corrupt_swarm_caps_keep_an_existing_dropin() { + let dir = tempfile::tempdir().expect("tempdir"); + let swarm = dir.path().join("swarm-resource-limits.json"); + let dropin = dir.path().join("hyperhive-limits.conf"); + std::fs::write(&swarm, TRUNCATED_LIMITS).expect("seed swarm caps"); + std::fs::write(&dropin, "MemoryMax=1G\n").expect("seed drop-in"); + let applied = limits_to_apply( + "sock", + &dir.path().join("resource-limits.json"), + &swarm, + &dropin, + &HiveEnv::default(), + ); + assert_eq!(applied, None); + } + + #[test] + fn a_swarm_cap_is_applied_over_the_operator_override() { + let dir = tempfile::tempdir().expect("tempdir"); + let limits = dir.path().join("resource-limits.json"); + let swarm = dir.path().join("swarm-resource-limits.json"); + std::fs::write(&limits, r#"{"sock": {"memory_max": "16G"}}"#).expect("seed limits"); + std::fs::write(&swarm, r#"{"sock": {"memory_max": "2G"}}"#).expect("seed swarm caps"); + let applied = limits_to_apply( + "sock", + &limits, + &swarm, + &dir.path().join("absent.conf"), + &HiveEnv::default(), + ); + assert_eq!(applied, Some(("200%".to_owned(), "2G".to_owned()))); + } + #[test] fn readable_limits_are_applied_over_an_existing_dropin() { let dir = tempfile::tempdir().expect("tempdir"); @@ -876,7 +931,13 @@ mod tests { let dropin = dir.path().join("hyperhive-limits.conf"); std::fs::write(&limits, r#"{"sock": {"memory_max": "1G"}}"#).expect("seed limits"); std::fs::write(&dropin, "MemoryMax=8G\n").expect("seed drop-in"); - let applied = limits_to_apply("sock", &limits, &dropin, &HiveEnv::default()); + let applied = limits_to_apply( + "sock", + &limits, + &absent_swarm(&dir), + &dropin, + &HiveEnv::default(), + ); assert_eq!(applied, Some(("200%".to_owned(), "1G".to_owned()))); } @@ -893,6 +954,7 @@ mod tests { let applied = stage_limits( "sock", &dir.path().join("resource-limits.json"), + &dir.path().join("swarm-resource-limits.json"), &dir.path().join("hyperhive-limits.conf"), &runtime, &HiveEnv::default(), @@ -903,6 +965,28 @@ mod tests { (applied.map(|(_, mem)| mem), copy) } + /// The agent's copy follows the swarm's cap, the one systemd enforces, over + /// the operator's override: a heap sized against the override would + /// outgrow the cgroup. + #[test] + fn the_agent_reads_the_swarm_cap_over_the_operator_override() { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write( + dir.path().join("resource-limits.json"), + r#"{"sock": {"memory_max": "16G"}}"#, + ) + .expect("seed limits"); + std::fs::write( + dir.path().join("swarm-resource-limits.json"), + r#"{"sock": {"memory_max": "2G"}}"#, + ) + .expect("seed swarm caps"); + assert_eq!( + stage(&dir), + (Some("2G".to_owned()), Some("2147483648\n".to_owned())) + ); + } + #[test] fn the_agent_reads_the_cap_its_dropin_carries() { let dir = tempfile::tempdir().expect("tempdir"); @@ -1035,6 +1119,7 @@ mod tests { let applied = stage_limits( "sock", &limits, + &dir.path().join("swarm-resource-limits.json"), &dropin, &runtime, &HiveEnv::default(), diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index 5d9839bc3..ea283fda3 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -426,8 +426,9 @@ async fn handle_set_resource_limits( let paths = Coordinator::agent_paths(name.as_str(), agent_dir)?; crate::lifecycle::write_dropins(name.as_str(), &hive, &paths).await?; - let (cpu, mem) = crate::resource_limits::effective( + let (cpu, mem) = crate::resource_limits::effective_with_swarm( &crate::resource_limits::resource_limits_path(), + &crate::resource_limits::swarm_limits_path(), name.as_str(), &hive.agent_cpu_quota, &hive.agent_memory_max, From ebeef4b84d267d3e2adde0ddd3310ce6069801b4 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 12 Oct 2026 11:26:18 +0200 Subject: [PATCH 2/6] swarm-controller: GET/PUT/DELETE /api/agents/{name}/limits An agent's CPU and memory caps, set from the swarm. Keyed by the agent alone, in a new `agent-limits` KV bucket that swarm-controller alone writes, the same shape as `agent-inference` (#5034): mara on #4268, "agents are not tied to a hive. dont tie the limits to the hive." PUT {"cpu_quota": "400%", "memory_max": "8G"} every field optional, replaces the stored value GET the stored caps, {} when there are none DELETE 204 PUT refuses an agent no hive's wanted state declares (as the inference route does), unknown fields (422), zero and anything systemd does not take, before the queue is asked, and stores memory in the spelling systemd takes (8gb -> 8G). The hive hands these values to systemd verbatim. The routes sit under /api/, behind the swarm UI vhost's authelia subrequest like every sibling; the core-toggle module-eval case pins that gate. The bucket's reader on the hive comes in the next commit. Contributes to #3865. --- nix/module-eval/core-toggle.nix | 16 + swarm-controller/README.md | 5 + swarm-controller/src/agent_limits.rs | 527 +++++++++++++++++++++++++ swarm-controller/src/main.rs | 6 + swarm-queue-client/Cargo.toml | 3 + swarm-queue-client/src/agent_limits.rs | 166 ++++++++ swarm-queue-client/src/lib.rs | 5 + 7 files changed, 728 insertions(+) create mode 100644 swarm-controller/src/agent_limits.rs create mode 100644 swarm-queue-client/src/agent_limits.rs diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index f99c1a11d..48326b2f4 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -286,6 +286,22 @@ let && !(lib.hasInfix "mint swarmctl" s) && lib.hasInfix "mint swarm-authelia-bridge" s; } + { + # swarm-controller checks no caller itself: this location's authelia + # subrequest is the operator gate for every route under `/api/`. The + # webhook location is the control, the one path to the same socket that + # skips the gate, so the first clause reads a string that can lack it. + name = "the swarm UI's API location asks authelia before reaching the controller"; + ok = + let + l = allLocal.services.nginx.virtualHosts."t.local".locations; + socket = allLocal.services.hyperhive.deploy.swarm-controller.socketPath; + in + lib.hasInfix "auth_request /__hive_authelia;" l."/api/".extraConfig + && lib.hasInfix socket l."/api/".proxyPass + && lib.hasInfix socket l."/webhook/forge/".proxyPass + && !(lib.hasInfix "auth_request" l."/webhook/forge/".extraConfig); + } { name = "a hive with matrix off serves no matrix discovery endpoint"; ok = diff --git a/swarm-controller/README.md b/swarm-controller/README.md index 036ff2836..4c7e12d4a 100644 --- a/swarm-controller/README.md +++ b/swarm-controller/README.md @@ -52,6 +52,11 @@ The swarm's control plane. The swarm UI and `swarmctl` are its clients. agent applies them itself. `GET` also answers the model and effort the agent last reported in its turn-state header, kept in memory → [`docs/swarm/inference.md`](../docs/swarm/inference.md). +- **Resource caps** — each agent's CPU and memory cap, one key per agent in + the `agent-limits` KV bucket, which this daemon alone writes + (`GET`/`PUT`/`DELETE /api/agents/{name}/limits`); the hive running the + agent reads it, and each cap replaces the hive's own answer for that field → + [`docs/scheduler/coordinator.md`](../docs/scheduler/coordinator.md#container-resource-limits). - **Relays** — each agent's terminal and turn-state header as SSE, agent icons, the cross-repo issue report, and the UI's quick links. diff --git a/swarm-controller/src/agent_limits.rs b/swarm-controller/src/agent_limits.rs new file mode 100644 index 000000000..478a92db6 --- /dev/null +++ b/swarm-controller/src/agent_limits.rs @@ -0,0 +1,527 @@ +//! `/api/agents/{name}/limits` — the CPU and memory caps the swarm sets for +//! one agent. +//! +//! Each agent's caps are one value under its own key of `agent-limits` +//! (`swarm_queue_client::agent_limits`), which this daemon alone writes. The +//! hive running the agent reads the key; on the hive, a cap replaces the +//! hive's own answer for that field, and a field left unset stays with the +//! hive. +//! +//! The hive hands these values to systemd verbatim, and a value systemd +//! rejects stops the agent's container from starting, so they are validated +//! here as well as on the hive. + +use axum::Json; +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use problem_details::ProblemDetails; +use serde::{Deserialize, Serialize}; +use swarm_queue_client::agent_limits::{self, AgentCaps}; +use utoipa::ToSchema; + +use crate::schedules::{live, queue, read, refused, write}; +use crate::{AppState, error_problem, wanted_error_status}; + +/// How many times a losing writer re-reads and re-applies, as in +/// `wanted::MAX_ATTEMPTS`. +const MAX_ATTEMPTS: usize = 5; + +/// One agent's caps. A field left out is not capped by the swarm. +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] +pub struct Caps { + /// systemd `CPUQuota=`: a percentage above zero, `"100%"` being one core. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schema(example = "400%")] + cpu_quota: Option, + /// systemd `MemoryMax=`: a size above zero with an optional `K`/`M`/`G`/`T` + /// suffix, a percentage of physical memory up to `100%`, or `infinity`. + /// Stored in the one spelling systemd accepts (`8gb` is stored as `8G`). + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schema(example = "8G")] + memory_max: Option, +} + +impl Caps { + /// The value to store, or why it is refused. + fn into_stored(self) -> Result { + Ok(AgentCaps { + cpu_quota: self.cpu_quota.as_deref().map(cpu_quota).transpose()?, + memory_max: self.memory_max.as_deref().map(memory_max).transpose()?, + }) + } +} + +impl From for Caps { + fn from(v: AgentCaps) -> Self { + Self { + cpu_quota: v.cpu_quota, + memory_max: v.memory_max, + } + } +} + +// The accepted shapes are hive-c0re's `resource_limits::validate_cpu_quota` / +// `validate_memory_max`, the operator's own input path for the same drop-in, +// with zero and an over-100% memory share refused as well. + +fn cpu_quota(value: &str) -> Result { + match value.strip_suffix('%') { + Some(n) if is_positive_number(n) => Ok(value.to_owned()), + _ => Err(format!( + "invalid cpu_quota {value:?}: expected a percentage above zero such as \"200%\" \ + (100% = one full core)" + )), + } +} + +fn memory_max(value: &str) -> Result { + if value == "infinity" { + return Ok(value.to_owned()); + } + let normalized = match value.strip_suffix('%') { + Some(n) => (is_positive_number(n) && n.parse::().is_ok_and(|p| p <= 100.0)) + .then(|| value.to_owned()), + None => byte_size(value), + }; + normalized.ok_or_else(|| { + format!( + "invalid memory_max {value:?}: expected a size above zero such as \"8G\", a \ + percentage up to \"100%\", or \"infinity\"" + ) + }) +} + +/// A byte count with an optional `K`/`M`/`G`/`T` multiplier, in any case and +/// with an optional trailing `B`, in the form systemd takes: `8gb` → `8G`, a +/// bare `1048576B` kept as it is. +fn byte_size(value: &str) -> Option { + const UNITS: [char; 8] = ['K', 'M', 'G', 'T', 'k', 'm', 'g', 't']; + let (body, b) = match value.strip_suffix(['B', 'b']) { + Some(body) => (body, true), + None => (value, false), + }; + let (mantissa, unit) = match body.strip_suffix(UNITS) { + Some(mantissa) => (mantissa, body[mantissa.len()..].to_ascii_uppercase()), + None if b => (body, "B".to_owned()), + None => (body, String::new()), + }; + is_positive_number(mantissa).then(|| format!("{mantissa}{unit}")) +} + +/// Digits with at most one `.` between digits, and not zero. No sign or +/// exponent, which systemd does not take either. +fn is_positive_number(value: &str) -> bool { + let (int, frac) = value.split_once('.').unwrap_or((value, "1")); + let digits = |s: &str| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()); + digits(int) && digits(frac) && value.bytes().any(|b| (b'1'..=b'9').contains(&b)) +} + +fn agent_name(name: &str) -> Result { + hive_types::Ident::parse(name) + .map(hive_types::Ident::into_string) + .map_err(|reason| (StatusCode::BAD_REQUEST, reason.to_owned())) +} + +fn queue_problem(e: &anyhow::Error) -> ProblemDetails { + tracing::warn!(error = %format!("{e:#}"), "agent limits: queue failure"); + error_problem(wanted_error_status(e), &format!("{e:#}")) +} + +async fn open( + client: &async_nats::Client, +) -> Result { + agent_limits::open_or_create(client) + .await + .map_err(|e| queue_problem(&anyhow::Error::new(e))) +} + +#[utoipa::path( + get, + path = "/api/agents/{name}/limits", + params(("name" = String, Path, description = "agent name")), + responses( + (status = 200, description = "the caps the swarm sets for the agent; `{}` when it \ + sets none", body = Caps), + (status = 400, description = "not an identifier (problem+json)", body = String), + (status = 503, description = "no swarm queue is configured on this host, or it is \ + not connected (problem+json)", body = String), + (status = 500, description = "the caps could not be read (problem+json)", + body = String), + ), + tag = "agents" +)] +pub(crate) async fn get_agent_limits( + State(state): State, + Path(name): Path, +) -> Result, ProblemDetails> { + let agent = agent_name(&name).map_err(refused)?; + let client = queue(&state).map_err(refused)?; + let store = open(&client).await?; + let (current, _) = read::(&store, &agent) + .await + .map_err(|e| queue_problem(&e))?; + Ok(Json(current.into())) +} + +/// Set the CPU and memory caps the swarm declares for one agent. +/// +/// The value replaces the stored one whole. Whichever hive runs the agent +/// applies it; a running agent picks the caps up only at its next spawn or +/// rebuild. +#[utoipa::path( + put, + path = "/api/agents/{name}/limits", + params(("name" = String, Path, description = "agent name")), + request_body = Caps, + responses( + (status = 200, description = "the caps as stored", body = Caps), + (status = 400, description = "not an identifier, not declared on any hive, or a cap \ + is not a value systemd takes or is zero (problem+json)", body = String), + (status = 422, description = "an unknown field"), + (status = 503, description = "no swarm queue is configured on this host, or it is \ + not connected (problem+json)", body = String), + (status = 500, description = "the caps could not be written (problem+json)", + body = String), + ), + tag = "agents" +)] +pub(crate) async fn put_agent_limits( + State(state): State, + Path(name): Path, + Json(req): Json, +) -> Result, ProblemDetails> { + let agent = agent_name(&name).map_err(refused)?; + let value = req + .into_stored() + .map_err(|why| error_problem(StatusCode::BAD_REQUEST, &why))?; + let client = queue(&state).map_err(refused)?; + if !live(&state).await?.contains(&agent) { + return Err(error_problem( + StatusCode::BAD_REQUEST, + &format!("no hive's wanted state declares an agent {agent:?} that is not destroyed"), + )); + } + let store = open(&client).await?; + for _ in 0..MAX_ATTEMPTS { + let (current, revision) = read::(&store, &agent) + .await + .map_err(|e| queue_problem(&e))?; + if current == value + || write(&store, &agent, &value, revision) + .await + .map_err(|e| queue_problem(&e))? + { + tracing::info!(%agent, ?value, "set agent limits"); + return Ok(Json(value.into())); + } + } + Err(queue_problem(&anyhow::anyhow!( + "gave up writing the limits of {agent} after {MAX_ATTEMPTS} conflicting writes" + ))) +} + +#[utoipa::path( + delete, + path = "/api/agents/{name}/limits", + params(("name" = String, Path, description = "agent name")), + responses( + (status = 204, description = "cleared; the hive's own answer applies again"), + (status = 400, description = "not an identifier (problem+json)", body = String), + (status = 503, description = "no swarm queue is configured on this host, or it is \ + not connected (problem+json)", body = String), + (status = 500, description = "the caps could not be cleared (problem+json)", + body = String), + ), + tag = "agents" +)] +pub(crate) async fn delete_agent_limits( + State(state): State, + Path(name): Path, +) -> Result { + let agent = agent_name(&name).map_err(refused)?; + let client = queue(&state).map_err(refused)?; + let store = open(&client).await?; + store + .delete(&agent) + .await + .map_err(|e| queue_problem(&anyhow::Error::new(e)))?; + tracing::info!(%agent, "cleared agent limits"); + Ok(StatusCode::NO_CONTENT) +} + +#[cfg(test)] +mod tests { + use super::{ + Caps, cpu_quota, delete_agent_limits, get_agent_limits, memory_max, put_agent_limits, + }; + use crate::test_nats::TestNats; + use crate::wanted::WantedWriter; + use axum::extract::{Path, State}; + use axum::http::StatusCode; + use std::sync::Arc; + use swarm_queue_client::wanted::AgentState; + + #[test] + fn cpu_quota_takes_a_percentage_above_zero() { + for ok in ["400%", "100%", "0.5%", "1500%"] { + assert_eq!(cpu_quota(ok).as_deref(), Ok(ok)); + } + for bad in [ + "0%", "0.0%", "-50%", "400", "4 00%", "%", "lots", "", "1e3%", "+5%", + ] { + assert!(cpu_quota(bad).is_err(), "{bad:?} was accepted"); + } + } + + #[test] + fn memory_max_takes_a_size_a_share_or_infinity_and_stores_the_systemd_spelling() { + for (given, stored) in [ + ("8G", "8G"), + ("8gb", "8G"), + ("512m", "512M"), + ("1048576", "1048576"), + ("1048576B", "1048576B"), + ("1.5T", "1.5T"), + ("50%", "50%"), + ("100%", "100%"), + ("infinity", "infinity"), + ] { + assert_eq!(memory_max(given).as_deref(), Ok(stored), "{given:?}"); + } + for bad in [ + "0", "0G", "0.0G", "0%", "-8G", "150%", "8X", "G", "8 G", "", "eight", "8GG", + ] { + assert!(memory_max(bad).is_err(), "{bad:?} was accepted"); + } + } + + /// One hive, `pr1ma`, on `client`. + fn state(client: &async_nats::Client) -> crate::AppState { + crate::AppState { + hives: Arc::new(vec![crate::HiveEntry { + name: "pr1ma".to_owned(), + domain: "pr1ma.example".to_owned(), + }]), + links: Arc::new(Vec::new()), + status: Some(Arc::new(crate::status::StatusReader::new( + client.clone(), + std::time::Duration::from_mins(1), + ))), + wanted: Some(Arc::new(WantedWriter::new(client.clone()))), + agent_status: None, + agent_icons: None, + reported_inference: None, + jobq: Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ))), + webhook_secret: None, + config_prs: Arc::default(), + swarm_name: None, + auth: None, + forge: Arc::default(), + create_gate: Arc::default(), + accounts: crate::linked_accounts::Accounts::Fake(Arc::default()), + try_outs: crate::review_classifier::TryOuts::default(), + } + } + + /// A state on a real server whose `pr1ma` declares `atlas`. + async fn declared(client: &async_nats::Client) -> crate::AppState { + WantedWriter::new(client.clone()) + .set("pr1ma", "atlas", AgentState::Up) + .await + .expect("declare atlas"); + state(client) + } + + fn caps(cpu: Option<&str>, mem: Option<&str>) -> Caps { + Caps { + cpu_quota: cpu.map(str::to_owned), + memory_max: mem.map(str::to_owned), + } + } + + async fn put( + state: &crate::AppState, + agent: &str, + value: Caps, + ) -> Result, problem_details::ProblemDetails> { + put_agent_limits( + State(state.clone()), + Path(agent.to_owned()), + axum::Json(value), + ) + .await + } + + async fn get(state: &crate::AppState, agent: &str) -> Caps { + get_agent_limits(State(state.clone()), Path(agent.to_owned())) + .await + .expect("caps are read") + .0 + } + + /// The raw value a hive reads for `agent`, or `None` for no key or a + /// deleted one. + async fn published(client: &async_nats::Client, agent: &str) -> Option { + let store = swarm_queue_client::agent_limits::open_or_create(client) + .await + .expect("bucket"); + store + .get(agent) + .await + .expect("read") + .map(|v| String::from_utf8(v.to_vec()).expect("utf-8")) + } + + /// Through the real handlers and a real `nats-server`: a cap lands under + /// the agent's own key as the hive reads it, `GET` answers it, and + /// `DELETE` takes it out again. + /// + /// The published bytes are pinned literally because hive-c0re's + /// `workers::swarm_limits` tests decode this same string; a change to + /// either side's reading of it fails one of the two. + #[tokio::test] + async fn the_route_writes_the_caps_a_hive_reads_and_delete_clears_them() { + let nats = TestNats::start(); + let client = nats.client().await; + let state = declared(&client).await; + + let answer = put(&state, "atlas", caps(Some("400%"), Some("8gb"))) + .await + .expect("caps are set"); + assert_eq!(answer.0, caps(Some("400%"), Some("8G"))); + assert_eq!( + published(&client, "atlas").await.as_deref(), + Some(r#"{"cpu_quota":"400%","memory_max":"8G"}"#) + ); + assert_eq!(get(&state, "atlas").await, caps(Some("400%"), Some("8G"))); + + assert_eq!( + delete_agent_limits(State(state.clone()), Path("atlas".to_owned())).await, + Ok(StatusCode::NO_CONTENT) + ); + assert_eq!(published(&client, "atlas").await, None); + assert_eq!(get(&state, "atlas").await, Caps::default()); + } + + /// A name no hive declares gets no key, so a typo cannot cap an agent + /// that does not exist yet. + #[tokio::test] + async fn an_agent_no_hive_declares_is_refused_and_nothing_is_written() { + let nats = TestNats::start(); + let client = nats.client().await; + let state = declared(&client).await; + + let problem = put(&state, "iris", caps(Some("400%"), None)) + .await + .expect_err("no hive declares iris"); + assert_eq!(problem.status, Some(StatusCode::BAD_REQUEST), "{problem:?}"); + assert_eq!(published(&client, "iris").await, None); + } + + /// A refused cap is answered before the queue is asked: the client here + /// was never connected, and would answer 503. + #[tokio::test] + async fn a_refused_cap_is_a_400_before_the_queue() { + let client = async_nats::ConnectOptions::new() + .retry_on_initial_connect() + .connect("127.0.0.1:1") + .await + .expect("returns without a connection"); + let state = state(&client); + + let problem = put(&state, "atlas", caps(Some("0%"), None)) + .await + .expect_err("a zero cap is refused"); + assert_eq!(problem.status, Some(StatusCode::BAD_REQUEST), "{problem:?}"); + + // The control: a valid cap on the same state does reach the queue. + let problem = put(&state, "atlas", caps(Some("400%"), None)) + .await + .expect_err("the queue is not connected"); + assert_eq!( + problem.status, + Some(StatusCode::SERVICE_UNAVAILABLE), + "{problem:?}" + ); + } + + const CAPPED: &str = r#"{"cpu_quota":"400%","memory_max":"8G"}"#; + + /// `atlas` capped, behind the real router. + async fn capped_router(client: &async_nats::Client) -> axum::Router { + let state = declared(client).await; + let _ = put(&state, "atlas", caps(Some("400%"), Some("8G"))) + .await + .expect("cap atlas"); + crate::build_app(state) + } + + async fn put_raw(app: &axum::Router, body: &'static str) -> StatusCode { + use tower::ServiceExt as _; + app.clone() + .oneshot( + axum::http::Request::put("/api/agents/atlas/limits") + .header(axum::http::header::CONTENT_TYPE, "application/json") + .body(axum::body::Body::from(body)) + .expect("request"), + ) + .await + .expect("router answers") + .status() + } + + /// A body that is not JSON is refused before the handler runs, and the + /// caps already set stay. The well-formed `{}` after it is the control: + /// the same router and request do reach the bucket. + #[tokio::test] + async fn a_malformed_body_is_a_400_and_the_caps_stay() { + let nats = TestNats::start(); + let client = nats.client().await; + let app = capped_router(&client).await; + + for body in [r#"{"cpu_quota":"#, "cpu_quota=400%", ""] { + assert_eq!( + put_raw(&app, body).await, + StatusCode::BAD_REQUEST, + "{body:?}" + ); + assert_eq!( + published(&client, "atlas").await.as_deref(), + Some(CAPPED), + "{body:?}" + ); + } + + assert_eq!(put_raw(&app, "{}").await, StatusCode::OK); + assert_eq!(published(&client, "atlas").await.as_deref(), Some("{}")); + } + + /// A field the body does not define is a 422 and writes nothing: a + /// misspelt `memory_max` must not read as "no memory cap". + #[tokio::test] + async fn an_unknown_field_is_a_422_and_the_caps_stay() { + let nats = TestNats::start(); + let client = nats.client().await; + let app = capped_router(&client).await; + + for body in [ + r#"{"cpu_quota":"100%","memory":"2G"}"#, + r#"{"limits":{"cpu_quota":"100%"}}"#, + ] { + assert_eq!( + put_raw(&app, body).await, + StatusCode::UNPROCESSABLE_ENTITY, + "{body}" + ); + assert_eq!( + published(&client, "atlas").await.as_deref(), + Some(CAPPED), + "{body}" + ); + } + } +} diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 63c3f03e0..cde67d31d 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -44,6 +44,7 @@ use utoipa_axum::{router::OpenApiRouter, routes}; mod agent_icon; mod agent_identity; mod agent_input; +mod agent_limits; mod agent_renewal; mod agent_state_stream; mod agent_status; @@ -3300,6 +3301,11 @@ fn build_app(state: AppState) -> axum::Router { inference::put_agent_inference, inference::delete_agent_inference )) + .routes(routes!( + agent_limits::get_agent_limits, + agent_limits::put_agent_limits, + agent_limits::delete_agent_limits + )) .routes(routes!(get_config_prs)) .routes(routes!(create_agent)) .routes(routes!(mint_agent_identity)) diff --git a/swarm-queue-client/Cargo.toml b/swarm-queue-client/Cargo.toml index 1123f9a35..5496599d2 100644 --- a/swarm-queue-client/Cargo.toml +++ b/swarm-queue-client/Cargo.toml @@ -47,6 +47,9 @@ agent-inference = ["async-nats/jetstream"] # `ToSchema` on the review classifier's shared types, for swarm-controller, # whose HTTP API serves the same shapes the queue carries. schema = ["dep:utoipa"] +# `agent_limits::read`, for hive-c0re, which reads the key of each agent it +# runs the same way. Explicit for the same reason as `notices`. +agent-limits = ["async-nats/jetstream"] [dependencies] # Bare (no `kv`/`jetstream`) unless a consumer opts into the `kv` feature diff --git a/swarm-queue-client/src/agent_limits.rs b/swarm-queue-client/src/agent_limits.rs new file mode 100644 index 000000000..91b857af8 --- /dev/null +++ b/swarm-queue-client/src/agent_limits.rs @@ -0,0 +1,166 @@ +//! The per-agent resource-cap KV bucket: one key per agent, holding the CPU +//! and memory caps the swarm sets for it. +//! +//! **swarm-controller is the only writer**, and creates the bucket on its +//! first write. Every field is optional: an absent field leaves the agent on +//! its hive's own answer, so an agent with no key runs exactly as it would +//! with no swarm. +//! +//! **The hive running the agent reads the key**, not the agent: the caps go +//! into the container's systemd drop-in, which the host writes. A hive reads +//! with one `DIRECT.GET` per agent +//! ([`crate::agent_limits::direct_get_subject`]), and its grant covers every +//! key of this bucket, read-only. Scoping it to the hive's own agents would +//! need a roster in the auth responder. +//! +//! Keyed by the agent alone, like [`crate::agent_inference`]: an agent can +//! move between hives, and its caps go with it. + +/// The KV bucket caps are published into, keyed by agent name. +pub const BUCKET: &str = "agent-limits"; + +/// The stream backing [`BUCKET`]. `create_key_value` prefixes `KV_`. +pub const STREAM: &str = "KV_agent-limits"; + +/// The subject `agent`'s entry is stored under. +#[must_use] +pub fn key_subject(agent: &str) -> String { + format!("$KV.{BUCKET}.{agent}") +} + +/// The request subject that reads `agent`'s current entry. +#[must_use] +pub fn direct_get_subject(agent: &str) -> String { + format!("$JS.API.DIRECT.GET.{STREAM}.{}", key_subject(agent)) +} + +/// One agent's caps: the value under `BUCKET`/``, as systemd values +/// carried verbatim. `None` is "not capped by the swarm", never "uncapped". +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct AgentCaps { + /// systemd `CPUQuota=`, e.g. `"400%"`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpu_quota: Option, + /// systemd `MemoryMax=`, e.g. `"8G"`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_max: Option, +} + +impl AgentCaps { + /// True when no field is set, which a hive reads the same as no key. + #[must_use] + pub fn is_empty(&self) -> bool { + self.cpu_quota.is_none() && self.memory_max.is_none() + } +} + +/// Open the caps bucket for writing, creating it if nothing has yet. +/// +/// **Controller-side only.** `history: 1`: a hive wants the current value. +#[cfg(feature = "kv")] +pub async fn open_or_create( + client: &async_nats::Client, +) -> Result { + let js = async_nats::jetstream::new(client.clone()); + crate::kv::open_or_create( + &js, + BUCKET, + "agent-limits", + "Resource caps the swarm controller sets for each agent".to_owned(), + ) + .await +} + +/// What reading one agent's key found. +#[cfg(feature = "agent-limits")] +#[derive(Debug)] +pub enum Read { + /// The bucket answered; an absent or deleted key has every field unset. + Current(AgentCaps), + /// Nothing answers for the bucket's stream: no cap has been written yet, + /// or `JetStream` is not serving it right now. Not an unset value, since + /// the second case says nothing about the agent's caps. + NoBucket, +} + +/// Read `agent`'s current caps with one direct get. +/// +/// # Errors +/// +/// A failed request or an undecodable value. A request the queue refused for +/// permissions reaches the client as a timeout, not as a refusal. +#[cfg(feature = "agent-limits")] +pub async fn read( + client: &async_nats::Client, + agent: &str, +) -> Result> { + use async_nats::jetstream::stream::DirectGetErrorKind; + + let js = async_nats::jetstream::new(client.clone()); + let stream = js.get_stream_no_info(STREAM).await?; + let message = match stream.direct_get_last_for_subject(key_subject(agent)).await { + Ok(message) => message, + Err(e) if matches!(e.kind(), DirectGetErrorKind::NotFound) => { + return Ok(Read::Current(AgentCaps::default())); + } + Err(e) if is_no_responders(&e) => return Ok(Read::NoBucket), + Err(e) => return Err(e.into()), + }; + // A KV delete or purge is stored as an empty marker carrying this header. + let removed = message + .headers + .get("KV-Operation") + .is_some_and(|op| matches!(op.as_str(), "DEL" | "PURGE")); + if removed { + return Ok(Read::Current(AgentCaps::default())); + } + Ok(Read::Current(serde_json::from_slice(&message.payload)?)) +} + +#[cfg(feature = "agent-limits")] +fn is_no_responders(e: &async_nats::jetstream::stream::DirectGetError) -> bool { + matches!( + e.kind(), + async_nats::jetstream::stream::DirectGetErrorKind::ErrorResponse(status, _) + if status == async_nats::StatusCode::NO_RESPONDERS + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The names the hive's grant in `swarm-nats-auth` is built from. + #[test] + fn names_match_the_queue_grant() { + assert_eq!(STREAM, format!("KV_{BUCKET}")); + assert_eq!(key_subject("iris"), "$KV.agent-limits.iris"); + assert_eq!( + direct_get_subject("iris"), + "$JS.API.DIRECT.GET.KV_agent-limits.$KV.agent-limits.iris" + ); + } + + /// An unset field is left out on the wire, and `{}` is every field unset. + #[test] + fn unset_fields_are_absent() { + let value = AgentCaps { + memory_max: Some("8G".to_owned()), + ..AgentCaps::default() + }; + assert_eq!( + serde_json::to_string(&value).unwrap(), + r#"{"memory_max":"8G"}"# + ); + let empty: AgentCaps = serde_json::from_str("{}").unwrap(); + assert!(empty.is_empty()); + } + + /// Fields a newer controller adds must not cost an older hive its caps. + #[test] + fn decoding_tolerates_unknown_fields() { + let decoded: AgentCaps = + serde_json::from_str(r#"{"cpu_quota":"400%","io_weight":"10"}"#).unwrap(); + assert_eq!(decoded.cpu_quota.as_deref(), Some("400%")); + } +} diff --git a/swarm-queue-client/src/lib.rs b/swarm-queue-client/src/lib.rs index fa5b991b6..6cd0da3a4 100644 --- a/swarm-queue-client/src/lib.rs +++ b/swarm-queue-client/src/lib.rs @@ -216,6 +216,11 @@ pub mod agent_schedules; /// unconditional; writing needs `kv` and reading needs `agent-inference`. pub mod agent_inference; +/// The bucket swarm-controller sets each agent's CPU and memory caps in — one +/// key per agent, read by the hive that runs it. The names are unconditional; +/// writing needs `kv` and reading needs `agent-limits`. +pub mod agent_limits; + #[cfg(feature = "kv")] mod kv; From fccf2ab8924a2b1557f76819ede5cb0e57329c82 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 12 Oct 2026 11:26:18 +0200 Subject: [PATCH 3/6] hive-c0re: read each agent's swarm caps from agent-limits Every 30 seconds `workers::swarm_limits` reads the `agent-limits` key of each agent this hive has and stores the result in swarm-resource-limits.json, where the drop-in, the heap ceiling and the dashboard read it. A cap therefore follows its agent to whichever hive runs it. Each value goes through the hive's own validate_cpu_quota / validate_memory_max before it is stored, since hive-priv renders it verbatim into the container's drop-in, and one bad value refuses the whole set (argus on #4268, carried over from the wanted-state consumer this replaces). A failed read, or a bucket that does not answer, leaves the stored caps as they are; three failures in a row raise a banner. swarm-nats-auth grants every hive `$JS.API.DIRECT.GET.KV_agent-limits.>`: read-only, and every agent's key, not just the hive's own. Scoping it to the hive's agents would need a roster in the auth responder, which policy.rs argues against. mara chose this on #4268 (c95293); narrowing it is a possible low-priority follow-up, not planned. A test pins that the grant reads every key, writes none, and reaches no other bucket. Contributes to #3865. --- docs/agent-lifecycle/persistence.md | 7 + docs/scheduler/coordinator.md | 42 +++++- docs/swarm/README.md | 3 + hive-c0re/Cargo.toml | 6 +- hive-c0re/src/swarm_status.rs | 6 + hive-c0re/src/workers/mod.rs | 4 +- hive-c0re/src/workers/swarm_limits.rs | 207 ++++++++++++++++++++++++++ swarm-nats-auth/src/policy.rs | 49 ++++++ 8 files changed, 320 insertions(+), 4 deletions(-) create mode 100644 hive-c0re/src/workers/swarm_limits.rs diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md index 3ca1043c9..851f7e73c 100644 --- a/docs/agent-lifecycle/persistence.md +++ b/docs/agent-lifecycle/persistence.md @@ -411,6 +411,13 @@ Contents: default for CPU. The `CPUWeight=` / `IOWeight=` shares in the same drop-in have **no** per-agent override — they're hive-wide only and come straight off `HiveEnv`, so this file has no field for them. +- `swarm-resource-limits.json` — the caps the swarm sets for this + hive's agents, same shape as `resource-limits.json`. Written only by + `workers::swarm_limits`, which copies each agent's key of the swarm's + `agent-limits` bucket here every 30 seconds. A field set here replaces + that field's answer from `resource-limits.json` and the hive-wide + default. Every value passes the same validators as the operator's + overrides first, and one bad value makes the worker change nothing here. The root agent has the meta dir RO-mounted at `/meta/`. diff --git a/docs/scheduler/coordinator.md b/docs/scheduler/coordinator.md index 20d4dab39..d6cf23ed3 100644 --- a/docs/scheduler/coordinator.md +++ b/docs/scheduler/coordinator.md @@ -452,14 +452,24 @@ regardless. rebuild, so changes take effect on the next lifecycle op without requiring a host rebuild. +The answer per agent and per field is, first match wins: + +1. the swarm's cap for the agent (see below); +2. the operator's per-agent override in `meta/resource-limits.json`; +3. `agentCpuQuota` / `agentMemoryMax`. + +A swarm cap replaces the hive's answer, so a hive can't raise an agent +above it. Like the rest of the drop-in, a changed swarm cap reaches a +running agent at its next spawn or rebuild. + The container can't read its own cap, so `write_dropins` also writes the effective `MemoryMax=` into the agent's runtime dir (`/run/hive/memory-max-bytes` inside) before every start: a byte count, or `infinity` for no cap. A percentage becomes bytes the way systemd resolves it, as that share of the host's physical memory rounded down to a page. A value systemd ignores (zero, over 100%) is `infinity`. When hive-c0re keeps -the last drop-in because it can't read the overrides file, the file carries -that drop-in's `MemoryMax=`. +the last drop-in because it can't read the overrides or the swarm caps file, +the file carries that drop-in's `MemoryMax=`. At boot the container derives two values from a byte count: `BUN_JSC_forceRAMSize` (75%, claude-code's heap ceiling) and `MemoryHigh=` on @@ -469,6 +479,34 @@ changed cap reaches them at the next container start without a rebuild. `hyperhive-memory-cap-check.service` logs a warning naming the path in the container's journal. The container's `MemoryMax=` still applies. +
Implementation: swarm caps + +swarm-controller keeps the caps in the `agent-limits` KV bucket, one key per +agent and no hive in it, so an agent that moves to another hive keeps its +caps. swarm-controller alone writes it: + +| method | path | body | +| -------- | ---------------------------- | --------------------------------------------------------------------------------------------- | +| `GET` | `/api/agents//limits` | —; answers the stored caps, `{}` when there are none | +| `PUT` | `/api/agents//limits` | `{"cpu_quota": "400%", "memory_max": "8G"}`, every field optional, replacing the stored value | +| `DELETE` | `/api/agents//limits` | —; answers 204 | + +`PUT` refuses an agent no hive's wanted state declares, unknown fields, zero +and anything systemd doesn't take, and stores memory the way systemd spells +it (`8gb` as `8G`). + +Every 30 seconds hive-c0re (`workers::swarm_limits`) reads the key of each +agent it has and stores the result in `meta/swarm-resource-limits.json`. Each +value passes the same validators as the operator's overrides first. One bad +value or a failed read leaves the file unchanged, and repeated failures raise a +dashboard banner. + +A hive's queue grant reads every key of the bucket +(`$JS.API.DIRECT.GET.KV_agent-limits.>`) and writes none. Scoping it to the +hive's own agents would need a roster in the auth responder. + +
+ The same drop-in carries `CPUWeight=` / `IOWeight=` from `agentCpuWeight` / `agentIoWeight`. Those are a different kind of setting: the quota and the memory max are **hard caps** that throttle diff --git a/docs/swarm/README.md b/docs/swarm/README.md index 8c89734ce..0535bd71c 100644 --- a/docs/swarm/README.md +++ b/docs/swarm/README.md @@ -25,6 +25,9 @@ grafana, the controller, authelia, …) → kept by the swarm and fired in each agent's container. → [`schedules.md`](schedules.md) - **inference settings** — the model and effort the swarm sets for an agent, over its own. → [`inference.md`](inference.md) +- **resource caps** — the CPU and memory cap the swarm sets for an agent, + over its hive's. → + [`coordinator.md`](../scheduler/coordinator.md#container-resource-limits) - **review classifier** — planned, not implemented yet: checks a PR against its repo's own yes/no rules and posts the result as a review. → [`review-classifier.md`](review-classifier.md) diff --git a/hive-c0re/Cargo.toml b/hive-c0re/Cargo.toml index ac74773ff..9e8d0f0c0 100644 --- a/hive-c0re/Cargo.toml +++ b/hive-c0re/Cargo.toml @@ -53,7 +53,11 @@ strum.workspace = true # Offering this hive's status to the swarm (`swarm_status`). The same crate # the swarm controller reads it with, and `kv` for the same reason: the # bucket's name and creation config belong to neither end of it alone. -swarm-queue-client = { workspace = true, features = ["kv", "notices"] } +swarm-queue-client = { workspace = true, features = [ + "kv", + "notices", + "agent-limits", +] } swarm-secret-client.workspace = true tokio.workspace = true tokio-stream.workspace = true diff --git a/hive-c0re/src/swarm_status.rs b/hive-c0re/src/swarm_status.rs index 13a1d455c..c4cfc54b6 100644 --- a/hive-c0re/src/swarm_status.rs +++ b/hive-c0re/src/swarm_status.rs @@ -103,6 +103,12 @@ pub fn spawn( shutdown.clone(), )); + // The swarm's resource caps, on the same connection again. + tokio::spawn(crate::workers::swarm_limits::follow( + client.clone(), + shutdown.clone(), + )); + let mut health = SweepHealth::new("swarm_status_publish", "warn", FAILURES_BEFORE_BANNER); loop { match publish(&client, &hive).await { diff --git a/hive-c0re/src/workers/mod.rs b/hive-c0re/src/workers/mod.rs index c3772e923..6f2ffa4c8 100644 --- a/hive-c0re/src/workers/mod.rs +++ b/hive-c0re/src/workers/mod.rs @@ -1,6 +1,7 @@ //! Background tasks and periodic sweeps: crash/login watcher, boot-time //! auto-update reconcile, the agent-sockets.json writer loop, the MCP socket listener -//! reconcile loop, knowledge-repo sync, and the swarm wanted-state pull. +//! reconcile loop, knowledge-repo sync, the swarm wanted-state pull, and the +//! swarm resource-cap read. //! Each submodule is re-exported //! at the crate root, so `crate::crash_watch::…` etc. keep working unchanged. @@ -9,4 +10,5 @@ pub mod auto_update; pub mod crash_watch; pub mod knowledge; pub mod mcp_sockets; +pub mod swarm_limits; pub mod wanted; diff --git a/hive-c0re/src/workers/swarm_limits.rs b/hive-c0re/src/workers/swarm_limits.rs new file mode 100644 index 000000000..ec3b09790 --- /dev/null +++ b/hive-c0re/src/workers/swarm_limits.rs @@ -0,0 +1,207 @@ +//! Reading the caps the swarm sets for this hive's agents. +//! +//! swarm-controller keeps one key per agent in the `agent-limits` bucket +//! ([`swarm_queue_client::agent_limits`]). Every [`READ_INTERVAL`] this reads +//! the key of each agent this hive has and stores what it finds in +//! `swarm-resource-limits.json`, where the drop-in and the dashboard read it. +//! A running agent gets a changed cap at its first spawn or rebuild after +//! that read. +//! +//! A failed read, or a bucket that does not answer, leaves the stored caps as +//! they are: dropping them would read exactly like the swarm clearing them. + +use std::collections::BTreeMap; +use std::time::Duration; + +use anyhow::{Context, Result}; +use swarm_queue_client::agent_limits::{self, AgentCaps, Read}; + +use crate::resource_limits::AgentLimits; +use crate::stats::sweep_health::{self, SweepHealth}; + +/// How often the caps are read. +pub const READ_INTERVAL: Duration = Duration::from_secs(30); + +/// At [`READ_INTERVAL`] this is ~90 seconds of failing reads. +const FAILURES_BEFORE_BANNER: u32 = 3; + +/// Read the caps now and every [`READ_INTERVAL`] until shutdown. +pub async fn follow(client: async_nats::Client, mut shutdown: tokio::sync::watch::Receiver) { + let mut health = SweepHealth::new("swarm_limits_read", "warn", FAILURES_BEFORE_BANNER); + loop { + match sync(&client).await { + Ok(()) => health.record_ok(), + Err(e) => { + tracing::warn!(error = ?e, "swarm limits: reading the caps failed"); + let err = format!("{e:#}"); + health.record_err(|ctx| { + let age = ctx.since_last_ok.map_or_else( + || "no success this session".to_owned(), + |d| format!("last ok {} ago", sweep_health::fmt_age(d)), + ); + format!( + "reading the swarm's resource caps is failing ({} consecutive, {age}) \ + — agents keep the caps last read: {err}", + ctx.consecutive + ) + }); + } + } + tokio::select! { + () = tokio::time::sleep(READ_INTERVAL) => {} + _ = shutdown.changed() => { + tracing::info!("swarm limits: shutdown signal received"); + return; + } + } + } +} + +/// Read every agent's key and store the result, unless a read fails or a +/// value is refused. +async fn sync(client: &async_nats::Client) -> Result<()> { + // An unconnected client does not fail a JetStream request, it hangs on it. + swarm_queue_client::ensure_connected(client)?; + let agents = crate::lifecycle::agents_for_meta_listing() + .await + .context("enumerating this hive's agents")?; + let mut declared = BTreeMap::new(); + for spec in agents { + let read = agent_limits::read(client, &spec.name) + .await + .map_err(|e| anyhow::anyhow!(e)) + .with_context(|| format!("reading the swarm's caps for {}", spec.name))?; + match read { + Read::Current(caps) => { + declared.insert(spec.name, caps); + } + Read::NoBucket => { + tracing::debug!("swarm limits: the agent-limits bucket does not answer"); + return Ok(()); + } + } + } + let caps = validated(&declared)?; + if caps != crate::resource_limits::read_swarm().context("reading the stored swarm caps")? { + crate::resource_limits::write_swarm(&caps).context("storing the swarm caps")?; + tracing::info!(?caps, "swarm limits: stored the swarm's resource caps"); + } + Ok(()) +} + +/// The caps to store for `declared`, by agent. An agent with no field set +/// gets no entry. +/// +/// Every value goes through the validators the operator's own input path +/// uses, because it is rendered verbatim into the container's systemd +/// drop-in. One bad value refuses the whole set. +fn validated(declared: &BTreeMap) -> Result> { + let mut out = BTreeMap::new(); + for (agent, caps) in declared { + if caps.is_empty() { + continue; + } + let limits = validated_caps(caps) + .map_err(|why| anyhow::anyhow!("the swarm set an invalid cap for {agent}: {why}"))?; + out.insert(agent.clone(), limits); + } + Ok(out) +} + +fn validated_caps(caps: &AgentCaps) -> Result { + if let Some(cpu) = &caps.cpu_quota { + crate::resource_limits::validate_cpu_quota(cpu)?; + } + Ok(AgentLimits { + cpu_quota: caps.cpu_quota.clone(), + memory_max: caps + .memory_max + .as_deref() + .map(crate::resource_limits::validate_memory_max) + .transpose()?, + }) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::validated; + use crate::resource_limits::AgentLimits; + use swarm_queue_client::agent_limits::AgentCaps; + + /// `atlas`'s answer with `atlas_key` as the value read for it, over a hive + /// chain that caps it at 800% / 16G. + fn atlas_with(atlas_key: &str) -> (String, String) { + let declared = BTreeMap::from([( + "atlas".to_owned(), + serde_json::from_str::(atlas_key).expect("decodes"), + )]); + let stored = validated(&declared).expect("valid caps"); + let hive_side = BTreeMap::from([( + "atlas".to_owned(), + AgentLimits { + cpu_quota: Some("800%".to_owned()), + memory_max: Some("16G".to_owned()), + }, + )]); + crate::resource_limits::effective_from_with_swarm( + &stored, &hive_side, "atlas", "200%", "4G", + ) + } + + /// The other half of swarm-controller's + /// `agent_limits::the_route_writes_the_caps_a_hive_reads_and_delete_clears_them`: + /// the first value is the bytes that route stores. The cap replaces the + /// hive's chain; a deleted key reads as `{}` and gives the chain back. + #[test] + fn the_caps_the_route_stores_replace_the_hive_chain_and_a_cleared_key_restores_it() { + assert_eq!( + atlas_with(r#"{"cpu_quota":"400%","memory_max":"8G"}"#), + ("400%".to_owned(), "8G".to_owned()) + ); + assert_eq!(atlas_with("{}"), ("800%".to_owned(), "16G".to_owned())); + } + + /// Per field: a swarm memory cap leaves CPU on the hive's chain. + #[test] + fn a_field_the_swarm_leaves_unset_stays_with_the_hive() { + assert_eq!( + atlas_with(r#"{"memory_max":"2G"}"#), + ("800%".to_owned(), "2G".to_owned()) + ); + } + + #[test] + fn an_agent_with_no_caps_gets_no_entry() { + let declared = BTreeMap::from([("atlas".to_owned(), AgentCaps::default())]); + assert!(validated(&declared).expect("valid").is_empty()); + } + + /// The values are rendered verbatim into a systemd drop-in, so a value + /// the hive's own validators refuse, a newline smuggling in a directive + /// among them, refuses the whole set, the valid cap beside it included. + #[test] + fn an_invalid_cap_is_refused_and_nothing_is_stored() { + for bad in [ + r#"{"cpu_quota":"400%\nExecStartPre=/bin/sh -c id #%"}"#, + r#"{"memory_max":"8G\nDevicePolicy=auto"}"#, + r#"{"cpu_quota":"lots"}"#, + r#"{"memory_max":"8X"}"#, + ] { + let declared = BTreeMap::from([ + ("atlas".to_owned(), AgentCaps::default()), + ( + "iris".to_owned(), + serde_json::from_str(r#"{"memory_max":"2G"}"#).expect("decodes"), + ), + ( + "sock".to_owned(), + serde_json::from_str(bad).expect("decodes"), + ), + ]); + let err = validated(&declared).expect_err(bad); + assert!(err.to_string().contains("sock"), "{err:#}"); + } + } +} diff --git a/swarm-nats-auth/src/policy.rs b/swarm-nats-auth/src/policy.rs index b9d6db695..86a4d7038 100644 --- a/swarm-nats-auth/src/policy.rs +++ b/swarm-nats-auth/src/policy.rs @@ -490,6 +490,17 @@ impl Policy { format!("$JS.API.STREAM.CREATE.{agent_status}"), format!("$KV.{}.{hive}.*", swarm_queue_client::agent_status::BUCKET), ]); + // Reading the caps the swarm sets for the agents this hive runs + // (`swarm_queue_client::agent_limits`): a direct get, so read-only, and + // no `STREAM.INFO` because the reader skips the lookup. + // + // Every agent's key, not just this hive's: the bucket is keyed by agent + // alone, and naming one hive's agents here needs the roster + // `hive_name`'s doc argues against. + subjects.push(format!( + "$JS.API.DIRECT.GET.{}.>", + swarm_queue_client::agent_limits::STREAM + )); subjects.extend( self.extra_hive_subjects .iter() @@ -712,6 +723,44 @@ mod tests { } } + /// A hive reads any agent's caps and can change none of them, and the + /// grant reaches no other bucket. + #[test] + fn a_hive_may_read_every_agents_caps_and_write_none() { + use swarm_queue_client::agent_limits::{STREAM, direct_get_subject, key_subject}; + + let p = policy() + .permissions("hive-alpha") + .expect("a hive is admitted"); + let may = |subject: &str| p.publish.iter().any(|g| covers(g, subject)); + + // Its own agents and agents running elsewhere alike. + for agent in ["atlas", "iris"] { + assert!(may(&direct_get_subject(agent)), "cannot read {agent}"); + } + // A KV put or delete is a publish to the key's subject; the rest + // reshape or empty the stream. + for forbidden in [ + key_subject("atlas"), + format!("$JS.API.STREAM.CREATE.{STREAM}"), + format!("$JS.API.STREAM.UPDATE.{STREAM}"), + format!("$JS.API.STREAM.DELETE.{STREAM}"), + format!("$JS.API.STREAM.PURGE.{STREAM}"), + format!("$JS.API.STREAM.MSG.DELETE.{STREAM}"), + format!("$JS.API.CONSUMER.CREATE.{STREAM}"), + ] { + assert!(!may(&forbidden), "hive alpha may publish {forbidden}"); + } + // The read reaches only this bucket. + for other in [ + swarm_queue_client::agent_inference::direct_get_subject("atlas"), + swarm_queue_client::agent_schedules::direct_get_subject("atlas"), + "$JS.API.DIRECT.GET.KV_hive-wanted-beta.$KV.hive-wanted-beta.beta".to_owned(), + ] { + assert!(!may(&other), "hive alpha may publish {other}"); + } + } + #[test] fn an_extra_subject_without_the_placeholder_is_refused() { // 🩸 The option exists to put a second stream inside ONE hive's From 028ca1bf1fb08289579e696162a8dceaaf7833ae Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 12 Oct 2026 12:03:49 +0200 Subject: [PATCH 4/6] swarm-controller: refuse a cap systemd would ignore; GET creates no bucket systemd ignores a CPUQuota= or MemoryMax= it can't parse ("Invalid ..., ignoring") and runs the unit without that limit. A swarm cap replaces the hive's answer, so such a value left the agent uncapped while the route, GET and the dashboard all said it was capped. argus on #4268 (R3) found values the route took that systemd ignores: 12.345%, 400.123%, 0.001%, 17000000T, 99999999999999999999T and 99999999999999999999%. The validators now accept a subset of what systemd parses, measured against `systemd-analyze verify` (systemd 260): - a percentage takes at most two decimals and is above zero, up to 100% for memory_max and 21474836.47% for cpu_quota (systemd keeps the quota in hundredths of a percent in a C int, not a u64) - a size is one parse_size term: digits, an optional fraction, then nothing, B, K, M, G or T, case-sensitive. The whole part and the fraction digits fit u64, the whole part (plus one with a fraction) fits u64 once scaled, and the byte count is at least 1 and below 2^64 - 1, as systemd checks them. P and E, which systemd reads, stay refused because hive-c0re's validator refuses them, and one value the hive refuses stops every cap update on that hive. Values are stored as given: lower-case suffixes and a B after a multiplier are refused rather than normalized, as systemd refuses them. The newline-injection values from the hive's test are in both bad lists and go through the router in a_cap_with_a_second_line_is_a_400_and_the_caps_stay. GET reads through a new agent_limits::open that returns None for a missing bucket, answered as {}, instead of creating the bucket. --- docs/scheduler/coordinator.md | 10 +- swarm-controller/src/agent_limits.rs | 294 +++++++++++++++++++------ swarm-queue-client/src/agent_limits.rs | 24 ++ swarm-queue-client/src/wanted.rs | 2 +- 4 files changed, 260 insertions(+), 70 deletions(-) diff --git a/docs/scheduler/coordinator.md b/docs/scheduler/coordinator.md index d6cf23ed3..5118956f7 100644 --- a/docs/scheduler/coordinator.md +++ b/docs/scheduler/coordinator.md @@ -491,9 +491,13 @@ caps. swarm-controller alone writes it: | `PUT` | `/api/agents//limits` | `{"cpu_quota": "400%", "memory_max": "8G"}`, every field optional, replacing the stored value | | `DELETE` | `/api/agents//limits` | —; answers 204 | -`PUT` refuses an agent no hive's wanted state declares, unknown fields, zero -and anything systemd doesn't take, and stores memory the way systemd spells -it (`8gb` as `8G`). +`PUT` refuses an agent no hive's wanted state declares, unknown fields, and +any value systemd would ignore, which would leave the agent uncapped. A +percentage has at most two decimals and is above zero, up to `100%` for +`memory_max` and `21474836.47%` for `cpu_quota`. A size is digits with an +optional fraction and an optional `B`, `K`, `M`, `G` or `T` (upper case), at +least one byte and below 2^64 - 1 bytes. The controller stores each value as +given. Every 30 seconds hive-c0re (`workers::swarm_limits`) reads the key of each agent it has and stores the result in `meta/swarm-resource-limits.json`. Each diff --git a/swarm-controller/src/agent_limits.rs b/swarm-controller/src/agent_limits.rs index 478a92db6..3d33a9002 100644 --- a/swarm-controller/src/agent_limits.rs +++ b/swarm-controller/src/agent_limits.rs @@ -7,9 +7,9 @@ //! hive's own answer for that field, and a field left unset stays with the //! hive. //! -//! The hive hands these values to systemd verbatim, and a value systemd -//! rejects stops the agent's container from starting, so they are validated -//! here as well as on the hive. +//! The hive hands these values to systemd verbatim, and systemd ignores a value +//! it can't parse, leaving the agent uncapped, so they are checked here against +//! systemd's own syntax (`Caps`) as well as on the hive. use axum::Json; use axum::extract::{Path, State}; @@ -27,16 +27,23 @@ use crate::{AppState, error_problem, wanted_error_status}; const MAX_ATTEMPTS: usize = 5; /// One agent's caps. A field left out is not capped by the swarm. +/// +/// Each value is stored as given, and only in a form systemd applies: a value +/// systemd ignores ("Invalid …, ignoring") would leave the agent uncapped, +/// since a swarm cap replaces the hive's answer. #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(deny_unknown_fields)] pub struct Caps { - /// systemd `CPUQuota=`: a percentage above zero, `"100%"` being one core. + /// systemd `CPUQuota=`: digits, optionally `.` and one or two digits, then + /// `%`; above zero and at most `21474836.47%`. `"100%"` is one core. #[serde(default, skip_serializing_if = "Option::is_none")] #[schema(example = "400%")] cpu_quota: Option, - /// systemd `MemoryMax=`: a size above zero with an optional `K`/`M`/`G`/`T` - /// suffix, a percentage of physical memory up to `100%`, or `infinity`. - /// Stored in the one spelling systemd accepts (`8gb` is stored as `8G`). + /// systemd `MemoryMax=`: `infinity`; a percentage of physical memory + /// written as for `cpu_quota`, above zero and at most `100%`; or a size: + /// digits, optionally `.` and digits, then nothing, `B` or one of + /// `K`/`M`/`G`/`T` (1024-based, upper case), at least one byte and below + /// 2^64 - 1 bytes. #[serde(default, skip_serializing_if = "Option::is_none")] #[schema(example = "8G")] memory_max: Option, @@ -61,60 +68,97 @@ impl From for Caps { } } -// The accepted shapes are hive-c0re's `resource_limits::validate_cpu_quota` / -// `validate_memory_max`, the operator's own input path for the same drop-in, -// with zero and an over-100% memory share refused as well. +// The accepted forms are a subset of what systemd parses (checked against +// `systemd-analyze verify`, systemd 260) and stricter than hive-c0re's own +// `validate_cpu_quota` / `validate_memory_max`, so the hive never refuses a +// stored value. + +/// systemd keeps `CPUQuota=` in hundredths of a percent in a C `int`. +const CPU_QUOTA_MAX_HUNDREDTHS: u64 = 2_147_483_647; fn cpu_quota(value: &str) -> Result { - match value.strip_suffix('%') { - Some(n) if is_positive_number(n) => Ok(value.to_owned()), + match hundredths(value) { + Some(h) if (1..=CPU_QUOTA_MAX_HUNDREDTHS).contains(&h) => Ok(value.to_owned()), _ => Err(format!( - "invalid cpu_quota {value:?}: expected a percentage above zero such as \"200%\" \ - (100% = one full core)" + "invalid cpu_quota {value:?}: expected a percentage above zero with at most two \ + decimals, such as \"200%\" (100% = one full core), up to \"21474836.47%\"" )), } } fn memory_max(value: &str) -> Result { - if value == "infinity" { - return Ok(value.to_owned()); + let applies = value == "infinity" + || if value.ends_with('%') { + hundredths(value).is_some_and(|h| (1..=10_000).contains(&h)) + } else { + size_bytes(value).is_some_and(|b| b >= 1 && b < u128::from(u64::MAX)) + }; + if applies { + Ok(value.to_owned()) + } else { + Err(format!( + "invalid memory_max {value:?}: expected a size above zero such as \"8G\" \ + (suffix B, K, M, G or T, upper case), a percentage up to \"100%\" with at \ + most two decimals, or \"infinity\"" + )) } - let normalized = match value.strip_suffix('%') { - Some(n) => (is_positive_number(n) && n.parse::().is_ok_and(|p| p <= 100.0)) - .then(|| value.to_owned()), - None => byte_size(value), - }; - normalized.ok_or_else(|| { - format!( - "invalid memory_max {value:?}: expected a size above zero such as \"8G\", a \ - percentage up to \"100%\", or \"infinity\"" - ) - }) } -/// A byte count with an optional `K`/`M`/`G`/`T` multiplier, in any case and -/// with an optional trailing `B`, in the form systemd takes: `8gb` → `8G`, a -/// bare `1048576B` kept as it is. -fn byte_size(value: &str) -> Option { - const UNITS: [char; 8] = ['K', 'M', 'G', 'T', 'k', 'm', 'g', 't']; - let (body, b) = match value.strip_suffix(['B', 'b']) { - Some(body) => (body, true), - None => (value, false), +/// `value` as systemd reads a percentage, in hundredths of a percent: digits, +/// optionally `.` and one or two digits, then `%`. `None` for anything else. +fn hundredths(value: &str) -> Option { + let number = value.strip_suffix('%')?; + let (int, frac) = match number.split_once('.') { + Some((int, frac)) if (1..=2).contains(&frac.len()) && digits(frac) => (int, frac), + Some(_) => return None, + None => (number, ""), }; - let (mantissa, unit) = match body.strip_suffix(UNITS) { - Some(mantissa) => (mantissa, body[mantissa.len()..].to_ascii_uppercase()), - None if b => (body, "B".to_owned()), - None => (body, String::new()), + if !digits(int) { + return None; + } + let frac = match frac.len() { + 0 => 0, + 1 => frac.parse::().ok()? * 10, + _ => frac.parse::().ok()?, }; - is_positive_number(mantissa).then(|| format!("{mantissa}{unit}")) + int.parse::().ok()?.checked_mul(100)?.checked_add(frac) } -/// Digits with at most one `.` between digits, and not zero. No sign or -/// exponent, which systemd does not take either. -fn is_positive_number(value: &str) -> bool { - let (int, frac) = value.split_once('.').unwrap_or((value, "1")); - let digits = |s: &str| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()); - digits(int) && digits(frac) && value.bytes().any(|b| (b'1'..=b'9').contains(&b)) +/// `value` as one term of systemd's `parse_size` reads it, in bytes: digits, +/// optionally `.` and digits, then nothing, `B`, or `K`/`M`/`G`/`T` (1024-based, +/// case-sensitive). `P` and `E`, which systemd also reads, are left out +/// because hive-c0re's validator refuses them. `None` for anything else, and for what +/// `parse_size` refuses as out of range: a whole part or fraction digits past +/// `u64`, or a whole part that, plus one when there is a fraction, is past +/// `u64` once scaled. +fn size_bytes(value: &str) -> Option { + let (number, exponent) = match value.char_indices().last()? { + (i, 'B') => (&value[..i], 0), + (i, c) => match "KMGT".find(c) { + Some(k) => (&value[..i], k + 1), + None => (value, 0), + }, + }; + let scale = 1024u128.pow(u32::try_from(exponent).ok()?); + let (int, frac) = number.split_once('.').unwrap_or((number, "")); + if !digits(int) || (number.contains('.') && !digits(frac)) { + return None; + } + let whole = u128::from(int.parse::().ok()?); + let frac_value = if frac.is_empty() { + 0 + } else { + u128::from(frac.parse::().ok()?) + }; + if whole + u128::from(frac_value > 0) > u128::from(u64::MAX) / scale { + return None; + } + let frac_bytes = frac_value * scale / 10u128.checked_pow(u32::try_from(frac.len()).ok()?)?; + Some(whole * scale + frac_bytes) +} + +fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) } fn agent_name(name: &str) -> Result { @@ -128,7 +172,7 @@ fn queue_problem(e: &anyhow::Error) -> ProblemDetails { error_problem(wanted_error_status(e), &format!("{e:#}")) } -async fn open( +async fn open_or_create( client: &async_nats::Client, ) -> Result { agent_limits::open_or_create(client) @@ -157,7 +201,12 @@ pub(crate) async fn get_agent_limits( ) -> Result, ProblemDetails> { let agent = agent_name(&name).map_err(refused)?; let client = queue(&state).map_err(refused)?; - let store = open(&client).await?; + let Some(store) = agent_limits::open(&client) + .await + .map_err(|e| queue_problem(&anyhow::Error::new(e)))? + else { + return Ok(Json(Caps::default())); + }; let (current, _) = read::(&store, &agent) .await .map_err(|e| queue_problem(&e))?; @@ -202,7 +251,7 @@ pub(crate) async fn put_agent_limits( &format!("no hive's wanted state declares an agent {agent:?} that is not destroyed"), )); } - let store = open(&client).await?; + let store = open_or_create(&client).await?; for _ in 0..MAX_ATTEMPTS { let (current, revision) = read::(&store, &agent) .await @@ -241,7 +290,7 @@ pub(crate) async fn delete_agent_limits( ) -> Result { let agent = agent_name(&name).map_err(refused)?; let client = queue(&state).map_err(refused)?; - let store = open(&client).await?; + let store = open_or_create(&client).await?; store .delete(&agent) .await @@ -262,35 +311,110 @@ mod tests { use std::sync::Arc; use swarm_queue_client::wanted::AgentState; + /// Of the `bad` values, systemd parses `""` (no quota) and `+5%`, and the + /// last adds a directive on a second line. It ignores every other one + /// ("Invalid CPU quota …, ignoring", checked with `systemd-analyze + /// verify`), which would leave the agent uncapped. #[test] fn cpu_quota_takes_a_percentage_above_zero() { - for ok in ["400%", "100%", "0.5%", "1500%"] { + for ok in [ + "400%", + "100%", + "0.5%", + "0.01%", + "1500%", + "400.12%", + "05%", + "21474836.47%", + ] { assert_eq!(cpu_quota(ok).as_deref(), Ok(ok)); } for bad in [ - "0%", "0.0%", "-50%", "400", "4 00%", "%", "lots", "", "1e3%", "+5%", + "0%", + "0.0%", + "0.00%", + "-50%", + "400", + "4 00%", + "%", + "lots", + "", + "1e3%", + "+5%", + "400.123%", + "0.001%", + "99999999999999999999%", + "21474836.48%", + "1.%", + ".5%", + "400%\nExecStartPre=/bin/sh -c id #%", ] { assert!(cpu_quota(bad).is_err(), "{bad:?} was accepted"); } } + /// As above for `MemoryMax=`. Of the `bad` values, systemd also parses `3P` + /// and `2E`, which hive-c0re's validator refuses, and `8 G` and `1.G`. #[test] - fn memory_max_takes_a_size_a_share_or_infinity_and_stores_the_systemd_spelling() { - for (given, stored) in [ - ("8G", "8G"), - ("8gb", "8G"), - ("512m", "512M"), - ("1048576", "1048576"), - ("1048576B", "1048576B"), - ("1.5T", "1.5T"), - ("50%", "50%"), - ("100%", "100%"), - ("infinity", "infinity"), + fn memory_max_takes_a_size_a_share_or_infinity_that_systemd_applies() { + for ok in [ + "8G", + "512M", + "4096K", + "1048576", + "1048576B", + "1.5B", + "0.5K", + "1.5T", + "1.23456789G", + "16383T", + "16777215T", + "18446744073709551614", + "18446744073709551614.5", + "1.00000000000000000001G", + "50%", + "12.34%", + "0.01%", + "100%", + "100.00%", + "infinity", ] { - assert_eq!(memory_max(given).as_deref(), Ok(stored), "{given:?}"); + assert_eq!(memory_max(ok).as_deref(), Ok(ok), "{ok:?}"); } for bad in [ - "0", "0G", "0.0G", "0%", "-8G", "150%", "8X", "G", "8 G", "", "eight", "8GG", + "0", + "0G", + "0.0G", + "0.5", + "0%", + "-8G", + "150%", + "100.01%", + "12.345%", + "8X", + "G", + "8 G", + "", + "eight", + "8GG", + "8gb", + "8g", + "8GB", + "4096k", + "1048576b", + "3P", + "2E", + ".5G", + "1.G", + "Infinity", + "17000000T", + "16777216T", + "16777215.5T", + "99999999999999999999T", + "18446744073709551615", + "18446744073709551616", + "1.123456789012345678901234567890G", + "8G\nDevicePolicy=auto", ] { assert!(memory_max(bad).is_err(), "{bad:?} was accepted"); } @@ -389,7 +513,7 @@ mod tests { let client = nats.client().await; let state = declared(&client).await; - let answer = put(&state, "atlas", caps(Some("400%"), Some("8gb"))) + let answer = put(&state, "atlas", caps(Some("400%"), Some("8G"))) .await .expect("caps are set"); assert_eq!(answer.0, caps(Some("400%"), Some("8G"))); @@ -407,6 +531,23 @@ mod tests { assert_eq!(get(&state, "atlas").await, Caps::default()); } + /// A read before any cap was set answers `{}` and leaves the bucket + /// uncreated. + #[tokio::test] + async fn a_read_before_any_cap_answers_empty_and_creates_no_bucket() { + let nats = TestNats::start(); + let client = nats.client().await; + let state = state(&client); + + assert_eq!(get(&state, "atlas").await, Caps::default()); + assert!( + swarm_queue_client::agent_limits::open(&client) + .await + .expect("the queue answers") + .is_none() + ); + } + /// A name no hive declares gets no key, so a typo cannot cap an agent /// that does not exist yet. #[tokio::test] @@ -500,6 +641,27 @@ mod tests { assert_eq!(published(&client, "atlas").await.as_deref(), Some("{}")); } + /// A value carrying a directive on a second line is a 400 through the real + /// router, and the caps already set stay. + #[tokio::test] + async fn a_cap_with_a_second_line_is_a_400_and_the_caps_stay() { + let nats = TestNats::start(); + let client = nats.client().await; + let app = capped_router(&client).await; + + for body in [ + r#"{"cpu_quota":"400%\nExecStartPre=/bin/sh -c id #%"}"#, + r#"{"memory_max":"8G\nDevicePolicy=auto"}"#, + ] { + assert_eq!(put_raw(&app, body).await, StatusCode::BAD_REQUEST, "{body}"); + assert_eq!( + published(&client, "atlas").await.as_deref(), + Some(CAPPED), + "{body}" + ); + } + } + /// A field the body does not define is a 422 and writes nothing: a /// misspelt `memory_max` must not read as "no memory cap". #[tokio::test] diff --git a/swarm-queue-client/src/agent_limits.rs b/swarm-queue-client/src/agent_limits.rs index 91b857af8..77cf3b7f0 100644 --- a/swarm-queue-client/src/agent_limits.rs +++ b/swarm-queue-client/src/agent_limits.rs @@ -71,6 +71,30 @@ pub async fn open_or_create( .await } +/// Open the caps bucket for reading, or `None` when nothing has created it +/// yet, which reads the same as no caps. +/// +/// **Controller-side**, for reads that must not create the bucket. +/// +/// # Errors +/// +/// [`crate::Error::OpenBucket`] for every failure other than the bucket not +/// existing. +#[cfg(feature = "kv")] +pub async fn open( + client: &async_nats::Client, +) -> Result, crate::Error> { + let js = async_nats::jetstream::new(client.clone()); + match js.get_key_value(BUCKET).await { + Ok(store) => Ok(Some(store)), + Err(e) if crate::wanted::is_bucket_not_found(&e) => Ok(None), + Err(source) => Err(crate::Error::OpenBucket { + bucket: BUCKET.to_owned(), + source, + }), + } +} + /// What reading one agent's key found. #[cfg(feature = "agent-limits")] #[derive(Debug)] diff --git a/swarm-queue-client/src/wanted.rs b/swarm-queue-client/src/wanted.rs index 451039c5d..16681441b 100644 --- a/swarm-queue-client/src/wanted.rs +++ b/swarm-queue-client/src/wanted.rs @@ -156,7 +156,7 @@ pub async fn open_read_only( /// `get_key_value` reports a missing bucket as `GetBucket` wrapping the /// stream lookup's `STREAM_NOT_FOUND`; every other kind is a real failure. #[cfg(feature = "kv")] -fn is_bucket_not_found(e: &async_nats::jetstream::context::KeyValueError) -> bool { +pub(crate) fn is_bucket_not_found(e: &async_nats::jetstream::context::KeyValueError) -> bool { use async_nats::jetstream::context::{GetStreamError, GetStreamErrorKind, KeyValueErrorKind}; e.kind() == KeyValueErrorKind::GetBucket From e14955f20ad1ed86001d1ed77d838834137ce366 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 12 Oct 2026 12:03:49 +0200 Subject: [PATCH 5/6] swarm-nats-auth: pin that a hive can't create a named consumer on agent-limits The grant test covered the bare consumer-create subject only, so a grant of `$JS.API.CONSUMER.CREATE.KV_agent-limits.>` would have passed it. It now also checks a named consumer, bare and with a key filter. Raised by the #4268 verify pass. --- swarm-nats-auth/src/policy.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/swarm-nats-auth/src/policy.rs b/swarm-nats-auth/src/policy.rs index 86a4d7038..e4a692772 100644 --- a/swarm-nats-auth/src/policy.rs +++ b/swarm-nats-auth/src/policy.rs @@ -748,6 +748,13 @@ mod tests { format!("$JS.API.STREAM.PURGE.{STREAM}"), format!("$JS.API.STREAM.MSG.DELETE.{STREAM}"), format!("$JS.API.CONSUMER.CREATE.{STREAM}"), + // A named consumer, bare and with a key filter, which a grant of + // `CONSUMER.CREATE..>` would cover. + format!("$JS.API.CONSUMER.CREATE.{STREAM}.c1"), + format!( + "$JS.API.CONSUMER.CREATE.{STREAM}.c1.{}", + key_subject("atlas") + ), ] { assert!(!may(&forbidden), "hive alpha may publish {forbidden}"); } From 05c9075825775ebc8156306875692493116fb157 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 12 Oct 2026 15:25:37 +0200 Subject: [PATCH 6/6] swarm-controller: refuse a percentage with a leading zero in its whole part systemd ignores `08%` and `085480%` for CPUQuota= and `08%` for MemoryMax= ("Invalid ... ignoring"), which leaves the agent uncapped. `hundredths` now refuses a whole part that starts with `0` unless it is exactly `0`, for both fields. `0.5%` and `0.01%` stay accepted. --- docs/scheduler/coordinator.md | 3 ++- swarm-controller/src/agent_limits.rs | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/scheduler/coordinator.md b/docs/scheduler/coordinator.md index 5118956f7..043c34b9e 100644 --- a/docs/scheduler/coordinator.md +++ b/docs/scheduler/coordinator.md @@ -493,7 +493,8 @@ caps. swarm-controller alone writes it: `PUT` refuses an agent no hive's wanted state declares, unknown fields, and any value systemd would ignore, which would leave the agent uncapped. A -percentage has at most two decimals and is above zero, up to `100%` for +percentage has at most two decimals, no leading zero in its whole part (it +refuses `08%`), and is above zero, up to `100%` for `memory_max` and `21474836.47%` for `cpu_quota`. A size is digits with an optional fraction and an optional `B`, `K`, `M`, `G` or `T` (upper case), at least one byte and below 2^64 - 1 bytes. The controller stores each value as diff --git a/swarm-controller/src/agent_limits.rs b/swarm-controller/src/agent_limits.rs index 3d33a9002..90351ab31 100644 --- a/swarm-controller/src/agent_limits.rs +++ b/swarm-controller/src/agent_limits.rs @@ -105,7 +105,9 @@ fn memory_max(value: &str) -> Result { } /// `value` as systemd reads a percentage, in hundredths of a percent: digits, -/// optionally `.` and one or two digits, then `%`. `None` for anything else. +/// optionally `.` and one or two digits, then `%`. `None` for anything else, +/// and for a whole part with a leading zero other than `0` itself: systemd +/// ignores `08%` and may read `010%` as octal. fn hundredths(value: &str) -> Option { let number = value.strip_suffix('%')?; let (int, frac) = match number.split_once('.') { @@ -113,7 +115,7 @@ fn hundredths(value: &str) -> Option { Some(_) => return None, None => (number, ""), }; - if !digits(int) { + if !digits(int) || (int.len() > 1 && int.starts_with('0')) { return None; } let frac = match frac.len() { @@ -324,7 +326,6 @@ mod tests { "0.01%", "1500%", "400.12%", - "05%", "21474836.47%", ] { assert_eq!(cpu_quota(ok).as_deref(), Ok(ok)); @@ -345,6 +346,11 @@ mod tests { "0.001%", "99999999999999999999%", "21474836.48%", + "05%", + "08%", + "085480%", + "010%", + "00.5%", "1.%", ".5%", "400%\nExecStartPre=/bin/sh -c id #%", @@ -391,6 +397,10 @@ mod tests { "150%", "100.01%", "12.345%", + "08%", + "085480%", + "010%", + "00.5%", "8X", "G", "8 G",