diff --git a/docs/persistence.md b/docs/persistence.md index 3bc7d87b..c5261e7f 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -309,6 +309,16 @@ Contents: (`{ "atlas": ["read_host_journal"] }`). Written by `capabilities::set_caps`; injected as `HIVE_CAPABILITIES` env var. Absent agents have no extra capabilities. +- `resource-limits.json` — per-agent container resource overrides + (`{ "sock": { "cpu_quota": "400%", "memory_max": "8G" } }`). + Written by `resource_limits::set_limits`; read where the systemd + drop-in is generated (`lifecycle::write_dropins`), **not** injected + into the container — these are host-side caps on the container, so + the capped party never sees or sets them. Fallback is per *field*: + an absent file, absent agent, or absent field falls back to the + hive-wide `services.hyperhive.agentCpuQuota` / `agentMemoryMax`, + so an agent can override only its memory and still track the hive + default for CPU. The root agent has the meta dir RO-mounted at `/meta/`. diff --git a/docs/tools/hivectl-cli.md b/docs/tools/hivectl-cli.md index 6031b7f5..28aaeda4 100644 --- a/docs/tools/hivectl-cli.md +++ b/docs/tools/hivectl-cli.md @@ -32,6 +32,7 @@ This document contains the help content for the `hivectl` command-line program. * [`hivectl agents destroy`↴](#hivectl-agents-destroy) * [`hivectl agents rebuild`↴](#hivectl-agents-rebuild) * [`hivectl agents set-parent`↴](#hivectl-agents-set-parent) +* [`hivectl agents set-limits`↴](#hivectl-agents-set-limits) * [`hivectl approvals`↴](#hivectl-approvals) * [`hivectl approvals pending`↴](#hivectl-approvals-pending) * [`hivectl approvals approve`↴](#hivectl-approvals-approve) @@ -349,6 +350,7 @@ Lifecycle actions on managed agent containers. Needs the hive-c0re daemon runnin * `destroy` — Tear down a sub-agent container, keeping its state by default. No undo * `rebuild` — Apply pending config to a managed container * `set-parent` — Move an agent in the topology tree — under a new parent, or to root +* `set-limits` — Declare an agent's CPU/memory limits, overriding the hive-wide defaults @@ -501,6 +503,26 @@ Move an agent in the topology tree — under a new parent, or to root +## `hivectl agents set-limits` + +Declare an agent's CPU/memory limits, overriding the hive-wide defaults. + +Replaces the agent's whole override entry rather than merging into it: any limit you don't pass returns to the hive-wide default. To change one and keep the other, pass both. + +**Usage:** `hivectl agents set-limits [OPTIONS] ` + +###### **Arguments:** + +* `` — Agent name + +###### **Options:** + +* `--cpu-quota ` — systemd `CPUQuota=` value, e.g. `400%` (100% = one full core) +* `--memory-max ` — systemd `MemoryMax=` value, e.g. `8G`, `50%`, or `infinity` +* `--reset` — Drop all overrides — the agent returns to the hive-wide defaults. Required to clear limits, so that a `set-limits` with a forgotten value can't silently reset the agent + + + ## `hivectl approvals` Operator approval queue: list, approve, or deny pending requests. diff --git a/docs/tools/hivectl.md b/docs/tools/hivectl.md index 15806453..ffadcd87 100644 --- a/docs/tools/hivectl.md +++ b/docs/tools/hivectl.md @@ -178,6 +178,34 @@ resume drains the backlog rather than dropping it. Points worth knowing: - Visible as ` paused` in `agents list`'s STATUS column, as a `paused` field on the JSON rows, and as a badge on the dashboard card. +### Per-agent resource limits + +```bash +hivectl agents set-limits sock --cpu-quota 400% --memory-max 8G +hivectl agents set-limits sock --memory-max 8G # CPU falls back to the hive default +hivectl agents set-limits sock --reset # drop all overrides +``` + +Overrides the hive-wide `services.hyperhive.agentCpuQuota` / +`agentMemoryMax` for one agent, persisted to +`meta/resource-limits.json` (see +[`persistence.md`](../persistence.md)). Values are systemd's +`CPUQuota=` / `MemoryMax=` syntax: a percentage (`400%` = four full +cores) for CPU; a size (`8G`), a percentage of physical RAM, or +`infinity` for memory. Both are validated before they're persisted — +they go into a systemd drop-in verbatim, and a typo there makes the +unit fail to start. + +**Declarative, not incremental**: each invocation replaces the agent's +whole entry. `set-limits sock --memory-max 8G` leaves `sock` with *only* +a memory override, reverting any previously-set CPU quota to the hive +default. To avoid a forgotten flag silently wiping an override, a bare +`set-limits ` with no flags is rejected — clearing requires the +explicit `--reset`. + +The command rewrites the container's drop-in and reloads systemd, so +new containers and restarts pick the values up immediately. + ## Choom Drop into an interactive Claude session inside an agent container. diff --git a/hive-c0re/src/agent_config/mod.rs b/hive-c0re/src/agent_config/mod.rs index 7ef28556..8fc2807b 100644 --- a/hive-c0re/src/agent_config/mod.rs +++ b/hive-c0re/src/agent_config/mod.rs @@ -1,10 +1,15 @@ //! Per-agent configuration registries: tool groups, capabilities, -//! topology (all JSON files under `/var/lib/hyperhive/meta/`) and the -//! shared wire-protocol size limits. Each submodule is re-exported at -//! the crate root, so `crate::topology::…` etc. keep working -//! unchanged. +//! resource limits, topology (all JSON files under +//! `/var/lib/hyperhive/meta/`) and the shared wire-protocol size +//! limits. Each submodule is re-exported at the crate root, so +//! `crate::topology::…` etc. keep working unchanged. +//! +//! Note the two similarly-named modules: [`limits`] caps inline +//! *message body* sizes on the sockets, while [`resource_limits`] holds +//! per-agent CPU/memory caps for the container drop-in. pub mod capabilities; pub mod limits; +pub mod resource_limits; pub mod tool_groups; pub mod topology; diff --git a/hive-c0re/src/agent_config/resource_limits.rs b/hive-c0re/src/agent_config/resource_limits.rs new file mode 100644 index 00000000..31be2d69 --- /dev/null +++ b/hive-c0re/src/agent_config/resource_limits.rs @@ -0,0 +1,312 @@ +//! Per-agent CPU/memory limit overrides. Stored at +//! `/var/lib/hyperhive/meta/resource-limits.json` alongside +//! `topology.json`, `tool-groups.json` and `capabilities.json`. +//! +//! Format: a JSON object mapping agent name to an object with optional +//! `cpu_quota` / `memory_max` strings, passed verbatim to systemd's +//! `CPUQuota=` / `MemoryMax=` in the per-container drop-in: +//! +//! ```json +//! { +//! "sock": { "cpu_quota": "400%", "memory_max": "8G" } +//! } +//! ``` +//! +//! Fallback is **per field**: an absent file, an absent agent, or an +//! absent field all fall back to the hive-wide +//! `services.hyperhive.agentCpuQuota` / `agentMemoryMax`. So an agent +//! can raise only its memory cap and keep tracking the hive default for +//! CPU — see [`effective`]. +//! +//! Read path: `lifecycle::host_config::write_dropins`, on every spawn +//! and every rebuild. +//! +//! Why host-side JSON and not an option in the agent's own `agent.nix`: +//! the drop-in lands on the *host's* `container@h-.service`, so +//! c0re would have to `nix eval` the agent's whole nixosConfiguration +//! just to read two strings. It is also the wrong trust boundary — +//! a resource *cap* should not be sourced from the capped party. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +const RESOURCE_LIMITS_FILE: &str = "resource-limits.json"; + +/// One agent's overrides. Both fields optional and independent; `None` +/// means "use the hive-wide default for this field". +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct AgentLimits { + /// systemd `CPUQuota=` value, e.g. `"400%"`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpu_quota: Option, + /// systemd `MemoryMax=` value, e.g. `"8G"`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_max: Option, +} + +impl AgentLimits { + /// True when neither field is set — such an entry is dropped rather + /// than persisted as an empty object. + #[must_use] + pub fn is_empty(&self) -> bool { + self.cpu_quota.is_none() && self.memory_max.is_none() + } +} + +#[must_use] +pub fn resource_limits_path() -> PathBuf { + crate::paths::meta_root().join(RESOURCE_LIMITS_FILE) +} + +/// Read the per-agent limit map. Returns an empty map when the file is +/// absent or unparsable — callers treat a missing entry as "hive-wide +/// defaults", which is also the safe failure mode for a malformed file. +#[must_use] +pub fn read() -> BTreeMap { + let path = resource_limits_path(); + let Ok(raw) = std::fs::read_to_string(&path) else { + return BTreeMap::new(); + }; + serde_json::from_str(&raw).unwrap_or_default() +} + +/// Look up one agent's overrides. Returns the all-`None` default when +/// the agent has no entry. +#[must_use] +pub fn limits_for(name: &str) -> AgentLimits { + read().get(name).cloned().unwrap_or_default() +} + +/// Resolve the effective values for an agent, filling each unset field +/// from the hive-wide default. This is the single place the fallback +/// rule lives; `write_dropins` calls it and passes the result straight +/// to systemd. +#[must_use] +pub fn effective(name: &str, hive_cpu_quota: &str, hive_memory_max: &str) -> (String, String) { + resolve(&limits_for(name), hive_cpu_quota, hive_memory_max) +} + +/// Pure core of [`effective`], split out so the fallback matrix is +/// testable without touching the filesystem. +#[must_use] +fn resolve(limits: &AgentLimits, hive_cpu_quota: &str, hive_memory_max: &str) -> (String, String) { + let cpu = limits + .cpu_quota + .clone() + .unwrap_or_else(|| hive_cpu_quota.to_owned()); + let mem = limits + .memory_max + .clone() + .unwrap_or_else(|| hive_memory_max.to_owned()); + (cpu, mem) +} + +/// Persist the full map. Sorted JSON output keeps meta-repo diffs +/// minimal. +/// +/// # Errors +/// +/// Returns an `io::Error` when the meta dir can't be created or the +/// file can't be written (permissions, disk full). Serialization +/// failure is surfaced as `InvalidData`, though it can't happen for +/// this type — it's a plain map of strings. +pub fn write(map: &BTreeMap) -> std::io::Result<()> { + let path = resource_limits_path(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let text = serde_json::to_string_pretty(map) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + std::fs::write(&path, format!("{text}\n")) +} + +/// Set one agent's overrides and persist. An entry with both fields +/// unset is removed rather than stored, so "reset to hive defaults" and +/// "never configured" are the same state on disk. +/// +/// # Errors +/// +/// Propagates whatever [`write`] fails with. An unreadable or malformed +/// existing file is *not* an error — [`read`] degrades to an empty map, +/// so this call rewrites the file from scratch. +pub fn set_limits(name: &str, limits: &AgentLimits) -> std::io::Result<()> { + let mut current = read(); + if limits.is_empty() { + current.remove(name); + } else { + current.insert(name.to_owned(), limits.clone()); + } + write(¤t) +} + +/// Validate a systemd `CPUQuota=` value. Percentages only, and values +/// above 100% are legitimate (one full core is 100%). +/// +/// Validated because the value is written verbatim into a drop-in: a +/// typo makes systemd reject the unit, which means the container stops +/// starting at all. Better to refuse at the CLI than to brick a spawn. +/// +/// # Errors +/// +/// Returns a human-readable message naming the offending value when it +/// isn't a percentage. The string is surfaced straight to the operator, +/// so it names the expected shape rather than just saying "invalid". +pub fn validate_cpu_quota(value: &str) -> Result<(), String> { + if is_percentage(value) { + return Ok(()); + } + Err(format!( + "invalid CPUQuota {value:?}: expected a percentage such as \"200%\" \ + (100% = one full core)" + )) +} + +/// Validate a systemd `MemoryMax=` value: a byte count with an optional +/// `K`/`M`/`G`/`T` suffix, a percentage of physical memory, or the +/// literal `infinity`. +/// +/// # Errors +/// +/// Returns a human-readable message naming the offending value and the +/// three accepted shapes. Same operator-facing contract as +/// [`validate_cpu_quota`]. +pub fn validate_memory_max(value: &str) -> Result<(), String> { + if value == "infinity" || is_percentage(value) || is_byte_size(value) { + return Ok(()); + } + Err(format!( + "invalid MemoryMax {value:?}: expected a size such as \"8G\", a percentage \ + such as \"50%\", or \"infinity\"" + )) +} + +/// A decimal number followed by `%`. +fn is_percentage(value: &str) -> bool { + value.strip_suffix('%').is_some_and(is_plain_number) +} + +/// A decimal number with an optional single-letter binary suffix. +fn is_byte_size(value: &str) -> bool { + let mantissa = value + .strip_suffix(['K', 'M', 'G', 'T', 'k', 'm', 'g', 't']) + .unwrap_or(value); + is_plain_number(mantissa) +} + +/// Digits, optionally followed by a single `.` and more digits. Hand +/// rolled rather than pulling in a regex dependency for two patterns; +/// deliberately rejects the exponent/sign forms `f64::from_str` accepts, +/// since systemd wouldn't take them either. +fn is_plain_number(value: &str) -> bool { + let mut parts = value.splitn(2, '.'); + let int = parts.next().unwrap_or_default(); + if int.is_empty() || !int.bytes().all(|b| b.is_ascii_digit()) { + return false; + } + match parts.next() { + None => true, + Some(frac) => !frac.is_empty() && frac.bytes().all(|b| b.is_ascii_digit()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const HIVE_CPU: &str = "200%"; + const HIVE_MEM: &str = "4G"; + + fn limits(cpu: Option<&str>, mem: Option<&str>) -> AgentLimits { + AgentLimits { + cpu_quota: cpu.map(ToOwned::to_owned), + memory_max: mem.map(ToOwned::to_owned), + } + } + + #[test] + fn unset_agent_gets_hive_defaults() { + let (cpu, mem) = resolve(&AgentLimits::default(), HIVE_CPU, HIVE_MEM); + assert_eq!(cpu, "200%"); + assert_eq!(mem, "4G"); + } + + #[test] + fn both_fields_override() { + let (cpu, mem) = resolve(&limits(Some("400%"), Some("8G")), HIVE_CPU, HIVE_MEM); + assert_eq!(cpu, "400%"); + assert_eq!(mem, "8G"); + } + + /// The point of per-field fallback: overriding memory must not drag + /// CPU along with it. + #[test] + fn partial_override_keeps_other_field_on_hive_default() { + let (cpu, mem) = resolve(&limits(None, Some("8G")), HIVE_CPU, HIVE_MEM); + assert_eq!(cpu, "200%", "cpu should still track the hive default"); + assert_eq!(mem, "8G"); + + let (cpu, mem) = resolve(&limits(Some("400%"), None), HIVE_CPU, HIVE_MEM); + assert_eq!(cpu, "400%"); + assert_eq!(mem, "4G", "memory should still track the hive default"); + } + + #[test] + fn empty_entry_is_reported_empty() { + assert!(AgentLimits::default().is_empty()); + assert!(!limits(None, Some("8G")).is_empty()); + assert!(!limits(Some("400%"), None).is_empty()); + } + + #[test] + fn malformed_json_reads_as_empty_map() { + let parsed: BTreeMap = + serde_json::from_str("{ not json").unwrap_or_default(); + assert!(parsed.is_empty()); + } + + /// Absent fields must deserialize to `None`, not fail — an entry + /// written by an older version with only one field must still load. + #[test] + fn partial_entry_deserializes() { + let map: BTreeMap = + serde_json::from_str(r#"{"sock":{"memory_max":"8G"}}"#).expect("parses"); + assert_eq!(map["sock"], limits(None, Some("8G"))); + } + + #[test] + fn empty_fields_are_not_serialized() { + let map = BTreeMap::from([("sock".to_owned(), limits(None, Some("8G")))]); + let text = serde_json::to_string(&map).expect("serializes"); + assert_eq!(text, r#"{"sock":{"memory_max":"8G"}}"#); + } + + #[test] + fn accepts_valid_cpu_quotas() { + for v in ["100%", "200%", "400%", "50%", "12.5%"] { + assert!(validate_cpu_quota(v).is_ok(), "{v} should be valid"); + } + } + + #[test] + fn rejects_invalid_cpu_quotas() { + for v in ["", "200", "%", "abc", "200%%", "-50%", "2e2%", "200 %"] { + assert!(validate_cpu_quota(v).is_err(), "{v} should be rejected"); + } + } + + #[test] + fn accepts_valid_memory_maxes() { + for v in [ + "8G", "512M", "1024", "2T", "4096K", "50%", "infinity", "1.5G", + ] { + assert!(validate_memory_max(v).is_ok(), "{v} should be valid"); + } + } + + #[test] + fn rejects_invalid_memory_maxes() { + for v in ["", "8GB", "G", "abc", "-8G", "8 G", "Infinity", "8Gi"] { + assert!(validate_memory_max(v).is_err(), "{v} should be rejected"); + } + } +} diff --git a/hive-c0re/src/container_view.rs b/hive-c0re/src/container_view.rs index 78be3b87..f312020e 100644 --- a/hive-c0re/src/container_view.rs +++ b/hive-c0re/src/container_view.rs @@ -62,11 +62,26 @@ pub struct ContainerView { /// a legitimate way to keep it idle when it next boots. #[serde(default)] pub paused: bool, + /// Effective systemd `CPUQuota=` for this container (e.g. `"400%"`) — + /// the per-agent override from `meta/resource-limits.json` when set, + /// otherwise the hive-wide `agentCpuQuota`. Always populated: there + /// is no "unset" state to render, only "same as everyone else". + /// Reflects what the *drop-in says*, which is what the next start + /// will enforce — not a live cgroup reading. + pub cpu_quota: String, + /// Effective systemd `MemoryMax=` for this container (e.g. `"8G"`). + /// Same resolution + caveat as [`ContainerView::cpu_quota`]. + pub memory_max: String, } /// Build the full container list. Wraps `lifecycle::list()` and /// resolves every per-agent attribute the dashboard surfaces. -pub async fn build_all() -> Vec { +/// +/// Takes `hive` because the effective resource limits are a per-field +/// fallback onto the hive-wide `agent_cpu_quota` / `agent_memory_max`, +/// and those live on [`HiveEnv`], not on disk. Both callers already +/// hold a `Coordinator`, so this is a parameter rather than a global. +pub async fn build_all(hive: &crate::coordinator::HiveEnv) -> Vec { let raw = lifecycle::list().await.unwrap_or_default(); let locked = read_meta_locked_revs(); // Pull the topology map once and look up each agent's parent below. @@ -108,6 +123,11 @@ pub async fn build_all() -> Vec { None }; let paused = Coordinator::is_paused(&logical); + let (cpu_quota, memory_max) = crate::resource_limits::effective( + logical.as_str(), + &hive.agent_cpu_quota, + &hive.agent_memory_max, + ); out.push(ContainerView { port: lifecycle::agent_web_port(logical.as_str()), running, @@ -119,6 +139,8 @@ pub async fn build_all() -> Vec { parent, active_model, paused, + cpu_quota, + memory_max, }); } out diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index e1cbd9ad..f9848e16 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -904,7 +904,7 @@ impl Coordinator { /// Cheap when nothing changed (one `nixos-container list` + a /// `HashMap` diff + zero emits). pub async fn rescan_containers_and_emit(self: &Arc) { - let fresh = container_view::build_all().await; + let fresh = container_view::build_all(&self.hive_env()).await; let mut last = self.last_containers.lock().await; let mut changed_or_new = Vec::new(); let mut removed = Vec::new(); diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index 39195f96..341b27f1 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -23,13 +23,17 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re validate(name)?; let container = container_name(name); set_nspawn_flags(&container, &paths.agent, &paths.claude, &paths.notes).await?; - set_resource_limits(&container, &hive.agent_cpu_quota, &hive.agent_memory_max).await?; + let (cpu_quota, memory_max) = + crate::resource_limits::effective(name, &hive.agent_cpu_quota, &hive.agent_memory_max); + set_resource_limits(&container, &cpu_quota, &memory_max).await?; systemd_daemon_reload().await } /// Write a systemd drop-in for `container@.service` that applies -/// our default resource caps. Goes under `/run/systemd/system/...` so it's -/// ephemeral (regenerated on every spawn / rebuild). +/// the agent's effective resource caps — its per-agent overrides from +/// `meta/resource-limits.json` where set, the hive-wide defaults +/// otherwise. Goes under `/run/systemd/system/...` so it's ephemeral +/// (regenerated on every spawn / rebuild). async fn set_resource_limits(container: &str, cpu_quota: &str, memory_max: &str) -> Result<()> { crate::priv_client::write_resource_limits(container, memory_max, cpu_quota).await } diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index f8e2870d..d869cd48 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -37,7 +37,7 @@ mod stores; mod webhook_secret; mod workers; -pub(crate) use agent_config::{capabilities, limits, tool_groups, topology}; +pub(crate) use agent_config::{capabilities, limits, resource_limits, tool_groups, topology}; pub(crate) use stats::{ container_stats, hive_stats, host_stats, otel_metrics, sweep_health, warnings, }; diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 66796212..c17a90bd 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -176,6 +176,12 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { if crate::capabilities::capabilities_path().exists() { git(&dir, &["add", "capabilities.json"]).await?; } + // Stage resource-limits.json when it exists. Created on first + // `set_limits` call; absent = every agent on the hive-wide + // CPU/memory defaults. + if crate::resource_limits::resource_limits_path().exists() { + git(&dir, &["add", "resource-limits.json"]).await?; + } // Stage roles.json when it exists. Written by topology::write_roles / // reconcile_roles on first role assignment or manager default seeding. // Without this, roles.json appears as untracked in the meta repo @@ -208,6 +214,7 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { f if f.starts_with("peer-ca-") && has_pem_ext(f) => Some("peer-ca"), "topology.json" => Some("topology"), "capabilities.json" => Some("capabilities"), + "resource-limits.json" => Some("resource-limits"), "tool-groups.json" => Some("tool-groups"), "roles.json" => Some("roles"), _ => None, @@ -473,6 +480,41 @@ pub async fn commit_capabilities(agent: &str, caps: &[String]) -> Result<()> { Ok(()) } +/// Write the resource-limits file for `agent` and commit it atomically +/// under `META_LOCK`. Same rationale as `commit_tool_groups`: the +/// working tree must never be left dirty for the next `prepare_deploy` +/// or `sync_agents` to trip over. +/// +/// Unlike the perm files this one is never injected into the container — +/// it's a host-side cap *on* the agent — but it lives in the same repo +/// so a limit change gets the same auditable one-commit-per-change trail. +/// +/// # Errors +/// +/// Returns an error if writing the JSON file fails or a git stage/commit +/// step fails. +pub async fn commit_resource_limits( + agent: &str, + limits: &crate::resource_limits::AgentLimits, +) -> Result<()> { + let _guard = META_LOCK.lock().await; + crate::resource_limits::set_limits(agent, limits) + .map_err(|e| anyhow::anyhow!("set resource limits for {agent}: {e}"))?; + let dir = crate::paths::meta_root(); + if crate::resource_limits::resource_limits_path().exists() { + git(&dir, &["add", "resource-limits.json"]).await?; + } + if paths_dirty(&dir, &["resource-limits.json"]).await? { + git_commit_paths( + &dir, + &format!("set resource limits for {agent}"), + &["resource-limits.json"], + ) + .await?; + } + Ok(()) +} + /// Write both perm files for `agent` (whichever are `Some`) and commit /// them in a SINGLE git commit under `META_LOCK` — the batch /// `POST /api/permissions` path. A `None` field leaves that file diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index 9aa00a37..63bc4381 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -162,7 +162,7 @@ async fn dispatch(req: &HostRequest, coord: Arc) -> HostResponse { HostResponse::dags(dags) } HostRequest::List => HostResponse::list(lifecycle::list().await?), - HostRequest::AgentStatus => handle_agent_status().await, + HostRequest::AgentStatus => handle_agent_status(&coord).await, // The hive domain + per-surface public URLs are injected into // c0re's service env by hive-c0re.nix; surface them so the // operator CLI can fill in this hive's own identity (the @@ -194,6 +194,19 @@ async fn dispatch(req: &HostRequest, coord: Arc) -> HostResponse { .map_err(anyhow::Error::msg)?; HostResponse::success() } + HostRequest::SetResourceLimits { + name, + cpu_quota, + memory_max, + } => { + handle_set_resource_limits( + &coord, + name, + cpu_quota.as_deref(), + memory_max.as_deref(), + ) + .await? + } HostRequest::MatrixCreateUser { name, password } => { handle_matrix_create_user(name, password.as_deref()).await? } @@ -324,8 +337,8 @@ async fn handle_set_paused( } /// Collect per-agent status rows for `hivectl status` and the dashboard. -async fn handle_agent_status() -> HostResponse { - let rows = crate::container_view::build_all() +async fn handle_agent_status(coord: &Arc) -> HostResponse { + let rows = crate::container_view::build_all(&coord.hive_env()) .await .into_iter() .map(|v| hive_sh4re::AgentStatusRow { @@ -376,6 +389,59 @@ fn agent_exists(name: &hive_types::Ident) -> Result { .with_context(|| format!("check agent state dir for {name}")) } +/// Validate + persist an agent's CPU/memory overrides, then re-apply the +/// drop-in so the change lands without waiting for a rebuild. +/// +/// Validation is here rather than only in `hivectl` because the values +/// are written verbatim into the systemd drop-in: a malformed +/// `CPUQuota=` makes systemd reject the unit, and the container stops +/// starting. Every client (CLI, dashboard, anything later) goes through +/// this path, so the guard belongs on this side of the socket. +/// +/// `None`/`None` removes the agent's entry, returning it to the +/// hive-wide defaults. +async fn handle_set_resource_limits( + coord: &Arc, + name: &hive_types::Ident, + cpu_quota: Option<&str>, + memory_max: Option<&str>, +) -> Result { + if let Some(value) = cpu_quota { + crate::resource_limits::validate_cpu_quota(value).map_err(anyhow::Error::msg)?; + } + if let Some(value) = memory_max { + crate::resource_limits::validate_memory_max(value).map_err(anyhow::Error::msg)?; + } + tracing::info!(%name, ?cpu_quota, ?memory_max, "set_resource_limits"); + let limits = crate::resource_limits::AgentLimits { + cpu_quota: cpu_quota.map(ToOwned::to_owned), + memory_max: memory_max.map(ToOwned::to_owned), + }; + // Goes through `meta::commit_resource_limits`, not the bare + // `resource_limits::set_limits`: the write has to be staged + + // committed under `META_LOCK` or it leaves the meta working tree + // dirty for the next `prepare_deploy` / `sync_agents` to trip over. + crate::meta::commit_resource_limits(name.as_str(), &limits).await?; + + // Re-apply the drop-in straight away — same three lines as the job + // queue's `WriteDropin` node. Without this the new values would sit + // in the JSON until the agent's next spawn or rebuild. + let agent_dir = crate::paths::agent_runtime_dir(name.as_str()); + let hive = coord.hive_env(); + let paths = Coordinator::agent_paths(name.as_str(), agent_dir); + crate::lifecycle::write_dropins(name.as_str(), &hive, &paths).await?; + + let (cpu, mem) = crate::resource_limits::effective( + name.as_str(), + &hive.agent_cpu_quota, + &hive.agent_memory_max, + ); + Ok(HostResponse::messages(vec![format!( + "{name}: CPUQuota={cpu} MemoryMax={mem} (restart the container if it is running \ + and the new caps need to take effect immediately)" + )])) +} + /// Guard: matrix provisioning needs the homeserver container running. async fn require_matrix_present() -> Result<()> { if crate::matrix::is_present().await { diff --git a/hive-c0re/src/stats/host_stats.rs b/hive-c0re/src/stats/host_stats.rs index 53985762..059a882c 100644 --- a/hive-c0re/src/stats/host_stats.rs +++ b/hive-c0re/src/stats/host_stats.rs @@ -253,6 +253,8 @@ mod tests { parent: None, active_model: None, paused: false, + cpu_quota: "200%".to_owned(), + memory_max: "4G".to_owned(), } } diff --git a/hive-host-sock/src/lib.rs b/hive-host-sock/src/lib.rs index ac8fbbb7..2b145b82 100644 --- a/hive-host-sock/src/lib.rs +++ b/hive-host-sock/src/lib.rs @@ -167,6 +167,23 @@ pub enum HostRequest { child: Ident, new_parent: Option, }, + /// Declare an agent's CPU/memory overrides for the per-container + /// systemd drop-in, persisted to `meta/resource-limits.json`. + /// + /// **Replace, not merge** — the pair given here becomes the agent's + /// entire entry, matching how tool-groups/capabilities are set. A + /// `None` field falls back to the hive-wide + /// `agentCpuQuota` / `agentMemoryMax`, so passing both as `None` + /// removes the entry entirely (reset to hive defaults). + /// + /// Values are passed verbatim to systemd, so the server validates + /// their shape before persisting: a malformed `CPUQuota=` makes + /// systemd reject the unit, which would stop the container starting. + SetResourceLimits { + name: Ident, + cpu_quota: Option, + memory_max: Option, + }, /// Stop managed containers hive-wide in one operator action /// (`hivectl stop`): agents plus the selected infra containers. `scope` /// selects which classes; an all-false scope means **everything** (the diff --git a/hivectl/src/agents.rs b/hivectl/src/agents.rs index 0c689a93..897634bd 100644 --- a/hivectl/src/agents.rs +++ b/hivectl/src/agents.rs @@ -201,5 +201,32 @@ pub(crate) async fn run_agents(socket: &Path, cmd: AgentsCmd) -> Result<()> { .await?, ) } + AgentsCmd::SetLimits { + name, + cpu_quota, + memory_max, + reset, + } => { + let name = crate::util::parse_ident(&name)?; + // `--reset` is the only way to reach an all-`None` request; + // clap rejects a bare `set-limits ` with neither flag, + // so a forgotten value can't silently clear the overrides. + let (cpu_quota, memory_max) = if reset { + (None, None) + } else { + (cpu_quota, memory_max) + }; + render( + crate::client::request( + socket, + HostRequest::SetResourceLimits { + name, + cpu_quota, + memory_max, + }, + ) + .await?, + ) + } } } diff --git a/hivectl/src/cli.rs b/hivectl/src/cli.rs index a87c0908..d81a965e 100644 --- a/hivectl/src/cli.rs +++ b/hivectl/src/cli.rs @@ -559,6 +559,30 @@ pub enum AgentsCmd { #[arg(long)] root: bool, }, + /// Declare an agent's CPU/memory limits, overriding the hive-wide defaults. + /// + /// Replaces the agent's whole override entry rather than merging into + /// it: any limit you don't pass returns to the hive-wide default. To + /// change one and keep the other, pass both. + SetLimits { + /// Agent name. + name: String, + /// systemd `CPUQuota=` value, e.g. `400%` (100% = one full core). + #[arg(long, conflicts_with = "reset")] + cpu_quota: Option, + /// systemd `MemoryMax=` value, e.g. `8G`, `50%`, or `infinity`. + #[arg(long, conflicts_with = "reset")] + memory_max: Option, + /// Drop all overrides — the agent returns to the hive-wide + /// defaults. Required to clear limits, so that a `set-limits` + /// with a forgotten value can't silently reset the agent. + #[arg( + long, + conflicts_with_all = ["cpu_quota", "memory_max"], + required_unless_present_any = ["cpu_quota", "memory_max"], + )] + reset: bool, + }, } /// Operator approval queue: list, approve, or deny pending requests.