hive-c0re: the per-agent resource limit override is gone
The hive no longer sets an agent's CPU or memory cap: caps come from the swarm, else the hive-wide defaults. The container load tab was the only hive UI that set them, and its figures are already exported as OTEL metrics, so with the tab gone nothing wrote meta/resource-limits.json.
Removed: the file's reader and writer, the `hivectl agent set-limits` verb with its host-socket request, the `GET /api/container-resources` and `POST /api/resource-limits/{name}` dashboard endpoints, and the `cpu_quota`/`memory_max` fields of the container view that only the tab read. A leftover resource-limits.json is ignored.
This commit is contained in:
iris
mara
parent
a2540be98b
commit
90e33cbd19
19 changed files with 161 additions and 828 deletions
|
|
@ -1,5 +1,8 @@
|
|||
//! Per-agent CPU/memory limit overrides. Stored at
|
||||
//! `/var/lib/hyperhive/meta/resource-limits.json` alongside
|
||||
//! CPU/memory limits for an agent's container: the caps the swarm controller
|
||||
//! declared, and the validators for the values the systemd drop-in takes.
|
||||
//!
|
||||
//! The swarm caps are stored at
|
||||
//! `/var/lib/hyperhive/meta/swarm-resource-limits.json` alongside
|
||||
//! `topology.json`, `tool-groups.json` and `capabilities.json`.
|
||||
//!
|
||||
//! Format: a JSON object mapping agent name to an object with optional
|
||||
|
|
@ -14,9 +17,9 @@
|
|||
//!
|
||||
//! Fallback is **per field**: an absent file, an absent agent, or an
|
||||
//! absent field all fall back to the hive-wide
|
||||
//! `services.hyperhive.agentCpuQuota` / `agentMemoryMax`. So an agent
|
||||
//! can raise only its memory cap and keep tracking the hive default for
|
||||
//! CPU — see [`effective`].
|
||||
//! `services.hyperhive.agentCpuQuota` / `agentMemoryMax`. So a swarm cap on
|
||||
//! memory alone leaves CPU on the hive default — see
|
||||
//! [`effective_from_with_swarm`].
|
||||
//!
|
||||
//! Read path: `lifecycle::host_config::write_dropins`, on every spawn
|
||||
//! and every rebuild.
|
||||
|
|
@ -30,10 +33,8 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const RESOURCE_LIMITS_FILE: &str = "resource-limits.json";
|
||||
|
||||
/// One agent's overrides. Both fields optional and independent; `None`
|
||||
/// means "use the hive-wide default for this field".
|
||||
/// One agent's caps. Both fields optional and independent; `None` means
|
||||
/// "use the hive-wide default for this field".
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct AgentLimits {
|
||||
/// systemd `CPUQuota=` value, e.g. `"400%"`.
|
||||
|
|
@ -44,84 +45,6 @@ pub struct AgentLimits {
|
|||
pub memory_max: Option<String>,
|
||||
}
|
||||
|
||||
impl AgentLimits {
|
||||
/// True when neither field is set — such an entry is dropped rather
|
||||
/// than persisted as an empty object.
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.cpu_quota.is_none() && self.memory_max.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn resource_limits_path() -> PathBuf {
|
||||
crate::paths::meta_root().join(RESOURCE_LIMITS_FILE)
|
||||
}
|
||||
|
||||
/// Read the per-agent limit map. An absent file is the empty map —
|
||||
/// callers treat a missing entry as "hive-wide defaults". A file that
|
||||
/// exists but can't be read or parsed is an error: an override can be
|
||||
/// tighter than the hive default, so reading it as empty is not a safe
|
||||
/// fallback.
|
||||
pub fn read() -> std::io::Result<BTreeMap<String, AgentLimits>> {
|
||||
super::read_map(&resource_limits_path())
|
||||
}
|
||||
|
||||
/// Resolve the effective values for an agent from the override file at
|
||||
/// `path` (normally [`resource_limits_path`]), filling each unset field
|
||||
/// from the hive-wide default. This is the single place the fallback
|
||||
/// rule lives; `write_dropins` calls it and passes the result straight
|
||||
/// to systemd. Errors as [`read`] does.
|
||||
///
|
||||
/// Use [`effective_from`] when resolving more than one agent in a row.
|
||||
pub fn effective(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
hive_cpu_quota: &str,
|
||||
hive_memory_max: &str,
|
||||
) -> std::io::Result<(String, String)> {
|
||||
let limits = super::read_map(path)?;
|
||||
Ok(effective_from(
|
||||
&limits,
|
||||
name,
|
||||
hive_cpu_quota,
|
||||
hive_memory_max,
|
||||
))
|
||||
}
|
||||
|
||||
/// [`effective`] against an already-loaded map — the multi-agent form.
|
||||
///
|
||||
/// `container_view::build_all` renders every agent on each SSE scan, so
|
||||
/// it loads the map once and calls this per agent rather than re-reading
|
||||
/// the same small file N times per scan.
|
||||
#[must_use]
|
||||
pub fn effective_from(
|
||||
limits: &BTreeMap<String, AgentLimits>,
|
||||
name: &str,
|
||||
hive_cpu_quota: &str,
|
||||
hive_memory_max: &str,
|
||||
) -> (String, String) {
|
||||
match limits.get(name) {
|
||||
Some(l) => resolve(l, hive_cpu_quota, hive_memory_max),
|
||||
None => (hive_cpu_quota.to_owned(), hive_memory_max.to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure core of [`effective`], split out so the fallback matrix is
|
||||
/// testable without touching the filesystem.
|
||||
#[must_use]
|
||||
fn resolve(limits: &AgentLimits, hive_cpu_quota: &str, hive_memory_max: &str) -> (String, String) {
|
||||
let cpu = limits
|
||||
.cpu_quota
|
||||
.clone()
|
||||
.unwrap_or_else(|| hive_cpu_quota.to_owned());
|
||||
let mem = limits
|
||||
.memory_max
|
||||
.clone()
|
||||
.unwrap_or_else(|| hive_memory_max.to_owned());
|
||||
(cpu, mem)
|
||||
}
|
||||
|
||||
/// Parse a systemd `MemoryMax=`-style byte-size value (`"4G"`, `"512M"`,
|
||||
/// `"1048576B"`, a bare byte count, optionally with a decimal like `"1.5G"`)
|
||||
/// into a raw byte count, accepting exactly the single-term forms systemd's
|
||||
|
|
@ -164,35 +87,7 @@ pub fn parse_bytes(value: &str) -> Option<u64> {
|
|||
u64::try_from(bytes).ok()
|
||||
}
|
||||
|
||||
/// Set one agent's overrides and persist the map atomically. An entry
|
||||
/// with both fields unset is removed rather than stored, so "reset to
|
||||
/// hive defaults" and "never configured" are the same state on disk.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Fails without writing when the existing file can't be read or parsed
|
||||
/// (`InvalidData` for a parse failure), and otherwise when the meta dir
|
||||
/// or the file can't be written.
|
||||
pub fn set_limits(name: &str, limits: &AgentLimits) -> std::io::Result<()> {
|
||||
set_limits_at(&resource_limits_path(), name, limits)
|
||||
}
|
||||
|
||||
fn set_limits_at(path: &Path, name: &str, limits: &AgentLimits) -> std::io::Result<()> {
|
||||
let mut current: BTreeMap<String, AgentLimits> = super::read_map(path)?;
|
||||
if limits.is_empty() {
|
||||
current.remove(name);
|
||||
} else {
|
||||
current.insert(name.to_owned(), limits.clone());
|
||||
}
|
||||
super::write_map(path, ¤t)
|
||||
}
|
||||
|
||||
/// Caps the swarm controller declared for this hive's agents.
|
||||
///
|
||||
/// A separate file from the operator's overrides above, not a merge into it:
|
||||
/// the two have different writers and different lifetimes, and a swarm
|
||||
/// declaration landing in the file an operator edits would make "reset to
|
||||
/// hive defaults" destroy something the operator never set.
|
||||
const SWARM_LIMITS_FILE: &str = "swarm-resource-limits.json";
|
||||
|
||||
#[must_use]
|
||||
|
|
@ -201,9 +96,9 @@ pub fn swarm_limits_path() -> PathBuf {
|
|||
}
|
||||
|
||||
/// Read the swarm-declared caps. An absent file is the empty map — no swarm
|
||||
/// opinion, so the hive's own chain decides. A file that exists but can't be
|
||||
/// read or parsed is an error, for the reason [`read`] gives: reading it as
|
||||
/// empty would silently drop a cap.
|
||||
/// opinion, so the hive-wide defaults decide. A file that exists but can't be
|
||||
/// read or parsed is an error: a cap can be tighter than the hive default, so
|
||||
/// reading it as empty would silently drop it.
|
||||
pub fn read_swarm() -> std::io::Result<BTreeMap<String, AgentLimits>> {
|
||||
super::read_map(&swarm_limits_path())
|
||||
}
|
||||
|
|
@ -216,55 +111,43 @@ pub fn write_swarm(map: &BTreeMap<String, AgentLimits>) -> std::io::Result<()> {
|
|||
super::write_map(&swarm_limits_path(), map)
|
||||
}
|
||||
|
||||
/// [`effective_from`] with the swarm's declaration on top.
|
||||
/// Resolve the effective values for an agent from the swarm caps in `swarm`:
|
||||
/// each field the swarm declared a cap for, and the hive-wide default for
|
||||
/// each it did not. This is the single place the fallback rule lives;
|
||||
/// `write_dropins` calls it and passes the result straight to systemd.
|
||||
///
|
||||
/// A swarm cap **replaces** the hive's answer for that field rather than
|
||||
/// joining the fallback chain as another tier: a tier a hive can out-set is
|
||||
/// not a cap. Per field, so a swarm memory ceiling leaves CPU on the hive's
|
||||
/// chain.
|
||||
///
|
||||
/// The chain itself is untouched — [`effective_from`] is called exactly as
|
||||
/// before and its result is what a missing swarm value falls back to.
|
||||
/// Use this form when resolving more than one agent in a row.
|
||||
#[must_use]
|
||||
pub fn effective_from_with_swarm(
|
||||
pub fn effective_from(
|
||||
swarm: &BTreeMap<String, AgentLimits>,
|
||||
limits: &BTreeMap<String, AgentLimits>,
|
||||
name: &str,
|
||||
hive_cpu_quota: &str,
|
||||
hive_memory_max: &str,
|
||||
) -> (String, String) {
|
||||
let (cpu, mem) = effective_from(limits, name, hive_cpu_quota, hive_memory_max);
|
||||
match swarm.get(name) {
|
||||
Some(s) => (
|
||||
s.cpu_quota.clone().unwrap_or(cpu),
|
||||
s.memory_max.clone().unwrap_or(mem),
|
||||
),
|
||||
None => (cpu, mem),
|
||||
}
|
||||
let caps = swarm.get(name);
|
||||
(
|
||||
caps.and_then(|c| c.cpu_quota.clone())
|
||||
.unwrap_or_else(|| hive_cpu_quota.to_owned()),
|
||||
caps.and_then(|c| c.memory_max.clone())
|
||||
.unwrap_or_else(|| hive_memory_max.to_owned()),
|
||||
)
|
||||
}
|
||||
|
||||
/// [`effective_from_with_swarm`] for a caller resolving a single agent, from
|
||||
/// the operator overrides at `limits_path` and the swarm caps at `swarm_path`.
|
||||
///
|
||||
/// Delegates to [`effective`] rather than re-deriving the hive's answer, so
|
||||
/// there stays exactly one place the fallback chain lives. Errors as [`read`]
|
||||
/// does, for either file.
|
||||
pub fn effective_with_swarm(
|
||||
limits_path: &Path,
|
||||
/// [`effective_from`] for a caller resolving a single agent, from the swarm
|
||||
/// caps at `swarm_path`. Errors as [`read_swarm`] does.
|
||||
pub fn effective(
|
||||
swarm_path: &Path,
|
||||
name: &str,
|
||||
hive_cpu_quota: &str,
|
||||
hive_memory_max: &str,
|
||||
) -> std::io::Result<(String, String)> {
|
||||
let (cpu, mem) = effective(limits_path, name, hive_cpu_quota, hive_memory_max)?;
|
||||
let swarm: BTreeMap<String, AgentLimits> = super::read_map(swarm_path)?;
|
||||
Ok(match swarm.get(name) {
|
||||
Some(s) => (
|
||||
s.cpu_quota.clone().unwrap_or(cpu),
|
||||
s.memory_max.clone().unwrap_or(mem),
|
||||
),
|
||||
None => (cpu, mem),
|
||||
})
|
||||
Ok(effective_from(
|
||||
&swarm,
|
||||
name,
|
||||
hive_cpu_quota,
|
||||
hive_memory_max,
|
||||
))
|
||||
}
|
||||
|
||||
/// Validate a systemd `CPUQuota=` value. Percentages only, and values
|
||||
|
|
@ -386,96 +269,66 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn unset_agent_gets_hive_defaults() {
|
||||
let (cpu, mem) = resolve(&AgentLimits::default(), HIVE_CPU, HIVE_MEM);
|
||||
assert_eq!(cpu, "200%");
|
||||
assert_eq!(mem, "4G");
|
||||
fn an_agent_the_swarm_says_nothing_about_gets_the_hive_defaults() {
|
||||
let swarm = BTreeMap::from([("other".to_owned(), limits(Some("100%"), Some("1G")))]);
|
||||
for map in [BTreeMap::new(), swarm] {
|
||||
assert_eq!(
|
||||
effective_from(&map, "sock", HIVE_CPU, HIVE_MEM),
|
||||
(HIVE_CPU.to_owned(), HIVE_MEM.to_owned())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_fields_override() {
|
||||
let (cpu, mem) = resolve(&limits(Some("400%"), Some("8G")), HIVE_CPU, HIVE_MEM);
|
||||
assert_eq!(cpu, "400%");
|
||||
assert_eq!(mem, "8G");
|
||||
fn a_swarm_cap_replaces_the_hive_default() {
|
||||
let swarm = BTreeMap::from([("sock".to_owned(), limits(Some("150%"), Some("2G")))]);
|
||||
assert_eq!(
|
||||
effective_from(&swarm, "sock", HIVE_CPU, HIVE_MEM),
|
||||
("150%".to_owned(), "2G".to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
/// The point of per-field fallback: overriding memory must not drag
|
||||
/// CPU along with it.
|
||||
/// The point of per-field fallback: capping memory must not drag CPU
|
||||
/// along with it.
|
||||
#[test]
|
||||
fn partial_override_keeps_other_field_on_hive_default() {
|
||||
let (cpu, mem) = resolve(&limits(None, Some("8G")), HIVE_CPU, HIVE_MEM);
|
||||
assert_eq!(cpu, "200%", "cpu should still track the hive default");
|
||||
assert_eq!(mem, "8G");
|
||||
|
||||
let (cpu, mem) = resolve(&limits(Some("400%"), None), HIVE_CPU, HIVE_MEM);
|
||||
assert_eq!(cpu, "400%");
|
||||
assert_eq!(mem, "4G", "memory should still track the hive default");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_entry_is_reported_empty() {
|
||||
assert!(AgentLimits::default().is_empty());
|
||||
assert!(!limits(None, Some("8G")).is_empty());
|
||||
assert!(!limits(Some("400%"), None).is_empty());
|
||||
fn a_swarm_cap_is_per_field() {
|
||||
let swarm = BTreeMap::from([("sock".to_owned(), limits(None, Some("2G")))]);
|
||||
assert_eq!(
|
||||
effective_from(&swarm, "sock", HIVE_CPU, HIVE_MEM),
|
||||
(HIVE_CPU.to_owned(), "2G".to_owned()),
|
||||
"cpu should still track the hive default"
|
||||
);
|
||||
let swarm = BTreeMap::from([("sock".to_owned(), limits(Some("150%"), None))]);
|
||||
assert_eq!(
|
||||
effective_from(&swarm, "sock", HIVE_CPU, HIVE_MEM),
|
||||
("150%".to_owned(), HIVE_MEM.to_owned()),
|
||||
"memory should still track the hive default"
|
||||
);
|
||||
}
|
||||
|
||||
const TRUNCATED: &str =
|
||||
"{\n \"sock\": { \"cpu_quota\": \"50%\", \"memory_max\": \"1G\" },\n \"iris\": { \"mem";
|
||||
|
||||
fn corrupt_file() -> (tempfile::TempDir, PathBuf) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join(RESOURCE_LIMITS_FILE);
|
||||
std::fs::write(&path, TRUNCATED).expect("seed");
|
||||
(dir, path)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effective_of_a_corrupt_file_is_an_error() {
|
||||
let (_dir, path) = corrupt_file();
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join(SWARM_LIMITS_FILE);
|
||||
std::fs::write(&path, TRUNCATED).expect("seed");
|
||||
let err = effective(&path, "sock", HIVE_CPU, HIVE_MEM)
|
||||
.expect_err("a corrupt file must not resolve to hive defaults");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_limits_leaves_a_corrupt_file_untouched() {
|
||||
let (_dir, path) = corrupt_file();
|
||||
let err = set_limits_at(&path, "ruth", &limits(Some("400%"), None))
|
||||
.expect_err("a corrupt file must not be overwritten");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(std::fs::read(&path).expect("read"), TRUNCATED.as_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_file_resolves_to_hive_defaults() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join(RESOURCE_LIMITS_FILE);
|
||||
let path = dir.path().join(SWARM_LIMITS_FILE);
|
||||
let (cpu, mem) = effective(&path, "sock", HIVE_CPU, HIVE_MEM).expect("missing file");
|
||||
assert_eq!((cpu.as_str(), mem.as_str()), (HIVE_CPU, HIVE_MEM));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_limits_keeps_other_agents_and_leaves_no_temp_file() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join(RESOURCE_LIMITS_FILE);
|
||||
set_limits_at(&path, "sock", &limits(None, Some("8G"))).expect("set on missing file");
|
||||
set_limits_at(&path, "iris", &limits(Some("50%"), None)).expect("set");
|
||||
let (cpu, mem) = effective(&path, "sock", HIVE_CPU, HIVE_MEM).expect("read");
|
||||
assert_eq!((cpu.as_str(), mem.as_str()), (HIVE_CPU, "8G"));
|
||||
let (cpu, _) = effective(&path, "iris", HIVE_CPU, HIVE_MEM).expect("read");
|
||||
assert_eq!(cpu, "50%");
|
||||
let entries: Vec<_> = std::fs::read_dir(dir.path())
|
||||
.expect("read_dir")
|
||||
.map(|e| e.expect("entry").file_name())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
entries,
|
||||
vec![std::ffi::OsString::from(RESOURCE_LIMITS_FILE)]
|
||||
);
|
||||
}
|
||||
|
||||
/// Absent fields must deserialize to `None`, not fail — an entry
|
||||
/// written by an older version with only one field must still load.
|
||||
/// with only one field must still load.
|
||||
#[test]
|
||||
fn partial_entry_deserializes() {
|
||||
let map: BTreeMap<String, AgentLimits> =
|
||||
|
|
@ -605,52 +458,4 @@ mod tests {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_swarm_cap_replaces_the_hive_answer_rather_than_joining_the_chain() {
|
||||
// The whole point of the ruling: a tier a hive can out-set is not a
|
||||
// cap. The agent override below is the strongest hive-side value
|
||||
// there is, and the swarm still wins.
|
||||
let hive_side = BTreeMap::from([("sock".to_owned(), limits(Some("800%"), Some("16G")))]);
|
||||
let swarm = BTreeMap::from([("sock".to_owned(), limits(Some("150%"), Some("2G")))]);
|
||||
assert_eq!(
|
||||
effective_from_with_swarm(&swarm, &hive_side, "sock", HIVE_CPU, HIVE_MEM),
|
||||
("150%".to_owned(), "2G".to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_swarm_cap_is_per_field_like_every_other_tier() {
|
||||
// A swarm that caps memory and says nothing about CPU leaves CPU on
|
||||
// the hive's chain — here the per-agent override, not the hive default.
|
||||
let hive_side = BTreeMap::from([("sock".to_owned(), limits(Some("800%"), None))]);
|
||||
let swarm = BTreeMap::from([("sock".to_owned(), limits(None, Some("2G")))]);
|
||||
assert_eq!(
|
||||
effective_from_with_swarm(&swarm, &hive_side, "sock", HIVE_CPU, HIVE_MEM),
|
||||
("800%".to_owned(), "2G".to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_swarm_map_leaves_the_existing_chain_bit_for_bit() {
|
||||
// The control that makes the two tests above mean something: the
|
||||
// wrapper must be a no-op when the swarm has said nothing, for every
|
||||
// shape the old chain distinguishes. If this drifts, the swarm tier
|
||||
// changed behaviour it was explicitly told not to change.
|
||||
let none = BTreeMap::new();
|
||||
let cases = [
|
||||
BTreeMap::new(),
|
||||
BTreeMap::from([("sock".to_owned(), limits(Some("400%"), Some("8G")))]),
|
||||
BTreeMap::from([("sock".to_owned(), limits(Some("400%"), None))]),
|
||||
BTreeMap::from([("sock".to_owned(), limits(None, Some("8G")))]),
|
||||
BTreeMap::from([("other".to_owned(), limits(Some("100%"), Some("1G")))]),
|
||||
];
|
||||
for hive_side in &cases {
|
||||
assert_eq!(
|
||||
effective_from_with_swarm(&none, hive_side, "sock", HIVE_CPU, HIVE_MEM),
|
||||
effective_from(hive_side, "sock", HIVE_CPU, HIVE_MEM),
|
||||
"the wrapper changed the answer with no swarm value present"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue