hive-c0re: fail on an unparseable resource-limits or topology file, write both atomically
resource-limits.json and topology.json were read with parse errors folded into an empty map, and written in place with std::fs::write. One truncated resource-limits.json followed by a single set_limits call rewrote the file with only that agent's entry, erasing every other agent's CPU and memory overrides without a log line. topology.json had the same shape: reconcile rebuilt it from the live set, losing pending (provisioned, never spawned) names. - agent_config::read_map / write_map are generic over the stored type. tool-groups and capabilities behave as before. - resource_limits::read / effective return an error for an existing but unreadable file; a missing file is still the empty map. set_limits fails without writing on such a file, and writes atomically. - topology: reconcile fails without writing on an unreadable file and writes atomically. all_agents logs the error and returns no agents, so a ManageRootAgent holder starts without cross-agent mounts. Read-path behaviour on an unreadable resource-limits.json, per caller: - write_dropins (every spawn / swap / WriteDropin): logs the error and keeps the limits drop-in already under /run; the agent still starts. With no drop-in yet (first start since boot) it writes the hive defaults, because no drop-in means an uncapped container. - render_flake: propagates, so sync_agents (and spawn/rebuild/destroy jobs) fail. An empty map would give tighter-capped agents the hive memoryMaxBytes. - container_view::build_all: logs the error each scan and renders the rows at the hive defaults (no ContainerView wire change). - set_resource_limits reply: propagates. Closes #4731
This commit is contained in:
parent
97cf8a1b2b
commit
6fac00dcc5
8 changed files with 332 additions and 119 deletions
|
|
@ -28,7 +28,7 @@
|
|||
//! a resource *cap* should not be sourced from the capped party.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const RESOURCE_LIMITS_FILE: &str = "resource-limits.json";
|
||||
|
||||
|
|
@ -58,28 +58,35 @@ pub fn resource_limits_path() -> PathBuf {
|
|||
crate::paths::meta_root().join(RESOURCE_LIMITS_FILE)
|
||||
}
|
||||
|
||||
/// Read the per-agent limit map. Returns an empty map when the file is
|
||||
/// absent or unparsable — callers treat a missing entry as "hive-wide
|
||||
/// defaults", which is also the safe failure mode for a malformed file.
|
||||
#[must_use]
|
||||
pub fn read() -> BTreeMap<String, AgentLimits> {
|
||||
let path = resource_limits_path();
|
||||
let Ok(raw) = std::fs::read_to_string(&path) else {
|
||||
return BTreeMap::new();
|
||||
};
|
||||
serde_json::from_str(&raw).unwrap_or_default()
|
||||
/// Read the per-agent limit map. An absent file is the empty map —
|
||||
/// callers treat a missing entry as "hive-wide defaults". A file that
|
||||
/// exists but can't be read or parsed is an error: an override can be
|
||||
/// tighter than the hive default, so reading it as empty is not a safe
|
||||
/// fallback.
|
||||
pub fn read() -> std::io::Result<BTreeMap<String, AgentLimits>> {
|
||||
super::read_map(&resource_limits_path())
|
||||
}
|
||||
|
||||
/// Resolve the effective values for an agent, filling each unset field
|
||||
/// Resolve the effective values for an agent from the override file at
|
||||
/// `path` (normally [`resource_limits_path`]), filling each unset field
|
||||
/// from the hive-wide default. This is the single place the fallback
|
||||
/// rule lives; `write_dropins` calls it and passes the result straight
|
||||
/// to systemd.
|
||||
/// to systemd. Errors as [`read`] does.
|
||||
///
|
||||
/// Reads the override file. Use [`effective_from`] when resolving more
|
||||
/// than one agent in a row.
|
||||
#[must_use]
|
||||
pub fn effective(name: &str, hive_cpu_quota: &str, hive_memory_max: &str) -> (String, String) {
|
||||
effective_from(&read(), name, hive_cpu_quota, hive_memory_max)
|
||||
/// Use [`effective_from`] when resolving more than one agent in a row.
|
||||
pub fn effective(
|
||||
path: &Path,
|
||||
name: &str,
|
||||
hive_cpu_quota: &str,
|
||||
hive_memory_max: &str,
|
||||
) -> std::io::Result<(String, String)> {
|
||||
let limits = super::read_map(path)?;
|
||||
Ok(effective_from(
|
||||
&limits,
|
||||
name,
|
||||
hive_cpu_quota,
|
||||
hive_memory_max,
|
||||
))
|
||||
}
|
||||
|
||||
/// [`effective`] against an already-loaded map — the multi-agent form.
|
||||
|
|
@ -200,42 +207,27 @@ pub fn parse_bytes(value: &str) -> Option<u64> {
|
|||
u64::try_from(bytes).ok()
|
||||
}
|
||||
|
||||
/// Persist the full map. Sorted JSON output keeps meta-repo diffs
|
||||
/// minimal.
|
||||
/// Set one agent's overrides and persist the map atomically. An entry
|
||||
/// with both fields unset is removed rather than stored, so "reset to
|
||||
/// hive defaults" and "never configured" are the same state on disk.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an `io::Error` when the meta dir can't be created or the
|
||||
/// file can't be written (permissions, disk full). Serialization
|
||||
/// failure is surfaced as `InvalidData`, though it can't happen for
|
||||
/// this type — it's a plain map of strings.
|
||||
pub fn write(map: &BTreeMap<String, AgentLimits>) -> std::io::Result<()> {
|
||||
let path = resource_limits_path();
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let text = serde_json::to_string_pretty(map)
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
std::fs::write(&path, format!("{text}\n"))
|
||||
/// Fails without writing when the existing file can't be read or parsed
|
||||
/// (`InvalidData` for a parse failure), and otherwise when the meta dir
|
||||
/// or the file can't be written.
|
||||
pub fn set_limits(name: &str, limits: &AgentLimits) -> std::io::Result<()> {
|
||||
set_limits_at(&resource_limits_path(), name, limits)
|
||||
}
|
||||
|
||||
/// Set one agent's overrides and persist. An entry with both fields
|
||||
/// unset is removed rather than stored, so "reset to hive defaults" and
|
||||
/// "never configured" are the same state on disk.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Propagates whatever the `write` function fails with. An unreadable or malformed
|
||||
/// existing file is *not* an error — the `read` function degrades to an empty map,
|
||||
/// so this call rewrites the file from scratch.
|
||||
pub fn set_limits(name: &str, limits: &AgentLimits) -> std::io::Result<()> {
|
||||
let mut current = read();
|
||||
fn set_limits_at(path: &Path, name: &str, limits: &AgentLimits) -> std::io::Result<()> {
|
||||
let mut current: BTreeMap<String, AgentLimits> = super::read_map(path)?;
|
||||
if limits.is_empty() {
|
||||
current.remove(name);
|
||||
} else {
|
||||
current.insert(name.to_owned(), limits.clone());
|
||||
}
|
||||
write(¤t)
|
||||
super::write_map(path, ¤t)
|
||||
}
|
||||
|
||||
/// Validate a systemd `CPUQuota=` value. Percentages only, and values
|
||||
|
|
@ -390,11 +382,59 @@ mod tests {
|
|||
assert!(!limits(Some("400%"), None).is_empty());
|
||||
}
|
||||
|
||||
const TRUNCATED: &str =
|
||||
"{\n \"sock\": { \"cpu_quota\": \"50%\", \"memory_max\": \"1G\" },\n \"iris\": { \"mem";
|
||||
|
||||
fn corrupt_file() -> (tempfile::TempDir, PathBuf) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join(RESOURCE_LIMITS_FILE);
|
||||
std::fs::write(&path, TRUNCATED).expect("seed");
|
||||
(dir, path)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_json_reads_as_empty_map() {
|
||||
let parsed: BTreeMap<String, AgentLimits> =
|
||||
serde_json::from_str("{ not json").unwrap_or_default();
|
||||
assert!(parsed.is_empty());
|
||||
fn effective_of_a_corrupt_file_is_an_error() {
|
||||
let (_dir, path) = corrupt_file();
|
||||
let err = effective(&path, "sock", HIVE_CPU, HIVE_MEM)
|
||||
.expect_err("a corrupt file must not resolve to hive defaults");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_limits_leaves_a_corrupt_file_untouched() {
|
||||
let (_dir, path) = corrupt_file();
|
||||
let err = set_limits_at(&path, "ruth", &limits(Some("400%"), None))
|
||||
.expect_err("a corrupt file must not be overwritten");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(std::fs::read(&path).expect("read"), TRUNCATED.as_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_file_resolves_to_hive_defaults() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join(RESOURCE_LIMITS_FILE);
|
||||
let (cpu, mem) = effective(&path, "sock", HIVE_CPU, HIVE_MEM).expect("missing file");
|
||||
assert_eq!((cpu.as_str(), mem.as_str()), (HIVE_CPU, HIVE_MEM));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_limits_keeps_other_agents_and_leaves_no_temp_file() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join(RESOURCE_LIMITS_FILE);
|
||||
set_limits_at(&path, "sock", &limits(None, Some("8G"))).expect("set on missing file");
|
||||
set_limits_at(&path, "iris", &limits(Some("50%"), None)).expect("set");
|
||||
let (cpu, mem) = effective(&path, "sock", HIVE_CPU, HIVE_MEM).expect("read");
|
||||
assert_eq!((cpu.as_str(), mem.as_str()), (HIVE_CPU, "8G"));
|
||||
let (cpu, _) = effective(&path, "iris", HIVE_CPU, HIVE_MEM).expect("read");
|
||||
assert_eq!(cpu, "50%");
|
||||
let entries: Vec<_> = std::fs::read_dir(dir.path())
|
||||
.expect("read_dir")
|
||||
.map(|e| e.expect("entry").file_name())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
entries,
|
||||
vec![std::ffi::OsString::from(RESOURCE_LIMITS_FILE)]
|
||||
);
|
||||
}
|
||||
|
||||
/// Absent fields must deserialize to `None`, not fail — an entry
|
||||
|
|
|
|||
Loading…
Reference in a new issue