budgeted agents throttle at 90% of the cap; alert on repeated oom kills
This commit is contained in:
atlas
parent
c014b48c22
commit
11393bc538
10 changed files with 201 additions and 27 deletions
|
|
@ -446,11 +446,12 @@ changed cap reaches them at the next container start without a rebuild.
|
|||
`hyperhive-memory-cap-check.service` logs a warning naming the path in the
|
||||
container's journal. The container's `MemoryMax=` still applies.
|
||||
|
||||
A container with a finite `MemoryMax=` also gets `MemorySwapMax=0` in the same
|
||||
drop-in, so a budgeted agent never swaps: paging it out to host swap would
|
||||
move the pressure onto the host. `MemoryMax=infinity` leaves swap alone. Like
|
||||
the rest of the drop-in, it reaches a running agent at its next spawn or
|
||||
rebuild.
|
||||
A container with a finite `MemoryMax=` also gets, in the same drop-in,
|
||||
`MemorySwapMax=0` and `MemoryHigh=` at 90% of the cap (bytes, rounded down).
|
||||
A budgeted agent never swaps: paging it out to host swap would move the
|
||||
pressure onto the host. Past 90% the kernel slows the container and reclaims
|
||||
from it, and at 100% it kills. `MemoryMax=infinity` sets neither. Like the
|
||||
rest of the drop-in, they reach a running agent at its next spawn or rebuild.
|
||||
|
||||
When the container reaches its cap, the kernel kills inside it, in this order:
|
||||
|
||||
|
|
@ -474,6 +475,8 @@ is the memory the container has swapped out: nonzero only for an agent with
|
|||
no budget, or one whose container started before its drop-in carried
|
||||
`MemorySwapMax=0`.
|
||||
|
||||
More than 3 kills of one agent in an hour raise the `hh-agent-oom-kills` alert.
|
||||
|
||||
<details><summary>Implementation: swarm caps</summary>
|
||||
|
||||
swarm-controller keeps the caps in the `agent-limits` KV bucket, one key per
|
||||
|
|
|
|||
|
|
@ -560,6 +560,7 @@ The swarm Grafana (`nix/host-modules/swarm-grafana.nix`) alerts on these:
|
|||
| `hh-hive-degraded` | `hyperhive.hive.degraded` is `1` for 2 minutes |
|
||||
| `hh-hive-warning` | any `hyperhive.hive.warning` series other than `disk_pressure` exists (`pending_logins` only for a hive that doesn't export `hyperhive.agent.needs_login`) for 2 minutes |
|
||||
| `hh-agent-needs-login` | `hyperhive.agent.needs_login` is `1` for an agent for 5 minutes, one alert per agent |
|
||||
| `hh-agent-oom-kills` | `hyperhive.container.memory.oom_kills` grew by more than 3 in the last hour for an agent, one alert per agent, no delay |
|
||||
| `hh-host-disk-high` | `hyperhive.hive.nix_store.utilization` stays above 0.85 for 10 minutes |
|
||||
| `hh-host-disk-critical` | `hyperhive.hive.nix_store.utilization` stays above 0.95 for 2 minutes |
|
||||
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ known operations; there is no arbitrary command pass-through:
|
|||
| `ReloadGatewayNginx` | `systemctl reload/start/reset-failed nginx` (host unit; the unit name is hard-coded, not a parameter) |
|
||||
| `WriteNspawnFlags` | write `/etc/nixos-containers/<container>.conf` (bind-mount list, network isolation vars, and `--ephemeral` after the root's salvage). hive-priv first copies a stopped container's root to the agent's `work/`, then renames it aside (it deletes nothing), following no link the container could have planted; for an agent whose closure record sets `private_users` it also adds `--private-users=pick`, mounts every bind `owneridmap`, and once chowns the agent's `state`, `harness`, `work` and `claude` dirs (never `config`, following no link) to the data owner |
|
||||
| `EnsureAgentSocketDir` | `mkdirat` `/run/hive-agent/<name>` `0751 root:root` under an `O_NOFOLLOW` parent fd; leaves an existing directory alone, refuses anything else; for an agent whose closure record sets `private_users` it then chowns the dir to the data owner |
|
||||
| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=`/`CPUWeight=`/`IOWeight=` systemd drop-in for agent container, with an `ExecStartPre=` that runs `hive-priv check-rootfs <container>` (no-follow check of the root paths the start script touches; see persistence.md), and a GC root on the build it names |
|
||||
| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=`/`MemoryHigh=`/`MemorySwapMax=`/`CPUWeight=`/`IOWeight=` systemd drop-in for agent container, with an `ExecStartPre=` that runs `hive-priv check-rootfs <container>` (no-follow check of the root paths the start script touches; see persistence.md), and a GC root on the build it names |
|
||||
| `RemoveServiceDropin` | remove `container@<name>.service.d/` drop-in on destroy, and the GC root of the hive-priv build it named |
|
||||
| `DaemonReload` | `systemctl daemon-reload` |
|
||||
| `ControlInfraContainer` | `systemctl <action> nginx.service` for the gateway — the `InfraContainer` enum is the allowlist, and serde rejects unknown names at the wire boundary (`hive-c0re`, the forge and the matrix have no variant, so no request can name them) |
|
||||
|
|
|
|||
|
|
@ -25,13 +25,14 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re
|
|||
validate(name)?;
|
||||
let container = container_name(name);
|
||||
set_nspawn_flags(&container, hive, &paths.agent, &paths.claude, &paths.notes).await?;
|
||||
if let Some((cpu_quota, memory_max)) = stage_limits(
|
||||
let host = HostMemory::read();
|
||||
if let Some((cpu_quota, memory_max, memory_high)) = stage_limits(
|
||||
name,
|
||||
&crate::resource_limits::swarm_limits_path(),
|
||||
&limits_dropin_path(&container),
|
||||
&paths.agent,
|
||||
hive,
|
||||
HostMemory::read(),
|
||||
host,
|
||||
)? {
|
||||
set_resource_limits(
|
||||
&container,
|
||||
|
|
@ -39,6 +40,7 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re
|
|||
&memory_max,
|
||||
hive.agent_cpu_weight,
|
||||
hive.agent_io_weight,
|
||||
memory_high,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
|
@ -94,6 +96,8 @@ impl HostMemory {
|
|||
/// systemd enforces on it, so this file is how it learns the value it sizes
|
||||
/// its heap and its subagents against. It is written on every call, from the
|
||||
/// `MemoryMax=` the agent will start under: the new one, or the kept drop-in's.
|
||||
///
|
||||
/// The limits to apply come with the `MemoryHigh=` byte count for their cap.
|
||||
fn stage_limits(
|
||||
agent_name: &str,
|
||||
swarm_path: &Path,
|
||||
|
|
@ -101,7 +105,7 @@ fn stage_limits(
|
|||
runtime_dir: &Path,
|
||||
hive: &HiveEnv,
|
||||
host: Option<HostMemory>,
|
||||
) -> Result<Option<(String, String)>> {
|
||||
) -> Result<Option<(String, String, Option<u64>)>> {
|
||||
let limits = limits_to_apply(agent_name, swarm_path, dropin_path, hive);
|
||||
let memory_max = match &limits {
|
||||
Some((_, memory_max)) => Some(memory_max.clone()),
|
||||
|
|
@ -109,7 +113,10 @@ fn stage_limits(
|
|||
};
|
||||
let cap = memory_max.and_then(|m| memory_cap_value(agent_name, &m, host));
|
||||
write_memory_cap(runtime_dir, cap.as_deref())?;
|
||||
Ok(limits)
|
||||
Ok(limits.map(|(cpu_quota, memory_max)| {
|
||||
let high = memory_high(agent_name, &memory_max, host);
|
||||
(cpu_quota, memory_max, high)
|
||||
}))
|
||||
}
|
||||
|
||||
/// The `MemoryMax=` the drop-in at `path` applies: its last assignment, as
|
||||
|
|
@ -169,6 +176,16 @@ fn memory_cap_value(
|
|||
)
|
||||
}
|
||||
|
||||
/// The `MemoryHigh=` byte count for an agent capped at `memory_max`: 90% of the
|
||||
/// cap, rounded down, so the kernel slows the container and reclaims from it
|
||||
/// before `MemoryMax=` kills in it. `None` when the agent has no cap.
|
||||
fn memory_high(agent_name: &str, memory_max: &str, host: Option<HostMemory>) -> Option<u64> {
|
||||
let cap = memory_cap_value(agent_name, memory_max, host)?
|
||||
.parse::<u64>()
|
||||
.ok()?;
|
||||
u64::try_from(u128::from(cap) * 9 / 10).ok()
|
||||
}
|
||||
|
||||
/// `percent` (the part before `%`) in hundredths of a percent, with systemd's
|
||||
/// `parse_permyriad` rules: digits, at most two decimals, at most 100%.
|
||||
fn permyriad(percent: &str) -> Option<u64> {
|
||||
|
|
@ -277,9 +294,15 @@ async fn set_resource_limits(
|
|||
memory_max: &str,
|
||||
cpu_weight: Option<u32>,
|
||||
io_weight: Option<u32>,
|
||||
memory_high: Option<u64>,
|
||||
) -> Result<()> {
|
||||
crate::priv_client::write_resource_limits(
|
||||
container, memory_max, cpu_quota, cpu_weight, io_weight,
|
||||
container,
|
||||
memory_max,
|
||||
cpu_quota,
|
||||
cpu_weight,
|
||||
io_weight,
|
||||
memory_high,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
|
@ -577,7 +600,7 @@ mod tests {
|
|||
use super::{
|
||||
BindMount, HostMemory, MEMORY_CAP_FILE, base_binds, bind_child_agent_dirs,
|
||||
bind_own_agent_dirs, holds_manage_root_agent, limits_to_apply, memory_cap_value,
|
||||
stage_limits,
|
||||
memory_high, stage_limits,
|
||||
};
|
||||
use crate::coordinator::HiveEnv;
|
||||
use hive_types::NixBuilds;
|
||||
|
|
@ -893,7 +916,7 @@ mod tests {
|
|||
)
|
||||
.expect("stage limits");
|
||||
let copy = std::fs::read_to_string(runtime.join(MEMORY_CAP_FILE)).ok();
|
||||
(applied.map(|(_, mem)| mem), copy)
|
||||
(applied.map(|(_, mem, _)| mem), copy)
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -969,6 +992,52 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memory_high_is_ninety_percent_of_the_cap_rounded_down() {
|
||||
for (memory_max, high) in [
|
||||
("4G", Some(3_865_470_566)),
|
||||
("1048576B", Some(943_718)),
|
||||
("50%", Some(7_372_803_686)),
|
||||
// No cap, or a value systemd ignores: nothing to throttle below.
|
||||
("infinity", None),
|
||||
("0", None),
|
||||
("150%", None),
|
||||
("lots", None),
|
||||
] {
|
||||
assert_eq!(
|
||||
memory_high("sock", memory_max, Some(HOST)),
|
||||
high,
|
||||
"{memory_max}"
|
||||
);
|
||||
}
|
||||
// A percentage needs the host's memory; a size does not.
|
||||
assert_eq!(memory_high("sock", "50%", None), None);
|
||||
assert_eq!(memory_high("sock", "8G", None), Some(7_730_941_132));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn staged_limits_carry_the_throttle_point_of_their_cap() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let swarm = swarm_file(&dir);
|
||||
let staged = |dir: &tempfile::TempDir| {
|
||||
stage_limits(
|
||||
"sock",
|
||||
&swarm,
|
||||
&dir.path().join("hyperhive-limits.conf"),
|
||||
&dir.path().join("runtime"),
|
||||
&HiveEnv::default(),
|
||||
Some(HOST),
|
||||
)
|
||||
.expect("stage limits")
|
||||
.map(|(_, mem, high)| (mem, high))
|
||||
};
|
||||
// The hive default, 4G.
|
||||
assert_eq!(staged(&dir), Some(("4G".to_owned(), Some(3_865_470_566))));
|
||||
// Presence control above; an uncapped agent has no throttle point.
|
||||
std::fs::write(&swarm, r#"{"sock": {"memory_max": "infinity"}}"#).expect("seed swarm caps");
|
||||
assert_eq!(staged(&dir), Some(("infinity".to_owned(), None)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_host_memory_systemd_would_read_is_found() {
|
||||
let host = HostMemory::read().expect("sysconf");
|
||||
|
|
@ -1027,7 +1096,7 @@ mod tests {
|
|||
let runtime = dir.path().join("runtime");
|
||||
let applied = stage_limits("sock", &swarm, &dropin, &runtime, &HiveEnv::default(), None)
|
||||
.expect("stage limits");
|
||||
assert_eq!(applied.map(|(_, mem)| mem), Some("50%".to_owned()));
|
||||
assert_eq!(applied.map(|(_, mem, _)| mem), Some("50%".to_owned()));
|
||||
assert!(!runtime.join(MEMORY_CAP_FILE).exists());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -292,9 +292,15 @@ pub async fn write_resource_limits(
|
|||
cpu_quota: &str,
|
||||
cpu_weight: Option<u32>,
|
||||
io_weight: Option<u32>,
|
||||
memory_high: Option<u64>,
|
||||
) -> Result<()> {
|
||||
ok(call(&resource_limits_request(
|
||||
container, memory_max, cpu_quota, cpu_weight, io_weight,
|
||||
container,
|
||||
memory_max,
|
||||
cpu_quota,
|
||||
cpu_weight,
|
||||
io_weight,
|
||||
memory_high,
|
||||
))
|
||||
.await?)
|
||||
}
|
||||
|
|
@ -305,6 +311,7 @@ fn resource_limits_request(
|
|||
cpu_quota: &str,
|
||||
cpu_weight: Option<u32>,
|
||||
io_weight: Option<u32>,
|
||||
memory_high: Option<u64>,
|
||||
) -> PrivRequest {
|
||||
PrivRequest::WriteResourceLimits {
|
||||
container: container.to_owned(),
|
||||
|
|
@ -312,6 +319,7 @@ fn resource_limits_request(
|
|||
cpu_quota: cpu_quota.to_owned(),
|
||||
cpu_weight,
|
||||
io_weight,
|
||||
memory_high,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -654,6 +662,7 @@ mod tests {
|
|||
"200%",
|
||||
Some(50),
|
||||
None,
|
||||
Some(3_865_470_566),
|
||||
));
|
||||
assert_eq!(got["op"], "write_resource_limits");
|
||||
assert_eq!(got["container"], "alice");
|
||||
|
|
@ -661,6 +670,7 @@ mod tests {
|
|||
assert_eq!(got["cpu_quota"], "200%");
|
||||
assert_eq!(got["cpu_weight"], 50);
|
||||
assert_eq!(got["io_weight"], Value::Null);
|
||||
assert_eq!(got["memory_high"], 3_865_470_566_u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -383,7 +383,8 @@ pub enum PrivRequest {
|
|||
|
||||
/// Write `/run/systemd/system/container@<container>.service.d/hyperhive-limits.conf`
|
||||
/// with `[Service]` carrying `MemoryMax=` / `CPUQuota=` (hard caps) and
|
||||
/// `CPUWeight=` / `IOWeight=` (cgroup v2 relative shares, contention-only),
|
||||
/// `MemoryHigh=` (throttle point below `MemoryMax=`), `CPUWeight=` / `IOWeight=`
|
||||
/// (cgroup v2 relative shares, contention-only),
|
||||
/// and an `ExecStartPre=` that runs `hive-priv check-rootfs <container>`
|
||||
/// to refuse a start over a root with links where the start script
|
||||
/// expects directories. The build that line names is rooted under
|
||||
|
|
@ -404,6 +405,12 @@ pub enum PrivRequest {
|
|||
/// `cpu_weight`.
|
||||
#[serde(default)]
|
||||
io_weight: Option<u32>,
|
||||
/// `MemoryHigh=` in bytes: the point where the kernel throttles the
|
||||
/// container and reclaims from it before `memory_max` kills in it.
|
||||
/// `None` omits the line, which is what a hive-c0re built before this
|
||||
/// field existed sends, and what an unbudgeted container gets.
|
||||
#[serde(default)]
|
||||
memory_high: Option<u64>,
|
||||
},
|
||||
|
||||
/// Remove `/run/systemd/system/container@<container>.service.d/` if present,
|
||||
|
|
|
|||
|
|
@ -156,6 +156,7 @@ pub(crate) fn write_resource_limits(
|
|||
cpu_quota: &str,
|
||||
cpu_weight: Option<u32>,
|
||||
io_weight: Option<u32>,
|
||||
memory_high: Option<u64>,
|
||||
) -> Result<(String, String)> {
|
||||
let logical = validate_container_system_name(container)?;
|
||||
validate_memory_max(memory_max)?;
|
||||
|
|
@ -176,6 +177,7 @@ pub(crate) fn write_resource_limits(
|
|||
cpu_quota,
|
||||
cpu_weight,
|
||||
io_weight,
|
||||
memory_high,
|
||||
);
|
||||
// Published whole: a `daemon-reload` between truncate and write would
|
||||
// otherwise load the unit with its limits and start guard missing.
|
||||
|
|
@ -226,6 +228,10 @@ const RESTART_SEC: u32 = 5;
|
|||
/// entirely, so a request from a hive-c0re built before the weights
|
||||
/// existed — or one whose nix option is `null` — renders no weight lines.
|
||||
///
|
||||
/// `memory_high` is the byte count of `MemoryHigh=`: the throttle point below
|
||||
/// the cap, where the kernel slows the container and reclaims from it. `None`
|
||||
/// omits the line.
|
||||
///
|
||||
/// A finite `MemoryMax=` comes with `MemorySwapMax=0`: at the budget the
|
||||
/// kernel kills inside the container instead of paging it out to host swap.
|
||||
/// `MemoryMax=infinity` is no budget and leaves swap alone.
|
||||
|
|
@ -236,12 +242,14 @@ fn limits_dropin_body(
|
|||
cpu_quota: &str,
|
||||
cpu_weight: Option<u32>,
|
||||
io_weight: Option<u32>,
|
||||
memory_high: Option<u64>,
|
||||
) -> String {
|
||||
// Built as two possibly-empty lines rather than pushed onto the
|
||||
// string: `format!` appended to a `String` trips clippy::pedantic's
|
||||
// `format_push_string`, and a `write!` would need an unwrap.
|
||||
let cpu_weight_line = cpu_weight.map_or_else(String::new, |w| format!("CPUWeight={w}\n"));
|
||||
let io_weight_line = io_weight.map_or_else(String::new, |w| format!("IOWeight={w}\n"));
|
||||
let high_line = memory_high.map_or_else(String::new, |b| format!("MemoryHigh={b}\n"));
|
||||
let swap_line = if memory_max == "infinity" {
|
||||
""
|
||||
} else {
|
||||
|
|
@ -257,6 +265,7 @@ fn limits_dropin_body(
|
|||
ExecStartPre={rootfs_check}\n\
|
||||
RestartSec={RESTART_SEC}\n\
|
||||
MemoryMax={memory_max}\n\
|
||||
{high_line}\
|
||||
{swap_line}\
|
||||
CPUQuota={cpu_quota}\n\
|
||||
{cpu_weight_line}{io_weight_line}"
|
||||
|
|
@ -892,6 +901,7 @@ mod tests {
|
|||
"4G",
|
||||
"200%",
|
||||
None,
|
||||
None,
|
||||
None
|
||||
),
|
||||
"[Unit]\n\
|
||||
|
|
@ -919,7 +929,7 @@ mod tests {
|
|||
("512M", "50%", Some(10), Some(10)),
|
||||
("infinity", "infinity", Some(10_000), None),
|
||||
] {
|
||||
let body = limits_dropin_body("/rt/x", CHECK, mem, cpu, cw, iw);
|
||||
let body = limits_dropin_body("/rt/x", CHECK, mem, cpu, cw, iw, None);
|
||||
let unit = body
|
||||
.split("[Service]")
|
||||
.next()
|
||||
|
|
@ -940,38 +950,59 @@ mod tests {
|
|||
#[test]
|
||||
fn swap_is_off_exactly_when_the_memory_is_budgeted() {
|
||||
for mem in ["4G", "512M", "50%", "8589934592"] {
|
||||
let body = limits_dropin_body("/rt/x", CHECK, mem, "200%", None, None);
|
||||
let body = limits_dropin_body("/rt/x", CHECK, mem, "200%", None, None, None);
|
||||
assert!(
|
||||
body.contains(&format!("\nMemoryMax={mem}\nMemorySwapMax=0\n")),
|
||||
"no swap line under a budget of {mem}: {body}"
|
||||
);
|
||||
assert_eq!(body.matches("MemorySwapMax=").count(), 1, "{body}");
|
||||
}
|
||||
let unbudgeted = limits_dropin_body("/rt/x", CHECK, "infinity", "200%", None, None);
|
||||
let unbudgeted = limits_dropin_body("/rt/x", CHECK, "infinity", "200%", None, None, None);
|
||||
assert!(
|
||||
unbudgeted.contains("\nMemoryMax=infinity\n"),
|
||||
"{unbudgeted}"
|
||||
);
|
||||
assert!(!unbudgeted.contains("MemorySwapMax"), "{unbudgeted}");
|
||||
}
|
||||
|
||||
/// `MemoryHigh=` is the byte count the caller sends, between `MemoryMax=`
|
||||
/// and the swap line, and absent when none is sent.
|
||||
#[test]
|
||||
fn memory_high_is_rendered_only_when_sent() {
|
||||
let with = limits_dropin_body(
|
||||
"/rt/x",
|
||||
CHECK,
|
||||
"4G",
|
||||
"200%",
|
||||
None,
|
||||
None,
|
||||
Some(3_865_470_566),
|
||||
);
|
||||
assert!(
|
||||
with.contains("\nMemoryMax=4G\nMemoryHigh=3865470566\nMemorySwapMax=0\n"),
|
||||
"{with}"
|
||||
);
|
||||
let without = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, None, None);
|
||||
assert!(!without.contains("MemoryHigh"), "{without}");
|
||||
}
|
||||
/// Weights are appended to the `[Service]` section, each omitted
|
||||
/// independently when `None`.
|
||||
#[test]
|
||||
fn weights_are_emitted_only_when_set() {
|
||||
let both = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), Some(80));
|
||||
let both = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), Some(80), None);
|
||||
assert!(
|
||||
both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"),
|
||||
"{both}"
|
||||
);
|
||||
|
||||
let cpu_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), None);
|
||||
let cpu_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), None, None);
|
||||
assert!(
|
||||
cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"),
|
||||
"{cpu_only}"
|
||||
);
|
||||
assert!(!cpu_only.contains("IOWeight"), "{cpu_only}");
|
||||
|
||||
let io_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, Some(80));
|
||||
let io_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, Some(80), None);
|
||||
assert!(
|
||||
io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"),
|
||||
"{io_only}"
|
||||
|
|
@ -984,7 +1015,7 @@ mod tests {
|
|||
#[test]
|
||||
fn the_rootfs_check_leads_the_service_section_whatever_the_caller_passes() {
|
||||
for (cw, iw) in [(None, None), (Some(80), Some(80))] {
|
||||
let body = limits_dropin_body("/rt/x", CHECK, "4G", "200%", cw, iw);
|
||||
let body = limits_dropin_body("/rt/x", CHECK, "4G", "200%", cw, iw, None);
|
||||
let service = body
|
||||
.split("[Service]\n")
|
||||
.nth(1)
|
||||
|
|
@ -1405,7 +1436,7 @@ mod tests {
|
|||
let errs = [
|
||||
handle_write_nspawn_flags(name, &[], &isolation, &[], &facts, None, None)
|
||||
.unwrap_err(),
|
||||
write_resource_limits(name, "4G", "200%", None, None).unwrap_err(),
|
||||
write_resource_limits(name, "4G", "200%", None, None, None).unwrap_err(),
|
||||
remove_service_dropin(name).unwrap_err(),
|
||||
];
|
||||
for e in errs {
|
||||
|
|
@ -1471,7 +1502,7 @@ mod tests {
|
|||
"$(id)",
|
||||
"%t",
|
||||
] {
|
||||
let err = write_resource_limits(PROBE, bad, "200%", None, None).unwrap_err();
|
||||
let err = write_resource_limits(PROBE, bad, "200%", None, None, None).unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("invalid memory_max"),
|
||||
"{bad:?}: {err:#}"
|
||||
|
|
@ -1491,7 +1522,7 @@ mod tests {
|
|||
"4G",
|
||||
"%t",
|
||||
] {
|
||||
let err = write_resource_limits(PROBE, "4G", bad, None, None).unwrap_err();
|
||||
let err = write_resource_limits(PROBE, "4G", bad, None, None, None).unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("invalid cpu_quota"),
|
||||
"{bad:?}: {err:#}"
|
||||
|
|
|
|||
|
|
@ -257,8 +257,14 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String,
|
|||
ref cpu_quota,
|
||||
cpu_weight,
|
||||
io_weight,
|
||||
memory_high,
|
||||
} => container_config::write_resource_limits(
|
||||
container, memory_max, cpu_quota, cpu_weight, io_weight,
|
||||
container,
|
||||
memory_max,
|
||||
cpu_quota,
|
||||
cpu_weight,
|
||||
io_weight,
|
||||
memory_high,
|
||||
),
|
||||
|
||||
PrivRequest::RemoveServiceDropin { ref container } => {
|
||||
|
|
|
|||
|
|
@ -1767,6 +1767,21 @@ in
|
|||
for = "5m";
|
||||
summary = "agent {{ $labels.agent }} on hive {{ $labels.hive }} has needed a claude login for 5 minutes; open its page in the dashboard to complete it";
|
||||
})
|
||||
# Processes the kernel OOM-killed in an agent's container, from
|
||||
# the counter hive-c0re exports (`otel_metrics.rs`). A budgeted
|
||||
# agent has no swap, so each kill is a subagent run, bash task or
|
||||
# the agent's claude that hit its RAM budget. More than three in
|
||||
# an hour is a budget that is too small for the work, not a
|
||||
# one-off peak. One alert instance per `(hive, agent)`.
|
||||
(metricThresholdRule {
|
||||
uid = "hh-agent-oom-kills";
|
||||
title = "agent OOM-killed repeatedly";
|
||||
windowSeconds = 3600;
|
||||
expr = ''sum by (hive, agent) (increase({__name__="hyperhive.container.memory.oom_kills", agent!=""}[1h]))'';
|
||||
threshold = 3;
|
||||
for = "0m";
|
||||
summary = "agent {{ $labels.agent }} on hive {{ $labels.hive }} had more than 3 processes OOM-killed in the last hour: it keeps hitting its RAM budget; see \"OOM kills by agent\" on the agents dashboard, and raise its memory cap if the work needs it";
|
||||
})
|
||||
# `hyperhive.hive.nix_store.utilization` is the used fraction of the
|
||||
# filesystem holding the nix store, the one that failed CI
|
||||
# with ENOSPC before the GC guardrails.
|
||||
|
|
|
|||
|
|
@ -998,6 +998,38 @@ let
|
|||
)
|
||||
&& lib.hasInfix "disk_pressure" (expr "hh-hive-warning");
|
||||
}
|
||||
{
|
||||
# The OOM-kill alert is one instance per (hive, agent), fires over three
|
||||
# kills in the last hour with no `for`, and selects the counter name
|
||||
# `otel_metrics.rs` spells in Rust. The needs-login rule is the presence
|
||||
# control: it goes through the same lookup, so a misspelled attribute path
|
||||
# fails both arms.
|
||||
# 🔬 Invert-proof: change `threshold = 3`, `for`, `[1h]` or `sum by (hive, agent)`
|
||||
# on `hh-agent-oom-kills`, or rename the counter in `otel_metrics.rs`.
|
||||
name = "the OOM-kill alert fires over three kills an hour per agent on the counter hive-c0re exports";
|
||||
ok =
|
||||
let
|
||||
groups =
|
||||
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
|
||||
rules = lib.concatMap (g: g.rules) groups;
|
||||
source = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs;
|
||||
rule = uid: lib.findFirst (r: r.uid == uid) null rules;
|
||||
oom = rule "hh-agent-oom-kills";
|
||||
query = uid: (lib.head (rule uid).data);
|
||||
threshold = lib.findFirst (q: q.refId == "C") null oom.data;
|
||||
in
|
||||
oom != null
|
||||
&& oom.for == "0m"
|
||||
&& (lib.head threshold.model.conditions).evaluator.params == [ 3 ]
|
||||
&& rule "hh-agent-needs-login" != null
|
||||
&& lib.hasInfix ''{__name__="hyperhive.container.memory.oom_kills"'' (query "hh-agent-oom-kills")
|
||||
.model.expr
|
||||
&& lib.hasInfix "[1h]" (query "hh-agent-oom-kills").model.expr
|
||||
&& lib.hasInfix "sum by (hive, agent)" (query "hh-agent-oom-kills").model.expr
|
||||
&& lib.hasInfix ''"hyperhive.container.memory.oom_kills"'' source
|
||||
&& (query "hh-agent-oom-kills").model.instant
|
||||
&& lib.hasInfix "{{ $labels.agent }}" oom.annotations.summary;
|
||||
}
|
||||
{
|
||||
# The held-deploys alert fires on any agent held for 30 minutes and
|
||||
# selects the gauge name `held_deploys.rs` spells in Rust. The disk rule
|
||||
|
|
|
|||
Loading…
Reference in a new issue