From 11393bc53850240146a39b3991ab08d02eb73c21 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 11 Nov 2026 02:38:53 +0100 Subject: [PATCH] budgeted agents throttle at 90% of the cap; alert on repeated oom kills --- .../4-implementation/scheduler/coordinator.md | 13 +-- .../scheduler/observability.md | 1 + .../trust-boundary/hive-c0re-privilege.md | 2 +- hive-c0re/src/lifecycle/host_config.rs | 85 +++++++++++++++++-- hive-c0re/src/priv_client.rs | 12 ++- hive-priv-sock/src/lib.rs | 9 +- hive-priv/src/container_config.rs | 51 ++++++++--- hive-priv/src/main.rs | 8 +- nix/host-modules/swarm-grafana.nix | 15 ++++ nix/module-eval/grafana.nix | 32 +++++++ 10 files changed, 201 insertions(+), 27 deletions(-) diff --git a/docs/4-implementation/scheduler/coordinator.md b/docs/4-implementation/scheduler/coordinator.md index d21b67e3a..d44363edd 100644 --- a/docs/4-implementation/scheduler/coordinator.md +++ b/docs/4-implementation/scheduler/coordinator.md @@ -446,11 +446,12 @@ changed cap reaches them at the next container start without a rebuild. `hyperhive-memory-cap-check.service` logs a warning naming the path in the container's journal. The container's `MemoryMax=` still applies. -A container with a finite `MemoryMax=` also gets `MemorySwapMax=0` in the same -drop-in, so a budgeted agent never swaps: paging it out to host swap would -move the pressure onto the host. `MemoryMax=infinity` leaves swap alone. Like -the rest of the drop-in, it reaches a running agent at its next spawn or -rebuild. +A container with a finite `MemoryMax=` also gets, in the same drop-in, +`MemorySwapMax=0` and `MemoryHigh=` at 90% of the cap (bytes, rounded down). +A budgeted agent never swaps: paging it out to host swap would move the +pressure onto the host. Past 90% the kernel slows the container and reclaims +from it, and at 100% it kills. `MemoryMax=infinity` sets neither. Like the +rest of the drop-in, they reach a running agent at its next spawn or rebuild. When the container reaches its cap, the kernel kills inside it, in this order: @@ -474,6 +475,8 @@ is the memory the container has swapped out: nonzero only for an agent with no budget, or one whose container started before its drop-in carried `MemorySwapMax=0`. +More than 3 kills of one agent in an hour raise the `hh-agent-oom-kills` alert. +
Implementation: swarm caps swarm-controller keeps the caps in the `agent-limits` KV bucket, one key per diff --git a/docs/4-implementation/scheduler/observability.md b/docs/4-implementation/scheduler/observability.md index 929c0160b..b4bc3e3fb 100644 --- a/docs/4-implementation/scheduler/observability.md +++ b/docs/4-implementation/scheduler/observability.md @@ -560,6 +560,7 @@ The swarm Grafana (`nix/host-modules/swarm-grafana.nix`) alerts on these: | `hh-hive-degraded` | `hyperhive.hive.degraded` is `1` for 2 minutes | | `hh-hive-warning` | any `hyperhive.hive.warning` series other than `disk_pressure` exists (`pending_logins` only for a hive that doesn't export `hyperhive.agent.needs_login`) for 2 minutes | | `hh-agent-needs-login` | `hyperhive.agent.needs_login` is `1` for an agent for 5 minutes, one alert per agent | +| `hh-agent-oom-kills` | `hyperhive.container.memory.oom_kills` grew by more than 3 in the last hour for an agent, one alert per agent, no delay | | `hh-host-disk-high` | `hyperhive.hive.nix_store.utilization` stays above 0.85 for 10 minutes | | `hh-host-disk-critical` | `hyperhive.hive.nix_store.utilization` stays above 0.95 for 2 minutes | diff --git a/docs/4-implementation/trust-boundary/hive-c0re-privilege.md b/docs/4-implementation/trust-boundary/hive-c0re-privilege.md index b6a1dfbf2..732fd1606 100644 --- a/docs/4-implementation/trust-boundary/hive-c0re-privilege.md +++ b/docs/4-implementation/trust-boundary/hive-c0re-privilege.md @@ -41,7 +41,7 @@ known operations; there is no arbitrary command pass-through: | `ReloadGatewayNginx` | `systemctl reload/start/reset-failed nginx` (host unit; the unit name is hard-coded, not a parameter) | | `WriteNspawnFlags` | write `/etc/nixos-containers/.conf` (bind-mount list, network isolation vars, and `--ephemeral` after the root's salvage). hive-priv first copies a stopped container's root to the agent's `work/`, then renames it aside (it deletes nothing), following no link the container could have planted; for an agent whose closure record sets `private_users` it also adds `--private-users=pick`, mounts every bind `owneridmap`, and once chowns the agent's `state`, `harness`, `work` and `claude` dirs (never `config`, following no link) to the data owner | | `EnsureAgentSocketDir` | `mkdirat` `/run/hive-agent/` `0751 root:root` under an `O_NOFOLLOW` parent fd; leaves an existing directory alone, refuses anything else; for an agent whose closure record sets `private_users` it then chowns the dir to the data owner | -| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=`/`CPUWeight=`/`IOWeight=` systemd drop-in for agent container, with an `ExecStartPre=` that runs `hive-priv check-rootfs ` (no-follow check of the root paths the start script touches; see persistence.md), and a GC root on the build it names | +| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=`/`MemoryHigh=`/`MemorySwapMax=`/`CPUWeight=`/`IOWeight=` systemd drop-in for agent container, with an `ExecStartPre=` that runs `hive-priv check-rootfs ` (no-follow check of the root paths the start script touches; see persistence.md), and a GC root on the build it names | | `RemoveServiceDropin` | remove `container@.service.d/` drop-in on destroy, and the GC root of the hive-priv build it named | | `DaemonReload` | `systemctl daemon-reload` | | `ControlInfraContainer` | `systemctl nginx.service` for the gateway — the `InfraContainer` enum is the allowlist, and serde rejects unknown names at the wire boundary (`hive-c0re`, the forge and the matrix have no variant, so no request can name them) | diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index f21abdd6f..5133b013f 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -25,13 +25,14 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re validate(name)?; let container = container_name(name); set_nspawn_flags(&container, hive, &paths.agent, &paths.claude, &paths.notes).await?; - if let Some((cpu_quota, memory_max)) = stage_limits( + let host = HostMemory::read(); + if let Some((cpu_quota, memory_max, memory_high)) = stage_limits( name, &crate::resource_limits::swarm_limits_path(), &limits_dropin_path(&container), &paths.agent, hive, - HostMemory::read(), + host, )? { set_resource_limits( &container, @@ -39,6 +40,7 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re &memory_max, hive.agent_cpu_weight, hive.agent_io_weight, + memory_high, ) .await?; } @@ -94,6 +96,8 @@ impl HostMemory { /// systemd enforces on it, so this file is how it learns the value it sizes /// its heap and its subagents against. It is written on every call, from the /// `MemoryMax=` the agent will start under: the new one, or the kept drop-in's. +/// +/// The limits to apply come with the `MemoryHigh=` byte count for their cap. fn stage_limits( agent_name: &str, swarm_path: &Path, @@ -101,7 +105,7 @@ fn stage_limits( runtime_dir: &Path, hive: &HiveEnv, host: Option, -) -> Result> { +) -> Result)>> { let limits = limits_to_apply(agent_name, swarm_path, dropin_path, hive); let memory_max = match &limits { Some((_, memory_max)) => Some(memory_max.clone()), @@ -109,7 +113,10 @@ fn stage_limits( }; let cap = memory_max.and_then(|m| memory_cap_value(agent_name, &m, host)); write_memory_cap(runtime_dir, cap.as_deref())?; - Ok(limits) + Ok(limits.map(|(cpu_quota, memory_max)| { + let high = memory_high(agent_name, &memory_max, host); + (cpu_quota, memory_max, high) + })) } /// The `MemoryMax=` the drop-in at `path` applies: its last assignment, as @@ -169,6 +176,16 @@ fn memory_cap_value( ) } +/// The `MemoryHigh=` byte count for an agent capped at `memory_max`: 90% of the +/// cap, rounded down, so the kernel slows the container and reclaims from it +/// before `MemoryMax=` kills in it. `None` when the agent has no cap. +fn memory_high(agent_name: &str, memory_max: &str, host: Option) -> Option { + let cap = memory_cap_value(agent_name, memory_max, host)? + .parse::() + .ok()?; + u64::try_from(u128::from(cap) * 9 / 10).ok() +} + /// `percent` (the part before `%`) in hundredths of a percent, with systemd's /// `parse_permyriad` rules: digits, at most two decimals, at most 100%. fn permyriad(percent: &str) -> Option { @@ -277,9 +294,15 @@ async fn set_resource_limits( memory_max: &str, cpu_weight: Option, io_weight: Option, + memory_high: Option, ) -> Result<()> { crate::priv_client::write_resource_limits( - container, memory_max, cpu_quota, cpu_weight, io_weight, + container, + memory_max, + cpu_quota, + cpu_weight, + io_weight, + memory_high, ) .await } @@ -577,7 +600,7 @@ mod tests { use super::{ BindMount, HostMemory, MEMORY_CAP_FILE, base_binds, bind_child_agent_dirs, bind_own_agent_dirs, holds_manage_root_agent, limits_to_apply, memory_cap_value, - stage_limits, + memory_high, stage_limits, }; use crate::coordinator::HiveEnv; use hive_types::NixBuilds; @@ -893,7 +916,7 @@ mod tests { ) .expect("stage limits"); let copy = std::fs::read_to_string(runtime.join(MEMORY_CAP_FILE)).ok(); - (applied.map(|(_, mem)| mem), copy) + (applied.map(|(_, mem, _)| mem), copy) } #[test] @@ -969,6 +992,52 @@ mod tests { ); } + #[test] + fn memory_high_is_ninety_percent_of_the_cap_rounded_down() { + for (memory_max, high) in [ + ("4G", Some(3_865_470_566)), + ("1048576B", Some(943_718)), + ("50%", Some(7_372_803_686)), + // No cap, or a value systemd ignores: nothing to throttle below. + ("infinity", None), + ("0", None), + ("150%", None), + ("lots", None), + ] { + assert_eq!( + memory_high("sock", memory_max, Some(HOST)), + high, + "{memory_max}" + ); + } + // A percentage needs the host's memory; a size does not. + assert_eq!(memory_high("sock", "50%", None), None); + assert_eq!(memory_high("sock", "8G", None), Some(7_730_941_132)); + } + + #[test] + fn staged_limits_carry_the_throttle_point_of_their_cap() { + let dir = tempfile::tempdir().expect("tempdir"); + let swarm = swarm_file(&dir); + let staged = |dir: &tempfile::TempDir| { + stage_limits( + "sock", + &swarm, + &dir.path().join("hyperhive-limits.conf"), + &dir.path().join("runtime"), + &HiveEnv::default(), + Some(HOST), + ) + .expect("stage limits") + .map(|(_, mem, high)| (mem, high)) + }; + // The hive default, 4G. + assert_eq!(staged(&dir), Some(("4G".to_owned(), Some(3_865_470_566)))); + // Presence control above; an uncapped agent has no throttle point. + std::fs::write(&swarm, r#"{"sock": {"memory_max": "infinity"}}"#).expect("seed swarm caps"); + assert_eq!(staged(&dir), Some(("infinity".to_owned(), None))); + } + #[test] fn the_host_memory_systemd_would_read_is_found() { let host = HostMemory::read().expect("sysconf"); @@ -1027,7 +1096,7 @@ mod tests { let runtime = dir.path().join("runtime"); let applied = stage_limits("sock", &swarm, &dropin, &runtime, &HiveEnv::default(), None) .expect("stage limits"); - assert_eq!(applied.map(|(_, mem)| mem), Some("50%".to_owned())); + assert_eq!(applied.map(|(_, mem, _)| mem), Some("50%".to_owned())); assert!(!runtime.join(MEMORY_CAP_FILE).exists()); } } diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index ad3de0ec0..03dfa5f53 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -292,9 +292,15 @@ pub async fn write_resource_limits( cpu_quota: &str, cpu_weight: Option, io_weight: Option, + memory_high: Option, ) -> Result<()> { ok(call(&resource_limits_request( - container, memory_max, cpu_quota, cpu_weight, io_weight, + container, + memory_max, + cpu_quota, + cpu_weight, + io_weight, + memory_high, )) .await?) } @@ -305,6 +311,7 @@ fn resource_limits_request( cpu_quota: &str, cpu_weight: Option, io_weight: Option, + memory_high: Option, ) -> PrivRequest { PrivRequest::WriteResourceLimits { container: container.to_owned(), @@ -312,6 +319,7 @@ fn resource_limits_request( cpu_quota: cpu_quota.to_owned(), cpu_weight, io_weight, + memory_high, } } @@ -654,6 +662,7 @@ mod tests { "200%", Some(50), None, + Some(3_865_470_566), )); assert_eq!(got["op"], "write_resource_limits"); assert_eq!(got["container"], "alice"); @@ -661,6 +670,7 @@ mod tests { assert_eq!(got["cpu_quota"], "200%"); assert_eq!(got["cpu_weight"], 50); assert_eq!(got["io_weight"], Value::Null); + assert_eq!(got["memory_high"], 3_865_470_566_u64); } #[test] diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index 7ce8b7d5a..e35e5b90b 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -383,7 +383,8 @@ pub enum PrivRequest { /// Write `/run/systemd/system/container@.service.d/hyperhive-limits.conf` /// with `[Service]` carrying `MemoryMax=` / `CPUQuota=` (hard caps) and - /// `CPUWeight=` / `IOWeight=` (cgroup v2 relative shares, contention-only), + /// `MemoryHigh=` (throttle point below `MemoryMax=`), `CPUWeight=` / `IOWeight=` + /// (cgroup v2 relative shares, contention-only), /// and an `ExecStartPre=` that runs `hive-priv check-rootfs ` /// to refuse a start over a root with links where the start script /// expects directories. The build that line names is rooted under @@ -404,6 +405,12 @@ pub enum PrivRequest { /// `cpu_weight`. #[serde(default)] io_weight: Option, + /// `MemoryHigh=` in bytes: the point where the kernel throttles the + /// container and reclaims from it before `memory_max` kills in it. + /// `None` omits the line, which is what a hive-c0re built before this + /// field existed sends, and what an unbudgeted container gets. + #[serde(default)] + memory_high: Option, }, /// Remove `/run/systemd/system/container@.service.d/` if present, diff --git a/hive-priv/src/container_config.rs b/hive-priv/src/container_config.rs index 8a1420e83..4132b7707 100644 --- a/hive-priv/src/container_config.rs +++ b/hive-priv/src/container_config.rs @@ -156,6 +156,7 @@ pub(crate) fn write_resource_limits( cpu_quota: &str, cpu_weight: Option, io_weight: Option, + memory_high: Option, ) -> Result<(String, String)> { let logical = validate_container_system_name(container)?; validate_memory_max(memory_max)?; @@ -176,6 +177,7 @@ pub(crate) fn write_resource_limits( cpu_quota, cpu_weight, io_weight, + memory_high, ); // Published whole: a `daemon-reload` between truncate and write would // otherwise load the unit with its limits and start guard missing. @@ -226,6 +228,10 @@ const RESTART_SEC: u32 = 5; /// entirely, so a request from a hive-c0re built before the weights /// existed — or one whose nix option is `null` — renders no weight lines. /// +/// `memory_high` is the byte count of `MemoryHigh=`: the throttle point below +/// the cap, where the kernel slows the container and reclaims from it. `None` +/// omits the line. +/// /// A finite `MemoryMax=` comes with `MemorySwapMax=0`: at the budget the /// kernel kills inside the container instead of paging it out to host swap. /// `MemoryMax=infinity` is no budget and leaves swap alone. @@ -236,12 +242,14 @@ fn limits_dropin_body( cpu_quota: &str, cpu_weight: Option, io_weight: Option, + memory_high: Option, ) -> String { // Built as two possibly-empty lines rather than pushed onto the // string: `format!` appended to a `String` trips clippy::pedantic's // `format_push_string`, and a `write!` would need an unwrap. let cpu_weight_line = cpu_weight.map_or_else(String::new, |w| format!("CPUWeight={w}\n")); let io_weight_line = io_weight.map_or_else(String::new, |w| format!("IOWeight={w}\n")); + let high_line = memory_high.map_or_else(String::new, |b| format!("MemoryHigh={b}\n")); let swap_line = if memory_max == "infinity" { "" } else { @@ -257,6 +265,7 @@ fn limits_dropin_body( ExecStartPre={rootfs_check}\n\ RestartSec={RESTART_SEC}\n\ MemoryMax={memory_max}\n\ + {high_line}\ {swap_line}\ CPUQuota={cpu_quota}\n\ {cpu_weight_line}{io_weight_line}" @@ -892,6 +901,7 @@ mod tests { "4G", "200%", None, + None, None ), "[Unit]\n\ @@ -919,7 +929,7 @@ mod tests { ("512M", "50%", Some(10), Some(10)), ("infinity", "infinity", Some(10_000), None), ] { - let body = limits_dropin_body("/rt/x", CHECK, mem, cpu, cw, iw); + let body = limits_dropin_body("/rt/x", CHECK, mem, cpu, cw, iw, None); let unit = body .split("[Service]") .next() @@ -940,38 +950,59 @@ mod tests { #[test] fn swap_is_off_exactly_when_the_memory_is_budgeted() { for mem in ["4G", "512M", "50%", "8589934592"] { - let body = limits_dropin_body("/rt/x", CHECK, mem, "200%", None, None); + let body = limits_dropin_body("/rt/x", CHECK, mem, "200%", None, None, None); assert!( body.contains(&format!("\nMemoryMax={mem}\nMemorySwapMax=0\n")), "no swap line under a budget of {mem}: {body}" ); assert_eq!(body.matches("MemorySwapMax=").count(), 1, "{body}"); } - let unbudgeted = limits_dropin_body("/rt/x", CHECK, "infinity", "200%", None, None); + let unbudgeted = limits_dropin_body("/rt/x", CHECK, "infinity", "200%", None, None, None); assert!( unbudgeted.contains("\nMemoryMax=infinity\n"), "{unbudgeted}" ); assert!(!unbudgeted.contains("MemorySwapMax"), "{unbudgeted}"); } + + /// `MemoryHigh=` is the byte count the caller sends, between `MemoryMax=` + /// and the swap line, and absent when none is sent. + #[test] + fn memory_high_is_rendered_only_when_sent() { + let with = limits_dropin_body( + "/rt/x", + CHECK, + "4G", + "200%", + None, + None, + Some(3_865_470_566), + ); + assert!( + with.contains("\nMemoryMax=4G\nMemoryHigh=3865470566\nMemorySwapMax=0\n"), + "{with}" + ); + let without = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, None, None); + assert!(!without.contains("MemoryHigh"), "{without}"); + } /// Weights are appended to the `[Service]` section, each omitted /// independently when `None`. #[test] fn weights_are_emitted_only_when_set() { - let both = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), Some(80)); + let both = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), Some(80), None); assert!( both.ends_with("CPUQuota=200%\nCPUWeight=80\nIOWeight=80\n"), "{both}" ); - let cpu_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), None); + let cpu_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", Some(80), None, None); assert!( cpu_only.ends_with("CPUQuota=200%\nCPUWeight=80\n"), "{cpu_only}" ); assert!(!cpu_only.contains("IOWeight"), "{cpu_only}"); - let io_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, Some(80)); + let io_only = limits_dropin_body("/rt/x", CHECK, "4G", "200%", None, Some(80), None); assert!( io_only.ends_with("CPUQuota=200%\nIOWeight=80\n"), "{io_only}" @@ -984,7 +1015,7 @@ mod tests { #[test] fn the_rootfs_check_leads_the_service_section_whatever_the_caller_passes() { for (cw, iw) in [(None, None), (Some(80), Some(80))] { - let body = limits_dropin_body("/rt/x", CHECK, "4G", "200%", cw, iw); + let body = limits_dropin_body("/rt/x", CHECK, "4G", "200%", cw, iw, None); let service = body .split("[Service]\n") .nth(1) @@ -1405,7 +1436,7 @@ mod tests { let errs = [ handle_write_nspawn_flags(name, &[], &isolation, &[], &facts, None, None) .unwrap_err(), - write_resource_limits(name, "4G", "200%", None, None).unwrap_err(), + write_resource_limits(name, "4G", "200%", None, None, None).unwrap_err(), remove_service_dropin(name).unwrap_err(), ]; for e in errs { @@ -1471,7 +1502,7 @@ mod tests { "$(id)", "%t", ] { - let err = write_resource_limits(PROBE, bad, "200%", None, None).unwrap_err(); + let err = write_resource_limits(PROBE, bad, "200%", None, None, None).unwrap_err(); assert!( err.to_string().contains("invalid memory_max"), "{bad:?}: {err:#}" @@ -1491,7 +1522,7 @@ mod tests { "4G", "%t", ] { - let err = write_resource_limits(PROBE, "4G", bad, None, None).unwrap_err(); + let err = write_resource_limits(PROBE, "4G", bad, None, None, None).unwrap_err(); assert!( err.to_string().contains("invalid cpu_quota"), "{bad:?}: {err:#}" diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 8148b7fbf..480c3fb8a 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -257,8 +257,14 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String, ref cpu_quota, cpu_weight, io_weight, + memory_high, } => container_config::write_resource_limits( - container, memory_max, cpu_quota, cpu_weight, io_weight, + container, + memory_max, + cpu_quota, + cpu_weight, + io_weight, + memory_high, ), PrivRequest::RemoveServiceDropin { ref container } => { diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index dfb4f6188..491606c3f 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -1767,6 +1767,21 @@ in for = "5m"; summary = "agent {{ $labels.agent }} on hive {{ $labels.hive }} has needed a claude login for 5 minutes; open its page in the dashboard to complete it"; }) + # Processes the kernel OOM-killed in an agent's container, from + # the counter hive-c0re exports (`otel_metrics.rs`). A budgeted + # agent has no swap, so each kill is a subagent run, bash task or + # the agent's claude that hit its RAM budget. More than three in + # an hour is a budget that is too small for the work, not a + # one-off peak. One alert instance per `(hive, agent)`. + (metricThresholdRule { + uid = "hh-agent-oom-kills"; + title = "agent OOM-killed repeatedly"; + windowSeconds = 3600; + expr = ''sum by (hive, agent) (increase({__name__="hyperhive.container.memory.oom_kills", agent!=""}[1h]))''; + threshold = 3; + for = "0m"; + summary = "agent {{ $labels.agent }} on hive {{ $labels.hive }} had more than 3 processes OOM-killed in the last hour: it keeps hitting its RAM budget; see \"OOM kills by agent\" on the agents dashboard, and raise its memory cap if the work needs it"; + }) # `hyperhive.hive.nix_store.utilization` is the used fraction of the # filesystem holding the nix store, the one that failed CI # with ENOSPC before the GC guardrails. diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 36f71ceaa..59a19c587 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -998,6 +998,38 @@ let ) && lib.hasInfix "disk_pressure" (expr "hh-hive-warning"); } + { + # The OOM-kill alert is one instance per (hive, agent), fires over three + # kills in the last hour with no `for`, and selects the counter name + # `otel_metrics.rs` spells in Rust. The needs-login rule is the presence + # control: it goes through the same lookup, so a misspelled attribute path + # fails both arms. + # 🔬 Invert-proof: change `threshold = 3`, `for`, `[1h]` or `sum by (hive, agent)` + # on `hh-agent-oom-kills`, or rename the counter in `otel_metrics.rs`. + name = "the OOM-kill alert fires over three kills an hour per agent on the counter hive-c0re exports"; + ok = + let + groups = + grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups; + rules = lib.concatMap (g: g.rules) groups; + source = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs; + rule = uid: lib.findFirst (r: r.uid == uid) null rules; + oom = rule "hh-agent-oom-kills"; + query = uid: (lib.head (rule uid).data); + threshold = lib.findFirst (q: q.refId == "C") null oom.data; + in + oom != null + && oom.for == "0m" + && (lib.head threshold.model.conditions).evaluator.params == [ 3 ] + && rule "hh-agent-needs-login" != null + && lib.hasInfix ''{__name__="hyperhive.container.memory.oom_kills"'' (query "hh-agent-oom-kills") + .model.expr + && lib.hasInfix "[1h]" (query "hh-agent-oom-kills").model.expr + && lib.hasInfix "sum by (hive, agent)" (query "hh-agent-oom-kills").model.expr + && lib.hasInfix ''"hyperhive.container.memory.oom_kills"'' source + && (query "hh-agent-oom-kills").model.instant + && lib.hasInfix "{{ $labels.agent }}" oom.annotations.summary; + } { # The held-deploys alert fires on any agent held for 30 minutes and # selects the gauge name `held_deploys.rs` spells in Rust. The disk rule