mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

budgeted agents throttle at 90% of the cap; alert on repeated oom kills

Browse source
This commit is contained in:
atlas 2026-11-11 02:38:53 +01:00
commit 11393bc538
1 changed files with 201 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

32
nix/module-eval/grafana.nix
View file

@ -998,6 +998,38 @@ let
)
&& lib.hasInfix "disk_pressure" (expr "hh-hive-warning");
}
{
# The OOM-kill alert is one instance per (hive, agent), fires over three
# kills in the last hour with no `for`, and selects the counter name
# `otel_metrics.rs` spells in Rust. The needs-login rule is the presence
# control: it goes through the same lookup, so a misspelled attribute path
# fails both arms.
# 🔬 Invert-proof: change `threshold = 3`, `for`, `[1h]` or `sum by (hive, agent)`
# on `hh-agent-oom-kills`, or rename the counter in `otel_metrics.rs`.
name = "the OOM-kill alert fires over three kills an hour per agent on the counter hive-c0re exports";
ok =
let
groups =
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
rules = lib.concatMap (g: g.rules) groups;
source = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs;
rule = uid: lib.findFirst (r: r.uid == uid) null rules;
oom = rule "hh-agent-oom-kills";
query = uid: (lib.head (rule uid).data);
threshold = lib.findFirst (q: q.refId == "C") null oom.data;
in
oom != null
&& oom.for == "0m"
&& (lib.head threshold.model.conditions).evaluator.params == [ 3 ]
&& rule "hh-agent-needs-login" != null
&& lib.hasInfix ''{__name__="hyperhive.container.memory.oom_kills"'' (query "hh-agent-oom-kills")
.model.expr
&& lib.hasInfix "[1h]" (query "hh-agent-oom-kills").model.expr
&& lib.hasInfix "sum by (hive, agent)" (query "hh-agent-oom-kills").model.expr
&& lib.hasInfix ''"hyperhive.container.memory.oom_kills"'' source
&& (query "hh-agent-oom-kills").model.instant
&& lib.hasInfix "{{ $labels.agent }}" oom.annotations.summary;
}
{
# The held-deploys alert fires on any agent held for 30 minutes and
# selects the gauge name `held_deploys.rs` spells in Rust. The disk rule