mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

export hyperhive.agent.needs_login and alert per agent

Browse source
This commit is contained in:
atlas 2026-11-09 10:35:16 +01:00
commit 9043be4a61
1 changed files with 117 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

28
nix/module-eval/grafana.nix
View file

@ -940,6 +940,34 @@ let
&& lib.hasInfix "/ 18000" expr
&& lib.hasInfix "max by (account, window)" expr;
}
{
# The needs-login alert is one instance per (hive, agent), fires after
# 5 minutes on the gauge hive-c0re exports, and `hh-hive-warning` leaves
# `pending_logins` to it so a login never pages twice. The disk rule is
# the presence control: it goes through the same lookup.
# 🔬 Invert-proof: change `for`, `max by (hive, agent)` or the metric on
# `hh-agent-needs-login`, drop `pending_logins` from `hh-hive-warning`,
# or rename the gauge in `otel_metrics.rs`.
name = "the needs-login alert is per agent, fires after 5 minutes and the hive warning rule skips pending_logins";
ok =
let
groups =
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
rules = lib.concatMap (g: g.rules) groups;
source = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs;
rule = uid: lib.findFirst (r: r.uid == uid) null rules;
login = rule "hh-agent-needs-login";
expr = uid: (lib.head (rule uid).data).model.expr;
in
login != null
&& login.for == "5m"
&& rule "hh-host-disk-high" != null
&& lib.hasInfix ''{__name__="hyperhive.agent.needs_login"}'' (expr "hh-agent-needs-login")
&& lib.hasInfix ''"hyperhive.agent.needs_login"'' source
&& lib.hasInfix "max by (hive, agent)" (expr "hh-agent-needs-login")
&& lib.hasInfix "pending_logins" (expr "hh-hive-warning")
&& lib.hasInfix "disk_pressure" (expr "hh-hive-warning");
}
{
# The held-deploys alert fires on any agent held for 30 minutes and
# selects the gauge name `held_deploys.rs` spells in Rust. The disk rule