mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

page pending_logins from hh-hive-warning for hives without the needs-login gauge

Browse source
This commit is contained in:
atlas 2026-11-09 10:56:50 +01:00
commit 1c3b37e46a
2 changed files with 22 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

9
nix/host-modules/swarm-grafana.nix
View file

@ -1720,13 +1720,14 @@ in
# pressure is left out: it has its own thresholds below, on
# the filesystem metric rather than the registry entry.
# Sweeps raise their warning only after several misses, so
# `for` is short. `pending_logins` is left out: it has no
# `agent` label, and `hh-agent-needs-login` below pages
# once per agent instead.
# `for` is short. `pending_logins` is dropped for a hive that exports
# `hyperhive.agent.needs_login`, where `hh-agent-needs-login`
# below pages once per agent; a hive on an older hive-c0re
# has no such series, so it keeps paging here.
(metricThresholdRule {
uid = "hh-hive-warning";
title = "hive warning raised";
expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!~"disk_pressure|pending_logins"})'';
expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!~"disk_pressure|pending_logins"} or ({__name__="hyperhive.hive.warning", kind="pending_logins"} unless on (hive) group by (hive) ({__name__="hyperhive.agent.needs_login"})))'';
threshold = 0;
for = "2m";
summary = "hive {{ $labels.hive }} has a {{ $labels.level }} warning raised: {{ $labels.kind }}; the message is the `hive warning raised` line in the hive-c0re log";

15
nix/module-eval/grafana.nix
View file

@ -942,13 +942,14 @@ let
}
{
# The needs-login alert is one instance per (hive, agent), fires after
# 5 minutes on the gauge hive-c0re exports, and `hh-hive-warning` leaves
# `pending_logins` to it so a login never pages twice. The disk rule is
# 5 minutes on the gauge hive-c0re exports, and `hh-hive-warning` drops
# `pending_logins` only for hives that export the gauge, so a login never
# pages twice and a hive on an older hive-c0re still pages. The disk rule is
# the presence control: it goes through the same lookup.
# 🔬 Invert-proof: change `for`, `max by (hive, agent)` or the metric on
# `hh-agent-needs-login`, drop `pending_logins` from `hh-hive-warning`,
# or rename the gauge in `otel_metrics.rs`.
name = "the needs-login alert is per agent, fires after 5 minutes and the hive warning rule skips pending_logins";
# `hh-agent-needs-login`, exclude `pending_logins` from `hh-hive-warning` for every hive
# instead of per hive (drop the `unless on (hive)` clause), or rename the gauge in `otel_metrics.rs`.
name = "the needs-login alert is per agent, fires after 5 minutes and the hive warning rule skips pending_logins only for hives with the gauge";
ok =
let
groups =
@ -965,7 +966,9 @@ let
&& lib.hasInfix ''{__name__="hyperhive.agent.needs_login"}'' (expr "hh-agent-needs-login")
&& lib.hasInfix ''"hyperhive.agent.needs_login"'' source
&& lib.hasInfix "max by (hive, agent)" (expr "hh-agent-needs-login")
&& lib.hasInfix "pending_logins" (expr "hh-hive-warning")
&& lib.hasInfix ''{__name__="hyperhive.hive.warning", kind="pending_logins"} unless on (hive) group by (hive) ({__name__="hyperhive.agent.needs_login"})'' (
expr "hh-hive-warning"
)
&& lib.hasInfix "disk_pressure" (expr "hh-hive-warning");
}
{