mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

export hyperhive.agent.needs_login and alert per agent

Browse source
This commit is contained in:
atlas 2026-11-09 10:35:16 +01:00
commit 9043be4a61
2 changed files with 117 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

21
nix/host-modules/swarm-grafana.nix
View file

@ -1720,18 +1720,29 @@ in
# pressure is left out: it has its own thresholds below, on
# the filesystem metric rather than the registry entry.
# Sweeps raise their warning only after several misses, so
# `for` is short. `pending_logins` pages too, on purpose:
# an agent that waits more than 2m for its claude login
# needs a human to act, which is what the banner it
# replaces was for.
# `for` is short. `pending_logins` is left out: it has no
# `agent` label, and `hh-agent-needs-login` below pages
# once per agent instead.
(metricThresholdRule {
uid = "hh-hive-warning";
title = "hive warning raised";
expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!="disk_pressure"})'';
expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!~"disk_pressure|pending_logins"})'';
threshold = 0;
for = "2m";
summary = "hive {{ $labels.hive }} has a {{ $labels.level }} warning raised: {{ $labels.kind }}; the message is the `hive warning raised` line in the hive-c0re log";
})
# `hyperhive.agent.needs_login` is `1` for a running agent
# without a usable claude login; stopped agents report
# nothing. One alert instance per `(hive, agent)`; `for`
# lets a login in progress finish first.
(metricThresholdRule {
uid = "hh-agent-needs-login";
title = "agent needs a claude login";
expr = ''max by (hive, agent) ({__name__="hyperhive.agent.needs_login"}) == 1'';
threshold = 0;
for = "5m";
summary = "agent {{ $labels.agent }} on hive {{ $labels.hive }} has needed a claude login for 5 minutes; open its page in the dashboard to complete it";
})
# `hyperhive.hive.nix_store.utilization` is the used fraction of the
# filesystem holding the nix store, the one that failed CI
# with ENOSPC before the GC guardrails.

28
nix/module-eval/grafana.nix
View file

@ -940,6 +940,34 @@ let
&& lib.hasInfix "/ 18000" expr
&& lib.hasInfix "max by (account, window)" expr;
}
{
# The needs-login alert is one instance per (hive, agent), fires after
# 5 minutes on the gauge hive-c0re exports, and `hh-hive-warning` leaves
# `pending_logins` to it so a login never pages twice. The disk rule is
# the presence control: it goes through the same lookup.
# 🔬 Invert-proof: change `for`, `max by (hive, agent)` or the metric on
# `hh-agent-needs-login`, drop `pending_logins` from `hh-hive-warning`,
# or rename the gauge in `otel_metrics.rs`.
name = "the needs-login alert is per agent, fires after 5 minutes and the hive warning rule skips pending_logins";
ok =
let
groups =
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
rules = lib.concatMap (g: g.rules) groups;
source = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs;
rule = uid: lib.findFirst (r: r.uid == uid) null rules;
login = rule "hh-agent-needs-login";
expr = uid: (lib.head (rule uid).data).model.expr;
in
login != null
&& login.for == "5m"
&& rule "hh-host-disk-high" != null
&& lib.hasInfix ''{__name__="hyperhive.agent.needs_login"}'' (expr "hh-agent-needs-login")
&& lib.hasInfix ''"hyperhive.agent.needs_login"'' source
&& lib.hasInfix "max by (hive, agent)" (expr "hh-agent-needs-login")
&& lib.hasInfix "pending_logins" (expr "hh-hive-warning")
&& lib.hasInfix "disk_pressure" (expr "hh-hive-warning");
}
{
# The held-deploys alert fires on any agent held for 30 minutes and
# selects the gauge name `held_deploys.rs` spells in Rust. The disk rule