mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

alert when claude usage runs ahead of the clock, per account

Browse source
This commit is contained in:
atlas 2026-11-07 20:32:09 +01:00 • committed by mara
commit 319dd6a808
1 changed files with 191 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

38
nix/module-eval/grafana.nix
View file

@ -871,6 +871,44 @@ let
]
&& query.instant;
}
{
# The usage-ahead-of-clock alert selects the two gauge names hive-agent
# spells in Rust, requires the `account` label, maps both windows to
# their lengths and fires above zero for an hour. The disk rule is the
# presence control: it goes through the same lookup.
# 🔬 Invert-proof: drop `account!=""`, change `threshold`/`for`, or a
# window length in `hh-claude-usage-ahead-of-clock`, or rename a gauge
# in `otel_turn_metrics.rs`.
name = "the claude usage alert is per account and window and fires above the clock share";
ok =
let
groups =
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
rules = lib.concatMap (g: g.rules) groups;
source = builtins.readFile ../../hive-agent/src/otel_turn_metrics.rs;
rule = lib.findFirst (r: r.uid == "hh-claude-usage-ahead-of-clock") null rules;
disk = lib.findFirst (r: r.uid == "hh-host-disk-high") null rules;
expr = (lib.head rule.data).model.expr;
condition = lib.findFirst (q: q.refId == "C") null rule.data;
in
disk != null
&& rule != null
&& rule.for == "1h"
&& lib.head (lib.head condition.model.conditions).evaluator.params == 0
&& (lib.head condition.model.conditions).evaluator.type == "gt"
&&
lib.all (metric: lib.hasInfix ''{__name__="${metric}"'' expr && lib.hasInfix ''"${metric}"'' source)
[
"hyperhive.agent.claude_usage.percent"
"hyperhive.agent.claude_usage.resets_at"
]
&& lib.hasInfix ''account!=""'' expr
&& lib.hasInfix ''window="seven_day"'' expr
&& lib.hasInfix "/ 604800" expr
&& lib.hasInfix ''window="five_hour"'' expr
&& lib.hasInfix "/ 18000" expr
&& lib.hasInfix "max by (account, window)" expr;
}
{
# The held-deploys alert fires on any agent held for 30 minutes and
# selects the gauge name `held_deploys.rs` spells in Rust. The disk rule