mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

alert when claude usage runs ahead of the clock, per account

Browse source
This commit is contained in:
atlas 2026-11-07 20:32:09 +01:00 • committed by mara
commit 319dd6a808
2 changed files with 191 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

23
nix/host-modules/swarm-grafana.nix
View file

@ -1782,6 +1782,29 @@ in
for = "10m";
summary = "the store client certificate {{ $labels.cert }} (hive {{ $labels.hive }}, agent {{ $labels.agent }}) is past 80% of its validity";
})
# Usage share ahead of the clock share, per claude account and
# window: percent used minus the percent of the window already
# elapsed, so above zero means ahead. The window length comes
# from the `window` label (`seven_day`, `five_hour`); other
# windows are not selected. `account` is stamped by hive-agent
# (`claude_usage_watch`); series without it (older agent builds)
# are not selected. `max by` collapses the agents that share an
# account. A `resets_at` in the past makes the elapsed share
# exceed 100, so a stale series cannot fire.
(metricThresholdRule {
uid = "hh-claude-usage-ahead-of-clock";
title = "claude usage ahead of the clock";
expr =
let
ahead =
window: seconds:
''max by (account, window) (last_over_time({__name__="hyperhive.agent.claude_usage.percent", window="${window}", account!=""}[15m])) - 100 * (1 - (max by (account, window) (last_over_time({__name__="hyperhive.agent.claude_usage.resets_at", window="${window}", account!=""}[15m])) - time()) / ${toString seconds})'';
in
"(${ahead "seven_day" 604800}) or (${ahead "five_hour" 18000})";
threshold = 0;
for = "1h";
summary = "claude account {{ $labels.account }} has used more of its {{ $labels.window }} usage window than the window has elapsed";
})
]
# One rule per declared hive rather than one grouped by
# `hive`, so a hive that sends nothing at all still has

38
nix/module-eval/grafana.nix
View file

@ -871,6 +871,44 @@ let
]
&& query.instant;
}
{
# The usage-ahead-of-clock alert selects the two gauge names hive-agent
# spells in Rust, requires the `account` label, maps both windows to
# their lengths and fires above zero for an hour. The disk rule is the
# presence control: it goes through the same lookup.
# 🔬 Invert-proof: drop `account!=""`, change `threshold`/`for`, or a
# window length in `hh-claude-usage-ahead-of-clock`, or rename a gauge
# in `otel_turn_metrics.rs`.
name = "the claude usage alert is per account and window and fires above the clock share";
ok =
let
groups =
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
rules = lib.concatMap (g: g.rules) groups;
source = builtins.readFile ../../hive-agent/src/otel_turn_metrics.rs;
rule = lib.findFirst (r: r.uid == "hh-claude-usage-ahead-of-clock") null rules;
disk = lib.findFirst (r: r.uid == "hh-host-disk-high") null rules;
expr = (lib.head rule.data).model.expr;
condition = lib.findFirst (q: q.refId == "C") null rule.data;
in
disk != null
&& rule != null
&& rule.for == "1h"
&& lib.head (lib.head condition.model.conditions).evaluator.params == 0
&& (lib.head condition.model.conditions).evaluator.type == "gt"
&&
lib.all (metric: lib.hasInfix ''{__name__="${metric}"'' expr && lib.hasInfix ''"${metric}"'' source)
[
"hyperhive.agent.claude_usage.percent"
"hyperhive.agent.claude_usage.resets_at"
]
&& lib.hasInfix ''account!=""'' expr
&& lib.hasInfix ''window="seven_day"'' expr
&& lib.hasInfix "/ 604800" expr
&& lib.hasInfix ''window="five_hour"'' expr
&& lib.hasInfix "/ 18000" expr
&& lib.hasInfix "max by (account, window)" expr;
}
{
# The held-deploys alert fires on any agent held for 30 minutes and
# selects the gauge name `held_deploys.rs` spells in Rust. The disk rule