alert when claude usage runs ahead of the clock, per account
This commit is contained in:
atlas
mara
parent
dab7768869
commit
319dd6a808
5 changed files with 191 additions and 28 deletions
|
|
@ -411,27 +411,27 @@ of their own — see [ACP-reported cost](#acp-reported-cost).
|
|||
|
||||
<!-- vale write-good.Passive = YES -->
|
||||
|
||||
| Metric | Unit | Kind | Attributes |
|
||||
| --------------------------------------------- | --------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` |
|
||||
| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` |
|
||||
| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) |
|
||||
| `hyperhive.agent.loose_ends.threads` | — | gauge | none |
|
||||
| `hyperhive.agent.loose_ends.reminders` | — | gauge | none |
|
||||
| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds |
|
||||
| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above |
|
||||
| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick |
|
||||
| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn |
|
||||
| `hyperhive.agent.claude_usage.resets_at` | `s` | gauge | `window` — unix seconds at which that window resets; same 5-minute poll |
|
||||
| `hyperhive.agent.context.used` | `{token}` | gauge | none — tokens in the context window at turn end, the numerator of the web UI's ctx% |
|
||||
| `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator |
|
||||
| `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below |
|
||||
| `hyperhive.agent.tool.calls` | — | counter | `tool` — the tool's name, or `Skill:<skill>` for a skill; counted when a turn ends |
|
||||
| `hyperhive.agent.bash.commands` | — | counter | `command` — the command name a bash task ran (the first real command, past `cd` and `sudo`), counted when a turn ends |
|
||||
| `hyperhive.agent.reminders` | — | counter | `event` (`scheduled` or `delivered`) |
|
||||
| `hyperhive.agent.first_turn.context` | `{token}` | gauge | none — input tokens of the first turn of a fresh session, the cold cost of the system prompt and `AGENTS.md` |
|
||||
| `hyperhive.agent.swarm_queue.connected` | — | gauge | none — `1` while the swarm-queue client holds a connection, `0` otherwise (including before the first connect), sampled every 30 seconds |
|
||||
| `hyperhive.agent.swarm_queue.publish_skipped` | — | counter | `site` (`terminal` or `agent_state`) — a publish dropped because the client had no connection |
|
||||
| Metric | Unit | Kind | Attributes |
|
||||
| --------------------------------------------- | --------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` |
|
||||
| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` |
|
||||
| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) |
|
||||
| `hyperhive.agent.loose_ends.threads` | — | gauge | none |
|
||||
| `hyperhive.agent.loose_ends.reminders` | — | gauge | none |
|
||||
| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds |
|
||||
| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above |
|
||||
| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick |
|
||||
| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them), `account` (see below) — polled every 5 minutes, **not** per turn |
|
||||
| `hyperhive.agent.claude_usage.resets_at` | `s` | gauge | `window`, `account` — unix seconds at which that window resets; same 5-minute poll |
|
||||
| `hyperhive.agent.context.used` | `{token}` | gauge | none — tokens in the context window at turn end, the numerator of the web UI's ctx% |
|
||||
| `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator |
|
||||
| `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below |
|
||||
| `hyperhive.agent.tool.calls` | — | counter | `tool` — the tool's name, or `Skill:<skill>` for a skill; counted when a turn ends |
|
||||
| `hyperhive.agent.bash.commands` | — | counter | `command` — the command name a bash task ran (the first real command, past `cd` and `sudo`), counted when a turn ends |
|
||||
| `hyperhive.agent.reminders` | — | counter | `event` (`scheduled` or `delivered`) |
|
||||
| `hyperhive.agent.first_turn.context` | `{token}` | gauge | none — input tokens of the first turn of a fresh session, the cold cost of the system prompt and `AGENTS.md` |
|
||||
| `hyperhive.agent.swarm_queue.connected` | — | gauge | none — `1` while the swarm-queue client holds a connection, `0` otherwise (including before the first connect), sampled every 30 seconds |
|
||||
| `hyperhive.agent.swarm_queue.publish_skipped` | — | counter | `site` (`terminal` or `agent_state`) — a publish dropped because the client had no connection |
|
||||
|
||||
An agent with no `subagent_roles/` directory exports no
|
||||
`subagent_role.lines` series. A role file deleted while the harness runs
|
||||
|
|
@ -446,6 +446,18 @@ session (API-key backends, not yet logged in) or an expired token skips the
|
|||
poll, so a gauge keeps its last value until the next successful poll — a
|
||||
`resets_at` in the past means the paired `percent` is stale.
|
||||
|
||||
`account` is the `oauthAccount.accountUuid` from the agent's `~/.claude.json`,
|
||||
an opaque id of the claude login that every agent on that login shares. The
|
||||
email address and tokens are never exported. When the file or the field is
|
||||
missing, the series carries no `account` label. Agent builds that predate the label export the
|
||||
series without it.
|
||||
|
||||
The swarm Grafana alert `hh-claude-usage-ahead-of-clock` fires per
|
||||
`(account, window)` when the percent used exceeds the percent of the window
|
||||
that has elapsed, for one hour. It knows `seven_day` (7 days) and `five_hour`
|
||||
(5 hours) and skips other windows and series without `account`. It clears when
|
||||
the clock catches up or the window resets.
|
||||
|
||||
The two `context` gauges hold the two numbers the web UI divides for its ctx%:
|
||||
the last turn's context tokens (input, cache read and cache creation), and the
|
||||
API-reported window, else the model's default. A turn that parsed no usage
|
||||
|
|
|
|||
|
|
@ -27,6 +27,8 @@ const USAGE_URL: &str = "https://api.anthropic.com/api/oauth/usage";
|
|||
const OAUTH_BETA: &str = "oauth-2025-04-20";
|
||||
/// The credentials file `claude` itself reads its OAuth session from.
|
||||
const CREDENTIALS_FILE: &str = ".credentials.json";
|
||||
/// claude's config file, beside the credentials directory.
|
||||
const CONFIG_FILE: &str = ".claude.json";
|
||||
|
||||
/// No `Debug`/`Display`, so the token cannot end up in a log line by
|
||||
/// accident.
|
||||
|
|
@ -64,6 +66,33 @@ struct Window {
|
|||
resets_at: Option<u64>,
|
||||
}
|
||||
|
||||
/// The opaque id of the claude login from the claude config file
|
||||
/// (`oauthAccount.accountUuid`). Only that one field is deserialised: the
|
||||
/// same object holds the email address, which must never reach a metric.
|
||||
fn account_from(bytes: &[u8]) -> Option<String> {
|
||||
#[derive(Deserialize)]
|
||||
struct Config {
|
||||
#[serde(rename = "oauthAccount")]
|
||||
account: Option<Account>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct Account {
|
||||
#[serde(rename = "accountUuid")]
|
||||
uuid: Option<String>,
|
||||
}
|
||||
let config = serde_json::from_slice::<Config>(bytes).ok()?;
|
||||
config.account?.uuid.filter(|u| !u.is_empty())
|
||||
}
|
||||
|
||||
/// `claude.json` sits next to the `.claude` directory. Read on every tick so
|
||||
/// a re-login is followed.
|
||||
async fn read_account(claude_dir: &Path) -> Option<String> {
|
||||
let bytes = tokio::fs::read(claude_dir.with_file_name(CONFIG_FILE))
|
||||
.await
|
||||
.ok()?;
|
||||
account_from(&bytes)
|
||||
}
|
||||
|
||||
/// Background loop. Detached task — runs for the harness's lifetime; every
|
||||
/// failure is logged and retried on the next tick, never fatal.
|
||||
pub async fn run() {
|
||||
|
|
@ -81,7 +110,8 @@ pub async fn run() {
|
|||
return;
|
||||
}
|
||||
};
|
||||
let creds = crate::paths::claude_dir().join(CREDENTIALS_FILE);
|
||||
let claude_dir = crate::paths::claude_dir();
|
||||
let creds = claude_dir.join(CREDENTIALS_FILE);
|
||||
let mut skip_logged = false;
|
||||
// Separate streak from `skip_logged`: a token can be usable while the
|
||||
// response still carries no recognisable window (endpoint change,
|
||||
|
|
@ -115,8 +145,14 @@ pub async fn run() {
|
|||
"claude usage watch: usage response had no recognisable windows"
|
||||
);
|
||||
}
|
||||
let account = read_account(&claude_dir).await;
|
||||
for w in windows {
|
||||
crate::otel_turn_metrics::record_claude_usage(&w.name, w.percent, w.resets_at);
|
||||
crate::otel_turn_metrics::record_claude_usage(
|
||||
account.as_deref(),
|
||||
&w.name,
|
||||
w.percent,
|
||||
w.resets_at,
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!("claude usage watch: {e}"),
|
||||
|
|
@ -250,6 +286,28 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
const CONFIG: &str = r#"{
|
||||
"oauthAccount": { "accountUuid": "0a1b-uuid", "emailAddress": "someone@example.com" },
|
||||
"projects": {}
|
||||
}"#;
|
||||
|
||||
#[test]
|
||||
fn account_is_the_uuid_and_never_the_email() {
|
||||
let id = account_from(CONFIG.as_bytes()).expect("account");
|
||||
assert_eq!(id, "0a1b-uuid");
|
||||
assert!(!id.contains('@'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_account_is_none() {
|
||||
assert_eq!(account_from(b"{}"), None);
|
||||
assert_eq!(
|
||||
account_from(br#"{"oauthAccount":{"accountUuid":""}}"#),
|
||||
None
|
||||
);
|
||||
assert_eq!(account_from(b"not json"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_object_body_yields_nothing() {
|
||||
assert!(windows(&Value::Null).is_empty());
|
||||
|
|
|
|||
|
|
@ -224,14 +224,30 @@ pub fn record_open_todos(count: u64) {
|
|||
inst.open_todos.record(count, &[]);
|
||||
}
|
||||
|
||||
/// Attributes of a usage series: `window`, plus `account` when the login is
|
||||
/// known. An unknown account omits the label rather than sending an empty one.
|
||||
fn claude_usage_attrs(account: Option<&str>, window: &str) -> Vec<KeyValue> {
|
||||
let mut attrs = vec![KeyValue::new("window", window.to_owned())];
|
||||
if let Some(account) = account.filter(|a| !a.is_empty()) {
|
||||
attrs.push(KeyValue::new("account", account.to_owned()));
|
||||
}
|
||||
attrs
|
||||
}
|
||||
|
||||
/// Record one subscription usage window (`five_hour`, `seven_day`, …):
|
||||
/// `percent` is 0-100, `resets_at` unix seconds. No-op when OTEL isn't
|
||||
/// configured, same as [`record`].
|
||||
pub fn record_claude_usage(window: &str, percent: f64, resets_at: Option<u64>) {
|
||||
/// `percent` is 0-100, `resets_at` unix seconds. `account` is the opaque
|
||||
/// id of the claude login, shared by every agent on that login. No-op when
|
||||
/// OTEL isn't configured, same as [`record`].
|
||||
pub fn record_claude_usage(
|
||||
account: Option<&str>,
|
||||
window: &str,
|
||||
percent: f64,
|
||||
resets_at: Option<u64>,
|
||||
) {
|
||||
let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else {
|
||||
return;
|
||||
};
|
||||
let attrs = [KeyValue::new("window", window.to_owned())];
|
||||
let attrs = claude_usage_attrs(account, window);
|
||||
inst.claude_usage_percent.record(percent, &attrs);
|
||||
if let Some(resets_at) = resets_at {
|
||||
inst.claude_usage_resets_at.record(resets_at, &attrs);
|
||||
|
|
@ -382,7 +398,9 @@ pub fn enabled() -> bool {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{TURN_DURATION_BOUNDS_MS, tool_call_counts};
|
||||
use opentelemetry::KeyValue;
|
||||
|
||||
use super::{TURN_DURATION_BOUNDS_MS, claude_usage_attrs, tool_call_counts};
|
||||
|
||||
#[test]
|
||||
fn a_turns_stored_tool_breakdown_becomes_per_tool_counts() {
|
||||
|
|
@ -411,4 +429,18 @@ mod tests {
|
|||
.is_some_and(|&b| b > 10_000.0)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_attrs_carry_account_only_when_known() {
|
||||
let has = |attrs: &[KeyValue], k: &str| attrs.iter().any(|kv| kv.key.as_str() == k);
|
||||
let known = claude_usage_attrs(Some("acct-1"), "seven_day");
|
||||
assert!(has(&known, "window"));
|
||||
assert!(
|
||||
known
|
||||
.iter()
|
||||
.any(|kv| kv.key.as_str() == "account" && kv.value.as_str() == "acct-1")
|
||||
);
|
||||
assert!(!has(&claude_usage_attrs(None, "seven_day"), "account"));
|
||||
assert!(!has(&claude_usage_attrs(Some(""), "seven_day"), "account"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1782,6 +1782,29 @@ in
|
|||
for = "10m";
|
||||
summary = "the store client certificate {{ $labels.cert }} (hive {{ $labels.hive }}, agent {{ $labels.agent }}) is past 80% of its validity";
|
||||
})
|
||||
# Usage share ahead of the clock share, per claude account and
|
||||
# window: percent used minus the percent of the window already
|
||||
# elapsed, so above zero means ahead. The window length comes
|
||||
# from the `window` label (`seven_day`, `five_hour`); other
|
||||
# windows are not selected. `account` is stamped by hive-agent
|
||||
# (`claude_usage_watch`); series without it (older agent builds)
|
||||
# are not selected. `max by` collapses the agents that share an
|
||||
# account. A `resets_at` in the past makes the elapsed share
|
||||
# exceed 100, so a stale series cannot fire.
|
||||
(metricThresholdRule {
|
||||
uid = "hh-claude-usage-ahead-of-clock";
|
||||
title = "claude usage ahead of the clock";
|
||||
expr =
|
||||
let
|
||||
ahead =
|
||||
window: seconds:
|
||||
''max by (account, window) (last_over_time({__name__="hyperhive.agent.claude_usage.percent", window="${window}", account!=""}[15m])) - 100 * (1 - (max by (account, window) (last_over_time({__name__="hyperhive.agent.claude_usage.resets_at", window="${window}", account!=""}[15m])) - time()) / ${toString seconds})'';
|
||||
in
|
||||
"(${ahead "seven_day" 604800}) or (${ahead "five_hour" 18000})";
|
||||
threshold = 0;
|
||||
for = "1h";
|
||||
summary = "claude account {{ $labels.account }} has used more of its {{ $labels.window }} usage window than the window has elapsed";
|
||||
})
|
||||
]
|
||||
# One rule per declared hive rather than one grouped by
|
||||
# `hive`, so a hive that sends nothing at all still has
|
||||
|
|
|
|||
|
|
@ -871,6 +871,44 @@ let
|
|||
]
|
||||
&& query.instant;
|
||||
}
|
||||
{
|
||||
# The usage-ahead-of-clock alert selects the two gauge names hive-agent
|
||||
# spells in Rust, requires the `account` label, maps both windows to
|
||||
# their lengths and fires above zero for an hour. The disk rule is the
|
||||
# presence control: it goes through the same lookup.
|
||||
# 🔬 Invert-proof: drop `account!=""`, change `threshold`/`for`, or a
|
||||
# window length in `hh-claude-usage-ahead-of-clock`, or rename a gauge
|
||||
# in `otel_turn_metrics.rs`.
|
||||
name = "the claude usage alert is per account and window and fires above the clock share";
|
||||
ok =
|
||||
let
|
||||
groups =
|
||||
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
|
||||
rules = lib.concatMap (g: g.rules) groups;
|
||||
source = builtins.readFile ../../hive-agent/src/otel_turn_metrics.rs;
|
||||
rule = lib.findFirst (r: r.uid == "hh-claude-usage-ahead-of-clock") null rules;
|
||||
disk = lib.findFirst (r: r.uid == "hh-host-disk-high") null rules;
|
||||
expr = (lib.head rule.data).model.expr;
|
||||
condition = lib.findFirst (q: q.refId == "C") null rule.data;
|
||||
in
|
||||
disk != null
|
||||
&& rule != null
|
||||
&& rule.for == "1h"
|
||||
&& lib.head (lib.head condition.model.conditions).evaluator.params == 0
|
||||
&& (lib.head condition.model.conditions).evaluator.type == "gt"
|
||||
&&
|
||||
lib.all (metric: lib.hasInfix ''{__name__="${metric}"'' expr && lib.hasInfix ''"${metric}"'' source)
|
||||
[
|
||||
"hyperhive.agent.claude_usage.percent"
|
||||
"hyperhive.agent.claude_usage.resets_at"
|
||||
]
|
||||
&& lib.hasInfix ''account!=""'' expr
|
||||
&& lib.hasInfix ''window="seven_day"'' expr
|
||||
&& lib.hasInfix "/ 604800" expr
|
||||
&& lib.hasInfix ''window="five_hour"'' expr
|
||||
&& lib.hasInfix "/ 18000" expr
|
||||
&& lib.hasInfix "max by (account, window)" expr;
|
||||
}
|
||||
{
|
||||
# The held-deploys alert fires on any agent held for 30 minutes and
|
||||
# selects the gauge name `held_deploys.rs` spells in Rust. The disk rule
|
||||
|
|
|
|||
Loading…
Reference in a new issue