diff --git a/docs/4-implementation/scheduler/observability.md b/docs/4-implementation/scheduler/observability.md index 0549c2f6..9aa7ccae 100644 --- a/docs/4-implementation/scheduler/observability.md +++ b/docs/4-implementation/scheduler/observability.md @@ -411,27 +411,27 @@ of their own — see [ACP-reported cost](#acp-reported-cost). -| Metric | Unit | Kind | Attributes | -| --------------------------------------------- | --------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` | -| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` | -| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) | -| `hyperhive.agent.loose_ends.threads` | — | gauge | none | -| `hyperhive.agent.loose_ends.reminders` | — | gauge | none | -| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds | -| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above | -| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick | -| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn | -| `hyperhive.agent.claude_usage.resets_at` | `s` | gauge | `window` — unix seconds at which that window resets; same 5-minute poll | -| `hyperhive.agent.context.used` | `{token}` | gauge | none — tokens in the context window at turn end, the numerator of the web UI's ctx% | -| `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator | -| `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below | -| `hyperhive.agent.tool.calls` | — | counter | `tool` — the tool's name, or `Skill:` for a skill; counted when a turn ends | -| `hyperhive.agent.bash.commands` | — | counter | `command` — the command name a bash task ran (the first real command, past `cd` and `sudo`), counted when a turn ends | -| `hyperhive.agent.reminders` | — | counter | `event` (`scheduled` or `delivered`) | -| `hyperhive.agent.first_turn.context` | `{token}` | gauge | none — input tokens of the first turn of a fresh session, the cold cost of the system prompt and `AGENTS.md` | -| `hyperhive.agent.swarm_queue.connected` | — | gauge | none — `1` while the swarm-queue client holds a connection, `0` otherwise (including before the first connect), sampled every 30 seconds | -| `hyperhive.agent.swarm_queue.publish_skipped` | — | counter | `site` (`terminal` or `agent_state`) — a publish dropped because the client had no connection | +| Metric | Unit | Kind | Attributes | +| --------------------------------------------- | --------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` | +| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` | +| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) | +| `hyperhive.agent.loose_ends.threads` | — | gauge | none | +| `hyperhive.agent.loose_ends.reminders` | — | gauge | none | +| `hyperhive.agent.todos.open` | — | gauge | none — un-acked todos in the harness-local todo store (reminders excluded), recorded every 30 seconds | +| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `AGENTS.md`-size watch's own ~15-minute tick, **not** per turn like the rows above | +| `hyperhive.agent.subagent_role.lines` | — | gauge | `role` (the file stem) — one series per role file in `subagent_roles/`, on the same ~15-minute tick | +| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them), `account` (see below) — polled every 5 minutes, **not** per turn | +| `hyperhive.agent.claude_usage.resets_at` | `s` | gauge | `window`, `account` — unix seconds at which that window resets; same 5-minute poll | +| `hyperhive.agent.context.used` | `{token}` | gauge | none — tokens in the context window at turn end, the numerator of the web UI's ctx% | +| `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator | +| `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below | +| `hyperhive.agent.tool.calls` | — | counter | `tool` — the tool's name, or `Skill:` for a skill; counted when a turn ends | +| `hyperhive.agent.bash.commands` | — | counter | `command` — the command name a bash task ran (the first real command, past `cd` and `sudo`), counted when a turn ends | +| `hyperhive.agent.reminders` | — | counter | `event` (`scheduled` or `delivered`) | +| `hyperhive.agent.first_turn.context` | `{token}` | gauge | none — input tokens of the first turn of a fresh session, the cold cost of the system prompt and `AGENTS.md` | +| `hyperhive.agent.swarm_queue.connected` | — | gauge | none — `1` while the swarm-queue client holds a connection, `0` otherwise (including before the first connect), sampled every 30 seconds | +| `hyperhive.agent.swarm_queue.publish_skipped` | — | counter | `site` (`terminal` or `agent_state`) — a publish dropped because the client had no connection | An agent with no `subagent_roles/` directory exports no `subagent_role.lines` series. A role file deleted while the harness runs @@ -446,6 +446,18 @@ session (API-key backends, not yet logged in) or an expired token skips the poll, so a gauge keeps its last value until the next successful poll — a `resets_at` in the past means the paired `percent` is stale. +`account` is the `oauthAccount.accountUuid` from the agent's `~/.claude.json`, +an opaque id of the claude login that every agent on that login shares. The +email address and tokens are never exported. When the file or the field is +missing, the series carries no `account` label. Agent builds that predate the label export the +series without it. + +The swarm Grafana alert `hh-claude-usage-ahead-of-clock` fires per +`(account, window)` when the percent used exceeds the percent of the window +that has elapsed, for one hour. It knows `seven_day` (7 days) and `five_hour` +(5 hours) and skips other windows and series without `account`. It clears when +the clock catches up or the window resets. + The two `context` gauges hold the two numbers the web UI divides for its ctx%: the last turn's context tokens (input, cache read and cache creation), and the API-reported window, else the model's default. A turn that parsed no usage diff --git a/hive-agent/src/claude_usage_watch.rs b/hive-agent/src/claude_usage_watch.rs index 66ba5b69..7e5c2590 100644 --- a/hive-agent/src/claude_usage_watch.rs +++ b/hive-agent/src/claude_usage_watch.rs @@ -27,6 +27,8 @@ const USAGE_URL: &str = "https://api.anthropic.com/api/oauth/usage"; const OAUTH_BETA: &str = "oauth-2025-04-20"; /// The credentials file `claude` itself reads its OAuth session from. const CREDENTIALS_FILE: &str = ".credentials.json"; +/// claude's config file, beside the credentials directory. +const CONFIG_FILE: &str = ".claude.json"; /// No `Debug`/`Display`, so the token cannot end up in a log line by /// accident. @@ -64,6 +66,33 @@ struct Window { resets_at: Option, } +/// The opaque id of the claude login from the claude config file +/// (`oauthAccount.accountUuid`). Only that one field is deserialised: the +/// same object holds the email address, which must never reach a metric. +fn account_from(bytes: &[u8]) -> Option { + #[derive(Deserialize)] + struct Config { + #[serde(rename = "oauthAccount")] + account: Option, + } + #[derive(Deserialize)] + struct Account { + #[serde(rename = "accountUuid")] + uuid: Option, + } + let config = serde_json::from_slice::(bytes).ok()?; + config.account?.uuid.filter(|u| !u.is_empty()) +} + +/// `claude.json` sits next to the `.claude` directory. Read on every tick so +/// a re-login is followed. +async fn read_account(claude_dir: &Path) -> Option { + let bytes = tokio::fs::read(claude_dir.with_file_name(CONFIG_FILE)) + .await + .ok()?; + account_from(&bytes) +} + /// Background loop. Detached task — runs for the harness's lifetime; every /// failure is logged and retried on the next tick, never fatal. pub async fn run() { @@ -81,7 +110,8 @@ pub async fn run() { return; } }; - let creds = crate::paths::claude_dir().join(CREDENTIALS_FILE); + let claude_dir = crate::paths::claude_dir(); + let creds = claude_dir.join(CREDENTIALS_FILE); let mut skip_logged = false; // Separate streak from `skip_logged`: a token can be usable while the // response still carries no recognisable window (endpoint change, @@ -115,8 +145,14 @@ pub async fn run() { "claude usage watch: usage response had no recognisable windows" ); } + let account = read_account(&claude_dir).await; for w in windows { - crate::otel_turn_metrics::record_claude_usage(&w.name, w.percent, w.resets_at); + crate::otel_turn_metrics::record_claude_usage( + account.as_deref(), + &w.name, + w.percent, + w.resets_at, + ); } } Err(e) => tracing::warn!("claude usage watch: {e}"), @@ -250,6 +286,28 @@ mod tests { ); } + const CONFIG: &str = r#"{ + "oauthAccount": { "accountUuid": "0a1b-uuid", "emailAddress": "someone@example.com" }, + "projects": {} + }"#; + + #[test] + fn account_is_the_uuid_and_never_the_email() { + let id = account_from(CONFIG.as_bytes()).expect("account"); + assert_eq!(id, "0a1b-uuid"); + assert!(!id.contains('@')); + } + + #[test] + fn missing_account_is_none() { + assert_eq!(account_from(b"{}"), None); + assert_eq!( + account_from(br#"{"oauthAccount":{"accountUuid":""}}"#), + None + ); + assert_eq!(account_from(b"not json"), None); + } + #[test] fn non_object_body_yields_nothing() { assert!(windows(&Value::Null).is_empty()); diff --git a/hive-agent/src/otel_turn_metrics.rs b/hive-agent/src/otel_turn_metrics.rs index 36b4022b..e07b2ec8 100644 --- a/hive-agent/src/otel_turn_metrics.rs +++ b/hive-agent/src/otel_turn_metrics.rs @@ -224,14 +224,30 @@ pub fn record_open_todos(count: u64) { inst.open_todos.record(count, &[]); } +/// Attributes of a usage series: `window`, plus `account` when the login is +/// known. An unknown account omits the label rather than sending an empty one. +fn claude_usage_attrs(account: Option<&str>, window: &str) -> Vec { + let mut attrs = vec![KeyValue::new("window", window.to_owned())]; + if let Some(account) = account.filter(|a| !a.is_empty()) { + attrs.push(KeyValue::new("account", account.to_owned())); + } + attrs +} + /// Record one subscription usage window (`five_hour`, `seven_day`, …): -/// `percent` is 0-100, `resets_at` unix seconds. No-op when OTEL isn't -/// configured, same as [`record`]. -pub fn record_claude_usage(window: &str, percent: f64, resets_at: Option) { +/// `percent` is 0-100, `resets_at` unix seconds. `account` is the opaque +/// id of the claude login, shared by every agent on that login. No-op when +/// OTEL isn't configured, same as [`record`]. +pub fn record_claude_usage( + account: Option<&str>, + window: &str, + percent: f64, + resets_at: Option, +) { let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { return; }; - let attrs = [KeyValue::new("window", window.to_owned())]; + let attrs = claude_usage_attrs(account, window); inst.claude_usage_percent.record(percent, &attrs); if let Some(resets_at) = resets_at { inst.claude_usage_resets_at.record(resets_at, &attrs); @@ -382,7 +398,9 @@ pub fn enabled() -> bool { #[cfg(test)] mod tests { - use super::{TURN_DURATION_BOUNDS_MS, tool_call_counts}; + use opentelemetry::KeyValue; + + use super::{TURN_DURATION_BOUNDS_MS, claude_usage_attrs, tool_call_counts}; #[test] fn a_turns_stored_tool_breakdown_becomes_per_tool_counts() { @@ -411,4 +429,18 @@ mod tests { .is_some_and(|&b| b > 10_000.0) ); } + + #[test] + fn usage_attrs_carry_account_only_when_known() { + let has = |attrs: &[KeyValue], k: &str| attrs.iter().any(|kv| kv.key.as_str() == k); + let known = claude_usage_attrs(Some("acct-1"), "seven_day"); + assert!(has(&known, "window")); + assert!( + known + .iter() + .any(|kv| kv.key.as_str() == "account" && kv.value.as_str() == "acct-1") + ); + assert!(!has(&claude_usage_attrs(None, "seven_day"), "account")); + assert!(!has(&claude_usage_attrs(Some(""), "seven_day"), "account")); + } } diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index ba7c5ff9..06a6601a 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -1782,6 +1782,29 @@ in for = "10m"; summary = "the store client certificate {{ $labels.cert }} (hive {{ $labels.hive }}, agent {{ $labels.agent }}) is past 80% of its validity"; }) + # Usage share ahead of the clock share, per claude account and + # window: percent used minus the percent of the window already + # elapsed, so above zero means ahead. The window length comes + # from the `window` label (`seven_day`, `five_hour`); other + # windows are not selected. `account` is stamped by hive-agent + # (`claude_usage_watch`); series without it (older agent builds) + # are not selected. `max by` collapses the agents that share an + # account. A `resets_at` in the past makes the elapsed share + # exceed 100, so a stale series cannot fire. + (metricThresholdRule { + uid = "hh-claude-usage-ahead-of-clock"; + title = "claude usage ahead of the clock"; + expr = + let + ahead = + window: seconds: + ''max by (account, window) (last_over_time({__name__="hyperhive.agent.claude_usage.percent", window="${window}", account!=""}[15m])) - 100 * (1 - (max by (account, window) (last_over_time({__name__="hyperhive.agent.claude_usage.resets_at", window="${window}", account!=""}[15m])) - time()) / ${toString seconds})''; + in + "(${ahead "seven_day" 604800}) or (${ahead "five_hour" 18000})"; + threshold = 0; + for = "1h"; + summary = "claude account {{ $labels.account }} has used more of its {{ $labels.window }} usage window than the window has elapsed"; + }) ] # One rule per declared hive rather than one grouped by # `hive`, so a hive that sends nothing at all still has diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 37c5acc2..7aed754e 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -871,6 +871,44 @@ let ] && query.instant; } + { + # The usage-ahead-of-clock alert selects the two gauge names hive-agent + # spells in Rust, requires the `account` label, maps both windows to + # their lengths and fires above zero for an hour. The disk rule is the + # presence control: it goes through the same lookup. + # 🔬 Invert-proof: drop `account!=""`, change `threshold`/`for`, or a + # window length in `hh-claude-usage-ahead-of-clock`, or rename a gauge + # in `otel_turn_metrics.rs`. + name = "the claude usage alert is per account and window and fires above the clock share"; + ok = + let + groups = + grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups; + rules = lib.concatMap (g: g.rules) groups; + source = builtins.readFile ../../hive-agent/src/otel_turn_metrics.rs; + rule = lib.findFirst (r: r.uid == "hh-claude-usage-ahead-of-clock") null rules; + disk = lib.findFirst (r: r.uid == "hh-host-disk-high") null rules; + expr = (lib.head rule.data).model.expr; + condition = lib.findFirst (q: q.refId == "C") null rule.data; + in + disk != null + && rule != null + && rule.for == "1h" + && lib.head (lib.head condition.model.conditions).evaluator.params == 0 + && (lib.head condition.model.conditions).evaluator.type == "gt" + && + lib.all (metric: lib.hasInfix ''{__name__="${metric}"'' expr && lib.hasInfix ''"${metric}"'' source) + [ + "hyperhive.agent.claude_usage.percent" + "hyperhive.agent.claude_usage.resets_at" + ] + && lib.hasInfix ''account!=""'' expr + && lib.hasInfix ''window="seven_day"'' expr + && lib.hasInfix "/ 604800" expr + && lib.hasInfix ''window="five_hour"'' expr + && lib.hasInfix "/ 18000" expr + && lib.hasInfix "max by (account, window)" expr; + } { # The held-deploys alert fires on any agent held for 30 minutes and # selects the gauge name `held_deploys.rs` spells in Rust. The disk rule