diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 648c0f64..586cf76e 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -355,22 +355,27 @@ When OTEL is enabled, the harness itself (`hive-agent`) exports one small set of metrics per claude turn, recorded the moment the turn ends (not polled). These are deliberately the fields Claude Code's own built-in export (see above) can't know about — the harness's own wall-clock timing, what woke the -turn, its own outcome classification, the loose-ends backlog, and session -boundaries. Token usage, cost, and tool-call counts are **not** duplicated -here; that's already covered by Claude's own export. +turn, its own outcome classification, the loose-ends backlog, session +boundaries, and how full the context window is. Token usage and tool-call +counts are **not** duplicated here; that's already covered by Claude's own +export. Cost is here only for agents on an ACP runtime, which have no export +of their own — see [ACP-reported cost](#acp-reported-cost). -| Metric | Unit | Kind | Attributes | -| ---------------------------------------- | ---- | --------- | ------------------------------------------------------------------------------------------------------------------ | -| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` | -| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` | -| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) | -| `hyperhive.agent.loose_ends.threads` | — | gauge | none | -| `hyperhive.agent.loose_ends.reminders` | — | gauge | none | -| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `CLAUDE.md`-size watch's own ~15-minute tick, **not** per turn like the rows above | -| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn | -| `hyperhive.agent.claude_usage.resets_at` | `s` | gauge | `window` — unix seconds at which that window resets; same 5-minute poll | +| Metric | Unit | Kind | Attributes | +| ---------------------------------------- | --------- | --------- | ------------------------------------------------------------------------------------------------------------------ | +| `hyperhive.agent.turn.duration` | `ms` | histogram | `wake_from`, `result_kind`, `model` | +| `hyperhive.agent.turn.count` | — | counter | `wake_from`, `result_kind`, `model` | +| `hyperhive.agent.session.count` | — | counter | `model` (incremented once per fresh, non-`--continue`'d session) | +| `hyperhive.agent.loose_ends.threads` | — | gauge | none | +| `hyperhive.agent.loose_ends.reminders` | — | gauge | none | +| `hyperhive.agent.claude_md.lines` | — | gauge | none — recorded from the `CLAUDE.md`-size watch's own ~15-minute tick, **not** per turn like the rows above | +| `hyperhive.agent.claude_usage.percent` | `%` | gauge | `window` (`five_hour`, `seven_day`, … as the usage endpoint names them) — polled every 5 minutes, **not** per turn | +| `hyperhive.agent.claude_usage.resets_at` | `s` | gauge | `window` — unix seconds at which that window resets; same 5-minute poll | +| `hyperhive.agent.context.used` | `{token}` | gauge | none — tokens in the context window at turn end, the numerator of the web UI's ctx% | +| `hyperhive.agent.context.size` | `{token}` | gauge | none — the context window those tokens fill, ctx%'s denominator | +| `hyperhive.agent.cost.usage` | — | counter | `model`, `currency` — ACP agents only, see below | For the two `claude_usage` gauges the harness polls the Claude subscription usage endpoint (`GET /api/oauth/usage`, the one behind claude's own `/usage`) @@ -381,6 +386,12 @@ session (API-key backends, not yet logged in) or an expired token skips the poll, so a gauge keeps its last value until the next successful poll — a `resets_at` in the past means the paired `percent` is stale. +The two `context` gauges hold the two numbers the web UI divides for its ctx%: +the last turn's context tokens (input, cache read and cache creation), and the +API-reported window, else the model's default. A turn that parsed no usage +leaves both where the previous turn put them. The dashboard's **Context +window used by agent** panel (health tab) divides one by the other. + Resource attributes (`service.name`, `agent`, `hive`, `swarm`) come from the same container-wide `OTEL_RESOURCE_ATTRIBUTES` as everything else in this section — nothing extra to configure. Cadence follows @@ -389,6 +400,32 @@ section — nothing extra to configure. Cadence follows the harness flushes the batched points to the collector, not how often it records them (every turn, always). +### ACP-reported cost + +An ACP agent reports what its session has cost so far in the `cost` field of +its `usage_update` notifications — a running total, not a per-turn figure +(opencode sums every assistant message in the session). The harness counts +what each report adds to the last one and exports that as +`hyperhive.agent.cost.usage`, so `increase()` over it gives the money spent +in a range. It follows a few rules: + +- A new session's first report counts in full, since the session started at + zero. +- A session loaded into a freshly started agent (after a harness restart or + an agent crash) has an unknown total until it reports one, so that first + report only sets the baseline — the turn it covers goes uncounted. +- A total lower than the previous one means the agent dropped part of the + session's history; it adds nothing and becomes the new baseline. +- Cost from a compaction the harness runs between turns counts toward the + next turn. + +The value is in whatever `currency` the agent names; opencode always sends +`USD`, and the `hyperhive · agents` dashboard's ACP cost panels (cost tab) +filter on it. A claude agent never records this metric: claude's own export +already carries its cost as `claude_code.cost.usage`, and keeping the two +apart means no turn is ever counted twice. Add both names together for a +whole-swarm spend figure. + ## Hive-scoped metrics (hive-c0re) diff --git a/hive-agent/src/main.rs b/hive-agent/src/main.rs index f118ccdd..a60120e0 100644 --- a/hive-agent/src/main.rs +++ b/hive-agent/src/main.rs @@ -1001,6 +1001,7 @@ async fn handle_turn( let ended_at = chrono::Utc::now().timestamp(); let duration_ms = i64::try_from(started_instant.elapsed().as_millis()).unwrap_or(i64::MAX); let (open_threads, open_reminders) = S::post_turn_counts(socket).await; + let context_window = bus.effective_context_window(&model_at_start); let row = serve_common::build_row(serve_common::TurnRowArgs { started_at, ended_at, @@ -1014,10 +1015,17 @@ async fn handle_turn( open_reminders_count: open_reminders, }); // Harness-only OTEL metrics (duration/wake_from/result_kind/loose-ends/ - // session boundaries) — independent of the sqlite sink below, and a - // cheap no-op when OTEL isn't configured. See `otel_turn_metrics`'s - // module doc for why token/cost/tool-count are deliberately not here. + // session boundaries/context fill) — independent of the sqlite sink + // below, and a cheap no-op when OTEL isn't configured. See + // `otel_turn_metrics`'s module doc for why token/tool-count are + // deliberately not here, and why cost is only ever an ACP agent's. otel_turn_metrics::record(&row, fresh_session); + if let Some(ctx) = bus.last_ctx_usage() { + otel_turn_metrics::record_context(ctx.context_tokens(), context_window); + } + for cost in hive_runtime::Runtime::take_reported_cost(session) { + otel_turn_metrics::record_reported_cost(&row.model, &cost); + } if let Some(stats) = stats { stats.record(&row); } diff --git a/hive-agent/src/otel_turn_metrics.rs b/hive-agent/src/otel_turn_metrics.rs index 2717becd..2039f725 100644 --- a/hive-agent/src/otel_turn_metrics.rs +++ b/hive-agent/src/otel_turn_metrics.rs @@ -6,8 +6,8 @@ //! only the harness-only concepts: wall-clock turn duration as *this harness* //! measures it (not Claude's own per-request latency), what woke the turn, //! the harness's own outcome classification, the loose-ends backlog at turn -//! end, and session boundaries (fresh vs. `--continue`'d). Scoped this way -//! per the tracker discussion on the "emit agent stats as OTEL metrics" issue. +//! end, session boundaries (fresh vs. `--continue`'d), context-window fill, +//! and the cost an ACP agent reports — claude's runtime reports none. //! //! Synchronous instruments, not observable ones: unlike hive-c0re's //! container-resource gauges (`hive-c0re/src/stats/otel_metrics.rs`), which @@ -62,6 +62,11 @@ struct Instruments { /// `window`. claude_usage_percent: Gauge, claude_usage_resets_at: Gauge, + context_used: Gauge, + context_size: Gauge, + /// Keyed by `model` and `currency`: an ACP agent reports cost in a + /// currency of its choosing. + reported_cost: Counter, } /// Lazily built on the first call to [`record`]. `None` when OTEL isn't @@ -129,6 +134,30 @@ pub fn record_claude_usage(window: &str, percent: f64, resets_at: Option) { } } +/// Record the context window's fill at turn end: `used` tokens of `size`, +/// the two numbers the web UI's ctx% divides. No-op when OTEL isn't +/// configured, same as [`record`]. +pub fn record_context(used: u64, size: u64) { + let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { + return; + }; + inst.context_used.record(used, &[]); + inst.context_size.record(size, &[]); +} + +/// Add what an ACP agent reported spending since the last call. No-op when +/// OTEL isn't configured, same as [`record`]. +pub fn record_reported_cost(model: &str, cost: &hive_runtime::ReportedCost) { + let Some(inst) = INSTRUMENTS.get_or_init(build).as_ref() else { + return; + }; + let attrs = [ + KeyValue::new("model", model.to_owned()), + KeyValue::new("currency", cost.currency.clone()), + ]; + inst.reported_cost.add(cost.amount, &attrs); +} + fn build() -> Option { if !enabled() { tracing::debug!("otel turn-metrics: no endpoint configured, exporter disabled"); @@ -161,6 +190,15 @@ fn build() -> Option { .u64_gauge("hyperhive.agent.claude_usage.resets_at") .with_unit("s") .build(), + context_used: meter + .u64_gauge("hyperhive.agent.context.used") + .with_unit("{token}") + .build(), + context_size: meter + .u64_gauge("hyperhive.agent.context.size") + .with_unit("{token}") + .build(), + reported_cost: meter.f64_counter("hyperhive.agent.cost.usage").build(), _provider: provider, }) } diff --git a/hive-runtime/src/acp/mod.rs b/hive-runtime/src/acp/mod.rs index dee41271..838df7bb 100644 --- a/hive-runtime/src/acp/mod.rs +++ b/hive-runtime/src/acp/mod.rs @@ -8,7 +8,7 @@ mod rpc; mod stream; -use std::collections::{HashMap, VecDeque}; +use std::collections::{BTreeMap, HashMap, VecDeque}; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::atomic::{AtomicBool, Ordering}; @@ -229,6 +229,67 @@ impl Choices { } } +/// An amount of money an agent reported, in an ISO 4217 `currency`. +#[derive(Debug, Clone, PartialEq)] +pub struct ReportedCost { + pub amount: f64, + pub currency: String, +} + +/// What the agent has reported spending, shared between the runtime and its +/// [`Live`] agent so an amount outlives a respawn until it is taken. +#[derive(Clone, Default)] +struct Spend(Arc>); + +#[derive(Default)] +struct Spent { + /// The attached session's running total at its last report; `None` until + /// it reports one. + total: Option, + /// The attached session is new, so its total started at zero. + fresh: bool, + /// Spent since the last [`Spend::take`], per currency. + untaken: BTreeMap, +} + +impl Spend { + /// Start following a newly attached session. A loaded session's total + /// before its first report is unknown, so that report only sets the + /// baseline. + fn attach(&self, fresh: bool) { + let mut spent = self.lock(); + spent.total = None; + spent.fresh = fresh; + } + + /// Fold in a running total the attached session reported. A total lower + /// than the last one means the agent dropped part of the session's + /// history, so it adds nothing. + fn observe(&self, total: ReportedCost) { + let mut spent = self.lock(); + let added = match &spent.total { + Some(last) if last.currency == total.currency => total.amount - last.amount, + None if spent.fresh => total.amount, + _ => 0.0, + }; + if added > 0.0 { + *spent.untaken.entry(total.currency.clone()).or_default() += added; + } + spent.total = Some(total); + } + + fn take(&self) -> Vec { + std::mem::take(&mut self.lock().untaken) + .into_iter() + .map(|(currency, amount)| ReportedCost { amount, currency }) + .collect() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Spent> { + self.0.lock().unwrap_or_else(PoisonError::into_inner) + } +} + fn find<'a>( options: &'a [stream::ConfigOption], category: &str, @@ -291,6 +352,7 @@ pub struct AcpRuntime { cancel_grace: Duration, compact_idle: Duration, choices: Choices, + spend: Spend, } /// The running agent process and the session loaded into it. @@ -305,6 +367,7 @@ struct Live { commands: Option>, /// The config options `loaded` last reported. choices: Choices, + spend: Spend, } impl Live { @@ -340,6 +403,7 @@ impl AcpRuntime

{ cancel_grace: CANCEL_GRACE, compact_idle: COMPACT_IDLE, choices: Choices::default(), + spend: Spend::default(), } } @@ -381,6 +445,7 @@ impl AcpRuntime

{ model: None, commands: None, choices: self.choices.clone(), + spend: self.spend.clone(), }) } @@ -414,6 +479,7 @@ impl AcpRuntime

{ live.model = stream::session_model(&response); live.offer(stream::config_options(&response).unwrap_or_default()); live.loaded = Some(id.clone()); + live.spend.attach(false); live.commands = None; discard_stale(live)?; return Ok((id, new)); @@ -436,6 +502,7 @@ impl AcpRuntime

{ live.model = stream::session_model(&response); live.offer(stream::config_options(&response).unwrap_or_default()); live.loaded = Some(id.clone()); + live.spend.attach(true); live.commands = None; write_id(&self.pending_file(), &id)?; Ok((id, true)) @@ -709,6 +776,10 @@ impl Runtime for AcpRuntime

{ Some(self.choices.clone()) } + fn take_reported_cost(&self) -> Vec { + self.spend.take() + } + /// Moves the session file aside; the agent keeps its own copy of the /// session, so nothing is lost. A new session whose first prompt was never /// answered is dropped too. @@ -761,8 +832,11 @@ fn deliver( } /// Keep the commands and config options an `update` for the loaded session -/// advertises. +/// advertises, and the cost it reports. fn keep_advertised(agent: &mut Live, update: &Value) { + if let Some(cost) = stream::reported_cost(update) { + agent.spend.observe(cost); + } if let Some(advertised) = stream::advertised_commands(update) { agent.commands = Some(advertised); } @@ -775,7 +849,7 @@ fn keep_advertised(agent: &mut Live, update: &Value) { /// replays (the caller already has it), and anything that arrived after the /// previous turn settled, which must not be shown as part of the next one. /// The commands and config options the agent advertises for the loaded -/// session are kept. +/// session, and the cost it reports, are kept. fn discard_stale(live: &mut Live) -> std::result::Result<(), AcpError> { while let Ok(incoming) = live.conn.incoming.try_recv() { between_turns(live, incoming)?; @@ -941,7 +1015,7 @@ mod tests { use hive_claude::{Config, NoopSink, PercentPolicy, Sink}; - use super::{AcpError, AcpRuntime, Choice, SessionChoices, TurnKind}; + use super::{AcpError, AcpRuntime, Choice, ReportedCost, SessionChoices, Spend, TurnKind}; use crate::{AcpCommand, Error, Runtime}; /// An ACP agent in plain `sh`. It answers `initialize`, numbers its @@ -971,7 +1045,9 @@ mod tests { /// and `m/plain`, and effort levels `low` (the default) and `high` on /// `m/think` only; it appends each `session/set_config_option` to /// `.sets` as `id=value`. With `REFUSE` set, it answers the first - /// `session/set_config_option` to that value with an error. + /// `session/set_config_option` to that value with an error. With `COST` + /// set, each prompt adds that many dollars to its session's total (from zero, + /// kept in `.cost.` across restarts) and reports it. const AGENT: &str = r#" n=0 prompt= model=m/think effort=low printf 'start\n' >> "$1.methods" @@ -1007,6 +1083,7 @@ while IFS= read -r line; do printf '{"jsonrpc":"2.0","id":%s,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":true,"mcpCapabilities":{"http":true}}}}\n' "$id" ;; session/new) n=$((n+1)) + rm -f "$1.cost.s$n" printf '{"jsonrpc":"2.0","id":%s,"result":{"sessionId":"s%s"%s}}\n' "$id" "$n" "$(options)" advertise "s$n" ;; session/load) @@ -1026,6 +1103,11 @@ while IFS= read -r line; do session/prompt) printf '%s\n' "$line" >> "$1" [ -n "$USED" ] && [ "$sid" = s1 ] && update "$sid" "{\"sessionUpdate\":\"usage_update\",\"used\":$USED,\"size\":1000}" + if [ -n "$COST" ]; then + total=$(( $(cat "$1.cost.$sid" 2>/dev/null || echo 0) + COST )) + printf '%s' "$total" > "$1.cost.$sid" + update "$sid" "{\"sessionUpdate\":\"usage_update\",\"cost\":{\"amount\":$total,\"currency\":\"USD\"}}" + fi case $line in *'"text":"/compact"'*) [ -n "$STALL_COMPACT" ] && prompt=$id && continue ;; esac @@ -1639,4 +1721,93 @@ done Some(choice("m/think", &["m/think", "m/plain"])) ); } + + fn usd(amount: f64) -> ReportedCost { + ReportedCost { + amount, + currency: "USD".into(), + } + } + + #[test] + fn a_new_session_s_first_report_is_all_spent() { + let spend = Spend::default(); + spend.attach(true); + spend.observe(usd(0.5)); + spend.observe(usd(1.25)); + assert_eq!(spend.take(), [usd(1.25)]); + assert_eq!(spend.take(), []); + spend.observe(usd(2.0)); + assert_eq!(spend.take(), [usd(0.75)]); + } + + #[test] + fn a_loaded_session_s_first_report_only_sets_the_baseline() { + let spend = Spend::default(); + spend.attach(false); + spend.observe(usd(10.0)); + assert_eq!(spend.take(), []); + spend.observe(usd(10.5)); + assert_eq!(spend.take(), [usd(0.5)]); + } + + #[test] + fn a_falling_total_adds_nothing_and_becomes_the_baseline() { + let spend = Spend::default(); + spend.attach(true); + spend.observe(usd(3.0)); + spend.observe(usd(1.0)); + spend.observe(usd(1.5)); + assert_eq!(spend.take(), [usd(3.5)]); + } + + #[test] + fn untaken_spend_survives_attaching_another_session() { + let spend = Spend::default(); + spend.attach(true); + spend.observe(usd(1.0)); + spend.attach(true); + spend.observe(usd(0.25)); + assert_eq!(spend.take(), [usd(1.25)]); + } + + #[test] + fn a_changed_currency_only_sets_a_new_baseline() { + let spend = Spend::default(); + spend.attach(true); + spend.observe(usd(1.0)); + let eur = |amount| ReportedCost { + amount, + currency: "EUR".into(), + }; + spend.observe(eur(0.75)); + spend.observe(eur(1.25)); + assert_eq!(spend.take(), [eur(0.5), usd(1.0)]); + } + + #[tokio::test] + async fn reported_cost_is_what_the_session_added_since_last_taken() { + let dir = tempfile::tempdir().unwrap(); + let config = config(dir.path()); + let env = [("COST", "2")]; + + let before = agent(dir.path(), "ok", &env, PercentPolicy::default()); + before.run(&config, "one", &NoopSink).await.unwrap(); + before.run(&config, "two", &NoopSink).await.unwrap(); + assert_eq!(before.take_reported_cost(), [usd(4.0)]); + assert_eq!(before.take_reported_cost(), []); + drop(before); + + let after = agent(dir.path(), "ok", &env, PercentPolicy::default()); + after.run(&config, "three", &NoopSink).await.unwrap(); + assert_eq!(after.take_reported_cost(), []); + after.run(&config, "four", &NoopSink).await.unwrap(); + assert_eq!(after.take_reported_cost(), [usd(2.0)]); + + after.archive().unwrap(); + after.run(&config, "five", &NoopSink).await.unwrap(); + assert_eq!(after.take_reported_cost(), [usd(2.0)]); + assert_eq!(count(dir.path(), "session/load"), 1); + assert_eq!(count(dir.path(), "session/new"), 2); + } } diff --git a/hive-runtime/src/acp/stream.rs b/hive-runtime/src/acp/stream.rs index d8e42a19..3fea981d 100644 --- a/hive-runtime/src/acp/stream.rs +++ b/hive-runtime/src/acp/stream.rs @@ -7,6 +7,8 @@ use std::collections::HashMap; use hive_claude::{Telemetry, TokenUsage}; use serde_json::{Map, Value, json}; +use super::ReportedCost; + /// Turns one prompt's `session/update` notifications into claude /// `stream-json` events. /// @@ -258,6 +260,19 @@ pub(super) fn config_options(value: &Value) -> Option> { ) } +/// The running total cost a `usage_update` reports for its session, or +/// `None` for any other `update` and for one that reports no cost. +pub(super) fn reported_cost(update: &Value) -> Option { + if update.get("sessionUpdate").and_then(Value::as_str) != Some("usage_update") { + return None; + } + let cost = update.get("cost")?; + Some(ReportedCost { + amount: cost.get("amount")?.as_f64()?, + currency: cost.get("currency")?.as_str()?.to_owned(), + }) +} + /// The options a `config_option_update` sets, or `None` for any other /// `update`. Each replaces the session's previous options. pub(super) fn updated_config_options(update: &Value) -> Option> { @@ -402,7 +417,8 @@ fn tool_result(id: &str, output: &str, is_error: bool) -> Value { #[cfg(test)] mod tests { use super::{ - ConfigOption, StreamMapper, canonical_tool_name, config_options, mcp_servers, session_model, + ConfigOption, ReportedCost, StreamMapper, canonical_tool_name, config_options, mcp_servers, + reported_cost, session_model, }; use serde_json::{Value, json}; @@ -537,6 +553,33 @@ mod tests { assert_eq!(t.model.as_deref(), Some("provider/model")); } + #[test] + fn a_usage_update_reports_its_cost_when_it_carries_one() { + assert_eq!( + reported_cost(&json!({ "sessionUpdate": "usage_update", "used": 5000, + "size": 262_144, "cost": { "amount": 0.045, "currency": "USD" } })), + Some(ReportedCost { + amount: 0.045, + currency: "USD".into() + }) + ); + assert_eq!( + reported_cost(&json!({ "sessionUpdate": "usage_update", "used": 5000, + "size": 262_144 })), + None + ); + assert_eq!( + reported_cost(&json!({ "sessionUpdate": "usage_update", + "cost": { "amount": 0.045 } })), + None + ); + assert_eq!( + reported_cost(&json!({ "sessionUpdate": "plan", + "cost": { "amount": 0.045, "currency": "USD" } })), + None + ); + } + #[test] fn updates_with_nothing_to_show_emit_nothing() { let out = feed( diff --git a/hive-runtime/src/claude.rs b/hive-runtime/src/claude.rs index adc04aac..6e959d3c 100644 --- a/hive-runtime/src/claude.rs +++ b/hive-runtime/src/claude.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use hive_claude::{CompactionPolicy, Config, InfiniteSession, Progress, SessionStore, Sink}; -use crate::{Canceller, Choices, Result, Runtime}; +use crate::{Canceller, Choices, ReportedCost, Result, Runtime}; /// `claude --print` turns on a titled [`InfiniteSession`], which owns /// resume-or-create and compaction. @@ -47,4 +47,8 @@ impl Runtime for ClaudeRuntime

{ fn choices(&self) -> Option { None } + + fn take_reported_cost(&self) -> Vec { + Vec::new() + } } diff --git a/hive-runtime/src/lib.rs b/hive-runtime/src/lib.rs index 1459f676..2315f0af 100644 --- a/hive-runtime/src/lib.rs +++ b/hive-runtime/src/lib.rs @@ -23,7 +23,7 @@ mod spec; pub use acp::{ AcpError, AcpRuntime, Canceller, Choice, Choices, PermissionAsk, PermissionPolicy, - SessionChoices, + ReportedCost, SessionChoices, }; pub use claude::ClaudeRuntime; pub use hive_claude::{ @@ -65,6 +65,11 @@ pub trait Runtime { /// pick. `None` for a runtime whose session offers none to read: claude's /// `--model` and `--effort` are passed through unchecked. fn choices(&self) -> Option; + + /// What the agent reported spending since the last call, one entry per + /// currency. Empty for a runtime whose agent reports no cost: claude + /// exports its own. + fn take_reported_cost(&self) -> Vec; } /// The runtime an agent was configured with, chosen at startup from a @@ -109,6 +114,13 @@ impl Runtime for AgentRuntime

{ Self::Acp(r) => r.choices(), } } + + fn take_reported_cost(&self) -> Vec { + match self { + Self::Claude(r) => r.take_reported_cost(), + Self::Acp(r) => r.take_reported_cost(), + } + } } /// Why a runtime operation did not complete. diff --git a/nix/host-modules/swarm-grafana/dashboards/agents.json b/nix/host-modules/swarm-grafana/dashboards/agents.json index 39f0ba95..6e0047d6 100644 --- a/nix/host-modules/swarm-grafana/dashboards/agents.json +++ b/nix/host-modules/swarm-grafana/dashboards/agents.json @@ -2126,6 +2126,233 @@ } } } + }, + "panel-65": { + "kind": "Panel", + "spec": { + "id": 65, + "title": "ACP agent cost by agent", + "description": "USD an ACP agent reported spending in the selected range, busiest first.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sort_desc(sum by (agent) (increase({__name__=\"hyperhive.agent.cost.usage\", hive=~\"$hive\", agent=~\"$agent\", currency=\"USD\"}[$__range])))", + "instant": true, + "legendFormat": "{{agent}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "bargauge", + "version": "", + "spec": { + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "text" + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "purple" + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "purple" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-66": { + "kind": "Panel", + "spec": { + "id": 66, + "title": "ACP agent cost rate by agent", + "description": "USD per hour an ACP agent reported spending.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sum by (agent) (rate({__name__=\"hyperhive.agent.cost.usage\", hive=~\"$hive\", agent=~\"$agent\", currency=\"USD\"}[$__rate_interval])) * 3600", + "legendFormat": "{{agent}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "interval": "10m" + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "decimals": 2, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisSoftMin": 0, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-34": { + "kind": "Panel", + "spec": { + "id": 34, + "title": "Context window used by agent", + "description": "Share of each agent's context window filled at its last turn end.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "max by (agent) ({__name__=\"hyperhive.agent.context.used\", hive=~\"$hive\", agent=~\"$agent\"}) / max by (agent) ({__name__=\"hyperhive.agent.context.size\", hive=~\"$hive\", agent=~\"$agent\"}) * 100", + "legendFormat": "{{agent}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisSoftMin": 0, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false, + "axisSoftMax": 100 + } + }, + "overrides": [] + } + } + } + } } }, "layout": { @@ -2373,6 +2600,32 @@ "name": "panel-64" } } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 30, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-65" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 30, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-66" + } + } } ] } @@ -2438,6 +2691,19 @@ "name": "panel-33" } } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 23, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-34" + } + } } ] }