agents board: a turns tab, and turn duration bounds past 10 s
The per-agent /stats page shows what the swarm agents board lacked about turns. The new tab reads the turn count and duration the harness already records: average duration, cache hit rate, tokens per turn, turns per hour by agent and by model, duration percentiles, and the wake source and result mixes. The duration histogram used the SDK default buckets, which stop at 10 s while a turn averages 24 s, so every p95 read a flat 10 s; it now has explicit bounds up to an hour.
This commit is contained in:
iris
parent
666f8975a2
commit
aecde1dc75
4 changed files with 898 additions and 1 deletions
|
|
@ -200,7 +200,7 @@ Grafana start — see [`credentials.md`](credentials.md).
|
|||
`nix/host-modules/swarm-grafana/dashboards/`; edits made in the UI aren't
|
||||
kept.
|
||||
|
||||
- `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first.
|
||||
- `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first. The **turns** tab holds what each agent's own `/stats` page shows about turns: average turn duration, cache hit rate and tokens per turn, turns per hour by agent and by model, the turn duration over time (median, 95th percentile and mean), and the mix of wake sources and results. The duration percentiles are right from the first deploy that includes the histogram bounds in `hive-agent`; before it, they read a flat 10 seconds.
|
||||
- `attic`: the swarm binary cache. Agents whose latest closure this hive can pull, and why not; the builder's pushes and push failures; closures hive-c0re pulled and deploy steps it refused; the cache's size and store-path count. It reads the closure-pullable gauge, the `swarm-builder` and `hive-c0re` logs, and the `hyperhive.attic.cache_bytes` and `hyperhive.attic.cache_nars` gauges, which a timer in the `swarm-attic` container (`swarm-attic-cache-size`) reads from atticd's database every five minutes and pushes through the container's forwarder. atticd itself exports no metrics and its logs don't reach the log store, so the board has no cache request or hit panel. Filter by hive and agent.
|
||||
- `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue.
|
||||
- `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents.
|
||||
|
|
|
|||
|
|
@ -43,6 +43,26 @@ use crate::turn_stats::TurnStatRow;
|
|||
/// two batch on the same rhythm.
|
||||
const EXPORT_INTERVAL: Duration = Duration::from_mins(1);
|
||||
|
||||
/// Bucket upper bounds of the turn-duration histogram, in milliseconds. The
|
||||
/// SDK's default bounds stop at 10 s, but a turn takes about 24 s on average,
|
||||
/// so with them every p95 read a flat 10 s. These run from half a second to an
|
||||
/// hour.
|
||||
const TURN_DURATION_BOUNDS_MS: [f64; 13] = [
|
||||
500.0,
|
||||
1_000.0,
|
||||
2_500.0,
|
||||
5_000.0,
|
||||
10_000.0,
|
||||
20_000.0,
|
||||
30_000.0,
|
||||
60_000.0,
|
||||
120_000.0,
|
||||
300_000.0,
|
||||
600_000.0,
|
||||
1_800_000.0,
|
||||
3_600_000.0,
|
||||
];
|
||||
|
||||
struct Instruments {
|
||||
/// Kept alive for the process lifetime — the `PeriodicReader` only
|
||||
/// exports while its provider lives.
|
||||
|
|
@ -198,6 +218,7 @@ fn build() -> Option<Instruments> {
|
|||
turn_duration_ms: meter
|
||||
.u64_histogram("hyperhive.agent.turn.duration")
|
||||
.with_unit("ms")
|
||||
.with_boundaries(TURN_DURATION_BOUNDS_MS.to_vec())
|
||||
.build(),
|
||||
turn_count: meter.u64_counter("hyperhive.agent.turn.count").build(),
|
||||
session_count: meter.u64_counter("hyperhive.agent.session.count").build(),
|
||||
|
|
@ -267,3 +288,20 @@ fn build_provider(interval: Duration) -> anyhow::Result<SdkMeterProvider> {
|
|||
pub fn enabled() -> bool {
|
||||
std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_ok_and(|s| !s.trim().is_empty())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::TURN_DURATION_BOUNDS_MS;
|
||||
|
||||
#[test]
|
||||
fn the_duration_bounds_ascend_and_reach_past_the_old_ten_second_ceiling() {
|
||||
// The SDK drops bounds that are not strictly ascending, which would
|
||||
// silently bring the default buckets (and the flat p95) back.
|
||||
assert!(TURN_DURATION_BOUNDS_MS.windows(2).all(|w| w[0] < w[1]));
|
||||
assert!(
|
||||
TURN_DURATION_BOUNDS_MS
|
||||
.last()
|
||||
.is_some_and(|&b| b > 10_000.0)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2623,6 +2623,701 @@
|
|||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-69": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 69,
|
||||
"title": "Average turn duration",
|
||||
"description": "Mean wall-clock time of a turn in the selected range, as the harness measures it.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sum(increase({__name__=\"hyperhive.agent.turn.duration_sum\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])) / sum(increase({__name__=\"hyperhive.agent.turn.duration_count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))",
|
||||
"instant": true,
|
||||
"legendFormat": "Average turn duration"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "stat",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ms",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "text"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-70": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 70,
|
||||
"title": "Cache hit rate",
|
||||
"description": "Cached input over all input-side tokens (input, cache reads and cache writes) in the selected range.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\", type=\"cacheRead\"}[$__range])) / sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\", type=~\"input|cacheRead|cacheCreation\"}[$__range]))",
|
||||
"instant": true,
|
||||
"legendFormat": "Cache hit rate"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "stat",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "text"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-71": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 71,
|
||||
"title": "Tokens per turn",
|
||||
"description": "Tokens of every type, subagents included, divided by turns in the selected range.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sum(increase({__name__=\"claude_code.token.usage\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])) / sum(increase({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range]))",
|
||||
"instant": true,
|
||||
"legendFormat": "Tokens per turn"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "stat",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "text"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-72": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 72,
|
||||
"title": "Turns per hour by agent",
|
||||
"description": "How many turns each agent runs per hour.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sum by (agent) (rate({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 3600",
|
||||
"legendFormat": "{{agent}}"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {
|
||||
"interval": "10m"
|
||||
}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "timeseries",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisSoftMin": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-73": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 73,
|
||||
"title": "Turns per hour by model",
|
||||
"description": "Turns per hour by the model that ran them; the model drives the token cost.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sum by (model) (rate({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 3600",
|
||||
"legendFormat": "{{model}}"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {
|
||||
"interval": "10m"
|
||||
}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "timeseries",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisSoftMin": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-74": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 74,
|
||||
"title": "Turn duration",
|
||||
"description": "Median, 95th percentile and mean turn duration.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate({__name__=\"hyperhive.agent.turn.duration_bucket\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])))",
|
||||
"legendFormat": "p50"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=\"hyperhive.agent.turn.duration_bucket\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])))",
|
||||
"legendFormat": "p95"
|
||||
}
|
||||
},
|
||||
"refId": "B",
|
||||
"hidden": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sum(rate({__name__=\"hyperhive.agent.turn.duration_sum\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) / sum(rate({__name__=\"hyperhive.agent.turn.duration_count\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval]))",
|
||||
"legendFormat": "mean"
|
||||
}
|
||||
},
|
||||
"refId": "C",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {
|
||||
"interval": "10m"
|
||||
}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "timeseries",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ms",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisSoftMin": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-75": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 75,
|
||||
"title": "Wake source mix",
|
||||
"description": "What woke the turns in the selected range: a peer, the operator, a reminder, a todo or the system.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sort_desc(sum by (wake_from) (increase({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))",
|
||||
"instant": true,
|
||||
"legendFormat": "{{wake_from}}"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "bargauge",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"displayMode": "basic",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showUnfilled": true,
|
||||
"valueMode": "text"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"value": null,
|
||||
"color": "blue"
|
||||
}
|
||||
]
|
||||
},
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "blue"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-76": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 76,
|
||||
"title": "Result mix",
|
||||
"description": "How the turns in the selected range ended, as the harness classifies them.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sort_desc(sum by (result_kind) (increase({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))",
|
||||
"instant": true,
|
||||
"legendFormat": "{{result_kind}}"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "bargauge",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"displayMode": "basic",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showUnfilled": true,
|
||||
"valueMode": "text"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"value": null,
|
||||
"color": "blue"
|
||||
}
|
||||
]
|
||||
},
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "blue"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-77": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 77,
|
||||
"title": "Turns that did not end ok",
|
||||
"description": "Turns per hour that ended in anything but ok: compacted, failed or an API stall.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sum by (result_kind) (rate({__name__=\"hyperhive.agent.turn.count\", hive=~\"$hive\", agent=~\"$agent\", result_kind!=\"ok\"}[$__rate_interval])) * 3600",
|
||||
"legendFormat": "{{result_kind}}"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {
|
||||
"interval": "10m"
|
||||
}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "timeseries",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 2,
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisSoftMin": 0,
|
||||
"drawStyle": "bars",
|
||||
"fillOpacity": 60,
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"layout": {
|
||||
|
|
@ -2733,6 +3428,136 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "TabsLayoutTab",
|
||||
"spec": {
|
||||
"title": "turns",
|
||||
"layout": {
|
||||
"kind": "GridLayout",
|
||||
"spec": {
|
||||
"items": [
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"width": 8,
|
||||
"height": 4,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-69"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 8,
|
||||
"y": 0,
|
||||
"width": 8,
|
||||
"height": 4,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-70"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 16,
|
||||
"y": 0,
|
||||
"width": 8,
|
||||
"height": 4,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-71"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 0,
|
||||
"y": 4,
|
||||
"width": 12,
|
||||
"height": 8,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-72"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 12,
|
||||
"y": 4,
|
||||
"width": 12,
|
||||
"height": 8,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-73"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 0,
|
||||
"y": 12,
|
||||
"width": 24,
|
||||
"height": 8,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-74"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 0,
|
||||
"y": 20,
|
||||
"width": 8,
|
||||
"height": 8,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-75"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 8,
|
||||
"y": 20,
|
||||
"width": 8,
|
||||
"height": 8,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-76"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 16,
|
||||
"y": 20,
|
||||
"width": 8,
|
||||
"height": 8,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-77"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "TabsLayoutTab",
|
||||
"spec": {
|
||||
|
|
|
|||
|
|
@ -502,6 +502,40 @@ let
|
|||
]
|
||||
&& (lib.head xf).spec.options.byField == "agent";
|
||||
}
|
||||
{
|
||||
# The agents board's turns tab reads the turn count and duration the harness
|
||||
# records, grouped by the labels it sets on them. A renamed instrument or
|
||||
# label there is a tab of empty panels, so each must still be spelled in the
|
||||
# harness; and the duration percentiles are only right while the histogram
|
||||
# has bounds past the SDK's 10 s default ceiling.
|
||||
name = "the agents dashboard's turns tab reads the turn metrics and labels the harness records";
|
||||
ok =
|
||||
let
|
||||
board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/agents.json);
|
||||
tab = lib.findFirst (t: t.spec.title == "turns") null board.spec.layout.spec.tabs;
|
||||
names = map (i: i.spec.element.name) tab.spec.layout.spec.items;
|
||||
exprs = lib.concatMap (
|
||||
n: map (q: q.spec.query.spec.expr) board.spec.elements.${n}.spec.data.spec.queries
|
||||
) names;
|
||||
recorded = builtins.readFile ../../hive-agent/src/otel_turn_metrics.rs;
|
||||
in
|
||||
tab != null
|
||||
&& lib.length names == 9
|
||||
&& lib.all (m: lib.hasInfix "\"${m}\"" recorded) [
|
||||
"hyperhive.agent.turn.count"
|
||||
"hyperhive.agent.turn.duration"
|
||||
"wake_from"
|
||||
"result_kind"
|
||||
"model"
|
||||
]
|
||||
&& lib.all (l: lib.any (e: lib.hasInfix l e) exprs) [
|
||||
"wake_from"
|
||||
"result_kind"
|
||||
"by (model)"
|
||||
"turn.duration_bucket"
|
||||
]
|
||||
&& lib.hasInfix "with_boundaries(TURN_DURATION_BOUNDS_MS" recorded;
|
||||
}
|
||||
{
|
||||
# The builds board reads the fields hive-c0re puts on every container-build
|
||||
# log line: the build id, agent and kind on each, the event on the start and
|
||||
|
|
|
|||
Loading…
Reference in a new issue