feat(#3494): provision the agents dashboard from the repo

Grafana served no dashboards: only datasources were provisioned, while
the module header already claimed dashboards were. This ships the agents
dashboard as a file provider and makes that sentence true.

The datasource uid is bound once and substituted into the dashboard at
build time. Committing the literal would make the dashboard a second
speller of a name the datasource already owns, and the drift failure is
silent -- panels render empty rather than erroring.

The shipped copy drops the `DS` datasource variable: it exists so an
operator can pick a store on manual import, and a provisioned dashboard
must not ask.
This commit is contained in:
atlas 2026-08-23 22:21:34 +02:00 committed by mara
commit 9fec1e1dca
2 changed files with 704 additions and 2 deletions

View file

@ -5,7 +5,9 @@
# restarted, reconfigured or broken without taking the TSDB down with it.
# They are a pair, not a unit.
#
# Dashboards are provisioned from config, deliberately not deployed from git.
# Dashboards are provisioned from files in this repo, deliberately not created
# in the UI: the store path is immutable, so Grafana is told not to accept UI
# edits it would lose on the next rebuild.
{
pkgs,
lib,
@ -24,6 +26,32 @@ let
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
# Spelled ONCE. The provisioned datasource declares it and every panel in the
# provisioned dashboards references it; a second literal would be free to
# drift, and the failure is a dashboard that renders with empty panels rather
# than an error.
datasourceUid = "swarm-victoriametrics";
# The shipped dashboards carry `@datasourceUid@` where a real deployment needs
# the uid above. They are substituted here rather than committed with the
# literal so the single binding stays single.
renderDashboard =
name:
pkgs.writeText name (
builtins.replaceStrings [ "@datasourceUid@" ] [ datasourceUid ] (
builtins.readFile (./swarm-grafana/dashboards + "/${name}")
)
);
# Grafana's file provider wants a DIRECTORY to scan, so the rendered files are
# collected into one.
dashboardDir = pkgs.linkFarm "hyperhive-grafana-dashboards" (
map (name: {
inherit name;
path = renderDashboard name;
}) [ "agents.json" ]
);
# Total on a null swarm domain for the same reason every sibling module is:
# the required-domain assertion in hive-network.nix should be what an
# operator sees, not a coercion error from here.
@ -569,13 +597,28 @@ in
{
name = "VictoriaMetrics";
type = "prometheus";
uid = "swarm-victoriametrics";
uid = datasourceUid;
url = cfg.datasourceUrl;
access = "proxy";
isDefault = true;
}
];
};
provision.dashboards.settings = {
apiVersion = 1;
providers = [
{
name = "hyperhive";
type = "file";
# Read-only in the UI: the store path is immutable, so an edit
# saved here would be silently discarded on the next rebuild.
# Better to refuse the edit than to lose it.
allowUiUpdates = false;
options.path = dashboardDir;
}
];
};
};
};
};

View file

@ -0,0 +1,659 @@
{
"uid": "hyperhive-agents",
"title": "hyperhive · agents",
"description": "Per-agent view: resource usage from hive-c0re's container sampler, activity from Claude Code's own export. Renamed from 'overview' — every producer we have today is per-agent, so a genuine swarm overview has nothing cross-cutting to summarise yet and should be built last, not first. Metric names are the OTLP DOTTED form, so every query uses the {__name__=\"...\"} selector — the underscore spelling matches nothing in this store. Cost/token totals use increase(), which is correct here: the claude_code.* counters are cumulative (forced in nix/agent-modules/otel.nix) but reset once per turn, since each turn is a new claude process.",
"tags": [
"hyperhive",
"agents"
],
"timezone": "utc",
"schemaVersion": 39,
"version": 1,
"editable": true,
"refresh": "1m",
"time": {
"from": "now-6h",
"to": "now"
},
"templating": {
"list": [
{
"name": "agent",
"label": "Agent",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"query": "label_values({__name__=\"hyperhive.container.cpu.percent\"}, agent)",
"refresh": 2,
"includeAll": true,
"allValue": ".*",
"multi": true,
"current": {
"text": "All",
"value": "$__all"
},
"sort": 1
}
]
},
"panels": [
{
"id": 1,
"type": "stat",
"title": "Agents reporting",
"description": "Distinct agents whose container metrics reached the store in the selected range. This counts what hive-c0re observes, not what the topology declares — a gap between the two is the interesting case.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "count(count by (agent) ({__name__=\"hyperhive.container.cpu.percent\", agent=~\"$agent\"}))",
"instant": true,
"legendFormat": "agents"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "text"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 2,
"type": "stat",
"title": "Cost",
"description": "sum(increase(claude_code.cost.usage)). Agent-reported, so it covers claude's own token spend only — nothing else in the swarm reports cost.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase({__name__=\"claude_code.cost.usage\", agent=~\"$agent\"}[$__range]))",
"instant": true,
"legendFormat": "cost"
}
],
"fieldConfig": {
"defaults": {
"unit": "currencyUSD",
"decimals": 2,
"color": {
"mode": "fixed",
"fixedColor": "text"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 3,
"type": "stat",
"title": "Tokens (excl. cache reads)",
"description": "Excludes type=cacheRead, which would otherwise BE this number: measured over 6h on this store, cacheRead was 273.5M of 275.8M total — 99.2%. Counting it makes the tile a cache-hit-rate readout wearing a token label, and it drowns the input/output volume that actually tracks work done. Remaining types: input, output, cacheCreation.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase({__name__=\"claude_code.token.usage\", agent=~\"$agent\", type!=\"cacheRead\"}[$__range]))",
"instant": true,
"legendFormat": "tokens"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 1,
"color": {
"mode": "fixed",
"fixedColor": "text"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 4,
"type": "stat",
"title": "CLI sessions started",
"description": "claude_code.session.count counts CLI STARTS, and the harness starts one claude process per turn — so this reads as turns, not as long-lived sessions. Named for what it measures rather than what the metric is called.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase({__name__=\"claude_code.session.count\", agent=~\"$agent\"}[$__range]))",
"instant": true,
"legendFormat": "sessions"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "text"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 10,
"type": "timeseries",
"title": "Cost rate by agent",
"description": "USD per hour, from rate() over the cumulative counter — reset-safe. A flat line at zero is an idle agent, not a broken exporter; check the CPU panel to tell them apart.\n\nThe 10m minimum step is load-bearing, not a style choice: agents export every 60s, and measured against this store a rate window of 1m or 2m resolves ONE agent instead of five. Any shorter and the panel quietly under-reports.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"interval": "10m",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (agent) (rate({__name__=\"claude_code.cost.usage\", agent=~\"$agent\"}[$__rate_interval])) * 3600",
"legendFormat": "{{agent}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "currencyUSD",
"decimals": 2,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 0,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 11,
"type": "timeseries",
"title": "Container CPU by agent",
"description": "hyperhive.container.cpu.percent, sampled by hive-c0re. Percent of one core.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 12
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (agent) ({__name__=\"hyperhive.container.cpu.percent\", agent=~\"$agent\"})",
"legendFormat": "{{agent}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"decimals": 1,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 0,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 20,
"type": "timeseries",
"title": "Container memory by agent",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 12
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (agent) ({__name__=\"container.memory.usage\", agent=~\"$agent\"})",
"legendFormat": "{{agent}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 0,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 21,
"type": "timeseries",
"title": "Tokens by model (excl. cache reads)",
"description": "Split by model rather than by agent on purpose — 'which model is the spend on' is the question this answers. Excludes type=cacheRead for the same reason as the tile above: at ~99% of volume it flattens every model into one line shaped like its cache. 10m minimum step for the same reason as the cost-rate panel.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"interval": "10m",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (model) (rate({__name__=\"claude_code.token.usage\", agent=~\"$agent\", type!=\"cacheRead\"}[$__rate_interval])) * 60",
"legendFormat": "{{model}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 0,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 30,
"type": "bargauge",
"title": "Container storage by agent",
"description": "Magnitude comparison, so a bar rather than seven lines. One fixed colour on purpose: the agents are nominal categories, and colouring them by size would encode the bar length twice.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 20
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (agent) ({__name__=\"hyperhive.container.storage.usage\", agent=~\"$agent\"})",
"instant": true,
"legendFormat": "{{agent}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"color": {
"mode": "fixed",
"fixedColor": "blue"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"displayMode": "basic",
"orientation": "horizontal",
"showUnfilled": true,
"valueMode": "text",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 31,
"type": "timeseries",
"title": "Lines of code changed",
"description": "claude_code.lines_of_code.count carries a `type` label (added / removed). 10m minimum step for the same reason as the cost-rate panel.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"interval": "10m",
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 20
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (type) (rate({__name__=\"claude_code.lines_of_code.count\", agent=~\"$agent\"}[$__rate_interval])) * 60",
"legendFormat": "{{type}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 0,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 32,
"type": "bargauge",
"title": "Active time by agent",
"description": "claude_code.active_time.total — seconds an agent was actually working, not wall-clock uptime.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 20
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (agent) (increase({__name__=\"claude_code.active_time.total\", agent=~\"$agent\"}[$__range]))",
"instant": true,
"legendFormat": "{{agent}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "purple"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "purple",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"displayMode": "basic",
"orientation": "horizontal",
"showUnfilled": true,
"valueMode": "text",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
}
]
}