From 9fec1e1dcae7030bd413349b5fcc7b58bc1bfd25 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Sun, 23 Aug 2026 22:21:34 +0200
Subject: [PATCH] feat(#3494): provision the agents dashboard from the repo

Grafana served no dashboards: only datasources were provisioned, while
the module header already claimed dashboards were. This ships the agents
dashboard as a file provider and makes that sentence true.

The datasource uid is bound once and substituted into the dashboard at
build time. Committing the literal would make the dashboard a second
speller of a name the datasource already owns, and the drift failure is
silent -- panels render empty rather than erroring.

The shipped copy drops the `DS` datasource variable: it exists so an
operator can pick a store on manual import, and a provisioned dashboard
must not ask.
---
 nix/host-modules/swarm-grafana.nix            |  47 +-
 .../swarm-grafana/dashboards/agents.json      | 659 ++++++++++++++++++
 2 files changed, 704 insertions(+), 2 deletions(-)
 create mode 100644 nix/host-modules/swarm-grafana/dashboards/agents.json

diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix
index cb1a1e38..62a9a58d 100644
--- a/nix/host-modules/swarm-grafana.nix
+++ b/nix/host-modules/swarm-grafana.nix
@@ -5,7 +5,9 @@
 # restarted, reconfigured or broken without taking the TSDB down with it.
 # They are a pair, not a unit.
 #
-# Dashboards are provisioned from config, deliberately not deployed from git.
+# Dashboards are provisioned from files in this repo, deliberately not created
+# in the UI: the store path is immutable, so Grafana is told not to accept UI
+# edits it would lose on the next rebuild.
 {
   pkgs,
   lib,
@@ -24,6 +26,32 @@ let
 
   caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
 
+  # Spelled ONCE. The provisioned datasource declares it and every panel in the
+  # provisioned dashboards references it; a second literal would be free to
+  # drift, and the failure is a dashboard that renders with empty panels rather
+  # than an error.
+  datasourceUid = "swarm-victoriametrics";
+
+  # The shipped dashboards carry `@datasourceUid@` where a real deployment needs
+  # the uid above. They are substituted here rather than committed with the
+  # literal so the single binding stays single.
+  renderDashboard =
+    name:
+    pkgs.writeText name (
+      builtins.replaceStrings [ "@datasourceUid@" ] [ datasourceUid ] (
+        builtins.readFile (./swarm-grafana/dashboards + "/${name}")
+      )
+    );
+
+  # Grafana's file provider wants a DIRECTORY to scan, so the rendered files are
+  # collected into one.
+  dashboardDir = pkgs.linkFarm "hyperhive-grafana-dashboards" (
+    map (name: {
+      inherit name;
+      path = renderDashboard name;
+    }) [ "agents.json" ]
+  );
+
   # Total on a null swarm domain for the same reason every sibling module is:
   # the required-domain assertion in hive-network.nix should be what an
   # operator sees, not a coercion error from here.
@@ -569,13 +597,28 @@ in
                 {
                   name = "VictoriaMetrics";
                   type = "prometheus";
-                  uid = "swarm-victoriametrics";
+                  uid = datasourceUid;
                   url = cfg.datasourceUrl;
                   access = "proxy";
                   isDefault = true;
                 }
               ];
             };
+
+            provision.dashboards.settings = {
+              apiVersion = 1;
+              providers = [
+                {
+                  name = "hyperhive";
+                  type = "file";
+                  # Read-only in the UI: the store path is immutable, so an edit
+                  # saved here would be silently discarded on the next rebuild.
+                  # Better to refuse the edit than to lose it.
+                  allowUiUpdates = false;
+                  options.path = dashboardDir;
+                }
+              ];
+            };
           };
         };
     };
diff --git a/nix/host-modules/swarm-grafana/dashboards/agents.json b/nix/host-modules/swarm-grafana/dashboards/agents.json
new file mode 100644
index 00000000..416288e3
--- /dev/null
+++ b/nix/host-modules/swarm-grafana/dashboards/agents.json
@@ -0,0 +1,659 @@
+{
+  "uid": "hyperhive-agents",
+  "title": "hyperhive · agents",
+  "description": "Per-agent view: resource usage from hive-c0re's container sampler, activity from Claude Code's own export. Renamed from 'overview' — every producer we have today is per-agent, so a genuine swarm overview has nothing cross-cutting to summarise yet and should be built last, not first. Metric names are the OTLP DOTTED form, so every query uses the {__name__=\"...\"} selector — the underscore spelling matches nothing in this store. Cost/token totals use increase(), which is correct here: the claude_code.* counters are cumulative (forced in nix/agent-modules/otel.nix) but reset once per turn, since each turn is a new claude process.",
+  "tags": [
+    "hyperhive",
+    "agents"
+  ],
+  "timezone": "utc",
+  "schemaVersion": 39,
+  "version": 1,
+  "editable": true,
+  "refresh": "1m",
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "templating": {
+    "list": [
+      {
+        "name": "agent",
+        "label": "Agent",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "@datasourceUid@"
+        },
+        "query": "label_values({__name__=\"hyperhive.container.cpu.percent\"}, agent)",
+        "refresh": 2,
+        "includeAll": true,
+        "allValue": ".*",
+        "multi": true,
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "sort": 1
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Agents reporting",
+      "description": "Distinct agents whose container metrics reached the store in the selected range. This counts what hive-c0re observes, not what the topology declares — a gap between the two is the interesting case.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "count(count by (agent) ({__name__=\"hyperhive.container.cpu.percent\", agent=~\"$agent\"}))",
+          "instant": true,
+          "legendFormat": "agents"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "text"
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Cost",
+      "description": "sum(increase(claude_code.cost.usage)). Agent-reported, so it covers claude's own token spend only — nothing else in the swarm reports cost.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum(increase({__name__=\"claude_code.cost.usage\", agent=~\"$agent\"}[$__range]))",
+          "instant": true,
+          "legendFormat": "cost"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "currencyUSD",
+          "decimals": 2,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "text"
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Tokens (excl. cache reads)",
+      "description": "Excludes type=cacheRead, which would otherwise BE this number: measured over 6h on this store, cacheRead was 273.5M of 275.8M total — 99.2%. Counting it makes the tile a cache-hit-rate readout wearing a token label, and it drowns the input/output volume that actually tracks work done. Remaining types: input, output, cacheCreation.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum(increase({__name__=\"claude_code.token.usage\", agent=~\"$agent\", type!=\"cacheRead\"}[$__range]))",
+          "instant": true,
+          "legendFormat": "tokens"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 1,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "text"
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 4,
+      "type": "stat",
+      "title": "CLI sessions started",
+      "description": "claude_code.session.count counts CLI STARTS, and the harness starts one claude process per turn — so this reads as turns, not as long-lived sessions. Named for what it measures rather than what the metric is called.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum(increase({__name__=\"claude_code.session.count\", agent=~\"$agent\"}[$__range]))",
+          "instant": true,
+          "legendFormat": "sessions"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "text"
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 10,
+      "type": "timeseries",
+      "title": "Cost rate by agent",
+      "description": "USD per hour, from rate() over the cumulative counter — reset-safe. A flat line at zero is an idle agent, not a broken exporter; check the CPU panel to tell them apart.\n\nThe 10m minimum step is load-bearing, not a style choice: agents export every 60s, and measured against this store a rate window of 1m or 2m resolves ONE agent instead of five. Any shorter and the panel quietly under-reports.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "interval": "10m",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (agent) (rate({__name__=\"claude_code.cost.usage\", agent=~\"$agent\"}[$__rate_interval])) * 3600",
+          "legendFormat": "{{agent}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "currencyUSD",
+          "decimals": 2,
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 0,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 11,
+      "type": "timeseries",
+      "title": "Container CPU by agent",
+      "description": "hyperhive.container.cpu.percent, sampled by hive-c0re. Percent of one core.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (agent) ({__name__=\"hyperhive.container.cpu.percent\", agent=~\"$agent\"})",
+          "legendFormat": "{{agent}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "decimals": 1,
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 0,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 20,
+      "type": "timeseries",
+      "title": "Container memory by agent",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (agent) ({__name__=\"container.memory.usage\", agent=~\"$agent\"})",
+          "legendFormat": "{{agent}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 0,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 21,
+      "type": "timeseries",
+      "title": "Tokens by model (excl. cache reads)",
+      "description": "Split by model rather than by agent on purpose — 'which model is the spend on' is the question this answers. Excludes type=cacheRead for the same reason as the tile above: at ~99% of volume it flattens every model into one line shaped like its cache. 10m minimum step for the same reason as the cost-rate panel.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "interval": "10m",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (model) (rate({__name__=\"claude_code.token.usage\", agent=~\"$agent\", type!=\"cacheRead\"}[$__rate_interval])) * 60",
+          "legendFormat": "{{model}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 0,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 30,
+      "type": "bargauge",
+      "title": "Container storage by agent",
+      "description": "Magnitude comparison, so a bar rather than seven lines. One fixed colour on purpose: the agents are nominal categories, and colouring them by size would encode the bar length twice.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 0,
+        "y": 20
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (agent) ({__name__=\"hyperhive.container.storage.usage\", agent=~\"$agent\"})",
+          "instant": true,
+          "legendFormat": "{{agent}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "blue"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "blue",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "displayMode": "basic",
+        "orientation": "horizontal",
+        "showUnfilled": true,
+        "valueMode": "text",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 31,
+      "type": "timeseries",
+      "title": "Lines of code changed",
+      "description": "claude_code.lines_of_code.count carries a `type` label (added / removed). 10m minimum step for the same reason as the cost-rate panel.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "interval": "10m",
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 8,
+        "y": 20
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (type) (rate({__name__=\"claude_code.lines_of_code.count\", agent=~\"$agent\"}[$__rate_interval])) * 60",
+          "legendFormat": "{{type}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 0,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 32,
+      "type": "bargauge",
+      "title": "Active time by agent",
+      "description": "claude_code.active_time.total — seconds an agent was actually working, not wall-clock uptime.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 16,
+        "y": 20
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (agent) (increase({__name__=\"claude_code.active_time.total\", agent=~\"$agent\"}[$__range]))",
+          "instant": true,
+          "legendFormat": "{{agent}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "decimals": 0,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "purple"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "purple",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "displayMode": "basic",
+        "orientation": "horizontal",
+        "showUnfilled": true,
+        "valueMode": "text",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      }
+    }
+  ]
+}