vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

review classifier: export request, verdict, check and run metrics, with a dashboard

Browse source 
The controller counts review requests, verdicts and per-check results and times each run; the classifier service counts retried asks. A provisioned Grafana dashboard charts them.
This commit is contained in:
atlas 2026-10-24 18:50:03 +02:00 • committed by mara
commit eaf26aaadd
13 changed files with 1390 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

1
nix/host-modules/swarm-grafana.nix
View file

@ -278,6 +278,7 @@ let
"logstore.json"
"metricstore.json"
"queue.json"
"review-classifier.json"
]
);

641
nix/host-modules/swarm-grafana/dashboards/review-classifier.json Normal file
View file

@ -0,0 +1,641 @@
{
"title": "hyperhive · review classifier results",
"uid": "hyperhive-review-classifier",
"description": "Review requests, verdicts, per-check results, retried asks and run duration.",
"editable": true,
"refresh": "1m",
"schemaVersion": 39,
"tags": ["hyperhive", "swarm", "review-classifier"],
"time": {
"from": "now-24h",
"to": "now"
},
"timezone": "utc",
"version": 1,
"panels": [
{
"id": 1,
"type": "stat",
"title": "Requests queued",
"description": "Review requests the queue stored in the window.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase({__name__=\"hyperhive.review_classifier.requests\", outcome=\"queued\"}[$__range])) or vector(0)",
"legendFormat": "Requests queued",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "text"
}
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 2,
"type": "stat",
"title": "Approved",
"description": "Runs that approved, including a re-post of an earlier approval.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase({__name__=\"hyperhive.review_classifier.verdicts\", outcome=\"approve\"}[$__range])) or vector(0)",
"legendFormat": "Approved",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "green"
}
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 3,
"type": "stat",
"title": "Changes requested",
"description": "Runs that requested changes, including a re-post.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase({__name__=\"hyperhive.review_classifier.verdicts\", outcome=\"request_changes\"}[$__range])) or vector(0)",
"legendFormat": "Changes requested",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "orange"
}
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 4,
"type": "stat",
"title": "No verdict",
"description": "Runs that posted a comment only: a failed run, a check with no answer, or rules that did not load.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase({__name__=\"hyperhive.review_classifier.verdicts\", outcome=\"no_verdict\"}[$__range])) or vector(0)",
"legendFormat": "No verdict",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "red"
}
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 5,
"type": "timeseries",
"title": "Requests by outcome",
"description": "Requests by what became of them: queued, started, reposted, deferred, queue_failed or unparsable.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (outcome) (increase({__name__=\"hyperhive.review_classifier.requests\"}[$__interval]))",
"legendFormat": "{{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 60,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0,
"stacking": {
"mode": "normal",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 6,
"type": "timeseries",
"title": "Verdicts over time",
"description": "The review event each run chose, including a re-post of an earlier verdict.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (outcome) (increase({__name__=\"hyperhive.review_classifier.verdicts\"}[$__interval]))",
"legendFormat": "{{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 60,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0,
"stacking": {
"mode": "normal",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 7,
"type": "piechart",
"title": "Verdict mix",
"description": "Share of each verdict in the window.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 12
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (outcome) (increase({__name__=\"hyperhive.review_classifier.verdicts\"}[$__range]))",
"legendFormat": "{{outcome}}",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"color": {
"mode": "palette-classic"
}
},
"overrides": []
},
"options": {
"pieType": "donut",
"displayLabels": ["percent"],
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": ["value"]
},
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
}
},
{
"id": 8,
"type": "table",
"title": "Check results by check",
"description": "Per check and result: match, no_match, unsure or no_answer, over the time range.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 16,
"x": 8,
"y": 12
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (check, result) (increase({__name__=\"hyperhive.review_classifier.checks\"}[$__range]))",
"legendFormat": "",
"instant": true,
"format": "table"
}
],
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true
},
"renameByName": {
"check": "Check",
"result": "Result",
"Value": "Count"
}
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"custom": {
"align": "auto"
}
},
"overrides": []
},
"options": {
"showHeader": true,
"sortBy": [
{
"displayName": "Check",
"desc": false
}
]
}
},
{
"id": 9,
"type": "timeseries",
"title": "Check results over time",
"description": "Classifications by result across every check.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 20
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (result) (increase({__name__=\"hyperhive.review_classifier.checks\"}[$__interval]))",
"legendFormat": "{{result}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 60,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0,
"stacking": {
"mode": "normal",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 10,
"type": "timeseries",
"title": "Asks retried",
"description": "Model asks repeated because the first answer failed validation.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 20
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum by (runtime) (increase({__name__=\"hyperhive.review_classifier.ask.retries\"}[$__interval]))",
"legendFormat": "{{runtime}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 60,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0,
"stacking": {
"mode": "normal",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 11,
"type": "timeseries",
"title": "Run duration p50 / p95",
"description": "Time from reading the rules to a posted review, across all outcomes.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 28
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "histogram_quantile(0.5, sum by (le) (rate({__name__=\"hyperhive.review_classifier.run.duration_bucket\"}[$__rate_interval])))",
"legendFormat": "p50"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=\"hyperhive.review_classifier.run.duration_bucket\"}[$__rate_interval])))",
"legendFormat": "p95"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
}
]
}