vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

add a grafana dashboard for the swarm binary cache

Browse source
This commit is contained in:
atlas 2026-11-01 17:43:59 +01:00 • committed by mara
commit b2694b0a92
2 changed files with 813 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
nix/host-modules/swarm-grafana.nix
View file

@ -269,6 +269,7 @@ let
})
[
"agents.json"
"attic.json"
"authelia.json"
"bao.json"
"builds.json"

811
nix/host-modules/swarm-grafana/dashboards/attic.json Normal file
View file

@ -0,0 +1,811 @@
{
"title": "hyperhive · binary cache",
"uid": "hyperhive-attic",
"description": "The swarm's binary cache: what the builder pushes, what each hive can pull, and the refusals.",
"editable": true,
"refresh": "1m",
"schemaVersion": 39,
"tags": ["hyperhive", "swarm", "attic", "cache"],
"time": {
"from": "now-24h",
"to": "now"
},
"timezone": "utc",
"version": 1,
"templating": {
"list": [
{
"name": "hive",
"label": "Hive",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"query": "label_values({__name__=\"hyperhive.agent.closure_pullable\"}, hive)",
"refresh": 2,
"includeAll": true,
"allValue": ".*",
"multi": true,
"current": {
"text": "All",
"value": "$__all"
},
"sort": 1
},
{
"name": "agent",
"label": "Agent",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"query": "label_values({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\"}, agent)",
"refresh": 2,
"includeAll": true,
"allValue": ".*",
"multi": false,
"current": {
"text": "All",
"value": "$__all"
},
"sort": 1
}
]
},
"panels": [
{
"id": 1,
"type": "row",
"title": "Pullable closures",
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"panels": []
},
{
"id": 2,
"type": "stat",
"title": "Agents pullable",
"description": "Agents whose latest closure this hive can pull and verify. Source: closure_pullable gauge.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 1
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(max by (hive, agent) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m]))) or vector(0)",
"instant": true,
"legendFormat": "pullable"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "green"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 3,
"type": "stat",
"title": "Agents not pullable",
"description": "Agents whose latest closure this hive cannot pull. Source: closure_pullable gauge.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 1
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "count(max by (hive, agent) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m])) == 0) or vector(0)",
"instant": true,
"legendFormat": "not pullable"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "red"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 4,
"type": "stat",
"title": "Closures pushed (range)",
"description": "Closures the builder pushed to the cache in the range. Source: swarm-builder logs.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 1
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure pushed to the cache\" | stats count_uniq(__CURSOR) as n",
"queryType": "stats",
"legendFormat": "pushed"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "green"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 5,
"type": "stat",
"title": "Push failures (range)",
"description": "Closures the builder failed to push in the range. Source: swarm-builder logs.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 1
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\" | stats count_uniq(__CURSOR) as n",
"queryType": "stats",
"legendFormat": "failed"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "red"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 6,
"type": "table",
"title": "Pull verdict per agent",
"description": "Per agent: whether the latest closure is pullable, and why not. Source: closure_pullable gauge.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 5
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent, reason) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m]))",
"instant": true,
"format": "table"
}
],
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true
},
"renameByName": {
"agent": "Agent",
"reason": "Reason",
"Value": "Pullable"
}
}
}
],
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto"
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Pullable"
},
"properties": [
{
"id": "mappings",
"value": [
{
"type": "value",
"options": {
"0": {
"text": "no",
"color": "red",
"index": 0
},
"1": {
"text": "yes",
"color": "green",
"index": 1
}
}
}
]
},
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
}
]
}
]
},
"options": {
"showHeader": true,
"sortBy": [
{
"displayName": "Pullable",
"desc": false
}
]
}
},
{
"id": 7,
"type": "timeseries",
"title": "Pullable agents over time",
"description": "Agents with a pullable closure against agents without. Source: closure_pullable gauge.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 5
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(max by (hive, agent) ({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"})) or vector(0)",
"legendFormat": "pullable"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "count(max by (hive, agent) ({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}) == 0) or vector(0)",
"legendFormat": "not pullable"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 10,
"type": "row",
"title": "Builder pushes",
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"panels": []
},
{
"id": 11,
"type": "timeseries",
"title": "Pushes over time",
"description": "Closures pushed and not pushed to the cache, per interval. Source: swarm-builder logs.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 15
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure pushed to the cache\" | stats count_uniq(__CURSOR) as n",
"queryType": "statsRange",
"legendFormat": "pushed"
},
{
"refId": "B",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\" | stats count_uniq(__CURSOR) as n",
"queryType": "statsRange",
"legendFormat": "not pushed"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 60,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 12,
"type": "timeseries",
"title": "Agent meta builds",
"description": "Agent builds from the meta flake by outcome, per interval. Source: swarm-builder logs.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 15
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"agent meta build built\" | stats count_uniq(__CURSOR) as n",
"queryType": "statsRange",
"legendFormat": "built"
},
{
"refId": "B",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" (\"agent meta build failed\" OR \"agent meta build not run\" OR \"meta build refused\") | stats count_uniq(__CURSOR) as n",
"queryType": "statsRange",
"legendFormat": "failed or refused"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 60,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 13,
"type": "logs",
"title": "Recent push failures",
"description": "The builder's push-failure lines with the reason, newest first. Source: swarm-builder logs.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 23
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\"",
"queryType": "instant"
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"prettifyLogMessage": false,
"enableLogDetails": true,
"sortOrder": "Descending",
"dedupStrategy": "none"
}
},
{
"id": 20,
"type": "row",
"title": "Hive pulls",
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 32
},
"panels": []
},
{
"id": 21,
"type": "stat",
"title": "Closures substituted (range)",
"description": "Closures hive-c0re pulled from the cache for a prebuild or create. Source: hive-c0re logs.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"h": 4,
"w": 24,
"x": 0,
"y": 33
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" (\"prebuild: substituted\" OR \"create: substituted\") | stats count_uniq(__CURSOR) as n",
"queryType": "stats",
"legendFormat": "substituted"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "green"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 23,
"type": "timeseries",
"title": "Substitutes and refusals",
"description": "Closures pulled and deploy steps refused, per interval. Source: hive-c0re logs.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 37
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" (\"prebuild: substituted\" OR \"create: substituted\") | stats count_uniq(__CURSOR) as n",
"queryType": "statsRange",
"legendFormat": "substituted"
},
{
"refId": "B",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" \"nothing was changed\" (\"the swarm has recorded no build\" OR \"closure cannot be pulled\") | stats count_uniq(__CURSOR) as n",
"queryType": "statsRange",
"legendFormat": "refused"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 60,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 24,
"type": "timeseries",
"title": "Probe verdict changes by reason",
"description": "Times the pull probe's verdict for an agent changed, by reason. Source: hive-c0re logs.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 45
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_AGENT:~\"$agent\" \"attic: can this hive pull\" F_REASON:* | stats by (F_REASON) count_uniq(__CURSOR) as n",
"queryType": "statsRange",
"legendFormat": "{{F_REASON}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 60,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
}
]
}