swarm-grafana: the forge and CI runs boards become one forge board with content, runtime and CI runs tabs
This commit is contained in:
iris
mara
parent
e30f5849d5
commit
bc5c36dba6
5 changed files with 2387 additions and 1303 deletions
|
|
@ -93,7 +93,7 @@ timer runs it again 5 minutes later.
|
|||
|
||||
## The board
|
||||
|
||||
`nix/host-modules/swarm-grafana/dashboards/ci.json`, with a repo and a workflow
|
||||
The **CI runs** tab of the `forge` board (`nix/host-modules/swarm-grafana/dashboards/forge.json`), with a repo and a workflow
|
||||
variable:
|
||||
|
||||
- mean and 90th percentile run time of successful runs, over time;
|
||||
|
|
|
|||
|
|
@ -208,7 +208,7 @@ kept.
|
|||
- `builds`: the container builds hive-c0re ran on every hive, in two tabs. **builds** has the count, the failures and the longest, builds over time by outcome, and a table of each build with its id, agent, kind, outcome and runtime. **output** shows every line of the build picked in the **Build** filter, with its start and finish lines. It reads the lines hive-c0re logs for each build, so a hive-c0re that doesn't log them leaves it empty. Filter by agent.
|
||||
- `controller`: the swarm controller's job graph by state, and the push and commit rates it sees on the forge.
|
||||
- `review-classifier`: review requests by outcome, the verdict mix, results per check, retried asks, and run duration p50 and p95.
|
||||
- `forge`: what the forge holds, and the forge process's CPU, memory and descriptors.
|
||||
- `forge`: the forge, in three tabs. **content** has what the forge holds (repositories, users, issues, releases, discussion volume and the issues by label). **runtime** has the forge process's CPU, memory, goroutines and descriptors. **CI runs** has Actions run time and queue wait per repo and workflow, with a per-PR table; the Repo and Workflow filters apply to that tab.
|
||||
- `gateway`: the gateway nginx on each hive's host: requests per second, connections by state, and accepted against handled connections.
|
||||
- `collector`: what the hive and swarm OTEL collectors accept, refuse, export and queue, with queue depth against capacity.
|
||||
- `logstore`, `metricstore`: health of the log store and the metrics store.
|
||||
|
|
|
|||
|
|
@ -273,7 +273,6 @@ let
|
|||
"authelia.json"
|
||||
"bao.json"
|
||||
"builds.json"
|
||||
"ci.json"
|
||||
"claude-usage.json"
|
||||
"collector.json"
|
||||
"controller.json"
|
||||
|
|
|
|||
|
|
@ -1,410 +0,0 @@
|
|||
{
|
||||
"uid": "hyperhive-swarm-ci",
|
||||
"title": "hyperhive · CI runs",
|
||||
"description": "Forge Actions run time and queue wait, one sample per finished run, per repo and workflow.",
|
||||
"tags": ["hyperhive", "ci"],
|
||||
"timezone": "utc",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": true,
|
||||
"refresh": "5m",
|
||||
"time": {
|
||||
"from": "now-30d",
|
||||
"to": "now"
|
||||
},
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "repo",
|
||||
"label": "Repo",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"query": "label_values({__name__=\"hyperhive.ci.run.duration\"}, repo)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"allValue": ".*",
|
||||
"multi": true,
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"sort": 1
|
||||
},
|
||||
{
|
||||
"name": "workflow",
|
||||
"label": "Workflow",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"query": "label_values({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\"}, workflow)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"allValue": ".*",
|
||||
"multi": true,
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"sort": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "Run duration, successful runs",
|
||||
"description": "Mean and 90th percentile of stopped minus started, over the runs that ended in each interval.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"interval": "1h",
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "avg(last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__interval]))",
|
||||
"legendFormat": "mean"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "quantile(0.9, last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__interval]))",
|
||||
"legendFormat": "p90"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "Queue wait",
|
||||
"description": "Started minus created: how long each run waited for a runner, mean and longest per interval.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"interval": "1h",
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "avg(last_over_time({__name__=\"hyperhive.ci.run.queue_wait\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
|
||||
"legendFormat": "mean"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max(last_over_time({__name__=\"hyperhive.ci.run.queue_wait\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
|
||||
"legendFormat": "longest"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 0,
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "Runs per day by outcome",
|
||||
"description": "Finished runs per UTC day. Cancelled runs are not counted.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"interval": "1d",
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "count by (outcome) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 80,
|
||||
"showPoints": "never",
|
||||
"stacking": {
|
||||
"mode": "normal",
|
||||
"group": "A"
|
||||
},
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "success"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "green"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "failure"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "red"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": ["sum"]
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "table",
|
||||
"title": "Duration per PR",
|
||||
"description": "Per PR (or ref, for runs not on a PR) over the time range: run time of its successful runs.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "avg by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__range]))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__range]))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "count by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__range]))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
},
|
||||
{
|
||||
"refId": "D",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "count by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"failure\"}[$__range]))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"id": "merge",
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {
|
||||
"Time": true
|
||||
},
|
||||
"renameByName": {
|
||||
"pr": "PR",
|
||||
"Value #A": "Mean (success)",
|
||||
"Value #B": "Longest (success)",
|
||||
"Value #C": "Runs",
|
||||
"Value #D": "Failed runs"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "sortBy",
|
||||
"options": {
|
||||
"sort": [
|
||||
{
|
||||
"field": "PR",
|
||||
"desc": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": "auto"
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": "^(Mean|Longest).*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": "^(Runs|Failed runs)$"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "decimals",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"cellHeight": "sm"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue