swarm-grafana: the forge and CI runs boards become one forge board with content, runtime and CI runs tabs
This commit is contained in:
iris
mara
parent
e30f5849d5
commit
bc5c36dba6
5 changed files with 2387 additions and 1303 deletions
|
|
@ -93,7 +93,7 @@ timer runs it again 5 minutes later.
|
||||||
|
|
||||||
## The board
|
## The board
|
||||||
|
|
||||||
`nix/host-modules/swarm-grafana/dashboards/ci.json`, with a repo and a workflow
|
The **CI runs** tab of the `forge` board (`nix/host-modules/swarm-grafana/dashboards/forge.json`), with a repo and a workflow
|
||||||
variable:
|
variable:
|
||||||
|
|
||||||
- mean and 90th percentile run time of successful runs, over time;
|
- mean and 90th percentile run time of successful runs, over time;
|
||||||
|
|
|
||||||
|
|
@ -208,7 +208,7 @@ kept.
|
||||||
- `builds`: the container builds hive-c0re ran on every hive, in two tabs. **builds** has the count, the failures and the longest, builds over time by outcome, and a table of each build with its id, agent, kind, outcome and runtime. **output** shows every line of the build picked in the **Build** filter, with its start and finish lines. It reads the lines hive-c0re logs for each build, so a hive-c0re that doesn't log them leaves it empty. Filter by agent.
|
- `builds`: the container builds hive-c0re ran on every hive, in two tabs. **builds** has the count, the failures and the longest, builds over time by outcome, and a table of each build with its id, agent, kind, outcome and runtime. **output** shows every line of the build picked in the **Build** filter, with its start and finish lines. It reads the lines hive-c0re logs for each build, so a hive-c0re that doesn't log them leaves it empty. Filter by agent.
|
||||||
- `controller`: the swarm controller's job graph by state, and the push and commit rates it sees on the forge.
|
- `controller`: the swarm controller's job graph by state, and the push and commit rates it sees on the forge.
|
||||||
- `review-classifier`: review requests by outcome, the verdict mix, results per check, retried asks, and run duration p50 and p95.
|
- `review-classifier`: review requests by outcome, the verdict mix, results per check, retried asks, and run duration p50 and p95.
|
||||||
- `forge`: what the forge holds, and the forge process's CPU, memory and descriptors.
|
- `forge`: the forge, in three tabs. **content** has what the forge holds (repositories, users, issues, releases, discussion volume and the issues by label). **runtime** has the forge process's CPU, memory, goroutines and descriptors. **CI runs** has Actions run time and queue wait per repo and workflow, with a per-PR table; the Repo and Workflow filters apply to that tab.
|
||||||
- `gateway`: the gateway nginx on each hive's host: requests per second, connections by state, and accepted against handled connections.
|
- `gateway`: the gateway nginx on each hive's host: requests per second, connections by state, and accepted against handled connections.
|
||||||
- `collector`: what the hive and swarm OTEL collectors accept, refuse, export and queue, with queue depth against capacity.
|
- `collector`: what the hive and swarm OTEL collectors accept, refuse, export and queue, with queue depth against capacity.
|
||||||
- `logstore`, `metricstore`: health of the log store and the metrics store.
|
- `logstore`, `metricstore`: health of the log store and the metrics store.
|
||||||
|
|
|
||||||
|
|
@ -273,7 +273,6 @@ let
|
||||||
"authelia.json"
|
"authelia.json"
|
||||||
"bao.json"
|
"bao.json"
|
||||||
"builds.json"
|
"builds.json"
|
||||||
"ci.json"
|
|
||||||
"claude-usage.json"
|
"claude-usage.json"
|
||||||
"collector.json"
|
"collector.json"
|
||||||
"controller.json"
|
"controller.json"
|
||||||
|
|
|
||||||
|
|
@ -1,410 +0,0 @@
|
||||||
{
|
|
||||||
"uid": "hyperhive-swarm-ci",
|
|
||||||
"title": "hyperhive · CI runs",
|
|
||||||
"description": "Forge Actions run time and queue wait, one sample per finished run, per repo and workflow.",
|
|
||||||
"tags": ["hyperhive", "ci"],
|
|
||||||
"timezone": "utc",
|
|
||||||
"schemaVersion": 39,
|
|
||||||
"version": 1,
|
|
||||||
"editable": true,
|
|
||||||
"refresh": "5m",
|
|
||||||
"time": {
|
|
||||||
"from": "now-30d",
|
|
||||||
"to": "now"
|
|
||||||
},
|
|
||||||
"templating": {
|
|
||||||
"list": [
|
|
||||||
{
|
|
||||||
"name": "repo",
|
|
||||||
"label": "Repo",
|
|
||||||
"type": "query",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"query": "label_values({__name__=\"hyperhive.ci.run.duration\"}, repo)",
|
|
||||||
"refresh": 2,
|
|
||||||
"includeAll": true,
|
|
||||||
"allValue": ".*",
|
|
||||||
"multi": true,
|
|
||||||
"current": {
|
|
||||||
"text": "All",
|
|
||||||
"value": "$__all"
|
|
||||||
},
|
|
||||||
"sort": 1
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "workflow",
|
|
||||||
"label": "Workflow",
|
|
||||||
"type": "query",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"query": "label_values({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\"}, workflow)",
|
|
||||||
"refresh": 2,
|
|
||||||
"includeAll": true,
|
|
||||||
"allValue": ".*",
|
|
||||||
"multi": true,
|
|
||||||
"current": {
|
|
||||||
"text": "All",
|
|
||||||
"value": "$__all"
|
|
||||||
},
|
|
||||||
"sort": 1
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"panels": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"type": "timeseries",
|
|
||||||
"title": "Run duration, successful runs",
|
|
||||||
"description": "Mean and 90th percentile of stopped minus started, over the runs that ended in each interval.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"interval": "1h",
|
|
||||||
"gridPos": {
|
|
||||||
"h": 9,
|
|
||||||
"w": 12,
|
|
||||||
"x": 0,
|
|
||||||
"y": 0
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "avg(last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__interval]))",
|
|
||||||
"legendFormat": "mean"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"refId": "B",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "quantile(0.9, last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__interval]))",
|
|
||||||
"legendFormat": "p90"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "s",
|
|
||||||
"color": {
|
|
||||||
"mode": "palette-classic"
|
|
||||||
},
|
|
||||||
"custom": {
|
|
||||||
"drawStyle": "line",
|
|
||||||
"lineWidth": 2,
|
|
||||||
"fillOpacity": 0,
|
|
||||||
"showPoints": "auto",
|
|
||||||
"spanNulls": false,
|
|
||||||
"axisSoftMin": 0
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"overrides": []
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"legend": {
|
|
||||||
"displayMode": "list",
|
|
||||||
"placement": "bottom",
|
|
||||||
"showLegend": true,
|
|
||||||
"calcs": []
|
|
||||||
},
|
|
||||||
"tooltip": {
|
|
||||||
"mode": "multi",
|
|
||||||
"sort": "desc"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 2,
|
|
||||||
"type": "timeseries",
|
|
||||||
"title": "Queue wait",
|
|
||||||
"description": "Started minus created: how long each run waited for a runner, mean and longest per interval.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"interval": "1h",
|
|
||||||
"gridPos": {
|
|
||||||
"h": 9,
|
|
||||||
"w": 12,
|
|
||||||
"x": 12,
|
|
||||||
"y": 0
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "avg(last_over_time({__name__=\"hyperhive.ci.run.queue_wait\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
|
|
||||||
"legendFormat": "mean"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"refId": "B",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "max(last_over_time({__name__=\"hyperhive.ci.run.queue_wait\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
|
|
||||||
"legendFormat": "longest"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "s",
|
|
||||||
"color": {
|
|
||||||
"mode": "palette-classic"
|
|
||||||
},
|
|
||||||
"custom": {
|
|
||||||
"drawStyle": "line",
|
|
||||||
"lineWidth": 2,
|
|
||||||
"fillOpacity": 0,
|
|
||||||
"showPoints": "auto",
|
|
||||||
"spanNulls": false,
|
|
||||||
"axisSoftMin": 0
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"overrides": []
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"legend": {
|
|
||||||
"displayMode": "list",
|
|
||||||
"placement": "bottom",
|
|
||||||
"showLegend": true,
|
|
||||||
"calcs": []
|
|
||||||
},
|
|
||||||
"tooltip": {
|
|
||||||
"mode": "multi",
|
|
||||||
"sort": "desc"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 3,
|
|
||||||
"type": "timeseries",
|
|
||||||
"title": "Runs per day by outcome",
|
|
||||||
"description": "Finished runs per UTC day. Cancelled runs are not counted.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"interval": "1d",
|
|
||||||
"gridPos": {
|
|
||||||
"h": 9,
|
|
||||||
"w": 12,
|
|
||||||
"x": 0,
|
|
||||||
"y": 9
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "count by (outcome) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__interval]))",
|
|
||||||
"legendFormat": "{{outcome}}"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "short",
|
|
||||||
"decimals": 0,
|
|
||||||
"color": {
|
|
||||||
"mode": "palette-classic"
|
|
||||||
},
|
|
||||||
"custom": {
|
|
||||||
"drawStyle": "bars",
|
|
||||||
"lineWidth": 1,
|
|
||||||
"fillOpacity": 80,
|
|
||||||
"showPoints": "never",
|
|
||||||
"stacking": {
|
|
||||||
"mode": "normal",
|
|
||||||
"group": "A"
|
|
||||||
},
|
|
||||||
"axisSoftMin": 0
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"overrides": [
|
|
||||||
{
|
|
||||||
"matcher": {
|
|
||||||
"id": "byName",
|
|
||||||
"options": "success"
|
|
||||||
},
|
|
||||||
"properties": [
|
|
||||||
{
|
|
||||||
"id": "color",
|
|
||||||
"value": {
|
|
||||||
"mode": "fixed",
|
|
||||||
"fixedColor": "green"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"matcher": {
|
|
||||||
"id": "byName",
|
|
||||||
"options": "failure"
|
|
||||||
},
|
|
||||||
"properties": [
|
|
||||||
{
|
|
||||||
"id": "color",
|
|
||||||
"value": {
|
|
||||||
"mode": "fixed",
|
|
||||||
"fixedColor": "red"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"legend": {
|
|
||||||
"displayMode": "list",
|
|
||||||
"placement": "bottom",
|
|
||||||
"showLegend": true,
|
|
||||||
"calcs": ["sum"]
|
|
||||||
},
|
|
||||||
"tooltip": {
|
|
||||||
"mode": "multi",
|
|
||||||
"sort": "desc"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 4,
|
|
||||||
"type": "table",
|
|
||||||
"title": "Duration per PR",
|
|
||||||
"description": "Per PR (or ref, for runs not on a PR) over the time range: run time of its successful runs.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"gridPos": {
|
|
||||||
"h": 9,
|
|
||||||
"w": 12,
|
|
||||||
"x": 12,
|
|
||||||
"y": 9
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "avg by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__range]))",
|
|
||||||
"format": "table",
|
|
||||||
"instant": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"refId": "B",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "max by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"success\"}[$__range]))",
|
|
||||||
"format": "table",
|
|
||||||
"instant": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"refId": "C",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "count by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\"}[$__range]))",
|
|
||||||
"format": "table",
|
|
||||||
"instant": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"refId": "D",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "count by (pr) (last_over_time({__name__=\"hyperhive.ci.run.duration\", repo=~\"$repo\", workflow=~\"$workflow\", outcome=\"failure\"}[$__range]))",
|
|
||||||
"format": "table",
|
|
||||||
"instant": true
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"transformations": [
|
|
||||||
{
|
|
||||||
"id": "merge",
|
|
||||||
"options": {}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "organize",
|
|
||||||
"options": {
|
|
||||||
"excludeByName": {
|
|
||||||
"Time": true
|
|
||||||
},
|
|
||||||
"renameByName": {
|
|
||||||
"pr": "PR",
|
|
||||||
"Value #A": "Mean (success)",
|
|
||||||
"Value #B": "Longest (success)",
|
|
||||||
"Value #C": "Runs",
|
|
||||||
"Value #D": "Failed runs"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sortBy",
|
|
||||||
"options": {
|
|
||||||
"sort": [
|
|
||||||
{
|
|
||||||
"field": "PR",
|
|
||||||
"desc": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"custom": {
|
|
||||||
"align": "auto"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"overrides": [
|
|
||||||
{
|
|
||||||
"matcher": {
|
|
||||||
"id": "byRegexp",
|
|
||||||
"options": "^(Mean|Longest).*"
|
|
||||||
},
|
|
||||||
"properties": [
|
|
||||||
{
|
|
||||||
"id": "unit",
|
|
||||||
"value": "s"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"matcher": {
|
|
||||||
"id": "byRegexp",
|
|
||||||
"options": "^(Runs|Failed runs)$"
|
|
||||||
},
|
|
||||||
"properties": [
|
|
||||||
{
|
|
||||||
"id": "decimals",
|
|
||||||
"value": 0
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"showHeader": true,
|
|
||||||
"cellHeight": "sm"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue