grafana: the ci runner board shows each runner, with a runner filter

The panels took the max or sum over the one swarm runner. A runner variable
(the series' service.instance.id) filters every panel, and uptime, job slots
and slot use show one series per runner, so a runner registered by hand shows
up beside the swarm's once its metrics reach the store.

nix/host-modules/swarm-grafana/dashboards/ci-runner.json

@@ -1,7 +1,7 @@
 {
   "title": "hyperhive · ci runner",
   "uid": "hyperhive-swarm-ci-runner",
-  "description": "The forge's Actions runner: how busy it is, how its jobs go, and whether it polls.",
+  "description": "The forge's Actions runners: how busy each is, how its jobs go, and whether it polls.",
   "tags": ["hyperhive", "swarm", "ci", "runner"],
   "editable": true,
   "refresh": "1m",
@@ -35,9 +35,9 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "max(gitea_runner_uptime_seconds)",
+          "expr": "max by (\"service.instance.id\") ({__name__=\"gitea_runner_uptime_seconds\", \"service.instance.id\"=~\"$runner\"})",
           "instant": true,
-          "legendFormat": "uptime"
+          "legendFormat": "{{service.instance.id}}"
         }
       ],
       "fieldConfig": {
@@ -94,9 +94,9 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "max(gitea_runner_capacity)",
+          "expr": "max by (\"service.instance.id\") ({__name__=\"gitea_runner_capacity\", \"service.instance.id\"=~\"$runner\"})",
           "instant": true,
-          "legendFormat": "slots"
+          "legendFormat": "{{service.instance.id}}"
         }
       ],
       "fieldConfig": {
@@ -153,7 +153,7 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum(increase(gitea_runner_job_total[$__range])) or vector(0)",
+          "expr": "sum(increase({__name__=\"gitea_runner_job_total\", \"service.instance.id\"=~\"$runner\"}[$__range])) or vector(0)",
           "instant": true,
           "legendFormat": "jobs"
         }
@@ -212,7 +212,7 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum(increase(gitea_runner_job_duration_seconds_sum[$__range])) / sum(increase(gitea_runner_job_duration_seconds_count[$__range]))",
+          "expr": "sum(increase({__name__=\"gitea_runner_job_duration_seconds_sum\", \"service.instance.id\"=~\"$runner\"}[$__range])) / sum(increase({__name__=\"gitea_runner_job_duration_seconds_count\", \"service.instance.id\"=~\"$runner\"}[$__range]))",
           "instant": true,
           "legendFormat": "mean"
         }
@@ -271,8 +271,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "max(gitea_runner_job_running)",
+          "expr": "max by (\"service.instance.id\") ({__name__=\"gitea_runner_job_running\", \"service.instance.id\"=~\"$runner\"})",
-          "legendFormat": "running"
+          "legendFormat": "{{service.instance.id}} running"
         },
         {
           "refId": "B",
@@ -280,8 +280,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "max(gitea_runner_job_capacity_utilization_ratio) * 100",
+          "expr": "max by (\"service.instance.id\") ({__name__=\"gitea_runner_job_capacity_utilization_ratio\", \"service.instance.id\"=~\"$runner\"}) * 100",
-          "legendFormat": "slots used %"
+          "legendFormat": "{{service.instance.id}} slots used %"
         }
       ],
       "fieldConfig": {
@@ -336,7 +336,7 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum by (status) (increase(gitea_runner_job_total[1h]))",
+          "expr": "sum by (status) (increase({__name__=\"gitea_runner_job_total\", \"service.instance.id\"=~\"$runner\"}[1h]))",
           "legendFormat": "{{status}}"
         }
       ],
@@ -397,7 +397,7 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum(increase(gitea_runner_job_duration_seconds_sum[1h])) / sum(increase(gitea_runner_job_duration_seconds_count[1h]))",
+          "expr": "sum(increase({__name__=\"gitea_runner_job_duration_seconds_sum\", \"service.instance.id\"=~\"$runner\"}[1h])) / sum(increase({__name__=\"gitea_runner_job_duration_seconds_count\", \"service.instance.id\"=~\"$runner\"}[1h]))",
           "legendFormat": "mean"
         },
         {
@@ -406,7 +406,7 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "histogram_quantile(0.95, sum by (le) (increase(gitea_runner_job_duration_seconds_bucket[1h])))",
+          "expr": "histogram_quantile(0.95, sum by (le) (increase({__name__=\"gitea_runner_job_duration_seconds_bucket\", \"service.instance.id\"=~\"$runner\"}[1h])))",
           "legendFormat": "p95"
         }
       ],
@@ -462,7 +462,7 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum by (result) (rate(gitea_runner_poll_fetch_total[$__rate_interval])) * 60",
+          "expr": "sum by (result) (rate({__name__=\"gitea_runner_poll_fetch_total\", \"service.instance.id\"=~\"$runner\"}[$__rate_interval])) * 60",
           "legendFormat": "{{result}}"
         }
       ],
@@ -497,5 +497,28 @@
         }
       }
     }
-  ]
+  ],
+  "templating": {
+    "list": [
+      {
+        "name": "runner",
+        "label": "Runner",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "@datasourceUid@"
+        },
+        "query": "label_values({__name__=\"gitea_runner_uptime_seconds\"}, \"service.instance.id\")",
+        "refresh": 2,
+        "includeAll": true,
+        "allValue": ".*",
+        "multi": true,
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "sort": 1
+      }
+    ]
+  }
 }