diff --git a/nix/host-modules/swarm-grafana/dashboards/ci-runner.json b/nix/host-modules/swarm-grafana/dashboards/ci-runner.json index 3f8f7fb7b..a0a09c672 100644 --- a/nix/host-modules/swarm-grafana/dashboards/ci-runner.json +++ b/nix/host-modules/swarm-grafana/dashboards/ci-runner.json @@ -1,7 +1,7 @@ { "title": "hyperhive ยท ci runner", "uid": "hyperhive-swarm-ci-runner", - "description": "The forge's Actions runner: how busy it is, how its jobs go, and whether it polls.", + "description": "The forge's Actions runners: how busy each is, how its jobs go, and whether it polls.", "tags": ["hyperhive", "swarm", "ci", "runner"], "editable": true, "refresh": "1m", @@ -35,9 +35,9 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "max(gitea_runner_uptime_seconds)", + "expr": "max by (\"service.instance.id\") ({__name__=\"gitea_runner_uptime_seconds\", \"service.instance.id\"=~\"$runner\"})", "instant": true, - "legendFormat": "uptime" + "legendFormat": "{{service.instance.id}}" } ], "fieldConfig": { @@ -94,9 +94,9 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "max(gitea_runner_capacity)", + "expr": "max by (\"service.instance.id\") ({__name__=\"gitea_runner_capacity\", \"service.instance.id\"=~\"$runner\"})", "instant": true, - "legendFormat": "slots" + "legendFormat": "{{service.instance.id}}" } ], "fieldConfig": { @@ -153,7 +153,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum(increase(gitea_runner_job_total[$__range])) or vector(0)", + "expr": "sum(increase({__name__=\"gitea_runner_job_total\", \"service.instance.id\"=~\"$runner\"}[$__range])) or vector(0)", "instant": true, "legendFormat": "jobs" } @@ -212,7 +212,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum(increase(gitea_runner_job_duration_seconds_sum[$__range])) / sum(increase(gitea_runner_job_duration_seconds_count[$__range]))", + "expr": "sum(increase({__name__=\"gitea_runner_job_duration_seconds_sum\", \"service.instance.id\"=~\"$runner\"}[$__range])) / sum(increase({__name__=\"gitea_runner_job_duration_seconds_count\", \"service.instance.id\"=~\"$runner\"}[$__range]))", "instant": true, "legendFormat": "mean" } @@ -271,8 +271,8 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "max(gitea_runner_job_running)", - "legendFormat": "running" + "expr": "max by (\"service.instance.id\") ({__name__=\"gitea_runner_job_running\", \"service.instance.id\"=~\"$runner\"})", + "legendFormat": "{{service.instance.id}} running" }, { "refId": "B", @@ -280,8 +280,8 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "max(gitea_runner_job_capacity_utilization_ratio) * 100", - "legendFormat": "slots used %" + "expr": "max by (\"service.instance.id\") ({__name__=\"gitea_runner_job_capacity_utilization_ratio\", \"service.instance.id\"=~\"$runner\"}) * 100", + "legendFormat": "{{service.instance.id}} slots used %" } ], "fieldConfig": { @@ -336,7 +336,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum by (status) (increase(gitea_runner_job_total[1h]))", + "expr": "sum by (status) (increase({__name__=\"gitea_runner_job_total\", \"service.instance.id\"=~\"$runner\"}[1h]))", "legendFormat": "{{status}}" } ], @@ -397,7 +397,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum(increase(gitea_runner_job_duration_seconds_sum[1h])) / sum(increase(gitea_runner_job_duration_seconds_count[1h]))", + "expr": "sum(increase({__name__=\"gitea_runner_job_duration_seconds_sum\", \"service.instance.id\"=~\"$runner\"}[1h])) / sum(increase({__name__=\"gitea_runner_job_duration_seconds_count\", \"service.instance.id\"=~\"$runner\"}[1h]))", "legendFormat": "mean" }, { @@ -406,7 +406,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "histogram_quantile(0.95, sum by (le) (increase(gitea_runner_job_duration_seconds_bucket[1h])))", + "expr": "histogram_quantile(0.95, sum by (le) (increase({__name__=\"gitea_runner_job_duration_seconds_bucket\", \"service.instance.id\"=~\"$runner\"}[1h])))", "legendFormat": "p95" } ], @@ -462,7 +462,7 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "sum by (result) (rate(gitea_runner_poll_fetch_total[$__rate_interval])) * 60", + "expr": "sum by (result) (rate({__name__=\"gitea_runner_poll_fetch_total\", \"service.instance.id\"=~\"$runner\"}[$__rate_interval])) * 60", "legendFormat": "{{result}}" } ], @@ -497,5 +497,28 @@ } } } - ] + ], + "templating": { + "list": [ + { + "name": "runner", + "label": "Runner", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "query": "label_values({__name__=\"gitea_runner_uptime_seconds\"}, \"service.instance.id\")", + "refresh": 2, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + } }