diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 7bc604ba..115e0dd1 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -213,6 +213,10 @@ kept. - `collector`: what the hive and swarm OTEL collectors accept, refuse, export and queue, with queue depth against capacity. - `logstore`, `metricstore`: health of the log store and the metrics store. - `grafana`: Grafana itself. Dashboards and data sources, alert instances firing, requests by status and their latency, alert rule evaluations and failures, sign-ins, database connections, memory and goroutines. +- `builder`: the swarm builder. Builds finished, by kind and outcome, their mean and 95th percentile time, the queue depth and the wait before a build starts. +- `ci-runner`: the forge's Actions runner. Its uptime and job slots, jobs finished by status, how many slots are in use, job time, and how often it polls the forge for work. +- `matrix`: the Matrix homeserver. Whether its client API answers, how fast, the share of the range it answered, and its database size. +- `notify-router`: the notification router. Forge events received by type and outcome, todos delivered, review requests that failed, time per event, and the backlog of the agents' queue. A new panel can read "No data" until its source exists: the push and commit counters on `controller` appear after the first push webhook, the refused diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index f9ca5d0d..1e5e33b9 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -272,7 +272,9 @@ let "attic.json" "authelia.json" "bao.json" + "builder.json" "builds.json" + "ci-runner.json" "claude-usage.json" "collector.json" "controller.json" @@ -281,7 +283,9 @@ let "gateway.json" "grafana.json" "logstore.json" + "matrix.json" "metricstore.json" + "notify-router.json" "queue.json" "review-classifier.json" ] diff --git a/nix/host-modules/swarm-grafana/dashboards/builder.json b/nix/host-modules/swarm-grafana/dashboards/builder.json new file mode 100644 index 00000000..774cbe32 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/builder.json @@ -0,0 +1,379 @@ +{ + "title": "hyperhive · builder", + "uid": "hyperhive-swarm-builder", + "description": "The swarm builder: builds by outcome, how long they take, and how long they wait.", + "tags": ["hyperhive", "swarm", "builder"], + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Builds in range", + "description": "Builds the swarm builder finished in the selected range.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"hyperhive.swarm_builder.builds\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "builds" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 2, + "type": "stat", + "title": "Mean build time", + "description": "Mean duration of a build in the selected range.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"hyperhive.swarm_builder.build.duration_sum\"}[$__range])) / sum(increase({__name__=\"hyperhive.swarm_builder.build.duration_count\"}[$__range]))", + "instant": true, + "legendFormat": "mean" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 1, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Queue depth", + "description": "Builds waiting for a free worker now.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum({__name__=\"hyperhive.swarm_builder.queue.depth\"})", + "instant": true, + "legendFormat": "waiting" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Mean queue wait", + "description": "Mean time a build waited before it started, in the range.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"hyperhive.swarm_builder.queue.wait_sum\"}[$__range])) / sum(increase({__name__=\"hyperhive.swarm_builder.queue.wait_count\"}[$__range]))", + "instant": true, + "legendFormat": "mean" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 3, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Builds by outcome", + "description": "Finished builds per hour, by kind and outcome.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (kind, outcome) (increase({__name__=\"hyperhive.swarm_builder.builds\"}[1h]))", + "legendFormat": "{{kind}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Build time", + "description": "Duration of a build in each hour, mean and 95th percentile.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"hyperhive.swarm_builder.build.duration_sum\"}[1h])) / sum(increase({__name__=\"hyperhive.swarm_builder.build.duration_count\"}[1h]))", + "legendFormat": "mean" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "histogram_quantile(0.95, sum by (le) (increase({__name__=\"hyperhive.swarm_builder.build.duration_bucket\"}[1h])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +} diff --git a/nix/host-modules/swarm-grafana/dashboards/ci-runner.json b/nix/host-modules/swarm-grafana/dashboards/ci-runner.json new file mode 100644 index 00000000..3f8f7fb7 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/ci-runner.json @@ -0,0 +1,501 @@ +{ + "title": "hyperhive · ci runner", + "uid": "hyperhive-swarm-ci-runner", + "description": "The forge's Actions runner: how busy it is, how its jobs go, and whether it polls.", + "tags": ["hyperhive", "swarm", "ci", "runner"], + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Uptime", + "description": "Time since the runner process started.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max(gitea_runner_uptime_seconds)", + "instant": true, + "legendFormat": "uptime" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 2, + "type": "stat", + "title": "Job slots", + "description": "Jobs the runner can run at once.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max(gitea_runner_capacity)", + "instant": true, + "legendFormat": "slots" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Jobs in range", + "description": "Jobs the runner finished in the selected range.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(gitea_runner_job_total[$__range])) or vector(0)", + "instant": true, + "legendFormat": "jobs" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Mean job time", + "description": "Mean duration of a job in the selected range.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(gitea_runner_job_duration_seconds_sum[$__range])) / sum(increase(gitea_runner_job_duration_seconds_count[$__range]))", + "instant": true, + "legendFormat": "mean" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 1, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Slots in use", + "description": "Jobs running now, and the share of the slots they use.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max(gitea_runner_job_running)", + "legendFormat": "running" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max(gitea_runner_job_capacity_utilization_ratio) * 100", + "legendFormat": "slots used %" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Jobs by status", + "description": "Finished jobs per hour, by status.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (status) (increase(gitea_runner_job_total[1h]))", + "legendFormat": "{{status}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Job time", + "description": "Duration of a job, mean and 95th percentile.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(gitea_runner_job_duration_seconds_sum[1h])) / sum(increase(gitea_runner_job_duration_seconds_count[1h]))", + "legendFormat": "mean" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "histogram_quantile(0.95, sum by (le) (increase(gitea_runner_job_duration_seconds_bucket[1h])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Polls for work", + "description": "Polls of the forge per minute, by result; the runner asks for jobs this way.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (result) (rate(gitea_runner_poll_fetch_total[$__rate_interval])) * 60", + "legendFormat": "{{result}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +} diff --git a/nix/host-modules/swarm-grafana/dashboards/matrix.json b/nix/host-modules/swarm-grafana/dashboards/matrix.json new file mode 100644 index 00000000..6ec398bf --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/matrix.json @@ -0,0 +1,434 @@ +{ + "title": "hyperhive · matrix", + "uid": "hyperhive-swarm-matrix", + "description": "The Matrix homeserver: whether it answers, how fast, and how big its database is.", + "tags": ["hyperhive", "swarm", "matrix"], + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Answers", + "description": "1 when the client API answers 2xx, 0 when it does not.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max({__name__=\"httpcheck.status\", \"http.status_class\"=\"2xx\"})", + "instant": true, + "legendFormat": "up" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 2, + "type": "stat", + "title": "Response time", + "description": "How long the client API took to answer the last check.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max({__name__=\"httpcheck.duration\"})", + "instant": true, + "legendFormat": "ms" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Database size", + "description": "Size of the homeserver's database.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max({__name__=\"hyperhive.matrix.database_bytes\"})", + "instant": true, + "legendFormat": "database" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Answered in range", + "description": "Share of the range in which the client API answered 2xx.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "avg_over_time(max({__name__=\"httpcheck.status\", \"http.status_class\"=\"2xx\"})[$__range:1m]) * 100", + "instant": true, + "legendFormat": "answered" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 99 + }, + { + "color": "green", + "value": 99.9 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Response time over time", + "description": "Time the client API took to answer each check.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max({__name__=\"httpcheck.duration\"})", + "legendFormat": "response time" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Database size over time", + "description": "Size of the homeserver's database.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max({__name__=\"hyperhive.matrix.database_bytes\"})", + "legendFormat": "database" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Answers over time", + "description": "1 while the client API answers 2xx, 0 when it does not.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max({__name__=\"httpcheck.status\", \"http.status_class\"=\"2xx\"})", + "legendFormat": "answers" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +} diff --git a/nix/host-modules/swarm-grafana/dashboards/notify-router.json b/nix/host-modules/swarm-grafana/dashboards/notify-router.json new file mode 100644 index 00000000..fda12927 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/notify-router.json @@ -0,0 +1,523 @@ +{ + "title": "hyperhive · notification router", + "uid": "hyperhive-swarm-notify-router", + "description": "The notification router: forge events in, todos out, and the backlog of each agent's queue.", + "tags": ["hyperhive", "swarm", "notify", "router"], + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Events in range", + "description": "Forge events the router received in the selected range.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"hyperhive.notify_router.events\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "events" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 2, + "type": "stat", + "title": "Todos delivered", + "description": "Todos the router put on agents' queues in the range.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"hyperhive.notify_router.todos\", outcome=\"delivered\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "delivered" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Review requests failed", + "description": "Review requests the router could not make in the range.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"hyperhive.notify_router.review_requests\", outcome=\"failed\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "failed" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Mean time per event", + "description": "Mean time the router took to handle one event, in the range.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"hyperhive.notify_router.event.duration_sum\"}[$__range])) / sum(increase({__name__=\"hyperhive.notify_router.event.duration_count\"}[$__range]))", + "instant": true, + "legendFormat": "mean" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 3, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Events by type and outcome", + "description": "Forge events per hour, by type and what the router did with them.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (event_type, outcome) (increase({__name__=\"hyperhive.notify_router.events\"}[1h]))", + "legendFormat": "{{event_type}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Todos and review requests", + "description": "Todos and review requests per hour, by outcome.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (op, outcome) (increase({__name__=\"hyperhive.notify_router.todos\"}[1h]))", + "legendFormat": "todo {{op}} {{outcome}}" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (outcome) (increase({__name__=\"hyperhive.notify_router.review_requests\"}[1h]))", + "legendFormat": "review request {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Time per event", + "description": "Time to handle one event, mean and 95th percentile.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase({__name__=\"hyperhive.notify_router.event.duration_sum\"}[1h])) / sum(increase({__name__=\"hyperhive.notify_router.event.duration_count\"}[1h]))", + "legendFormat": "mean" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "histogram_quantile(0.95, sum by (le) (increase({__name__=\"hyperhive.notify_router.event.duration_bucket\"}[1h])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Queue backlog", + "description": "Messages waiting, awaiting an ack, and redelivered on the agents' queue.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum({__name__=\"hyperhive.notify_router.consumer.pending\"})", + "legendFormat": "pending" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum({__name__=\"hyperhive.notify_router.consumer.ack_pending\"})", + "legendFormat": "waiting for ack" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum({__name__=\"hyperhive.notify_router.consumer.redelivered\"})", + "legendFormat": "redelivered" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +}