diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 1ef6fa80..362a7a90 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -212,6 +212,7 @@ kept. - `gateway`: the gateway nginx on each hive's host: requests per second, connections by state, and accepted against handled connections. - `collector`: what the hive and swarm OTEL collectors accept, refuse, export and queue, with queue depth against capacity. - `logstore`, `metricstore`: health of the log store and the metrics store. +- `grafana`: Grafana itself. Dashboards and data sources, alert instances firing, requests by status and their latency, alert rule evaluations and failures, sign-ins, database connections, memory and goroutines. A new panel can read "No data" until its source exists: the push and commit counters on `controller` appear after the first push webhook, the refused diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index ca17fd04..f9ca5d0d 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -279,6 +279,7 @@ let "egress.json" "forge.json" "gateway.json" + "grafana.json" "logstore.json" "metricstore.json" "queue.json" diff --git a/nix/host-modules/swarm-grafana/dashboards/grafana.json b/nix/host-modules/swarm-grafana/dashboards/grafana.json new file mode 100644 index 00000000..86f42ae3 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/grafana.json @@ -0,0 +1,649 @@ +{ + "title": "hyperhive ยท grafana", + "uid": "hyperhive-swarm-grafana", + "description": "The swarm's Grafana: what it serves, how fast, its alerting and its embedded authorization store.", + "tags": ["hyperhive", "swarm", "grafana"], + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Version", + "description": "The Grafana version running.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "grafana_build_info", + "instant": true, + "legendFormat": "{{version}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 2, + "type": "stat", + "title": "Dashboards", + "description": "Dashboards Grafana holds.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "grafana_stat_totals_dashboard{\"service.name\"=\"grafana\"}", + "instant": true, + "legendFormat": "dashboards" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Data sources", + "description": "Data sources configured.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(grafana_stat_totals_datasource{\"service.name\"=\"grafana\"})", + "instant": true, + "legendFormat": "data sources" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Alert instances firing", + "description": "Alert instances in the alerting state.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(grafana_alerting_alerts{\"service.name\"=\"grafana\", state=\"alerting\"}) or vector(0)", + "instant": true, + "legendFormat": "alerting" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Requests by status", + "description": "HTTP requests Grafana served per second, by status code.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (status_code) (rate(grafana_http_request_duration_seconds_count{\"service.name\"=\"grafana\"}[$__rate_interval]))", + "legendFormat": "{{status_code}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Request latency", + "description": "Request duration, p50 and p95.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(grafana_http_request_duration_seconds_bucket{\"service.name\"=\"grafana\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(grafana_http_request_duration_seconds_bucket{\"service.name\"=\"grafana\"}[$__rate_interval])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Alert rule evaluations", + "description": "Rule evaluations per second, and how many of them failed.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(rate(grafana_alerting_rule_evaluations_total{\"service.name\"=\"grafana\"}[$__rate_interval]))", + "legendFormat": "evaluations" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(rate(grafana_alerting_rule_evaluation_failures_total{\"service.name\"=\"grafana\"}[$__rate_interval]))", + "legendFormat": "failures" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "decimals": 2, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Logins", + "description": "Sign-ins per second by method.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(rate(grafana_api_login_oauth_total{\"service.name\"=\"grafana\"}[$__rate_interval]))", + "legendFormat": "oauth" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(rate(grafana_api_login_post_total{\"service.name\"=\"grafana\"}[$__rate_interval]))", + "legendFormat": "password" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "decimals": 3, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Database connections", + "description": "Connections to Grafana's own database.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "grafana_database_conn_open{\"service.name\"=\"grafana\"}", + "legendFormat": "open" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "grafana_database_conn_in_use{\"service.name\"=\"grafana\"}", + "legendFormat": "in use" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "grafana_database_conn_idle{\"service.name\"=\"grafana\"}", + "legendFormat": "idle" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Memory and goroutines", + "description": "Resident memory (bytes) and goroutines of the Grafana process.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "process_resident_memory_bytes{\"service.name\"=\"grafana\"}", + "legendFormat": "resident memory" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "go_goroutines{\"service.name\"=\"grafana\"}", + "legendFormat": "goroutines" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +}