diff --git a/docs/2-handbook/swarm/services.md b/docs/2-handbook/swarm/services.md index 1ddd6179..c8b54e62 100644 --- a/docs/2-handbook/swarm/services.md +++ b/docs/2-handbook/swarm/services.md @@ -212,6 +212,7 @@ kept. - `review-classifier`: review requests by outcome, the verdict mix, results per check, retried asks, and run duration p50 and p95. - `forge`: the forge, in three tabs. **content** has what the forge holds (repositories, users, issues, releases, discussion volume and the issues by label). **runtime** has the forge process's CPU, memory, goroutines and descriptors. **CI runs** has Actions run time and queue wait per repo and workflow, with a per-PR table; the Repo and Workflow filters apply to that tab. - `gateway`: the gateway nginx on each hive's host: requests per second, connections by state, and accepted against handled connections. +- `host`: the machine under each hive, from the collector's host metrics: CPU by state, load, memory, swap and paging, disk and network throughput, filesystem usage, the Nix store's share and the busiest agent containers. Filter by hive and host. - `collector`: what the hive and swarm OTEL collectors accept, refuse, export and queue, with queue depth against capacity. - `logstore`, `metricstore`: health of the log store and the metrics store. - `grafana`: Grafana itself. Dashboards and data sources, alert instances firing, requests by status and their latency, alert rule evaluations and failures, sign-ins, database connections, memory and goroutines. diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 06a6601a..67677dbf 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -282,6 +282,7 @@ let "forge.json" "gateway.json" "grafana.json" + "host.json" "logstore.json" "matrix.json" "metricstore.json" diff --git a/nix/host-modules/swarm-grafana/dashboards/host.json b/nix/host-modules/swarm-grafana/dashboards/host.json new file mode 100644 index 00000000..35e86423 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/host.json @@ -0,0 +1,1003 @@ +{ + "uid": "hyperhive-host", + "title": "hyperhive ยท host", + "description": "CPU, memory, swap, disk, network and filesystem of each hive host.", + "tags": ["hyperhive", "host", "system"], + "timezone": "utc", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "1m", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "hive", + "label": "Hive", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "query": "label_values({__name__=\"system.memory.usage\"}, hive)", + "refresh": 2, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + }, + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "query": "label_values({__name__=\"system.memory.usage\", hive=~\"$hive\"}, \"host.name\")", + "refresh": 2, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "CPU utilisation by state", + "description": "Share of CPU time per state, all cores of the selected hosts.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (state) (rate({__name__=\"system.cpu.time\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])) / scalar(sum(rate({__name__=\"system.cpu.time\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])))", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { + "mode": "palette-classic" + }, + "max": 1, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 30, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Load average", + "description": "Runnable plus uninterruptible tasks averaged over 1, 5 and 15 minutes.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (\"host.name\") ({__name__=\"system.cpu.load_average.1m\", hive=~\"$hive\", \"host.name\"=~\"$host\"})", + "legendFormat": "{{host.name}} 1m" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (\"host.name\") ({__name__=\"system.cpu.load_average.5m\", hive=~\"$hive\", \"host.name\"=~\"$host\"})", + "legendFormat": "{{host.name}} 5m" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (\"host.name\") ({__name__=\"system.cpu.load_average.15m\", hive=~\"$hive\", \"host.name\"=~\"$host\"})", + "legendFormat": "{{host.name}} 15m" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Memory", + "description": "Used, cached and free memory.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (state) ({__name__=\"system.memory.usage\", hive=~\"$hive\", \"host.name\"=~\"$host\", state=~\"used|cached|free\"})", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 30, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 4, + "type": "timeseries", + "title": "Swap used", + "description": "Swap in use, per swap device.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (device) ({__name__=\"system.paging.usage\", hive=~\"$hive\", \"host.name\"=~\"$host\", state=\"used\"})", + "legendFormat": "{{device}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Paging in and out", + "description": "Pages moved between memory and swap or disk per second.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (direction, type) (rate({__name__=\"system.paging.operations\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m]))", + "legendFormat": "{{direction}} {{type}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Disk throughput", + "description": "Bytes read and written per second, per device.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (device, direction) (rate({__name__=\"system.disk.io\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m]))", + "legendFormat": "{{device}} {{direction}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Disk operations", + "description": "Read and write operations per second, per device.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (device, direction) (rate({__name__=\"system.disk.operations\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m]))", + "legendFormat": "{{device}} {{direction}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "iops", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Network throughput", + "description": "Bytes received and transmitted per second, per interface.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (device, direction) (rate({__name__=\"system.network.io\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m]))", + "legendFormat": "{{device}} {{direction}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Network errors", + "description": "Packet errors per second, per interface.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (device, direction) (rate({__name__=\"system.network.errors\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m]))", + "legendFormat": "{{device}} {{direction}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "pps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Filesystem usage", + "description": "Used share of each mounted filesystem.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (mountpoint) ({__name__=\"system.filesystem.usage\", hive=~\"$hive\", \"host.name\"=~\"$host\", state=\"used\"}) / sum by (mountpoint) ({__name__=\"system.filesystem.usage\", hive=~\"$hive\", \"host.name\"=~\"$host\", state=~\"used|free\"})", + "legendFormat": "{{mountpoint}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { + "mode": "palette-classic" + }, + "max": 1, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "stat", + "title": "Nix store utilisation", + "description": "Used share of the filesystem holding the Nix store.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 12, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (\"host.name\") ({__name__=\"hyperhive.hive.nix_store.utilization\", hive=~\"$hive\", \"host.name\"=~\"$host\"})", + "legendFormat": "{{host.name}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { + "mode": "thresholds" + }, + "min": 0, + "max": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 12, + "type": "stat", + "title": "Uptime", + "description": "Time since the host booted.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (\"host.name\") ({__name__=\"system.uptime\", hive=~\"$hive\", \"host.name\"=~\"$host\"})", + "legendFormat": "{{host.name}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 13, + "type": "stat", + "title": "Processes", + "description": "Processes on the host in any state.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (\"host.name\") ({__name__=\"system.processes.count\", hive=~\"$hive\", \"host.name\"=~\"$host\"})", + "legendFormat": "{{host.name}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Processes by status", + "description": "Process count per status.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 36 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (status) ({__name__=\"system.processes.count\", hive=~\"$hive\", \"host.name\"=~\"$host\"})", + "legendFormat": "{{status}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 30, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Top 10 containers by memory", + "description": "Agent containers using the most memory.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "topk(10, sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", \"host.name\"=~\"$host\"}))", + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 16, + "type": "timeseries", + "title": "Top 10 containers by CPU", + "description": "Agent containers using the most CPU, in cores.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "topk(10, sum by (agent) (rate({__name__=\"container.cpu.time\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])))", + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +} diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 7aed754e..1dab6b43 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -624,6 +624,23 @@ let ] ) tables; } + { + # The host board ships, and every query filters on both template + # variables: a panel missing one shows another hive's or host's numbers + # under this selection. + name = "the host dashboard ships, and every panel query filters on hive and host"; + ok = + let + provider = lib.head grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.dashboards.settings.providers; + shipped = lib.attrNames provider.options.path.entries; + board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/host.json); + exprs = lib.concatMap (p: map (t: t.expr) p.targets) board.panels; + in + lib.elem "host.json" shipped + && lib.length exprs > 0 + && lib.all (e: lib.hasInfix "hive=~\"$hive\"" e && lib.hasInfix "\"host.name\"=~\"$host\"" e) exprs + && lib.any (e: lib.hasInfix "system.cpu.time" e) exprs; + } { # The egress board ships, and its queries read the two squid log lines as # squid writes them: the would_deny line's field layout, and the default