diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 0ed83350..e4294da1 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -166,22 +166,6 @@ Grafana start — see [`secrets.md`](secrets.md). -**Dashboards.** Grafana provisions these from -`nix/host-modules/swarm-grafana/dashboards/`; edits made in the UI aren't -kept. - -- `agents`: per-agent resource usage and Claude Code activity. -- `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue. -- `claude-usage`: Claude use by model, effort, token type and source. -- `controller`: the swarm controller's job graph by state, and the push and commit rates it sees on the forge. -- `forge`: what the forge holds, and the forge process's CPU, memory and descriptors. -- `collector`: what the OTEL collectors accept, refuse, export and queue, with queue depth against capacity. -- `logstore`, `metricstore`: health of the log store and the metrics store. - -A new panel can read "No data" until its source exists: the push and commit -counters on `controller` appear after the first push webhook, and the refused -and failure series on `collector` appear once something fails. - **Where the data comes from.** The swarm's OTEL collector, below. Neither container is reachable except through the gateway: each listens diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 979b3924..dbbe9c37 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -198,8 +198,6 @@ let "authelia.json" "bao.json" "claude-usage.json" - "collector.json" - "controller.json" "forge.json" "logstore.json" "metricstore.json" diff --git a/nix/host-modules/swarm-grafana/dashboards/collector.json b/nix/host-modules/swarm-grafana/dashboards/collector.json deleted file mode 100644 index b2b720df..00000000 --- a/nix/host-modules/swarm-grafana/dashboards/collector.json +++ /dev/null @@ -1,648 +0,0 @@ -{ - "title": "hyperhive \u00b7 collector (self-telemetry)", - "uid": "hyperhive-swarm-collector", - "description": "What the OTEL collectors accept, refuse, export and queue.", - "editable": true, - "refresh": "1m", - "schemaVersion": 39, - "tags": ["hyperhive", "swarm", "otel", "collector"], - "time": { - "from": "now-24h", - "to": "now" - }, - "timezone": "utc", - "version": 1, - "panels": [ - { - "id": 1, - "type": "stat", - "title": "Refused or failed (range)", - "description": "Items receivers refused or failed to take in.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__range])) or vector(0)", - "instant": true, - "legendFormat": "refused" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - }, - "x-zero-is-healthy": true - }, - { - "id": 2, - "type": "stat", - "title": "Export failures (range)", - "description": "Items exporters failed to send or enqueue.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__range])) or vector(0)", - "instant": true, - "legendFormat": "failed" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - }, - "x-zero-is-healthy": true - }, - { - "id": 3, - "type": "stat", - "title": "Fullest export queue", - "description": "Highest queue fill across exporters.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "max(otelcol_exporter_queue_size{\"service.name\"=\"collector\"} / otelcol_exporter_queue_capacity{\"service.name\"=\"collector\"})", - "instant": true, - "legendFormat": "fill" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.5 - }, - { - "color": "red", - "value": 0.9 - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 4, - "type": "stat", - "title": "Memory (RSS)", - "description": "Largest resident memory across collectors.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "max({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"collector\"})", - "instant": true, - "legendFormat": "rss" - } - ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "decimals": 1, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 5, - "type": "timeseries", - "title": "Accepted per second", - "description": "Items receivers accept, per signal.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 4 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_metric_points(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))", - "legendFormat": "metric points" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_log_records(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))", - "legendFormat": "log records" - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_spans(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))", - "legendFormat": "spans" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 1, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 6, - "type": "timeseries", - "title": "Refused or failed per second", - "description": "Items refused or failed at each receiver.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 4 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum by (receiver) (rate({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))", - "legendFormat": "{{receiver}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 2, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 7, - "type": "timeseries", - "title": "Sent per second", - "description": "Items each exporter delivers.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))", - "legendFormat": "{{exporter}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 1, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 8, - "type": "timeseries", - "title": "Export failures per second", - "description": "Items each exporter fails to send or enqueue.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum by (exporter) (rate({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))", - "legendFormat": "{{exporter}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 2, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 9, - "type": "timeseries", - "title": "Export queue depth", - "description": "Batches waiting in each exporter's queue, and its capacity.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "max by (exporter) (otelcol_exporter_queue_size{\"service.name\"=\"collector\"})", - "legendFormat": "{{exporter}} size" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "max by (exporter) (otelcol_exporter_queue_capacity{\"service.name\"=\"collector\"})", - "legendFormat": "{{exporter}} capacity" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 10, - "type": "timeseries", - "title": "Collector memory", - "description": "Largest resident memory and live Go heap across collectors.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "max({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"collector\"})", - "legendFormat": "rss" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "max(otelcol_process_runtime_heap_alloc_bytes{\"service.name\"=\"collector\"})", - "legendFormat": "heap" - } - ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "decimals": 1, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - } - ] -} diff --git a/nix/host-modules/swarm-grafana/dashboards/controller.json b/nix/host-modules/swarm-grafana/dashboards/controller.json deleted file mode 100644 index 52547d49..00000000 --- a/nix/host-modules/swarm-grafana/dashboards/controller.json +++ /dev/null @@ -1,491 +0,0 @@ -{ - "title": "hyperhive \u00b7 swarm controller (jobq + vcs)", - "uid": "hyperhive-swarm-controller", - "description": "The swarm controller's job graph and the forge activity it sees.", - "editable": true, - "refresh": "1m", - "schemaVersion": 39, - "tags": ["hyperhive", "swarm", "controller", "jobq", "vcs"], - "time": { - "from": "now-24h", - "to": "now" - }, - "timezone": "utc", - "version": 1, - "panels": [ - { - "id": 1, - "type": "stat", - "title": "Running nodes", - "description": "Job nodes executing now, at any depth.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum({__name__=\"jobq.nodes\", \"service.name\"=\"swarm-controller\", state=\"Running\"}) or vector(0)", - "instant": true, - "legendFormat": "running" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 2, - "type": "stat", - "title": "Pending nodes", - "description": "Job nodes waiting on dependencies or resources.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum({__name__=\"jobq.nodes\", \"service.name\"=\"swarm-controller\", state=\"Pending\"}) or vector(0)", - "instant": true, - "legendFormat": "pending" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 3, - "type": "stat", - "title": "Failed root groups", - "description": "Root job groups in the Failed state.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum({__name__=\"jobq.roots\", \"service.name\"=\"swarm-controller\", state=\"Failed\"}) or vector(0)", - "instant": true, - "legendFormat": "failed" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - }, - "x-zero-is-healthy": true - }, - { - "id": 4, - "type": "stat", - "title": "Pushes (range)", - "description": "Push deliveries seen in the selected range.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase({__name__=\"hyperhive.vcs.push.count\", \"service.name\"=\"swarm-controller\"}[$__range])) or vector(0)", - "instant": true, - "legendFormat": "pushes" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 5, - "type": "timeseries", - "title": "Nodes by state", - "description": "Job nodes at any depth, per lifecycle state.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 4 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum by (state) ({__name__=\"jobq.nodes\", \"service.name\"=\"swarm-controller\"})", - "legendFormat": "{{state}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 6, - "type": "timeseries", - "title": "Root groups by state", - "description": "Root job groups, per lifecycle state.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 4 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum by (state) ({__name__=\"jobq.roots\", \"service.name\"=\"swarm-controller\"})", - "legendFormat": "{{state}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 7, - "type": "timeseries", - "title": "Pushes and commits", - "description": "Push deliveries and commits per minute.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(rate({__name__=\"hyperhive.vcs.push.count\", \"service.name\"=\"swarm-controller\"}[$__rate_interval])) * 60", - "legendFormat": "pushes" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(rate({__name__=\"hyperhive.vcs.commit.count\", \"service.name\"=\"swarm-controller\"}[$__rate_interval])) * 60", - "legendFormat": "commits" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 8, - "type": "timeseries", - "title": "Commits by repo", - "description": "Commits per hour, per repository.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sort_desc(sum by (repo) (rate({__name__=\"hyperhive.vcs.commit.count\", \"service.name\"=\"swarm-controller\"}[$__rate_interval])) * 3600)", - "legendFormat": "{{repo}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - } - ] -} diff --git a/nix/host-modules/swarm-grafana/dashboards/forge.json b/nix/host-modules/swarm-grafana/dashboards/forge.json index 40f90859..6fe20382 100644 --- a/nix/host-modules/swarm-grafana/dashboards/forge.json +++ b/nix/host-modules/swarm-grafana/dashboards/forge.json @@ -457,515 +457,6 @@ "values": false } } - }, - { - "id": 39, - "type": "row", - "title": "Forgejo runtime", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "panels": [] - }, - { - "id": 41, - "type": "stat", - "title": "Uptime", - "description": "Time since the forge process started.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 36 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "time() - process_start_time_seconds{\"service.name\"=\"forgejo\"}", - "instant": true, - "legendFormat": "uptime" - } - ], - "fieldConfig": { - "defaults": { - "unit": "s", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 42, - "type": "stat", - "title": "Memory (RSS)", - "description": "Resident memory of the forge process.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 36 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "process_resident_memory_bytes{\"service.name\"=\"forgejo\"}", - "instant": true, - "legendFormat": "rss" - } - ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "decimals": 1, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 43, - "type": "stat", - "title": "Goroutines", - "description": "Goroutines running in the forge process.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 36 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "go_goroutines{\"service.name\"=\"forgejo\"}", - "instant": true, - "legendFormat": "goroutines" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 44, - "type": "stat", - "title": "Open descriptors", - "description": "File descriptors held against the process limit.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 36 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "process_open_fds{\"service.name\"=\"forgejo\"} / process_max_fds{\"service.name\"=\"forgejo\"}", - "instant": true, - "legendFormat": "used" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "decimals": 1, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 45, - "type": "timeseries", - "title": "CPU", - "description": "CPU seconds used per second.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "rate({__name__=~\"process_cpu_seconds(_total)?\", \"service.name\"=\"forgejo\"}[$__rate_interval])", - "legendFormat": "cpu" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 46, - "type": "timeseries", - "title": "Memory", - "description": "Resident memory and live Go heap.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "process_resident_memory_bytes{\"service.name\"=\"forgejo\"}", - "legendFormat": "rss" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "go_memstats_heap_alloc_bytes{\"service.name\"=\"forgejo\"}", - "legendFormat": "heap" - } - ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "decimals": 1, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 47, - "type": "timeseries", - "title": "Goroutines and threads", - "description": "Goroutines and OS threads in the process.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "go_goroutines{\"service.name\"=\"forgejo\"}", - "legendFormat": "goroutines" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "go_threads{\"service.name\"=\"forgejo\"}", - "legendFormat": "threads" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 48, - "type": "timeseries", - "title": "Open descriptors", - "description": "File descriptors held, against the limit.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "process_open_fds{\"service.name\"=\"forgejo\"}", - "legendFormat": "open" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "process_max_fds{\"service.name\"=\"forgejo\"}", - "legendFormat": "limit" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } } ] }