diff --git a/docs/swarm/services.md b/docs/swarm/services.md index ed13700d..0ed83350 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -174,7 +174,7 @@ kept. - `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue. - `claude-usage`: Claude use by model, effort, token type and source. - `controller`: the swarm controller's job graph by state, and the push and commit rates it sees on the forge. -- `forge` and `forgejo`: what the forge holds, and the forge process's CPU, memory and descriptors. +- `forge`: what the forge holds, and the forge process's CPU, memory and descriptors. - `collector`: what the OTEL collectors accept, refuse, export and queue, with queue depth against capacity. - `logstore`, `metricstore`: health of the log store and the metrics store. diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index b62f3e0d..979b3924 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -201,7 +201,6 @@ let "collector.json" "controller.json" "forge.json" - "forgejo.json" "logstore.json" "metricstore.json" "queue.json" diff --git a/nix/host-modules/swarm-grafana/dashboards/forge.json b/nix/host-modules/swarm-grafana/dashboards/forge.json index 6fe20382..40f90859 100644 --- a/nix/host-modules/swarm-grafana/dashboards/forge.json +++ b/nix/host-modules/swarm-grafana/dashboards/forge.json @@ -457,6 +457,515 @@ "values": false } } + }, + { + "id": 39, + "type": "row", + "title": "Forgejo runtime", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "panels": [] + }, + { + "id": 41, + "type": "stat", + "title": "Uptime", + "description": "Time since the forge process started.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 36 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "time() - process_start_time_seconds{\"service.name\"=\"forgejo\"}", + "instant": true, + "legendFormat": "uptime" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 42, + "type": "stat", + "title": "Memory (RSS)", + "description": "Resident memory of the forge process.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 36 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "process_resident_memory_bytes{\"service.name\"=\"forgejo\"}", + "instant": true, + "legendFormat": "rss" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 43, + "type": "stat", + "title": "Goroutines", + "description": "Goroutines running in the forge process.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 36 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "go_goroutines{\"service.name\"=\"forgejo\"}", + "instant": true, + "legendFormat": "goroutines" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 44, + "type": "stat", + "title": "Open descriptors", + "description": "File descriptors held against the process limit.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 36 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "process_open_fds{\"service.name\"=\"forgejo\"} / process_max_fds{\"service.name\"=\"forgejo\"}", + "instant": true, + "legendFormat": "used" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 1, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 45, + "type": "timeseries", + "title": "CPU", + "description": "CPU seconds used per second.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "rate({__name__=~\"process_cpu_seconds(_total)?\", \"service.name\"=\"forgejo\"}[$__rate_interval])", + "legendFormat": "cpu" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 46, + "type": "timeseries", + "title": "Memory", + "description": "Resident memory and live Go heap.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "process_resident_memory_bytes{\"service.name\"=\"forgejo\"}", + "legendFormat": "rss" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "go_memstats_heap_alloc_bytes{\"service.name\"=\"forgejo\"}", + "legendFormat": "heap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 47, + "type": "timeseries", + "title": "Goroutines and threads", + "description": "Goroutines and OS threads in the process.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "go_goroutines{\"service.name\"=\"forgejo\"}", + "legendFormat": "goroutines" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "go_threads{\"service.name\"=\"forgejo\"}", + "legendFormat": "threads" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 48, + "type": "timeseries", + "title": "Open descriptors", + "description": "File descriptors held, against the limit.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "process_open_fds{\"service.name\"=\"forgejo\"}", + "legendFormat": "open" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "process_max_fds{\"service.name\"=\"forgejo\"}", + "legendFormat": "limit" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } } ] } diff --git a/nix/host-modules/swarm-grafana/dashboards/forgejo.json b/nix/host-modules/swarm-grafana/dashboards/forgejo.json deleted file mode 100644 index 661a47fc..00000000 --- a/nix/host-modules/swarm-grafana/dashboards/forgejo.json +++ /dev/null @@ -1,513 +0,0 @@ -{ - "title": "hyperhive \u00b7 forge (runtime)", - "uid": "hyperhive-swarm-forge-runtime", - "description": "The forge process: uptime, CPU, memory and descriptors.", - "editable": true, - "refresh": "1m", - "schemaVersion": 39, - "tags": ["hyperhive", "swarm", "forge", "runtime"], - "time": { - "from": "now-24h", - "to": "now" - }, - "timezone": "utc", - "version": 1, - "panels": [ - { - "id": 1, - "type": "stat", - "title": "Uptime", - "description": "Time since the forge process started.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "time() - process_start_time_seconds{\"service.name\"=\"forgejo\"}", - "instant": true, - "legendFormat": "uptime" - } - ], - "fieldConfig": { - "defaults": { - "unit": "s", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 2, - "type": "stat", - "title": "Memory (RSS)", - "description": "Resident memory of the forge process.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "process_resident_memory_bytes{\"service.name\"=\"forgejo\"}", - "instant": true, - "legendFormat": "rss" - } - ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "decimals": 1, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 3, - "type": "stat", - "title": "Goroutines", - "description": "Goroutines running in the forge process.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "go_goroutines{\"service.name\"=\"forgejo\"}", - "instant": true, - "legendFormat": "goroutines" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 4, - "type": "stat", - "title": "Open descriptors", - "description": "File descriptors held against the process limit.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "process_open_fds{\"service.name\"=\"forgejo\"} / process_max_fds{\"service.name\"=\"forgejo\"}", - "instant": true, - "legendFormat": "used" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "decimals": 1, - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - } - }, - { - "id": 5, - "type": "timeseries", - "title": "CPU", - "description": "CPU seconds used per second.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 4 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "rate({__name__=~\"process_cpu_seconds(_total)?\", \"service.name\"=\"forgejo\"}[$__rate_interval])", - "legendFormat": "cpu" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 6, - "type": "timeseries", - "title": "Memory", - "description": "Resident memory and live Go heap.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 4 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "process_resident_memory_bytes{\"service.name\"=\"forgejo\"}", - "legendFormat": "rss" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "go_memstats_heap_alloc_bytes{\"service.name\"=\"forgejo\"}", - "legendFormat": "heap" - } - ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "decimals": 1, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 7, - "type": "timeseries", - "title": "Goroutines and threads", - "description": "Goroutines and OS threads in the process.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "go_goroutines{\"service.name\"=\"forgejo\"}", - "legendFormat": "goroutines" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "go_threads{\"service.name\"=\"forgejo\"}", - "legendFormat": "threads" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 8, - "type": "timeseries", - "title": "Open descriptors", - "description": "File descriptors held, against the limit.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "process_open_fds{\"service.name\"=\"forgejo\"}", - "legendFormat": "open" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "process_max_fds{\"service.name\"=\"forgejo\"}", - "legendFormat": "limit" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - } - ] -}