collector dashboard: query the service.name the collectors emit, lint dashboard names

.forgejo/workflows/ci.yml

@@ -71,6 +71,15 @@ jobs:
         # nix-shell-for-one-tool shape the vale jobs below use.
         run: nix shell --inputs-from . nixpkgs#jq --command sh scripts/check-dashboard-descriptions.sh
 
+  dashboard-service-names:
+    name: dashboard-service-name lint
+    if: vars.PUBLIC_FORGE != 'true'
+    runs-on: ${{ vars.PUBLIC_FORGE == 'true' && 'nixos' || 'hive-ci' }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: lint
+        run: sh scripts/check-dashboard-service-names.sh
+
   shellcheck:
     name: shellcheck
     if: vars.PUBLIC_FORGE != 'true'

nix/host-modules/swarm-grafana/dashboards/collector.json

@@ -1,7 +1,7 @@
 {
   "title": "hyperhive \u00b7 collector (self-telemetry)",
   "uid": "hyperhive-swarm-collector",
-  "description": "What the OTEL collectors accept, refuse, export and queue.",
+  "description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.",
   "editable": true,
   "refresh": "1m",
   "schemaVersion": 39,
@@ -35,7 +35,7 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum(increase({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__range])) or vector(0)",
+          "expr": "sum(increase({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__range])) or vector(0)",
           "instant": true,
           "legendFormat": "refused"
         }
@@ -99,7 +99,7 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum(increase({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__range])) or vector(0)",
+          "expr": "sum(increase({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__range])) or vector(0)",
           "instant": true,
           "legendFormat": "failed"
         }
@@ -163,7 +163,7 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "max(otelcol_exporter_queue_size{\"service.name\"=\"collector\"} / otelcol_exporter_queue_capacity{\"service.name\"=\"collector\"})",
+          "expr": "max(otelcol_exporter_queue_size{\"service.name\"=\"otelcol-contrib\"} / otelcol_exporter_queue_capacity{\"service.name\"=\"otelcol-contrib\"})",
           "instant": true,
           "legendFormat": "fill"
         }
@@ -230,7 +230,7 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "max({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"collector\"})",
+          "expr": "max({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"otelcol-contrib\"})",
           "instant": true,
           "legendFormat": "rss"
         }
@@ -289,8 +289,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_metric_points(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
+          "expr": "sum by (\"server.port\") (rate({__name__=~\"otelcol_receiver_accepted_metric_points(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
-          "legendFormat": "metric points"
+          "legendFormat": "{{server.port}} metric points"
         },
         {
           "refId": "B",
@@ -298,8 +298,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_log_records(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
+          "expr": "sum by (\"server.port\") (rate({__name__=~\"otelcol_receiver_accepted_log_records(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
-          "legendFormat": "log records"
+          "legendFormat": "{{server.port}} log records"
         },
         {
           "refId": "C",
@@ -307,8 +307,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_spans(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
+          "expr": "sum by (\"server.port\") (rate({__name__=~\"otelcol_receiver_accepted_spans(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
-          "legendFormat": "spans"
+          "legendFormat": "{{server.port}} spans"
         }
       ],
       "fieldConfig": {
@@ -364,8 +364,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum by (receiver) (rate({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
+          "expr": "sum by (receiver, \"server.port\") (rate({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
-          "legendFormat": "{{receiver}}"
+          "legendFormat": "{{server.port}} {{receiver}}"
         }
       ],
       "fieldConfig": {
@@ -421,8 +421,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
+          "expr": "sum by (exporter, \"server.port\") (rate({__name__=~\"otelcol_exporter_sent_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
-          "legendFormat": "{{exporter}}"
+          "legendFormat": "{{server.port}} {{exporter}}"
         }
       ],
       "fieldConfig": {
@@ -478,8 +478,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "sum by (exporter) (rate({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
+          "expr": "sum by (exporter, \"server.port\") (rate({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
-          "legendFormat": "{{exporter}}"
+          "legendFormat": "{{server.port}} {{exporter}}"
         }
       ],
       "fieldConfig": {
@@ -535,8 +535,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "max by (exporter) (otelcol_exporter_queue_size{\"service.name\"=\"collector\"})",
+          "expr": "max by (exporter, \"server.port\") (otelcol_exporter_queue_size{\"service.name\"=\"otelcol-contrib\"})",
-          "legendFormat": "{{exporter}} size"
+          "legendFormat": "{{server.port}} {{exporter}} size"
         },
         {
           "refId": "B",
@@ -544,8 +544,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "max by (exporter) (otelcol_exporter_queue_capacity{\"service.name\"=\"collector\"})",
+          "expr": "max by (exporter, \"server.port\") (otelcol_exporter_queue_capacity{\"service.name\"=\"otelcol-contrib\"})",
-          "legendFormat": "{{exporter}} capacity"
+          "legendFormat": "{{server.port}} {{exporter}} capacity"
         }
       ],
       "fieldConfig": {
@@ -600,8 +600,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "max({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"collector\"})",
+          "expr": "max by (\"server.port\") ({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"otelcol-contrib\"})",
-          "legendFormat": "rss"
+          "legendFormat": "{{server.port}} rss"
         },
         {
           "refId": "B",
@@ -609,8 +609,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "max(otelcol_process_runtime_heap_alloc_bytes{\"service.name\"=\"collector\"})",
+          "expr": "max by (\"server.port\") (otelcol_process_runtime_heap_alloc_bytes{\"service.name\"=\"otelcol-contrib\"})",
-          "legendFormat": "heap"
+          "legendFormat": "{{server.port}} heap"
         }
       ],
       "fieldConfig": {

scripts/check-dashboard-service-names.sh

@@ -0,0 +1,67 @@
+#!/bin/sh
+# CI lint: every literal `"service.name"="<x>"` selector in a shipped grafana
+# dashboard must name a service.name that something emits. A dashboard
+# querying a name nobody sets renders empty panels with no error.
+#
+# `known` lists the emitted names, one `name|file|anchor` per line. The
+# anchor is a fixed string that must still appear in that file, so an entry
+# cannot outlive the code that sets the name. Regex selectors
+# (`"service.name"=~"$service"`) are template variables and are not checked.
+#
+set -eu
+
+dashboards_dir="nix/host-modules/swarm-grafana/dashboards"
+
+# bao: `collectorServiceName` upserted on the store's collector.
+# authelia, forgejo: `serviceName` of the journal forwarder in each module.
+# swarm-controller: the resource of the controller's own metrics.
+# otelcol-contrib: the unset default of the collector binary, which the hive
+#   collector and swarm-otel both run (`pkgs.opentelemetry-collector-contrib`)
+#   and which stamps their self-metrics.
+known='bao|nix/host-modules/swarm-bao.nix|collectorServiceName = "bao"
+authelia|nix/host-modules/swarm-authelia.nix|serviceName = "authelia"
+forgejo|nix/host-modules/hive-forge/default.nix|serviceName = "forgejo"
+swarm-controller|swarm-controller/src/vcs_metrics.rs|with_service_name("swarm-controller")
+otelcol-contrib|nix/host-modules/otel.nix|package = pkgs.opentelemetry-collector-contrib'
+
+violations=0
+
+while IFS='|' read -r name file anchor; do
+  if ! grep -qF -- "$anchor" "$file" 2>/dev/null; then
+    violations=$((violations + 1))
+    printf '::error file=%s::service.name "%s" is listed as emitted but %s no longer contains: %s\n' \
+      "$file" "$name" "$file" "$anchor"
+  fi
+done <<EOF2
+$known
+EOF2
+
+names="$(printf '%s\n' "$known" | cut -d'|' -f1)"
+checked=0
+
+for f in "$dashboards_dir"/*.json; do
+  # The expr strings escape the quotes, so the raw file holds
+  # service.name backslash-quote-equals-backslash-quote name.
+  used_names="$(grep -oE '\\"service\.name\\"=\\"[^\\]+\\"' "$f" | sed -E 's/.*=\\"([^\\]+)\\"$/\1/' | sort -u || true)"
+  while IFS= read -r used; do
+    [ -n "$used" ] || continue
+    checked=$((checked + 1))
+    if ! printf '%s\n' "$names" | grep -qxF -- "$used"; then
+      violations=$((violations + 1))
+      printf '::error file=%s::queries service.name "%s", which nothing emits (known: %s)\n' \
+        "$f" "$used" "$(printf '%s' "$names" | tr '\n' ' ')"
+    fi
+  done <<EOF3
+$used_names
+EOF3
+done
+
+if [ "$checked" -eq 0 ]; then
+  echo 'check-dashboard-service-names: found zero service.name selectors — extractor or tree changed' >&2
+  exit 1
+fi
+if [ "$violations" -gt 0 ]; then
+  printf 'check-dashboard-service-names: %s violation(s)\n' "$violations" >&2
+  exit 1
+fi
+echo "check-dashboard-service-names: $checked selector name(s) checked, clean" >&2

scripts/pre-push

@@ -39,5 +39,11 @@ if ! nix shell --inputs-from . nixpkgs#jq --command sh "$repo_root/scripts/check
   exit 1
 fi
 
+echo "pre-push: running dashboard-service-name lint..." >&2
+if ! sh "$repo_root/scripts/check-dashboard-service-names.sh"; then
+  echo "pre-push: dashboard-service-name lint FAILED — fix before pushing" >&2
+  exit 1
+fi
+
 echo "pre-push: lints passed ✓" >&2
 exit 0