collector dashboard: query the service.name the collectors emit, lint dashboard names
This commit is contained in:
atlas
mara
parent
067a260b85
commit
181c42dae1
4 changed files with 107 additions and 25 deletions
|
|
@ -71,6 +71,15 @@ jobs:
|
|||
# nix-shell-for-one-tool shape the vale jobs below use.
|
||||
run: nix shell --inputs-from . nixpkgs#jq --command sh scripts/check-dashboard-descriptions.sh
|
||||
|
||||
dashboard-service-names:
|
||||
name: dashboard-service-name lint
|
||||
if: vars.PUBLIC_FORGE != 'true'
|
||||
runs-on: ${{ vars.PUBLIC_FORGE == 'true' && 'nixos' || 'hive-ci' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: lint
|
||||
run: sh scripts/check-dashboard-service-names.sh
|
||||
|
||||
shellcheck:
|
||||
name: shellcheck
|
||||
if: vars.PUBLIC_FORGE != 'true'
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"title": "hyperhive \u00b7 collector (self-telemetry)",
|
||||
"uid": "hyperhive-swarm-collector",
|
||||
"description": "What the OTEL collectors accept, refuse, export and queue.",
|
||||
"description": "What the OTEL collectors accept, refuse, export and queue. Port 8888 is the hive, 8889 swarm-otel.",
|
||||
"editable": true,
|
||||
"refresh": "1m",
|
||||
"schemaVersion": 39,
|
||||
|
|
@ -35,7 +35,7 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum(increase({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__range])) or vector(0)",
|
||||
"expr": "sum(increase({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__range])) or vector(0)",
|
||||
"instant": true,
|
||||
"legendFormat": "refused"
|
||||
}
|
||||
|
|
@ -99,7 +99,7 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum(increase({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__range])) or vector(0)",
|
||||
"expr": "sum(increase({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__range])) or vector(0)",
|
||||
"instant": true,
|
||||
"legendFormat": "failed"
|
||||
}
|
||||
|
|
@ -163,7 +163,7 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max(otelcol_exporter_queue_size{\"service.name\"=\"collector\"} / otelcol_exporter_queue_capacity{\"service.name\"=\"collector\"})",
|
||||
"expr": "max(otelcol_exporter_queue_size{\"service.name\"=\"otelcol-contrib\"} / otelcol_exporter_queue_capacity{\"service.name\"=\"otelcol-contrib\"})",
|
||||
"instant": true,
|
||||
"legendFormat": "fill"
|
||||
}
|
||||
|
|
@ -230,7 +230,7 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"collector\"})",
|
||||
"expr": "max({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"otelcol-contrib\"})",
|
||||
"instant": true,
|
||||
"legendFormat": "rss"
|
||||
}
|
||||
|
|
@ -289,8 +289,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_metric_points(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
||||
"legendFormat": "metric points"
|
||||
"expr": "sum by (\"server.port\") (rate({__name__=~\"otelcol_receiver_accepted_metric_points(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server.port}} metric points"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
|
|
@ -298,8 +298,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_log_records(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
||||
"legendFormat": "log records"
|
||||
"expr": "sum by (\"server.port\") (rate({__name__=~\"otelcol_receiver_accepted_log_records(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server.port}} log records"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
|
|
@ -307,8 +307,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_spans(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
||||
"legendFormat": "spans"
|
||||
"expr": "sum by (\"server.port\") (rate({__name__=~\"otelcol_receiver_accepted_spans(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server.port}} spans"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
|
@ -364,8 +364,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum by (receiver) (rate({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{receiver}}"
|
||||
"expr": "sum by (receiver, \"server.port\") (rate({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server.port}} {{receiver}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
|
@ -421,8 +421,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{exporter}}"
|
||||
"expr": "sum by (exporter, \"server.port\") (rate({__name__=~\"otelcol_exporter_sent_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server.port}} {{exporter}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
|
@ -478,8 +478,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum by (exporter) (rate({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{exporter}}"
|
||||
"expr": "sum by (exporter, \"server.port\") (rate({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"otelcol-contrib\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server.port}} {{exporter}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
|
@ -535,8 +535,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max by (exporter) (otelcol_exporter_queue_size{\"service.name\"=\"collector\"})",
|
||||
"legendFormat": "{{exporter}} size"
|
||||
"expr": "max by (exporter, \"server.port\") (otelcol_exporter_queue_size{\"service.name\"=\"otelcol-contrib\"})",
|
||||
"legendFormat": "{{server.port}} {{exporter}} size"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
|
|
@ -544,8 +544,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max by (exporter) (otelcol_exporter_queue_capacity{\"service.name\"=\"collector\"})",
|
||||
"legendFormat": "{{exporter}} capacity"
|
||||
"expr": "max by (exporter, \"server.port\") (otelcol_exporter_queue_capacity{\"service.name\"=\"otelcol-contrib\"})",
|
||||
"legendFormat": "{{server.port}} {{exporter}} capacity"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
|
@ -600,8 +600,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"collector\"})",
|
||||
"legendFormat": "rss"
|
||||
"expr": "max by (\"server.port\") ({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"otelcol-contrib\"})",
|
||||
"legendFormat": "{{server.port}} rss"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
|
|
@ -609,8 +609,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max(otelcol_process_runtime_heap_alloc_bytes{\"service.name\"=\"collector\"})",
|
||||
"legendFormat": "heap"
|
||||
"expr": "max by (\"server.port\") (otelcol_process_runtime_heap_alloc_bytes{\"service.name\"=\"otelcol-contrib\"})",
|
||||
"legendFormat": "{{server.port}} heap"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
|
|
|||
67
scripts/check-dashboard-service-names.sh
Normal file
67
scripts/check-dashboard-service-names.sh
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
#!/bin/sh
|
||||
# CI lint: every literal `"service.name"="<x>"` selector in a shipped grafana
|
||||
# dashboard must name a service.name that something emits. A dashboard
|
||||
# querying a name nobody sets renders empty panels with no error.
|
||||
#
|
||||
# `known` lists the emitted names, one `name|file|anchor` per line. The
|
||||
# anchor is a fixed string that must still appear in that file, so an entry
|
||||
# cannot outlive the code that sets the name. Regex selectors
|
||||
# (`"service.name"=~"$service"`) are template variables and are not checked.
|
||||
#
|
||||
set -eu
|
||||
|
||||
dashboards_dir="nix/host-modules/swarm-grafana/dashboards"
|
||||
|
||||
# bao: `collectorServiceName` upserted on the store's collector.
|
||||
# authelia, forgejo: `serviceName` of the journal forwarder in each module.
|
||||
# swarm-controller: the resource of the controller's own metrics.
|
||||
# otelcol-contrib: the unset default of the collector binary, which the hive
|
||||
# collector and swarm-otel both run (`pkgs.opentelemetry-collector-contrib`)
|
||||
# and which stamps their self-metrics.
|
||||
known='bao|nix/host-modules/swarm-bao.nix|collectorServiceName = "bao"
|
||||
authelia|nix/host-modules/swarm-authelia.nix|serviceName = "authelia"
|
||||
forgejo|nix/host-modules/hive-forge/default.nix|serviceName = "forgejo"
|
||||
swarm-controller|swarm-controller/src/vcs_metrics.rs|with_service_name("swarm-controller")
|
||||
otelcol-contrib|nix/host-modules/otel.nix|package = pkgs.opentelemetry-collector-contrib'
|
||||
|
||||
violations=0
|
||||
|
||||
while IFS='|' read -r name file anchor; do
|
||||
if ! grep -qF -- "$anchor" "$file" 2>/dev/null; then
|
||||
violations=$((violations + 1))
|
||||
printf '::error file=%s::service.name "%s" is listed as emitted but %s no longer contains: %s\n' \
|
||||
"$file" "$name" "$file" "$anchor"
|
||||
fi
|
||||
done <<EOF2
|
||||
$known
|
||||
EOF2
|
||||
|
||||
names="$(printf '%s\n' "$known" | cut -d'|' -f1)"
|
||||
checked=0
|
||||
|
||||
for f in "$dashboards_dir"/*.json; do
|
||||
# The expr strings escape the quotes, so the raw file holds
|
||||
# service.name backslash-quote-equals-backslash-quote name.
|
||||
used_names="$(grep -oE '\\"service\.name\\"=\\"[^\\]+\\"' "$f" | sed -E 's/.*=\\"([^\\]+)\\"$/\1/' | sort -u || true)"
|
||||
while IFS= read -r used; do
|
||||
[ -n "$used" ] || continue
|
||||
checked=$((checked + 1))
|
||||
if ! printf '%s\n' "$names" | grep -qxF -- "$used"; then
|
||||
violations=$((violations + 1))
|
||||
printf '::error file=%s::queries service.name "%s", which nothing emits (known: %s)\n' \
|
||||
"$f" "$used" "$(printf '%s' "$names" | tr '\n' ' ')"
|
||||
fi
|
||||
done <<EOF3
|
||||
$used_names
|
||||
EOF3
|
||||
done
|
||||
|
||||
if [ "$checked" -eq 0 ]; then
|
||||
echo 'check-dashboard-service-names: found zero service.name selectors — extractor or tree changed' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$violations" -gt 0 ]; then
|
||||
printf 'check-dashboard-service-names: %s violation(s)\n' "$violations" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "check-dashboard-service-names: $checked selector name(s) checked, clean" >&2
|
||||
|
|
@ -39,5 +39,11 @@ if ! nix shell --inputs-from . nixpkgs#jq --command sh "$repo_root/scripts/check
|
|||
exit 1
|
||||
fi
|
||||
|
||||
echo "pre-push: running dashboard-service-name lint..." >&2
|
||||
if ! sh "$repo_root/scripts/check-dashboard-service-names.sh"; then
|
||||
echo "pre-push: dashboard-service-name lint FAILED — fix before pushing" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "pre-push: lints passed ✓" >&2
|
||||
exit 0
|
||||
|
|
|
|||
Loading…
Reference in a new issue