652b02b53f
Three dashboards over metrics already in VictoriaMetrics: the swarm controller's job graph and forge push/commit counters, the forge process's runtime, and the OTEL collectors' self-telemetry.
648 lines
16 KiB
JSON
648 lines
16 KiB
JSON
{
|
|
"title": "hyperhive \u00b7 collector (self-telemetry)",
|
|
"uid": "hyperhive-swarm-collector",
|
|
"description": "What the OTEL collectors accept, refuse, export and queue.",
|
|
"editable": true,
|
|
"refresh": "1m",
|
|
"schemaVersion": 39,
|
|
"tags": ["hyperhive", "swarm", "otel", "collector"],
|
|
"time": {
|
|
"from": "now-24h",
|
|
"to": "now"
|
|
},
|
|
"timezone": "utc",
|
|
"version": 1,
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "stat",
|
|
"title": "Refused or failed (range)",
|
|
"description": "Items receivers refused or failed to take in.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum(increase({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__range])) or vector(0)",
|
|
"instant": true,
|
|
"legendFormat": "refused"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
},
|
|
"x-zero-is-healthy": true
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "Export failures (range)",
|
|
"description": "Items exporters failed to send or enqueue.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum(increase({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__range])) or vector(0)",
|
|
"instant": true,
|
|
"legendFormat": "failed"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
},
|
|
"x-zero-is-healthy": true
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "Fullest export queue",
|
|
"description": "Highest queue fill across exporters.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "max(otelcol_exporter_queue_size{\"service.name\"=\"collector\"} / otelcol_exporter_queue_capacity{\"service.name\"=\"collector\"})",
|
|
"instant": true,
|
|
"legendFormat": "fill"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 0.5
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 0.9
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "Memory (RSS)",
|
|
"description": "Largest resident memory across collectors.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "max({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"collector\"})",
|
|
"instant": true,
|
|
"legendFormat": "rss"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"decimals": 1,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "text",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "timeseries",
|
|
"title": "Accepted per second",
|
|
"description": "Items receivers accept, per signal.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 4
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_metric_points(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
|
"legendFormat": "metric points"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_log_records(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
|
"legendFormat": "log records"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum(rate({__name__=~\"otelcol_receiver_accepted_spans(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
|
"legendFormat": "spans"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 1,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"axisSoftMin": 0
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true,
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "timeseries",
|
|
"title": "Refused or failed per second",
|
|
"description": "Items refused or failed at each receiver.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 4
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum by (receiver) (rate({__name__=~\"otelcol_receiver_(refused|failed)_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
|
"legendFormat": "{{receiver}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 2,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"axisSoftMin": 0
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true,
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "timeseries",
|
|
"title": "Sent per second",
|
|
"description": "Items each exporter delivers.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 12
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
|
"legendFormat": "{{exporter}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 1,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"axisSoftMin": 0
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true,
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 8,
|
|
"type": "timeseries",
|
|
"title": "Export failures per second",
|
|
"description": "Items each exporter fails to send or enqueue.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 12
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum by (exporter) (rate({__name__=~\"otelcol_exporter_(send|enqueue)_failed_(metric_points|log_records|spans)(_total)?\", \"service.name\"=\"collector\"}[$__rate_interval]))",
|
|
"legendFormat": "{{exporter}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 2,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"axisSoftMin": 0
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true,
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 9,
|
|
"type": "timeseries",
|
|
"title": "Export queue depth",
|
|
"description": "Batches waiting in each exporter's queue, and its capacity.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 20
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "max by (exporter) (otelcol_exporter_queue_size{\"service.name\"=\"collector\"})",
|
|
"legendFormat": "{{exporter}} size"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "max by (exporter) (otelcol_exporter_queue_capacity{\"service.name\"=\"collector\"})",
|
|
"legendFormat": "{{exporter}} capacity"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"axisSoftMin": 0
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true,
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 10,
|
|
"type": "timeseries",
|
|
"title": "Collector memory",
|
|
"description": "Largest resident memory and live Go heap across collectors.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 20
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "max({__name__=~\"otelcol_process_memory_rss(_bytes)?\", \"service.name\"=\"collector\"})",
|
|
"legendFormat": "rss"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "max(otelcol_process_runtime_heap_alloc_bytes{\"service.name\"=\"collector\"})",
|
|
"legendFormat": "heap"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"decimals": 1,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"axisSoftMin": 0
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true,
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|