gateway: send nginx stub_status to VictoriaMetrics, add a gateway dashboard
The gateway nginx serves stub_status on a loopback server of its own, 127.0.0.1:8085 (services.hyperhive.gateway.statusPort). The hive-tier collector reads it with the contrib nginx receiver, which the pinned otelcol-contrib 0.151.0 ships, so no exporter daemon is added. The receiver dials http:// only (a unix: or http+unix: endpoint fails with "unsupported protocol scheme"), so the listener is loopback TCP with allow 127.0.0.1 / deny all rather than a unix socket. The new gateway dashboard plots nginx.requests, nginx.connections_current by state, and nginx.connections_accepted against handled, per hive. Part of #5342.
This commit is contained in:
atlas
mara
parent
e59a2a34fb
commit
f70bbda7d7
8 changed files with 460 additions and 1 deletions
391
nix/host-modules/swarm-grafana/dashboards/gateway.json
Normal file
391
nix/host-modules/swarm-grafana/dashboards/gateway.json
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
{
|
||||
"title": "hyperhive · gateway (nginx)",
|
||||
"uid": "hyperhive-gateway",
|
||||
"description": "The gateway nginx on each hive's host: request rate and connections.",
|
||||
"editable": true,
|
||||
"refresh": "1m",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["hyperhive", "gateway", "nginx"],
|
||||
"time": {
|
||||
"from": "now-24h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "utc",
|
||||
"version": 1,
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "hive",
|
||||
"label": "Hive",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"query": "label_values({__name__=\"nginx.requests\"}, hive)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"allValue": ".*",
|
||||
"multi": true,
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"sort": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Requests/s",
|
||||
"description": "Requests per second across the selected gateways.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum(rate({__name__=\"nginx.requests\", hive=~\"$hive\"}[$__rate_interval])) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "text",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "none",
|
||||
"colorMode": "value",
|
||||
"textMode": "value",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Active connections",
|
||||
"description": "Open client connections now, across the selected gateways.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum({__name__=\"nginx.connections_current\", hive=~\"$hive\", state=\"active\"}) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "text",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "none",
|
||||
"colorMode": "value",
|
||||
"textMode": "value",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Dropped connections",
|
||||
"description": "Connections accepted but not handled over the time range.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "(sum(increase({__name__=\"nginx.connections_accepted\", hive=~\"$hive\"}[$__range])) - sum(increase({__name__=\"nginx.connections_handled\", hive=~\"$hive\"}[$__range]))) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "text",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "none",
|
||||
"colorMode": "value",
|
||||
"textMode": "value",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Requests per second",
|
||||
"description": "Requests per second, per hive.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum by (hive) (rate({__name__=\"nginx.requests\", hive=~\"$hive\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{hive}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Connections by state",
|
||||
"description": "Client connections per state; active is reading + writing + waiting.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 4
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum by (hive, state) ({__name__=\"nginx.connections_current\", hive=~\"$hive\"})",
|
||||
"legendFormat": "{{hive}} {{state}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Accepted vs handled",
|
||||
"description": "Connections per second. A gap between the two lines is connections dropped.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum by (hive) (rate({__name__=\"nginx.connections_accepted\", hive=~\"$hive\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{hive}} accepted"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum by (hive) (rate({__name__=\"nginx.connections_handled\", hive=~\"$hive\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{hive}} handled"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "cps",
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Reference in a new issue