gateway: send nginx stub_status to VictoriaMetrics, add a gateway dashboard
The gateway nginx serves stub_status on a loopback server of its own, 127.0.0.1:8085 (services.hyperhive.gateway.statusPort). The hive-tier collector reads it with the contrib nginx receiver, which the pinned otelcol-contrib 0.151.0 ships, so no exporter daemon is added. The receiver dials http:// only (a unix: or http+unix: endpoint fails with "unsupported protocol scheme"), so the listener is loopback TCP with allow 127.0.0.1 / deny all rather than a unix socket. The new gateway dashboard plots nginx.requests, nginx.connections_current by state, and nginx.connections_accepted against handled, per hive. Part of #5342.
This commit is contained in:
atlas
mara
parent
e59a2a34fb
commit
f70bbda7d7
8 changed files with 460 additions and 1 deletions
|
|
@ -404,6 +404,24 @@ in
|
|||
# `error` threshold and this changes where lines are filed, not
|
||||
# which lines exist.
|
||||
logError = "syslog:server=unix:/dev/log,nohostname,tag=nginx";
|
||||
|
||||
# Read by the hive collector's `nginx` receiver (../otel.nix). Its own
|
||||
# loopback server, outside the vhost tree, so it never answers on the
|
||||
# gateway's public listeners.
|
||||
appendHttpConfig = ''
|
||||
server {
|
||||
listen 127.0.0.1:${toString cfg.statusPort};
|
||||
access_log off;
|
||||
location = /stub_status {
|
||||
stub_status;
|
||||
allow 127.0.0.1;
|
||||
deny all;
|
||||
}
|
||||
location / {
|
||||
return 404;
|
||||
}
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
# ⚠️ NO `SupplementaryGroups = [ "hive-core" ]` on nginx, and its
|
||||
|
|
|
|||
|
|
@ -254,6 +254,21 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
statusPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8085;
|
||||
description = ''
|
||||
Loopback TCP port nginx serves its `stub_status` counters on, at
|
||||
`http://127.0.0.1:''${statusPort}/stub_status`. The hive's collector
|
||||
({option}`services.hyperhive.otel.enable`) reads it with its `nginx`
|
||||
receiver.
|
||||
|
||||
TCP rather than a unix socket because that receiver can only dial
|
||||
`http://`. Anything on the host can read these counters; they are
|
||||
connection and request totals, nothing per-client.
|
||||
'';
|
||||
};
|
||||
|
||||
tls = {
|
||||
certDir = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.path;
|
||||
|
|
|
|||
|
|
@ -334,6 +334,7 @@ in
|
|||
hiveName = config.services.hyperhive.hiveName;
|
||||
listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
|
||||
swarmName = "otlphttp/swarm";
|
||||
gatewayCfg = config.services.hyperhive.gateway;
|
||||
authName = "oauth2client/swarm";
|
||||
|
||||
# A hive always authenticates to the swarm's collector as itself, so
|
||||
|
|
@ -555,6 +556,15 @@ in
|
|||
job_name = job;
|
||||
static_configs = [ { targets = [ target ]; } ];
|
||||
}) otel.scrapeTargets;
|
||||
}
|
||||
# This host's gateway nginx, from the `stub_status` server
|
||||
# ./hive-gateway/default.nix adds. The receiver dials `http://`
|
||||
# only, which is why that server is on loopback TCP.
|
||||
// lib.optionalAttrs gatewayCfg.enable {
|
||||
nginx = {
|
||||
endpoint = "http://127.0.0.1:${toString gatewayCfg.statusPort}/stub_status";
|
||||
collection_interval = "60s";
|
||||
};
|
||||
};
|
||||
|
||||
# One destination, and it is the swarm's collector. This tier
|
||||
|
|
@ -662,7 +672,8 @@ in
|
|||
"otlp"
|
||||
"hostmetrics"
|
||||
]
|
||||
++ lib.optional (otel.scrapeTargets != { }) "prometheus";
|
||||
++ lib.optional (otel.scrapeTargets != { }) "prometheus"
|
||||
++ lib.optional gatewayCfg.enable "nginx";
|
||||
# Applies to EVERY receiver above, not just `hostmetrics` — a
|
||||
# processor sits in the pipeline, not on a receiver. That is
|
||||
# what we want here: samples an agent pushed over `otlp` and
|
||||
|
|
|
|||
|
|
@ -201,6 +201,7 @@ let
|
|||
"collector.json"
|
||||
"controller.json"
|
||||
"forge.json"
|
||||
"gateway.json"
|
||||
"logstore.json"
|
||||
"metricstore.json"
|
||||
"queue.json"
|
||||
|
|
|
|||
391
nix/host-modules/swarm-grafana/dashboards/gateway.json
Normal file
391
nix/host-modules/swarm-grafana/dashboards/gateway.json
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
{
|
||||
"title": "hyperhive · gateway (nginx)",
|
||||
"uid": "hyperhive-gateway",
|
||||
"description": "The gateway nginx on each hive's host: request rate and connections.",
|
||||
"editable": true,
|
||||
"refresh": "1m",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["hyperhive", "gateway", "nginx"],
|
||||
"time": {
|
||||
"from": "now-24h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "utc",
|
||||
"version": 1,
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "hive",
|
||||
"label": "Hive",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"query": "label_values({__name__=\"nginx.requests\"}, hive)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"allValue": ".*",
|
||||
"multi": true,
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"sort": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Requests/s",
|
||||
"description": "Requests per second across the selected gateways.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum(rate({__name__=\"nginx.requests\", hive=~\"$hive\"}[$__rate_interval])) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "text",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "none",
|
||||
"colorMode": "value",
|
||||
"textMode": "value",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Active connections",
|
||||
"description": "Open client connections now, across the selected gateways.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum({__name__=\"nginx.connections_current\", hive=~\"$hive\", state=\"active\"}) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "text",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "none",
|
||||
"colorMode": "value",
|
||||
"textMode": "value",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Dropped connections",
|
||||
"description": "Connections accepted but not handled over the time range.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "(sum(increase({__name__=\"nginx.connections_accepted\", hive=~\"$hive\"}[$__range])) - sum(increase({__name__=\"nginx.connections_handled\", hive=~\"$hive\"}[$__range]))) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "text",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "none",
|
||||
"colorMode": "value",
|
||||
"textMode": "value",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Requests per second",
|
||||
"description": "Requests per second, per hive.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum by (hive) (rate({__name__=\"nginx.requests\", hive=~\"$hive\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{hive}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Connections by state",
|
||||
"description": "Client connections per state; active is reading + writing + waiting.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 4
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum by (hive, state) ({__name__=\"nginx.connections_current\", hive=~\"$hive\"})",
|
||||
"legendFormat": "{{hive}} {{state}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Accepted vs handled",
|
||||
"description": "Connections per second. A gap between the two lines is connections dropped.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum by (hive) (rate({__name__=\"nginx.connections_accepted\", hive=~\"$hive\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{hive}} accepted"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "sum by (hive) (rate({__name__=\"nginx.connections_handled\", hive=~\"$hive\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{hive}} handled"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "cps",
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -141,6 +141,26 @@ let
|
|||
name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs";
|
||||
ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers;
|
||||
}
|
||||
{
|
||||
# The receiver, the pipeline entry and the nginx server it reads have to
|
||||
# agree on one address, and none of them may exist without a gateway on
|
||||
# the host. `hiveOtelLeaf` is the control: it runs no gateway, and the
|
||||
# clause asserting that keeps the control from going vacuous.
|
||||
name = "the hive collector reads the gateway's stub_status, and only where a gateway runs";
|
||||
ok =
|
||||
let
|
||||
nginxConf = hiveOtel.services.nginx.appendHttpConfig;
|
||||
leafSettings = hiveOtelLeaf.services.opentelemetry-collector.settings;
|
||||
in
|
||||
hiveOtel.services.hyperhive.gateway.enable
|
||||
&& hiveOtelSettings.receivers.nginx.endpoint == "http://127.0.0.1:8085/stub_status"
|
||||
&& builtins.elem "nginx" hiveOtelPipelines.metrics.receivers
|
||||
&& lib.hasInfix "listen 127.0.0.1:8085;" nginxConf
|
||||
&& lib.hasInfix "location = /stub_status {" nginxConf
|
||||
&& !hiveOtelLeaf.services.hyperhive.gateway.enable
|
||||
&& !(leafSettings.receivers ? nginx)
|
||||
&& !(builtins.elem "nginx" leafSettings.service.pipelines.metrics.receivers);
|
||||
}
|
||||
{
|
||||
# `metrics.address` is the spelling that looks right and is rejected by
|
||||
# this collector version. The first clause is the control: without it a
|
||||
|
|
|
|||
Loading…
Reference in a new issue