gateway: send nginx stub_status to VictoriaMetrics, add a gateway dashboard

The gateway nginx serves stub_status on a loopback server of its own, 127.0.0.1:8085 (services.hyperhive.gateway.statusPort). The hive-tier collector reads it with the contrib nginx receiver, which the pinned otelcol-contrib 0.151.0 ships, so no exporter daemon is added. The receiver dials http:// only (a unix: or http+unix: endpoint fails with "unsupported protocol scheme"), so the listener is loopback TCP with allow 127.0.0.1 / deny all rather than a unix socket. The new gateway dashboard plots nginx.requests, nginx.connections_current by state, and nginx.connections_accepted against handled, per hive. Part of #5342.
2026-10-16 02:40:26 +02:00 · 2026-10-16 02:40:26 +02:00 · f70bbda7d7
commit f70bbda7d7
parent e59a2a34fb
8 changed files with 460 additions and 1 deletions
--- a/docs/networking/gateway.md
+++ b/docs/networking/gateway.md
@ -129,6 +129,8 @@ nginx is the one external entry point. The per-agent web-port range (`8100`–`8

 The dashboard port (`services.hyperhive.c0re.dashboardPort`, default 7000) binds `127.0.0.1` only, so remote dashboard access goes through the gateway. The dashboard can stop, rebuild and destroy agents; don't expose it without a reverse proxy in front.

+nginx serves its `stub_status` counters on `127.0.0.1:<statusPort>` (`services.hyperhive.gateway.statusPort`, default 8085) at `/stub_status`, from a server of its own outside the vhost tree. The hive's collector reads them with its `nginx` receiver, and the `gateway` Grafana dashboard plots them. The receiver only dials `http://`, which is why this is loopback TCP rather than a unix socket.
+
 ## `HIVE_FORGE_URL`: agents reach the forge via the gateway by domain

 Agents poll `HIVE_FORGE_URL` for Forgejo notifications and run every `hive-forge` call against it. `nix/host-modules/hive-c0re/environment.nix` sets it to `http://<swarm.forge.domain>` (default `forge.<swarm-domain>` — a swarm runs one forge). Agents run in a private netns and can't reach the host's loopback, so they resolve that name through the bridge dnsmasq to the bridge IP and reach nginx on port 80 (the bridge firewall opens 80 and 443) — the same path an operator's browser takes.
--- a/docs/swarm/services.md
+++ b/docs/swarm/services.md
@ -176,6 +176,7 @@ kept.
 - `claude-usage`: Claude use by model, effort, token type and source.
 - `controller`: the swarm controller's job graph by state, and the push and commit rates it sees on the forge.
 - `forge`: what the forge holds, and the forge process's CPU, memory and descriptors.
+- `gateway`: the gateway nginx on each hive's host: requests per second, connections by state, and accepted against handled connections.
 - `collector`: what the OTEL collectors accept, refuse, export and queue, with queue depth against capacity.
 - `logstore`, `metricstore`: health of the log store and the metrics store.

--- a/nix/host-modules/hive-gateway/default.nix
+++ b/nix/host-modules/hive-gateway/default.nix
@ -404,6 +404,24 @@ in
      # `error` threshold and this changes where lines are filed, not
      # which lines exist.
      logError = "syslog:server=unix:/dev/log,nohostname,tag=nginx";
+
+      # Read by the hive collector's `nginx` receiver (../otel.nix). Its own
+      # loopback server, outside the vhost tree, so it never answers on the
+      # gateway's public listeners.
+      appendHttpConfig = ''
+        server {
+          listen 127.0.0.1:${toString cfg.statusPort};
+          access_log off;
+          location = /stub_status {
+            stub_status;
+            allow 127.0.0.1;
+            deny all;
+          }
+          location / {
+            return 404;
+          }
+        }
+      '';
    };

    # ⚠️ NO `SupplementaryGroups = [ "hive-core" ]` on nginx, and its
--- a/nix/host-modules/hive-gateway/options.nix
+++ b/nix/host-modules/hive-gateway/options.nix
@ -254,6 +254,21 @@ in
      '';
    };

+    statusPort = lib.mkOption {
+      type = lib.types.port;
+      default = 8085;
+      description = ''
+        Loopback TCP port nginx serves its `stub_status` counters on, at
+        `http://127.0.0.1:''${statusPort}/stub_status`. The hive's collector
+        ({option}`services.hyperhive.otel.enable`) reads it with its `nginx`
+        receiver.
+
+        TCP rather than a unix socket because that receiver can only dial
+        `http://`. Anything on the host can read these counters; they are
+        connection and request totals, nothing per-client.
+      '';
+    };
+
    tls = {
      certDir = lib.mkOption {
        type = lib.types.nullOr lib.types.path;
--- a/nix/host-modules/otel.nix
+++ b/nix/host-modules/otel.nix
@ -334,6 +334,7 @@ in
      hiveName = config.services.hyperhive.hiveName;
      listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
      swarmName = "otlphttp/swarm";
+      gatewayCfg = config.services.hyperhive.gateway;
      authName = "oauth2client/swarm";

      # A hive always authenticates to the swarm's collector as itself, so
@ -555,6 +556,15 @@ in
              job_name = job;
              static_configs = [ { targets = [ target ]; } ];
            }) otel.scrapeTargets;
+          }
+          # This host's gateway nginx, from the `stub_status` server
+          # ./hive-gateway/default.nix adds. The receiver dials `http://`
+          # only, which is why that server is on loopback TCP.
+          // lib.optionalAttrs gatewayCfg.enable {
+            nginx = {
+              endpoint = "http://127.0.0.1:${toString gatewayCfg.statusPort}/stub_status";
+              collection_interval = "60s";
+            };
          };

          # One destination, and it is the swarm's collector. This tier
@ -662,7 +672,8 @@ in
              "otlp"
              "hostmetrics"
            ]
-            ++ lib.optional (otel.scrapeTargets != { }) "prometheus";
+            ++ lib.optional (otel.scrapeTargets != { }) "prometheus"
+            ++ lib.optional gatewayCfg.enable "nginx";
            # Applies to EVERY receiver above, not just `hostmetrics` — a
            # processor sits in the pipeline, not on a receiver. That is
            # what we want here: samples an agent pushed over `otlp` and
--- a/nix/host-modules/swarm-grafana.nix
+++ b/nix/host-modules/swarm-grafana.nix
@ -201,6 +201,7 @@ let
        "collector.json"
        "controller.json"
        "forge.json"
+        "gateway.json"
        "logstore.json"
        "metricstore.json"
        "queue.json"
--- a/nix/host-modules/swarm-grafana/dashboards/gateway.json
+++ b/nix/host-modules/swarm-grafana/dashboards/gateway.json
@ -0,0 +1,391 @@
+{
+  "title": "hyperhive · gateway (nginx)",
+  "uid": "hyperhive-gateway",
+  "description": "The gateway nginx on each hive's host: request rate and connections.",
+  "editable": true,
+  "refresh": "1m",
+  "schemaVersion": 39,
+  "tags": ["hyperhive", "gateway", "nginx"],
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timezone": "utc",
+  "version": 1,
+  "templating": {
+    "list": [
+      {
+        "name": "hive",
+        "label": "Hive",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "@datasourceUid@"
+        },
+        "query": "label_values({__name__=\"nginx.requests\"}, hive)",
+        "refresh": 2,
+        "includeAll": true,
+        "allValue": ".*",
+        "multi": true,
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "sort": 1
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Requests/s",
+      "description": "Requests per second across the selected gateways.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum(rate({__name__=\"nginx.requests\", hive=~\"$hive\"}[$__rate_interval])) or vector(0)",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "decimals": 1,
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "text",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Active connections",
+      "description": "Open client connections now, across the selected gateways.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 8,
+        "x": 8,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum({__name__=\"nginx.connections_current\", hive=~\"$hive\", state=\"active\"}) or vector(0)",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "text",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Dropped connections",
+      "description": "Connections accepted but not handled over the time range.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 8,
+        "x": 16,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "(sum(increase({__name__=\"nginx.connections_accepted\", hive=~\"$hive\"}[$__range])) - sum(increase({__name__=\"nginx.connections_handled\", hive=~\"$hive\"}[$__range]))) or vector(0)",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "text",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 4,
+      "type": "timeseries",
+      "title": "Requests per second",
+      "description": "Requests per second, per hive.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (hive) (rate({__name__=\"nginx.requests\", hive=~\"$hive\"}[$__rate_interval]))",
+          "legendFormat": "{{hive}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "Connections by state",
+      "description": "Client connections per state; active is reading + writing + waiting.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (hive, state) ({__name__=\"nginx.connections_current\", hive=~\"$hive\"})",
+          "legendFormat": "{{hive}} {{state}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 6,
+      "type": "timeseries",
+      "title": "Accepted vs handled",
+      "description": "Connections per second. A gap between the two lines is connections dropped.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 12
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (hive) (rate({__name__=\"nginx.connections_accepted\", hive=~\"$hive\"}[$__rate_interval]))",
+          "legendFormat": "{{hive}} accepted"
+        },
+        {
+          "refId": "B",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (hive) (rate({__name__=\"nginx.connections_handled\", hive=~\"$hive\"}[$__rate_interval]))",
+          "legendFormat": "{{hive}} handled"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "cps",
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    }
+  ]
+}
--- a/nix/module-eval/hive-otel.nix
+++ b/nix/module-eval/hive-otel.nix
@ -141,6 +141,26 @@ let
      name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs";
      ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers;
    }
+    {
+      # The receiver, the pipeline entry and the nginx server it reads have to
+      # agree on one address, and none of them may exist without a gateway on
+      # the host. `hiveOtelLeaf` is the control: it runs no gateway, and the
+      # clause asserting that keeps the control from going vacuous.
+      name = "the hive collector reads the gateway's stub_status, and only where a gateway runs";
+      ok =
+        let
+          nginxConf = hiveOtel.services.nginx.appendHttpConfig;
+          leafSettings = hiveOtelLeaf.services.opentelemetry-collector.settings;
+        in
+        hiveOtel.services.hyperhive.gateway.enable
+        && hiveOtelSettings.receivers.nginx.endpoint == "http://127.0.0.1:8085/stub_status"
+        && builtins.elem "nginx" hiveOtelPipelines.metrics.receivers
+        && lib.hasInfix "listen 127.0.0.1:8085;" nginxConf
+        && lib.hasInfix "location = /stub_status {" nginxConf
+        && !hiveOtelLeaf.services.hyperhive.gateway.enable
+        && !(leafSettings.receivers ? nginx)
+        && !(builtins.elem "nginx" leafSettings.service.pipelines.metrics.receivers);
+    }
    {
      # `metrics.address` is the spelling that looks right and is rejected by
      # this collector version. The first clause is the control: without it a