From f70bbda7d7e3e10f651990b84bfc30f94f5ba1e8 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Fri, 16 Oct 2026 02:40:26 +0200
Subject: [PATCH] gateway: send nginx stub_status to VictoriaMetrics, add a
 gateway dashboard

The gateway nginx serves stub_status on a loopback server of its own,
127.0.0.1:8085 (services.hyperhive.gateway.statusPort). The hive-tier
collector reads it with the contrib nginx receiver, which the pinned
otelcol-contrib 0.151.0 ships, so no exporter daemon is added. The
receiver dials http:// only (a unix: or http+unix: endpoint fails with
"unsupported protocol scheme"), so the listener is loopback TCP with
allow 127.0.0.1 / deny all rather than a unix socket.

The new gateway dashboard plots nginx.requests, nginx.connections_current
by state, and nginx.connections_accepted against handled, per hive.

Part of #5342.
---
 docs/networking/gateway.md                    |   2 +
 docs/swarm/services.md                        |   1 +
 nix/host-modules/hive-gateway/default.nix     |  18 +
 nix/host-modules/hive-gateway/options.nix     |  15 +
 nix/host-modules/otel.nix                     |  13 +-
 nix/host-modules/swarm-grafana.nix            |   1 +
 .../swarm-grafana/dashboards/gateway.json     | 391 ++++++++++++++++++
 nix/module-eval/hive-otel.nix                 |  20 +
 8 files changed, 460 insertions(+), 1 deletion(-)
 create mode 100644 nix/host-modules/swarm-grafana/dashboards/gateway.json

diff --git a/docs/networking/gateway.md b/docs/networking/gateway.md
index 7851968b..60ff6e72 100644
--- a/docs/networking/gateway.md
+++ b/docs/networking/gateway.md
@@ -129,6 +129,8 @@ nginx is the one external entry point. The per-agent web-port range (`8100`–`8
 
 The dashboard port (`services.hyperhive.c0re.dashboardPort`, default 7000) binds `127.0.0.1` only, so remote dashboard access goes through the gateway. The dashboard can stop, rebuild and destroy agents; don't expose it without a reverse proxy in front.
 
+nginx serves its `stub_status` counters on `127.0.0.1:<statusPort>` (`services.hyperhive.gateway.statusPort`, default 8085) at `/stub_status`, from a server of its own outside the vhost tree. The hive's collector reads them with its `nginx` receiver, and the `gateway` Grafana dashboard plots them. The receiver only dials `http://`, which is why this is loopback TCP rather than a unix socket.
+
 ## `HIVE_FORGE_URL`: agents reach the forge via the gateway by domain
 
 Agents poll `HIVE_FORGE_URL` for Forgejo notifications and run every `hive-forge` call against it. `nix/host-modules/hive-c0re/environment.nix` sets it to `http://<swarm.forge.domain>` (default `forge.<swarm-domain>` — a swarm runs one forge). Agents run in a private netns and can't reach the host's loopback, so they resolve that name through the bridge dnsmasq to the bridge IP and reach nginx on port 80 (the bridge firewall opens 80 and 443) — the same path an operator's browser takes.
diff --git a/docs/swarm/services.md b/docs/swarm/services.md
index 3ecf2421..1d353e17 100644
--- a/docs/swarm/services.md
+++ b/docs/swarm/services.md
@@ -176,6 +176,7 @@ kept.
 - `claude-usage`: Claude use by model, effort, token type and source.
 - `controller`: the swarm controller's job graph by state, and the push and commit rates it sees on the forge.
 - `forge`: what the forge holds, and the forge process's CPU, memory and descriptors.
+- `gateway`: the gateway nginx on each hive's host: requests per second, connections by state, and accepted against handled connections.
 - `collector`: what the OTEL collectors accept, refuse, export and queue, with queue depth against capacity.
 - `logstore`, `metricstore`: health of the log store and the metrics store.
 
diff --git a/nix/host-modules/hive-gateway/default.nix b/nix/host-modules/hive-gateway/default.nix
index 1a3321e2..19166321 100644
--- a/nix/host-modules/hive-gateway/default.nix
+++ b/nix/host-modules/hive-gateway/default.nix
@@ -404,6 +404,24 @@ in
       # `error` threshold and this changes where lines are filed, not
       # which lines exist.
       logError = "syslog:server=unix:/dev/log,nohostname,tag=nginx";
+
+      # Read by the hive collector's `nginx` receiver (../otel.nix). Its own
+      # loopback server, outside the vhost tree, so it never answers on the
+      # gateway's public listeners.
+      appendHttpConfig = ''
+        server {
+          listen 127.0.0.1:${toString cfg.statusPort};
+          access_log off;
+          location = /stub_status {
+            stub_status;
+            allow 127.0.0.1;
+            deny all;
+          }
+          location / {
+            return 404;
+          }
+        }
+      '';
     };
 
     # ⚠️ NO `SupplementaryGroups = [ "hive-core" ]` on nginx, and its
diff --git a/nix/host-modules/hive-gateway/options.nix b/nix/host-modules/hive-gateway/options.nix
index c67c2908..2391d13e 100644
--- a/nix/host-modules/hive-gateway/options.nix
+++ b/nix/host-modules/hive-gateway/options.nix
@@ -254,6 +254,21 @@ in
       '';
     };
 
+    statusPort = lib.mkOption {
+      type = lib.types.port;
+      default = 8085;
+      description = ''
+        Loopback TCP port nginx serves its `stub_status` counters on, at
+        `http://127.0.0.1:''${statusPort}/stub_status`. The hive's collector
+        ({option}`services.hyperhive.otel.enable`) reads it with its `nginx`
+        receiver.
+
+        TCP rather than a unix socket because that receiver can only dial
+        `http://`. Anything on the host can read these counters; they are
+        connection and request totals, nothing per-client.
+      '';
+    };
+
     tls = {
       certDir = lib.mkOption {
         type = lib.types.nullOr lib.types.path;
diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix
index df609ea0..49699439 100644
--- a/nix/host-modules/otel.nix
+++ b/nix/host-modules/otel.nix
@@ -334,6 +334,7 @@ in
       hiveName = config.services.hyperhive.hiveName;
       listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
       swarmName = "otlphttp/swarm";
+      gatewayCfg = config.services.hyperhive.gateway;
       authName = "oauth2client/swarm";
 
       # A hive always authenticates to the swarm's collector as itself, so
@@ -555,6 +556,15 @@ in
               job_name = job;
               static_configs = [ { targets = [ target ]; } ];
             }) otel.scrapeTargets;
+          }
+          # This host's gateway nginx, from the `stub_status` server
+          # ./hive-gateway/default.nix adds. The receiver dials `http://`
+          # only, which is why that server is on loopback TCP.
+          // lib.optionalAttrs gatewayCfg.enable {
+            nginx = {
+              endpoint = "http://127.0.0.1:${toString gatewayCfg.statusPort}/stub_status";
+              collection_interval = "60s";
+            };
           };
 
           # One destination, and it is the swarm's collector. This tier
@@ -662,7 +672,8 @@ in
               "otlp"
               "hostmetrics"
             ]
-            ++ lib.optional (otel.scrapeTargets != { }) "prometheus";
+            ++ lib.optional (otel.scrapeTargets != { }) "prometheus"
+            ++ lib.optional gatewayCfg.enable "nginx";
             # Applies to EVERY receiver above, not just `hostmetrics` — a
             # processor sits in the pipeline, not on a receiver. That is
             # what we want here: samples an agent pushed over `otlp` and
diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix
index 73c35e24..99a6be73 100644
--- a/nix/host-modules/swarm-grafana.nix
+++ b/nix/host-modules/swarm-grafana.nix
@@ -201,6 +201,7 @@ let
         "collector.json"
         "controller.json"
         "forge.json"
+        "gateway.json"
         "logstore.json"
         "metricstore.json"
         "queue.json"
diff --git a/nix/host-modules/swarm-grafana/dashboards/gateway.json b/nix/host-modules/swarm-grafana/dashboards/gateway.json
new file mode 100644
index 00000000..1a9ee6a5
--- /dev/null
+++ b/nix/host-modules/swarm-grafana/dashboards/gateway.json
@@ -0,0 +1,391 @@
+{
+  "title": "hyperhive · gateway (nginx)",
+  "uid": "hyperhive-gateway",
+  "description": "The gateway nginx on each hive's host: request rate and connections.",
+  "editable": true,
+  "refresh": "1m",
+  "schemaVersion": 39,
+  "tags": ["hyperhive", "gateway", "nginx"],
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timezone": "utc",
+  "version": 1,
+  "templating": {
+    "list": [
+      {
+        "name": "hive",
+        "label": "Hive",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "@datasourceUid@"
+        },
+        "query": "label_values({__name__=\"nginx.requests\"}, hive)",
+        "refresh": 2,
+        "includeAll": true,
+        "allValue": ".*",
+        "multi": true,
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "sort": 1
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Requests/s",
+      "description": "Requests per second across the selected gateways.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum(rate({__name__=\"nginx.requests\", hive=~\"$hive\"}[$__rate_interval])) or vector(0)",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "decimals": 1,
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "text",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Active connections",
+      "description": "Open client connections now, across the selected gateways.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 8,
+        "x": 8,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum({__name__=\"nginx.connections_current\", hive=~\"$hive\", state=\"active\"}) or vector(0)",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "text",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Dropped connections",
+      "description": "Connections accepted but not handled over the time range.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 8,
+        "x": 16,
+        "y": 0
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "(sum(increase({__name__=\"nginx.connections_accepted\", hive=~\"$hive\"}[$__range])) - sum(increase({__name__=\"nginx.connections_handled\", hive=~\"$hive\"}[$__range]))) or vector(0)",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "text",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "graphMode": "none",
+        "colorMode": "value",
+        "textMode": "value",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      }
+    },
+    {
+      "id": 4,
+      "type": "timeseries",
+      "title": "Requests per second",
+      "description": "Requests per second, per hive.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (hive) (rate({__name__=\"nginx.requests\", hive=~\"$hive\"}[$__rate_interval]))",
+          "legendFormat": "{{hive}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "Connections by state",
+      "description": "Client connections per state; active is reading + writing + waiting.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (hive, state) ({__name__=\"nginx.connections_current\", hive=~\"$hive\"})",
+          "legendFormat": "{{hive}} {{state}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 6,
+      "type": "timeseries",
+      "title": "Accepted vs handled",
+      "description": "Connections per second. A gap between the two lines is connections dropped.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "@datasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 12
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (hive) (rate({__name__=\"nginx.connections_accepted\", hive=~\"$hive\"}[$__rate_interval]))",
+          "legendFormat": "{{hive}} accepted"
+        },
+        {
+          "refId": "B",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "@datasourceUid@"
+          },
+          "expr": "sum by (hive) (rate({__name__=\"nginx.connections_handled\", hive=~\"$hive\"}[$__rate_interval]))",
+          "legendFormat": "{{hive}} handled"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "cps",
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    }
+  ]
+}
diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix
index 1f291b4a..3d2defcc 100644
--- a/nix/module-eval/hive-otel.nix
+++ b/nix/module-eval/hive-otel.nix
@@ -141,6 +141,26 @@ let
       name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs";
       ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers;
     }
+    {
+      # The receiver, the pipeline entry and the nginx server it reads have to
+      # agree on one address, and none of them may exist without a gateway on
+      # the host. `hiveOtelLeaf` is the control: it runs no gateway, and the
+      # clause asserting that keeps the control from going vacuous.
+      name = "the hive collector reads the gateway's stub_status, and only where a gateway runs";
+      ok =
+        let
+          nginxConf = hiveOtel.services.nginx.appendHttpConfig;
+          leafSettings = hiveOtelLeaf.services.opentelemetry-collector.settings;
+        in
+        hiveOtel.services.hyperhive.gateway.enable
+        && hiveOtelSettings.receivers.nginx.endpoint == "http://127.0.0.1:8085/stub_status"
+        && builtins.elem "nginx" hiveOtelPipelines.metrics.receivers
+        && lib.hasInfix "listen 127.0.0.1:8085;" nginxConf
+        && lib.hasInfix "location = /stub_status {" nginxConf
+        && !hiveOtelLeaf.services.hyperhive.gateway.enable
+        && !(leafSettings.receivers ? nginx)
+        && !(builtins.elem "nginx" leafSettings.service.pipelines.metrics.receivers);
+    }
     {
       # `metrics.address` is the spelling that looks right and is rejected by
       # this collector version. The first clause is the control: without it a