From 228842b9d822fdd24375af23c90b76f70c736462 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Mon, 24 Aug 2026 17:06:07 +0200
Subject: [PATCH] feat(swarm-grafana): scrape Grafana's own metrics over a
 loopback listener

Grafana was the one swarm service that exports Prometheus metrics and had no
scrape target, so the swarm's metrics UI was invisible to the metrics it
displays.

It serves on a unix socket and claims no TCP port, which is deliberate and
stays that way; a prometheus target has to be a host:port, so nginx re-serves
the one endpoint a scraper needs on loopback. An exact-match location, not a
prefix: widening it would re-serve the whole UI without authorization.

The metrics section is now stated explicitly rather than inherited, because a
scrape target depends on it and a changed upstream default would take the
endpoint away while nginx kept answering.
---
 nix/host-modules/swarm-grafana.nix | 62 ++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix
index 4b5258a9..93c8ff93 100644
--- a/nix/host-modules/swarm-grafana.nix
+++ b/nix/host-modules/swarm-grafana.nix
@@ -167,6 +167,33 @@ in
       '';
     };
 
+    metricsPort = lib.mkOption {
+      type = lib.types.port;
+      default = 9095;
+      description = ''
+        Loopback port on which the gateway's nginx re-serves Grafana's
+        `/metrics`, and nothing else, so the swarm's collector can scrape it.
+
+        ⚠️ **This is nginx's port, not Grafana's.** Grafana still claims none —
+        see {option}`services.hyperhive.swarm.grafana.socketDir` for why that
+        matters. A prometheus scrape target is a `host:port`, and it cannot
+        address a unix socket; rather than undo the socket decision, the one
+        endpoint a scraper needs gets a listener of its own.
+
+        Bound to loopback and unauthenticated, which is the same posture every
+        other entry in
+        {option}`services.hyperhive.swarm.otel.scrapeTargets` has: those
+        targets are trusted by *proximity* rather than by credential.
+        Deliberately **not** the published `grafana.<domain>` vhost, which
+        would put an authorization decision in front of a scrape.
+
+        The number itself is arbitrary and free today;
+        `state/eval-port-collisions.sh` is what keeps it that way, since a
+        second claim on a port in this shared namespace produces no bind error
+        and nothing in any log.
+      '';
+    };
+
     datasourceUrl = lib.mkOption {
       type = lib.types.str;
       default = "http://127.0.0.1:${toString vmCfg.port}";
@@ -350,6 +377,34 @@ in
       };
     };
 
+    # The scrape listener: Grafana's `/metrics` and nothing else, on loopback.
+    #
+    # No `serverName` and no TLS on purpose — this listener exists for exactly
+    # one client on the same host, so there is no name to match and nothing to
+    # verify. It is the only vhost here that is not reachable from outside the
+    # machine, and that is what lets it skip the authorization the published
+    # one carries.
+    #
+    # ⚠️ `= /metrics` is an EXACT match, not a prefix: it is the difference
+    # between exposing one endpoint and re-serving the whole of Grafana on an
+    # unauthenticated port. Widening it would be a silent authorization bypass,
+    # since nothing else about this block would have to change.
+    services.nginx.virtualHosts."grafana-metrics" = {
+      listen = [
+        {
+          addr = "127.0.0.1";
+          port = cfg.metricsPort;
+        }
+      ];
+      locations."= /metrics".proxyPass = "http://unix:${socketPath}:/metrics";
+    };
+
+    # Declared here rather than in the collector's module, per that option's
+    # rule: an entry exists where the service that named it runs.
+    services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.swarm.otel.enable {
+      grafana = "127.0.0.1:${toString cfg.metricsPort}";
+    };
+
     # Order the container after the host CA service so the bind source below
     # exists before nspawn sets the mount up.
     systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
@@ -602,6 +657,13 @@ in
                 check_for_updates = false;
               };
 
+              # Stated rather than inherited. This is upstream's default, but
+              # a scrape target now depends on it, and a default that changes
+              # silently takes the target with it — the endpoint disappears,
+              # nginx keeps answering, and the scrape fails with a 404 far
+              # from anything that mentions Grafana.
+              metrics.enabled = true;
+
               users.auto_assign_org_role = cfg.oidc.role;
 
               # ⚠️ `$__file{}`, and required rather than optional: nixpkgs