diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 4b5258a9..93c8ff93 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -167,6 +167,33 @@ in ''; }; + metricsPort = lib.mkOption { + type = lib.types.port; + default = 9095; + description = '' + Loopback port on which the gateway's nginx re-serves Grafana's + `/metrics`, and nothing else, so the swarm's collector can scrape it. + + ⚠️ **This is nginx's port, not Grafana's.** Grafana still claims none — + see {option}`services.hyperhive.swarm.grafana.socketDir` for why that + matters. A prometheus scrape target is a `host:port`, and it cannot + address a unix socket; rather than undo the socket decision, the one + endpoint a scraper needs gets a listener of its own. + + Bound to loopback and unauthenticated, which is the same posture every + other entry in + {option}`services.hyperhive.swarm.otel.scrapeTargets` has: those + targets are trusted by *proximity* rather than by credential. + Deliberately **not** the published `grafana.` vhost, which + would put an authorization decision in front of a scrape. + + The number itself is arbitrary and free today; + `state/eval-port-collisions.sh` is what keeps it that way, since a + second claim on a port in this shared namespace produces no bind error + and nothing in any log. + ''; + }; + datasourceUrl = lib.mkOption { type = lib.types.str; default = "http://127.0.0.1:${toString vmCfg.port}"; @@ -350,6 +377,34 @@ in }; }; + # The scrape listener: Grafana's `/metrics` and nothing else, on loopback. + # + # No `serverName` and no TLS on purpose — this listener exists for exactly + # one client on the same host, so there is no name to match and nothing to + # verify. It is the only vhost here that is not reachable from outside the + # machine, and that is what lets it skip the authorization the published + # one carries. + # + # ⚠️ `= /metrics` is an EXACT match, not a prefix: it is the difference + # between exposing one endpoint and re-serving the whole of Grafana on an + # unauthenticated port. Widening it would be a silent authorization bypass, + # since nothing else about this block would have to change. + services.nginx.virtualHosts."grafana-metrics" = { + listen = [ + { + addr = "127.0.0.1"; + port = cfg.metricsPort; + } + ]; + locations."= /metrics".proxyPass = "http://unix:${socketPath}:/metrics"; + }; + + # Declared here rather than in the collector's module, per that option's + # rule: an entry exists where the service that named it runs. + services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.swarm.otel.enable { + grafana = "127.0.0.1:${toString cfg.metricsPort}"; + }; + # Order the container after the host CA service so the bind source below # exists before nspawn sets the mount up. systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; @@ -602,6 +657,13 @@ in check_for_updates = false; }; + # Stated rather than inherited. This is upstream's default, but + # a scrape target now depends on it, and a default that changes + # silently takes the target with it — the endpoint disappears, + # nginx keeps answering, and the scrape fails with a 404 far + # from anything that mentions Grafana. + metrics.enabled = true; + users.auto_assign_org_role = cfg.oidc.role; # ⚠️ `$__file{}`, and required rather than optional: nixpkgs