docs/networking/gateway.md

@@ -52,6 +52,8 @@ The gateway always terminates TLS. Which certificate it serves depends on what y
 
 `tls.certDir` together with `tls.acme.enable` fails an assertion — pick one.
 
+⚠️ One vhost class has no TLS: a vhost bound to loopback for a local consumer. `grafana-metrics` (`nix/host-modules/swarm-grafana.nix`) is the one in the tree: it listens on `127.0.0.1` only and serves `= /metrics` from grafana's unix socket for this host's collector. The published `grafana.<swarm>` vhost answers `/metrics` with 403. Everything with a routable name follows the table above.
+
 ### ACME / Let's Encrypt (`tls.acme`)
 
 The simplest production path for a public domain:

docs/swarm/services.md

@@ -180,11 +180,6 @@ collector's authelia client: authelia refuses any other session or client
 there. It refuses every other path, the store's write and admin
 API included, with 403. The bare domain redirects to vmui.
 
-The collector scrapes Grafana's own metrics at
-`https://<swarm.grafana.domain>/metrics`, so a collector on a host without
-Grafana still gets them. The gateway admits only the collector's client to
-that path; any other session gets `deny`.
-
 The collector scrapes the store's own metrics the same way, at
 `https://<swarm.victoriametrics.domain>/metrics`, so a collector on a host
 that doesn't run the store still gets them. On this name the gateway admits

nix/host-modules/glue-otel-scrape-targets.nix

@@ -1,20 +1,21 @@
-# Glue: declare the loopback scrape target of nats on the swarm's OTEL
+# Glue: declare the loopback scrape targets of grafana and nats on the swarm's
-# collector, wherever it shares a host with it. Authelia and grafana are
+# OTEL collector, wherever one of them shares a host with it. Authelia is a
-# `publishedScrapeTargets` jobs (./swarm-otel-service.nix); a loopback job of
+# `publishedScrapeTargets` job (./swarm-otel-service.nix); a loopback job of
 # the same name fails ./swarm-otel.nix's job-name assertion.
 #
-# ONE PAIRING — nats ← the collector. Deleting this leaves the collector not
+# ONE FILE, TWO PAIRINGS — each service ← the collector, with the same shape.
-# scraping nats: no error, no metrics.
+# Deleting this leaves the collector scraping none of them: no error, no
+# metrics.
 #
-# The target is gated on the service AND the collector being enabled here. The
+# Each target is gated on the service AND the collector being enabled here. The
 # loopback address is only honest when both run on this host; a target nobody
-# can read asserts a collection that is not happening. Both default from
+# can read asserts a collection that is not happening. Both services default
-# `deploy.allSwarmServices`, but `mkDefault` is an invitation to override, so
+# from `deploy.allSwarmServices`, but `mkDefault` is an invitation to override,
-# co-location is a property of the auto-deployed topology, not the modules.
+# so co-location is a property of the auto-deployed topology, not the modules.
 #
 # ⚠️ Silent when split: separate hosts are separate evaluations with no shared
-# context, so a host running nats and no collector declares nothing and
+# context, so a host running a service and no collector declares nothing and
-# the collector's host never learns of it. No error, no warning — a
+# the collector's host never learns of the service. No error, no warning — a
 # healthy exporter and an empty dashboard. Not guardable from here.
 #
 # ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart, and only
@@ -32,6 +33,7 @@ let
 in
 {
   config.services.hyperhive.swarm.otel.scrapeTargets = {
+    grafana = withCollector "grafana" "127.0.0.1:${toString swarmCfg.grafana.metricsPort}";
     nats = withCollector "nats" "127.0.0.1:${toString swarmCfg.nats.metricsPort}";
   };
 }

nix/host-modules/swarm-grafana-service.nix

@@ -1,7 +1,7 @@
-# The swarm's metrics UI as every hive sees it: the name it is served under
+# The swarm's metrics UI as every hive sees it: the name it is served under,
-# and the OIDC client it is registered as, identical on every host. What the
+# the port its `/metrics` is re-served on, and the OIDC client it is
-# host running it decides, and the container itself, are in
+# registered as, identical on every host. What the host running it decides,
-# ./swarm-grafana.nix.
+# and the container itself, are in ./swarm-grafana.nix.
 {
   lib,
   config,
@@ -20,11 +20,12 @@ in
   # `enable` is `services.hyperhive.deploy.grafana.enable` — see
   # ./deploy.nix. Whether this host runs the swarm's Grafana is a
   # deployment decision, and `swarm.*` has to be identical on every host.
-  # Here is what the service IS to every hive: the name it is served under
+  # Here is what the service IS to every hive: the name it is served under,
-  # and the OIDC client it is registered as. What the host running it
+  # the port its `/metrics` is re-served on, and the OIDC client it is
-  # decides — which build it runs, where its datasources point, which plugins
+  # registered as. What the host running it decides — which build it runs,
-  # are in its store path, and the socket directory it shares with nginx — is
+  # where its datasources point, which plugins are in its store path, and
-  # `deploy.grafana`, in ./swarm-grafana.nix.
+  # the socket directory it shares with nginx — is `deploy.grafana`, in
+  # ./swarm-grafana.nix.
   options.services.hyperhive.swarm.grafana = {
     machine = lib.mkOption {
       type = lib.types.str;
@@ -50,13 +51,35 @@ in
         ⚠️ Changing this changes the OAuth redirect URI, which authelia
         matches exactly. Both sides move together because both derive from
         this option; an operator who pins one by hand breaks the login.
+      '';
+    };
 
-        The swarm collector scrapes Grafana's own metrics at
+    metricsPort = lib.mkOption {
-        `https://<this name>/metrics`, from whichever host it runs on.
+      type = lib.types.port;
-        Grafana serves that endpoint to anyone, so the gateway puts it
+      default = 9095;
-        behind authelia, whose rules admit only the collector's OAuth2
+      description = ''
-        client there: see `publishedScrapeTargets` in
+        Loopback port on which the gateway's nginx re-serves Grafana's
-        ./swarm-otel-service.nix.
+        `/metrics`, and nothing else, so the swarm's collector can scrape it.
+
+        ⚠️ **This is nginx's port, not Grafana's.** Grafana still claims none —
+        see {option}`services.hyperhive.deploy.grafana.socketDir` for why that
+        matters. A prometheus scrape target is a `host:port`, and it cannot
+        address a unix socket; rather than undo the socket decision, the one
+        endpoint a scraper needs gets a listener of its own.
+
+        Bound to loopback and unauthenticated, which is the same posture every
+        other entry in
+        {option}`services.hyperhive.swarm.otel.scrapeTargets` has: those
+        targets are trusted by *proximity* rather than by credential.
+        Deliberately **not** the published `grafana.<domain>` vhost, which
+        would put an authorization decision in front of a scrape: that vhost
+        answers `/metrics` with 403, because Grafana serves it without
+        authentication.
+
+        The number itself is arbitrary and free today;
+        `state/eval-port-collisions.sh` is what keeps it that way, since a
+        second claim on a port in this shared namespace produces no bind error
+        and nothing in any log.
       '';
     };
 

nix/host-modules/swarm-grafana.nix

@@ -20,7 +20,6 @@ let
   networkCfg = config.services.hyperhive.network;
   hyperhiveCfg = config.services.hyperhive;
   gatewayCfg = hyperhiveCfg.gateway;
-  autheliaDomain = hyperhiveCfg.swarm.authelia.domain;
   baoCfg = hyperhiveCfg.swarm.bao;
   baoDeploy = deployCfg.bao;
 
@@ -460,43 +459,12 @@ in
       listen = gatewayCfg.lib.listen;
       extraConfig = gatewayCfg.lib.securityHeaders;
       # Grafana serves its Prometheus exposition to anyone, signed in or not
-      # (`metrics.enabled` below, no `basic_auth_*`), so this location's auth
+      # (`metrics.enabled` below, no `basic_auth_*`), so the published name
-      # is the only gate on it. `=` is enough: nginx matches the decoded,
+      # refuses it; the scrape goes through the loopback `grafana-metrics`
-      # normalised path, so `//metrics`, `/./metrics`, `/%6detrics` and
+      # vhost instead. `=` is enough: nginx matches the decoded, normalised
-      # `/metrics?x` all land here, and Grafana itself answers `/metrics/` with
+      # path, so `//metrics`, `/./metrics`, `/%6detrics` and `/metrics?x` all
-      # a login redirect.
+      # land here, and Grafana itself answers `/metrics/` with a login redirect.
-      #
+      locations."= /metrics".return = "403";
-      # The collector requests exactly this URL, declared as
-      # `publishedScrapeTargets.grafana` in ./swarm-otel-service.nix; the two
-      # must agree, since that string is also the token's audience. Authelia's
-      # rules for it admit the collector's client and deny every other subject
-      # (`metricsRules` in ./swarm-authelia.nix).
-      locations."= /metrics" = {
-        proxyPass = "http://unix:${socketPath}:/metrics";
-        extraConfig = ''
-          auth_request /__grafana_metrics_authz;
-        '';
-      };
-      # The subrequest, same shape as the forge's `= /__forge_metrics_authz`
-      # (./hive-forge/default.nix): no `error_page 401 =302`, so a scraper
-      # gets the 401 rather than a login page parsed as metrics.
-      locations."= /__grafana_metrics_authz" = {
-        proxyPass = "https://${autheliaDomain}/api/authz/auth-request";
-        # nixpkgs appends its OWN `Host $host` after extraConfig, which would
-        # override verifiedProxyTo's — see hive-gateway/vhost-lib.nix.
-        recommendedProxySettings = false;
-        extraConfig = ''
-          internal;
-          ${gatewayCfg.lib.verifiedProxyTo autheliaDomain}
-          proxy_pass_request_body off;
-          proxy_set_header Content-Length "";
-          proxy_set_header X-Original-Method $request_method;
-          proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
-          proxy_set_header X-Forwarded-Proto $scheme;
-          proxy_set_header X-Forwarded-Host $http_host;
-          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        '';
-      };
       locations."/" = {
         proxyPass = "http://unix:${socketPath}:/";
         proxyWebsockets = true;
@@ -512,6 +480,29 @@ in
       };
     };
 
+    # The scrape listener: Grafana's `/metrics` and nothing else, on loopback.
+    #
+    # No `serverName` and no TLS on purpose — this listener exists for exactly
+    # one client on the same host, so there is no name to match and nothing to
+    # verify. It is the only vhost here that is not reachable from outside the
+    # machine, and that is what lets it skip the authorization the published
+    # one carries.
+    #
+    # ⚠️ `= /metrics` is an EXACT match, not a prefix: it is the difference
+    # between exposing one endpoint and re-serving the whole of Grafana on an
+    # unauthenticated port. Widening it would be a silent authorization bypass,
+    # since nothing else about this block would have to change. The published
+    # vhost above refuses `/metrics`, so this listener is the only way to it.
+    services.nginx.virtualHosts."grafana-metrics" = {
+      listen = [
+        {
+          addr = "127.0.0.1";
+          port = cfg.metricsPort;
+        }
+      ];
+      locations."= /metrics".proxyPass = "http://unix:${socketPath}:/metrics";
+    };
+
     # Order the container after the host CA service so the bind source below
     # exists before nspawn sets the mount up.
     systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;

nix/host-modules/swarm-otel-service.nix

@@ -362,7 +362,5 @@ in
     victoriametrics = "https://${vmCfg.domain}/metrics";
     # The vhost location is in ./swarm-victorialogs.nix.
     victorialogs = "https://${hyperhiveCfg.swarm.victorialogs.domain}/metrics";
-    # The vhost location is in ./swarm-grafana.nix.
-    grafana = "https://${hyperhiveCfg.swarm.grafana.domain}/metrics";
   };
 }

nix/module-eval/forge-placement.nix

@@ -138,15 +138,16 @@ let
         && !(refusedOver bare "deploy.forgejo.sso.clientSecretFile");
     }
     {
-      # Authelia and grafana are published targets; ./swarm-otel-identity.nix
+      # Authelia is a published target; ./swarm-otel-identity.nix covers it.
-      # covers them.
+      name = "a host running the collector, authelia, grafana and nats scrapes grafana and nats on loopback, authelia not";
-      name = "a host running the collector, authelia, grafana and nats scrapes nats on loopback, authelia and grafana not";
       ok =
         let
           t = servicesHere.services.hyperhive.swarm.otel.scrapeTargets;
           port = n: toString servicesHere.services.hyperhive.swarm.${n}.metricsPort;
         in
-        t.nats == "127.0.0.1:${port "nats"}" && !(t ? authelia) && !(t ? grafana);
+        t.grafana == "127.0.0.1:${port "grafana"}"
+        && t.nats == "127.0.0.1:${port "nats"}"
+        && !(t ? authelia);
     }
     {
       name = "hosting the swarm's shared services runs the forge";

nix/module-eval/grafana.nix

@@ -109,6 +109,12 @@ let
       && lib.hasInfix option a.message
     ) m.assertions;
   cases = [
+    {
+      name = "grafana declares no scrape target on a host with no collector";
+      ok =
+        !(grafanaOldPath.services.hyperhive.deploy.swarm-otel.enable)
+        && !(grafanaOldPath.services.hyperhive.swarm.otel.scrapeTargets ? grafana);
+    }
     {
       # The socket's directory is its access control, so it must stay dedicated
       # to this one socket. `readOnly` is what makes that inexpressible: a
@@ -118,30 +124,22 @@ let
       ok =
         !(builtins.tryEval grafanaSocketDirSet.services.hyperhive.deploy.grafana.socketDir).success
         &&
-          grafanaOldPath.services.nginx.virtualHosts.${grafanaOldPath.services.hyperhive.swarm.grafana.domain}.locations."= /metrics".proxyPass
+          grafanaOldPath.services.nginx.virtualHosts."grafana-metrics".locations."= /metrics".proxyPass
           == "http://unix:/run/swarm-grafana/grafana.sock:/metrics";
     }
     {
-      # Grafana serves `/metrics` unauthenticated, so the published name gates
+      # Grafana serves `/metrics` unauthenticated, so the published name must
-      # it on the collector's auth subrequest: no `error_page 401 =302` (a
+      # answer it itself; the scrape reads the loopback vhost above instead.
-      # scraper would parse a login page as metrics) and no `/__hive_authelia`.
       # `=` is what catches the normalised variants and still leaves the rest
       # of Grafana proxied.
-      name = "grafana's published vhost serves /metrics only behind the collector's auth subrequest";
+      name = "grafana's published vhost refuses /metrics and still proxies everything else";
       ok =
         let
           g = grafanaOldPath.services.hyperhive.swarm.grafana;
           locs = grafanaOldPath.services.nginx.virtualHosts.${g.domain}.locations;
-          checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig);
-          authz = locs."= /__grafana_metrics_authz";
         in
-        locs ? "= /metrics"
+        locs."= /metrics".return == "403"
-        && locs."= /metrics".return == null
+        && locs."= /metrics".proxyPass == null
-        && checksOf locs."= /metrics" == [ "auth_request /__grafana_metrics_authz;" ]
-        && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
-        && authz.proxyPass == "https://auth.t.local/api/authz/auth-request"
-        && lib.hasInfix "internal;" authz.extraConfig
-        && !(grafanaOldPath.services.nginx.virtualHosts ? "grafana-metrics")
         && lib.hasPrefix "http://unix:" locs."/".proxyPass;
     }
     {

nix/module-eval/swarm-otel-identity.nix

@@ -108,7 +108,6 @@ let
 
   forgeMetricsUrl = "https://forge.t.local/metrics";
   autheliaMetricsUrl = "https://auth.t.local/metrics";
-  grafanaMetricsUrl = "https://grafana.t.local/metrics";
   logsMetricsUrl = "https://logs.t.local/metrics";
   vmMetricsUrl = "https://metrics.t.local/metrics";
 
@@ -375,11 +374,10 @@ let
         lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [
           autheliaMetricsUrl
           forgeMetricsUrl
-          grafanaMetricsUrl
           logsMetricsUrl
           vmMetricsUrl
         ]
-        && lib.length targets == 10
+        && lib.length targets == 8
         && lib.all (t: policy collector t == "one_factor") targets
         && lib.all (t: policy "user:someone" t == "deny") targets
         && lib.all (t: policy "group:admins" t == "deny") targets;
@@ -427,7 +425,7 @@ let
           ];
           matched = row: builtins.match re row.path != null;
         in
-        lib.length metricsRules == 8
+        lib.length metricsRules == 6
         && lib.all (r: r.resources == [ re ]) metricsRules
         && re == "^/metrics([?].*)?$"
         && lib.all (row: matched row == (row.nginx == "= /metrics")) table;
@@ -564,63 +562,6 @@ let
         && lib.length jobs == 1
         && (lib.head jobs).static_configs == [ { targets = [ "logs.t.local" ]; } ];
     }
-    {
-      name = "a collector on a host without grafana scrapes it by name";
-      ok =
-        let
-          s = otelSettings otelRemoteAuthelia;
-          job = lib.findFirst (j: j.job_name == "grafana") null s.receivers.prometheus.config.scrape_configs;
-        in
-        !otelRemoteAuthelia.services.hyperhive.deploy.grafana.enable
-        && job != null
-        && job.static_configs == [ { targets = [ "grafana.t.local" ]; } ]
-        && job.metrics_path == "/metrics"
-        && job.oauth2.endpoint_params.audience == grafanaMetricsUrl;
-    }
-    {
-      # The `deny` that confines grafana's `/metrics` is scoped to that path,
-      # and comes after the collector's rule: every other path on the name
-      # still reaches `default_policy`, as it did with no rule at all.
-      name = "authelia confines grafana's /metrics to the collector and leaves its other paths alone";
-      ok =
-        let
-          h = autheliaOnly.services.hyperhive;
-          collector = "oauth2:client:${h.swarm.otel.clientId}";
-          rules =
-            autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules;
-          grafanaRules = lib.filter (r: r.domain == "grafana.t.local") rules;
-          policy = subject: autheliaPolicy autheliaOnly subject "grafana.t.local";
-        in
-        map (r: r.policy) grafanaRules == [
-          "one_factor"
-          "deny"
-        ]
-        && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) grafanaRules
-        && (lib.head grafanaRules).subject == [ collector ]
-        && policy collector "/metrics" == "one_factor"
-        && policy "user:someone" "/metrics" == "deny"
-        && policy "user:someone" "/metrics?x=1" == "deny"
-        && policy "group:admins" "/metrics" == "deny"
-        && policy "user:someone" "/" == "one_factor"
-        && policy "user:someone" "/login/generic_oauth" == "one_factor";
-    }
-    {
-      # Single host: grafana is scraped once, by its published name.
-      name = "a host running grafana and the collector scrapes it once, by name";
-      ok =
-        let
-          o = allLocal.services.hyperhive.swarm.otel;
-          jobs =
-            lib.filter (j: j.job_name == "grafana")
-              (otelSettings allLocal).receivers.prometheus.config.scrape_configs;
-        in
-        allLocal.services.hyperhive.deploy.grafana.enable
-        && allLocal.services.hyperhive.deploy.swarm-otel.enable
-        && !(o.scrapeTargets ? grafana)
-        && (o.publishedScrapeTargets.grafana or null) == grafanaMetricsUrl
-        && lib.length jobs == 1
-        && (lib.head jobs).static_configs == [ { targets = [ "grafana.t.local" ]; } ];
-    }
     {
       # Read against the roster the fixture declares rather than against
       # names spelled here: an arm naming `h1` passes on a single-hive