swarm-otel: scrape VictoriaMetrics through its published name

The store's own /metrics was scraped over loopback, declared under
deploy.victoriametrics.enable, so a collector on another host never had
a job for it. Declare it as a published target on every host instead,
keyed on swarm.victoriametrics.domain, and serve it on the metrics vhost
behind the existing authz subrequest. Authelia's per-target rules (one
collector rule, then deny) now cover it too.

The loopback entry goes, so a single-host swarm scrapes the store once.

docs/swarm/services.md

@@ -172,8 +172,9 @@ Neither container is reachable except through the gateway: both bind
 loopback, and VictoriaMetrics authenticates nothing itself. The
 `metrics.<swarm-domain>` vhost forwards only the store's read API (the
 Prometheus query, series, label and metadata routes, plus the vmui web UI)
-without a credential. It forwards the collector's OTLP push only with an
+without a credential. It forwards the collector's OTLP push, and the
-authelia token, and refuses every other path, the store's write and admin
+collector's scrape of the store's own `/metrics`, only with an authelia
+token, and refuses every other path, the store's write and admin
 API included, with 403. The bare domain redirects to vmui.
 
 ### Logs (VictoriaLogs)

nix/host-modules/swarm-otel-service.nix

@@ -358,5 +358,7 @@ in
     forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics";
     # The vhost location is in ./swarm-authelia.nix.
     authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics";
+    # The vhost location is in ./swarm-victoriametrics.nix.
+    victoriametrics = "https://${vmCfg.domain}/metrics";
   };
 }

nix/host-modules/swarm-victoriametrics.nix

@@ -116,22 +116,6 @@ in
     services.hyperhive.gateway.enable = lib.mkDefault true;
     services.hyperhive.gateway.dns.enable = lib.mkDefault true;
 
-    # This store publishes its own health as prometheus metrics on the same
-    # listener it serves queries on, so the swarm's collector can scrape it
-    # with no exporter and no extra port.
-    #
-    # Declared here rather than in the collector's module because that is the
-    # rule the option carries: an entry exists only where the service that
-    # named it runs, which is what keeps scraper and target on one host by
-    # construction instead of by luck.
-    #
-    # The loopback literal is safe for this option and NOT for the write path,
-    # which is the distinction that matters now that the two differ. A scrape
-    # target is only ever read by a collector on this host, so loopback states
-    # a fact. The collector's push goes to the swarm name through the gateway,
-    # because the collector need not be here at all.
-    services.hyperhive.swarm.otel.scrapeTargets.victoriametrics = "127.0.0.1:${toString cfg.port}";
-
     services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
       listen = gatewayCfg.lib.listen;
       extraConfig = gatewayCfg.lib.securityHeaders;
@@ -146,8 +130,8 @@ in
         # index page links mostly to paths refused above, so send it to vmui.
         "= /".return = "302 /vmui/";
 
-        # The swarm collector's ingest route, and the only authenticated thing
+        # The swarm collector's ingest route. `=` so it outranks the `/`
-        # on this vhost. `=` so it outranks the `/` prefix above.
+        # prefix above.
         #
         # ⚠️ No `error_page 401 =302` here, and its absence is the point: a
         # redirect is right for a browser and wrong for a pusher, which would
@@ -160,7 +144,24 @@ in
           '';
         };
 
-        # The subrequest. Same target and header set as the sibling log
+        # The store's own health, for the swarm collector wherever it runs:
+        # served on the query listener, so no exporter and no extra port. `=`
+        # for the same reason as the push route, and no `error_page` for the
+        # same reason: a scraper would parse a login page as metrics.
+        #
+        # The collector requests exactly this URL, declared as
+        # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
+        # the two must agree, since that string is also the token's audience.
+        "= /metrics" = {
+          proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
+          extraConfig = ''
+            auth_request /__metrics_push_authz;
+          '';
+        };
+
+        # The subrequest for both gated locations above. It carries the
+        # requested URL, and authelia decides per URL, so the scrape needs no
+        # subrequest of its own. Same target and header set as the sibling log
         # store's, which took them from `swarm-ui.nix` — `X-Original-URL` and
         # `X-Original-Method` are what authelia's auth-request implementation
         # reads, and the address it compares the token's audience against.

nix/module-eval/swarm-otel-identity.nix

@@ -106,6 +106,7 @@ let
 
   forgeMetricsUrl = "https://forge.t.local/metrics";
   autheliaMetricsUrl = "https://auth.t.local/metrics";
+  vmMetricsUrl = "https://metrics.t.local/metrics";
 
   # The policy authelia applies to `subject` requesting `https://<domain><path>`,
   # where `path` is what authelia matches `resources` against: the cleaned path
@@ -251,6 +252,42 @@ let
         && authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request"
         && lib.hasInfix "internal;" authz.extraConfig;
     }
+    {
+      # The store's half of the same move, on a collector host that does not
+      # run the store.
+      name = "a collector on a host without the metrics store scrapes the store by name";
+      ok =
+        let
+          s = otelSettings otelRemoteAuthelia;
+          job = lib.findFirst (
+            j: j.job_name == "victoriametrics"
+          ) null s.receivers.prometheus.config.scrape_configs;
+        in
+        !otelRemoteAuthelia.services.hyperhive.deploy.victoriametrics.enable
+        && job != null
+        && job.static_configs == [ { targets = [ "metrics.t.local" ]; } ]
+        && job.metrics_path == "/metrics"
+        && job.oauth2.endpoint_params.audience == vmMetricsUrl;
+    }
+    {
+      # The metrics vhost is default-deny; `/metrics` reaches the store only
+      # through its own exact location, gated like the push route. The `/`
+      # clauses pin that the catch-all still refuses every other path.
+      name = "the metrics store's vhost serves /metrics only behind the collector's auth subrequest";
+      ok =
+        let
+          h = allLocal.services.hyperhive;
+          locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations;
+          m = locs."= /metrics" or null;
+        in
+        m != null
+        && m.proxyPass == "http://127.0.0.1:${toString h.swarm.victoriametrics.port}/metrics"
+        && lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig
+        && !(lib.hasInfix "error_page" m.extraConfig)
+        && lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig
+        && locs."/".return == "403"
+        && locs."/".proxyPass == null;
+    }
     {
       # Every published target admits the collector's client and refuses any
       # other authenticated subject, with or without a query string. Read
@@ -283,6 +320,7 @@ let
         lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [
           autheliaMetricsUrl
           forgeMetricsUrl
+          vmMetricsUrl
         ]
         && lib.length targets == 4
         && lib.all (t: policy collector t == "one_factor") targets
@@ -367,6 +405,23 @@ let
         && lib.length jobs == 1
         && (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ];
     }
+    {
+      # Same for the metrics store: one job, by its published name.
+      name = "a host running the metrics store and the collector scrapes the store once, by name";
+      ok =
+        let
+          o = allLocal.services.hyperhive.swarm.otel;
+          jobs =
+            lib.filter (j: j.job_name == "victoriametrics")
+              (otelSettings allLocal).receivers.prometheus.config.scrape_configs;
+        in
+        allLocal.services.hyperhive.deploy.victoriametrics.enable
+        && allLocal.services.hyperhive.deploy.swarm-otel.enable
+        && !(o.scrapeTargets ? victoriametrics)
+        && (o.publishedScrapeTargets.victoriametrics or null) == vmMetricsUrl
+        && lib.length jobs == 1
+        && (lib.head jobs).static_configs == [ { targets = [ "metrics.t.local" ]; } ];
+    }
     {
       # The case above reads the options; this one reads the assertion they
       # feed, which is what refuses an operator's `nixos-rebuild`.