docs: swarm services publish VictoriaMetrics' /metrics to the collector

No access rule covered the push route on `metrics.<swarm>`, so a request
there fell to `default_policy = one_factor`: any logged-in session cookie
for the swarm could write samples into VictoriaMetrics.

The name now gets two rules with no `resources`: the collector's client
at `one_factor`, then `deny`. Only the push route and `/metrics` on that
vhost send an authelia subrequest (the read API, Grafana's datasource
included, is served with no check), and the collector is the one caller
of both, so a domain-wide pair covers them with no path or query regex
to slip past. It replaces the per-path `/metrics` pair the published
target would otherwise derive for that name. Both rules sit ahead of the
apex and store-UI rules, and authelia matches `domain` exactly, so they
shadow no other name.

Authelia 4.39.20, read from source: a rule with no `domain` wildcard
matches with `strings.EqualFold` (access_control_domain.go), an empty
`resources` or `subject` list matches everything (access_control_rule.go),
and the first matching rule wins (authorizer.go).

Fixes #5218

docs/swarm/services.md

@@ -172,10 +172,18 @@ Neither container is reachable except through the gateway: both bind
 loopback, and VictoriaMetrics authenticates nothing itself. The
 `metrics.<swarm-domain>` vhost forwards only the store's read API (the
 Prometheus query, series, label and metadata routes, plus the vmui web UI)
-without a credential. It forwards the collector's OTLP push only with an
-authelia token, and refuses every other path, the store's write and admin
+without a credential. It forwards the collector's OTLP push, and the
+collector's scrape of the store's own `/metrics`, only for the swarm
+collector's authelia client: authelia refuses any other session or client
+there. It refuses every other path, the store's write and admin
 API included, with 403. The bare domain redirects to vmui.
 
+The collector scrapes the store's own metrics the same way, at
+`https://<swarm.victoriametrics.domain>/metrics`, so a collector on a host
+that doesn't run the store still gets them. On this name the gateway admits
+only the collector's client to `/metrics` and the push route; any other
+session gets `deny`.
+
 ### Logs (VictoriaLogs)
 
 Each hive ships its journals to one VictoriaLogs at `logs.<swarm-domain>`,

nix/host-modules/swarm-authelia.nix

@@ -61,7 +61,7 @@ let
   collectorClientId = config.services.hyperhive.swarm.otel.clientId;
   collectorRegistered = lib.any (c: c.id == collectorClientId) cfg.oidc.clients;
 
-  # Every published metrics endpoint: the collector's client, and nobody else.
+  # `target`, for the collector's client and nobody else.
   #
   # The `deny` is the protection, and it has to be a rule of its own. A rule
   # with a `subject` does not match a subject outside it: authelia skips it and
@@ -71,38 +71,52 @@ let
   # between a browser session and this data: `authn_strategies` on the authz
   # endpoint also accepts `CookieSession`, and a cookie carries no audience.
   # Until a collector is registered, the `deny` is the only rule.
-  #
+  collectorOnly =
+    target:
+    lib.optional collectorRegistered (
+      target
+      // {
+        policy = "one_factor";
+        subject = [ "oauth2:client:${collectorClientId}" ];
+      }
+    )
+    ++ [ (target // { policy = "deny"; }) ];
+
+  # The metrics store's name, every path. Only its push route and `/metrics`
+  # consult authelia (its read API is served with no check), and the
+  # collector is the one caller of both, so the rules take no `resources`
+  # and leave no path or query form to fall through to `default_policy`.
+  # ⚠️ An authenticated location added to that vhost for anyone else is
+  # refused here until it gets a rule ahead of the `deny`.
+  metricsStoreDomain = config.services.hyperhive.swarm.victoriametrics.domain;
+  metricsStoreRules = collectorOnly { domain = metricsStoreDomain; };
+
   # One pair per `swarm.otel.publishedScrapeTargets` entry, read off the same
   # URL the collector requests and its token is minted for, so a target
   # cannot be published without the rules that confine it to the collector.
-  metricsRules = lib.concatLists (
-    lib.mapAttrsToList (
-      job: url:
-      let
-        # The split `parsePublished` in ./swarm-otel.nix makes for the scrape
-        # job; the two must agree on what the URL's host and path are.
-        parts = builtins.match "https://([^/]+)(/.*)" url;
-        target = {
-          domain = builtins.elemAt parts 0;
-          # ⚠️ Authelia matches `resources` against the cleaned path PLUS the
-          # query string, while nginx's `= <path>` location ignores the query.
-          # A bare `^<path>$` therefore misses `<path>?x`, which the location
-          # still serves, and that request falls through to `default_policy`.
-          resources = [ "^${lib.escapeRegex (builtins.elemAt parts 1)}([?].*)?$" ];
-        };
-      in
-      if parts == null then
-        throw "swarm.otel.publishedScrapeTargets.${job} = \"${url}\" is not an https URL with a path; authelia's access rule for it cannot be derived."
-      else
-        lib.optional collectorRegistered (
-          target
-          // {
-            policy = "one_factor";
-            subject = [ "oauth2:client:${collectorClientId}" ];
+  # An entry on the metrics store's name is covered by `metricsStoreRules`.
+  metricsRules = lib.filter (r: r.domain != metricsStoreDomain) (
+    lib.concatLists (
+      lib.mapAttrsToList (
+        job: url:
+        let
+          # The split `parsePublished` in ./swarm-otel.nix makes for the scrape
+          # job; the two must agree on what the URL's host and path are.
+          parts = builtins.match "https://([^/]+)(/.*)" url;
+        in
+        if parts == null then
+          throw "swarm.otel.publishedScrapeTargets.${job} = \"${url}\" is not an https URL with a path; authelia's access rule for it cannot be derived."
+        else
+          collectorOnly {
+            domain = builtins.elemAt parts 0;
+            # ⚠️ Authelia matches `resources` against the cleaned path PLUS the
+            # query string, while nginx's `= <path>` location ignores the query.
+            # A bare `^<path>$` therefore misses `<path>?x`, which the location
+            # still serves, and that request falls through to `default_policy`.
+            resources = [ "^${lib.escapeRegex (builtins.elemAt parts 1)}([?].*)?$" ];
           }
-        )
-        ++ [ (target // { policy = "deny"; }) ]
-    ) config.services.hyperhive.swarm.otel.publishedScrapeTargets
+      ) config.services.hyperhive.swarm.otel.publishedScrapeTargets
+    )
   );
 
   # Upstream's `services.authelia.instances.<name>` derives the unit,
@@ -1138,9 +1152,10 @@ in
                 # shadowed by a broader domain rule added later.
                 rules =
                   # The collector's client where one is registered, then
-                  # `deny` — see `metricsRules` above, which is where the
+                  # `deny` — see `collectorOnly` above, which is where the
                   # reasoning for both halves lives.
                   metricsRules
+                  ++ metricsStoreRules
                   # Also guarded on the domain being set: without it a null
                   # apex would render a rule matching the string "null".
                   ++ lib.optional (deployCfg.swarm-controller.enable && swarmDomain != null) {

nix/host-modules/swarm-otel-service.nix

@@ -358,5 +358,7 @@ in
     forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics";
     # The vhost location is in ./swarm-authelia.nix.
     authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics";
+    # The vhost location is in ./swarm-victoriametrics.nix.
+    victoriametrics = "https://${vmCfg.domain}/metrics";
   };
 }

nix/host-modules/swarm-victoriametrics.nix

@@ -116,22 +116,6 @@ in
     services.hyperhive.gateway.enable = lib.mkDefault true;
     services.hyperhive.gateway.dns.enable = lib.mkDefault true;
 
-    # This store publishes its own health as prometheus metrics on the same
-    # listener it serves queries on, so the swarm's collector can scrape it
-    # with no exporter and no extra port.
-    #
-    # Declared here rather than in the collector's module because that is the
-    # rule the option carries: an entry exists only where the service that
-    # named it runs, which is what keeps scraper and target on one host by
-    # construction instead of by luck.
-    #
-    # The loopback literal is safe for this option and NOT for the write path,
-    # which is the distinction that matters now that the two differ. A scrape
-    # target is only ever read by a collector on this host, so loopback states
-    # a fact. The collector's push goes to the swarm name through the gateway,
-    # because the collector need not be here at all.
-    services.hyperhive.swarm.otel.scrapeTargets.victoriametrics = "127.0.0.1:${toString cfg.port}";
-
     services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
       listen = gatewayCfg.lib.listen;
       extraConfig = gatewayCfg.lib.securityHeaders;
@@ -146,8 +130,8 @@ in
         # index page links mostly to paths refused above, so send it to vmui.
         "= /".return = "302 /vmui/";
 
-        # The swarm collector's ingest route, and the only authenticated thing
-        # on this vhost. `=` so it outranks the `/` prefix above.
+        # The swarm collector's ingest route. `=` so it outranks the `/`
+        # prefix above.
         #
         # ⚠️ No `error_page 401 =302` here, and its absence is the point: a
         # redirect is right for a browser and wrong for a pusher, which would
@@ -160,10 +144,29 @@ in
           '';
         };
 
-        # The subrequest. Same target and header set as the sibling log
-        # store's, which took them from `swarm-ui.nix` — `X-Original-URL` and
-        # `X-Original-Method` are what authelia's auth-request implementation
-        # reads, and the address it compares the token's audience against.
+        # The store's own health, for the swarm collector wherever it runs:
+        # served on the query listener, so no exporter and no extra port. `=`
+        # for the same reason as the push route, and no `error_page` for the
+        # same reason: a scraper would parse a login page as metrics.
+        #
+        # The collector requests exactly this URL, declared as
+        # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
+        # the two must agree, since that string is also the token's audience.
+        "= /metrics" = {
+          proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
+          extraConfig = ''
+            auth_request /__metrics_push_authz;
+          '';
+        };
+
+        # The subrequest for both gated locations above. Authelia admits the
+        # collector's client anywhere on this name and refuses every other
+        # subject (`metricsStoreRules` in ./swarm-authelia.nix), so the scrape
+        # needs no subrequest of its own. Same target and header set as the
+        # sibling log store's, which took them from `swarm-ui.nix` —
+        # `X-Original-URL` and `X-Original-Method` are what authelia's
+        # auth-request implementation reads, and the address it compares the
+        # token's audience against.
         "= /__metrics_push_authz" = {
           proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
           # nixpkgs appends its OWN `Host $host` after extraConfig, which

nix/module-eval/swarm-otel-identity.nix

@@ -106,6 +106,7 @@ let
 
   forgeMetricsUrl = "https://forge.t.local/metrics";
   autheliaMetricsUrl = "https://auth.t.local/metrics";
+  vmMetricsUrl = "https://metrics.t.local/metrics";
 
   # The policy authelia applies to `subject` requesting `https://<domain><path>`,
   # where `path` is what authelia matches `resources` against: the cleaned path
@@ -251,6 +252,93 @@ let
         && authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request"
         && lib.hasInfix "internal;" authz.extraConfig;
     }
+    {
+      # The store's half of the same move, on a collector host that does not
+      # run the store.
+      name = "a collector on a host without the metrics store scrapes the store by name";
+      ok =
+        let
+          s = otelSettings otelRemoteAuthelia;
+          job = lib.findFirst (
+            j: j.job_name == "victoriametrics"
+          ) null s.receivers.prometheus.config.scrape_configs;
+        in
+        !otelRemoteAuthelia.services.hyperhive.deploy.victoriametrics.enable
+        && job != null
+        && job.static_configs == [ { targets = [ "metrics.t.local" ]; } ]
+        && job.metrics_path == "/metrics"
+        && job.oauth2.endpoint_params.audience == vmMetricsUrl;
+    }
+    {
+      # The metrics vhost is default-deny; `/metrics` reaches the store only
+      # through its own exact location, gated like the push route. The `/`
+      # clauses pin that the catch-all still refuses every other path.
+      name = "the metrics store's vhost serves /metrics only behind the collector's auth subrequest";
+      ok =
+        let
+          h = allLocal.services.hyperhive;
+          locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations;
+          m = locs."= /metrics" or null;
+        in
+        m != null
+        && m.proxyPass == "http://127.0.0.1:${toString h.swarm.victoriametrics.port}/metrics"
+        && lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig
+        && !(lib.hasInfix "error_page" m.extraConfig)
+        && lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig
+        && locs."/".return == "403"
+        && locs."/".proxyPass == null;
+    }
+    {
+      # The metrics store's name admits the collector's client on every path
+      # and refuses every other subject. The rules carry no `resources`, so no
+      # path or query spelling misses them and reaches `default_policy`, and
+      # they come before the apex rule. That is only right while the push
+      # route and `/metrics` are the vhost's sole authelia-gated locations,
+      # which the last clause pins.
+      name = "authelia admits only the collector anywhere on the metrics store's name";
+      ok =
+        let
+          h = allLocal.services.hyperhive;
+          collector = "oauth2:client:${h.swarm.otel.clientId}";
+          rules =
+            allLocal.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules;
+          indexed = lib.imap0 (i: r: r // { inherit i; }) rules;
+          storeRules = lib.filter (r: r.domain == "metrics.t.local") indexed;
+          apex = lib.findFirst (r: r.domain == "t.local") null indexed;
+          policy = subject: autheliaPolicy allLocal subject "metrics.t.local";
+          paths = [
+            "/opentelemetry/api/v1/push"
+            "/metrics"
+            "/metrics?x=1"
+            "/api/v1/query"
+          ];
+          locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations;
+        in
+        map (r: removeAttrs r [ "i" ]) storeRules == [
+          {
+            domain = "metrics.t.local";
+            policy = "one_factor";
+            subject = [ collector ];
+          }
+          {
+            domain = "metrics.t.local";
+            policy = "deny";
+          }
+        ]
+        && apex != null
+        && lib.all (r: r.i < apex.i) storeRules
+        && lib.all (p: policy collector p == "one_factor") paths
+        && lib.all (p: policy "user:someone" p == "deny") paths
+        && lib.all (p: policy "group:admins" p == "deny") paths
+        &&
+          lib.hasInfix "auth_request /__metrics_push_authz;"
+            locs."= /opentelemetry/api/v1/push".extraConfig
+        &&
+          lib.attrNames (lib.filterAttrs (_: l: lib.hasInfix "auth_request" (l.extraConfig or "")) locs) == [
+            "= /metrics"
+            "= /opentelemetry/api/v1/push"
+          ];
+    }
     {
       # Every published target admits the collector's client and refuses any
       # other authenticated subject, with or without a query string. Read
@@ -283,8 +371,9 @@ let
         lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [
           autheliaMetricsUrl
           forgeMetricsUrl
+          vmMetricsUrl
         ]
-        && lib.length targets == 4
+        && lib.length targets == 6
         && lib.all (t: policy collector t == "one_factor") targets
         && lib.all (t: policy "user:someone" t == "deny") targets
         && lib.all (t: policy "group:admins" t == "deny") targets;
@@ -367,6 +456,23 @@ let
         && lib.length jobs == 1
         && (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ];
     }
+    {
+      # Same for the metrics store: one job, by its published name.
+      name = "a host running the metrics store and the collector scrapes the store once, by name";
+      ok =
+        let
+          o = allLocal.services.hyperhive.swarm.otel;
+          jobs =
+            lib.filter (j: j.job_name == "victoriametrics")
+              (otelSettings allLocal).receivers.prometheus.config.scrape_configs;
+        in
+        allLocal.services.hyperhive.deploy.victoriametrics.enable
+        && allLocal.services.hyperhive.deploy.swarm-otel.enable
+        && !(o.scrapeTargets ? victoriametrics)
+        && (o.publishedScrapeTargets.victoriametrics or null) == vmMetricsUrl
+        && lib.length jobs == 1
+        && (lib.head jobs).static_configs == [ { targets = [ "metrics.t.local" ]; } ];
+    }
     {
       # The case above reads the options; this one reads the assertion they
       # feed, which is what refuses an operator's `nixos-rebuild`.