swarm-otel: scrape VictoriaLogs through its published name

VictoriaLogs' own metrics were scraped over loopback, declared only on
the host running the store, so a collector on any other host had no job
for it. Its target is now declared on every host as
`https://<swarm.victorialogs.domain>/metrics`, next to forgejo's and
authelia's, and the store's vhost serves `= /metrics` behind a bare
`auth_request` to authelia. The rules derived from
`publishedScrapeTargets` admit the collector's client there and deny every
other subject. The loopback entry is gone, so a single host still scrapes
the store once, now by name.

On that vhost `/metrics` was reachable through the `/` location by any
authelia session. The exact-match location now takes it, so a session
gets a `deny` on `/metrics` and nothing else on the name changes.

Part of #5205

nix/host-modules/swarm-otel-service.nix

@@ -360,5 +360,7 @@ in
     authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics";
     # The vhost location is in ./swarm-victoriametrics.nix.
     victoriametrics = "https://${vmCfg.domain}/metrics";
+    # The vhost location is in ./swarm-victorialogs.nix.
+    victorialogs = "https://${hyperhiveCfg.swarm.victorialogs.domain}/metrics";
   };
 }

nix/host-modules/swarm-victorialogs.nix

@@ -68,8 +68,8 @@ let
   };
   baoCfg = hyperhiveCfg.swarm.bao;
 
-  # Shared host netns, like every sibling swarm container: the collector
+  # Shared host netns, like every sibling swarm container: Grafana reaches
-  # and Grafana reach this at 127.0.0.1:<port>.
+  # this at 127.0.0.1:<port>.
   privateNetwork = false;
 in
 {
@@ -215,18 +215,6 @@ in
       limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${deployCfg.victorialogs.authRate};
     '';
 
-    # This store publishes its own health as prometheus metrics on the same
-    # listener it serves queries on, so the swarm's collector scrapes it with
-    # no exporter and no extra port — same arrangement as the metrics store.
-    #
-    # Declared here rather than in the collector's module because that is the
-    # rule the option carries: an entry exists only where the service that
-    # named it runs. ⚠️ That constrains the TARGET, not the scraper — a
-    # deployment that splits this container away from the collector's host
-    # silently drops the entry, and no assertion can see it, because separate
-    # hosts are separate evaluations.
-    services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}";
-
     # The gateway name, inside `deployCfg.victorialogs.enable` — same
     # "only the host that runs the service may claim the name" guard every
     # sibling swarm-service module uses (`swarm-grafana.nix`,
@@ -313,6 +301,23 @@ in
             '';
           };
 
+          # The store's own health, which it publishes as prometheus metrics
+          # on the listener it serves queries on. The swarm collector requests
+          # exactly this URL, declared as `publishedScrapeTargets.victorialogs`
+          # in ./swarm-otel-service.nix; the two must agree, since that string
+          # is also the token's audience.
+          #
+          # Bare `auth_request` to authelia, as on the ingest route above, and
+          # never `/__swarm_logs_auth`, which admits an agent's bearer:
+          # authelia's rules for this URL admit the collector's client and
+          # deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
+          "= /metrics" = {
+            proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
+            extraConfig = ''
+              auth_request /__hive_authelia;
+            '';
+          };
+
           # `rewrite … last` re-matches locations, so the subrequest lands in
           # one of the two internal locations named by the map.
           "= /__swarm_logs_auth" = {

nix/module-eval/swarm-otel-identity.nix

@@ -106,6 +106,7 @@ let
 
   forgeMetricsUrl = "https://forge.t.local/metrics";
   autheliaMetricsUrl = "https://auth.t.local/metrics";
+  logsMetricsUrl = "https://logs.t.local/metrics";
   vmMetricsUrl = "https://metrics.t.local/metrics";
 
   # The policy authelia applies to `subject` requesting `https://<domain><path>`,
@@ -371,9 +372,10 @@ let
         lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [
           autheliaMetricsUrl
           forgeMetricsUrl
+          logsMetricsUrl
           vmMetricsUrl
         ]
-        && lib.length targets == 6
+        && lib.length targets == 8
         && lib.all (t: policy collector t == "one_factor") targets
         && lib.all (t: policy "user:someone" t == "deny") targets
         && lib.all (t: policy "group:admins" t == "deny") targets;
@@ -421,7 +423,7 @@ let
           ];
           matched = row: builtins.match re row.path != null;
         in
-        lib.length metricsRules == 4
+        lib.length metricsRules == 6
         && lib.all (r: r.resources == [ re ]) metricsRules
         && re == "^/metrics([?].*)?$"
         && lib.all (row: matched row == (row.nginx == "= /metrics")) table;
@@ -479,6 +481,85 @@ let
       name = "a single-host swarm passes the loopback-vs-published job name assertion";
       ok = !(refusedOver allLocal "is declared as both a loopback");
     }
+    {
+      name = "a collector on a host without the log store scrapes it by name";
+      ok =
+        let
+          s = otelSettings otelRemoteAuthelia;
+          job = lib.findFirst (
+            j: j.job_name == "victorialogs"
+          ) null s.receivers.prometheus.config.scrape_configs;
+        in
+        !otelRemoteAuthelia.services.hyperhive.deploy.victorialogs.enable
+        && job != null
+        && job.static_configs == [ { targets = [ "logs.t.local" ]; } ]
+        && job.metrics_path == "/metrics"
+        && job.oauth2.endpoint_params.audience == logsMetricsUrl;
+    }
+    {
+      # `= /metrics` must carry the bare authelia check and nothing else: the
+      # agents' `/__swarm_logs_auth` would admit an agent's bearer, and the
+      # browser block's `error_page 401 =302` hands a scraper a login page.
+      # The other routes keep their own checks.
+      name = "the log store's vhost serves /metrics behind authelia alone, its other routes unchanged";
+      ok =
+        let
+          h = allLocal.services.hyperhive;
+          locs = allLocal.services.nginx.virtualHosts.${h.swarm.victorialogs.domain}.locations;
+          checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig);
+        in
+        h.deploy.victorialogs.enable
+        && locs ? "= /metrics"
+        && locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.victorialogs.port}/metrics"
+        && checksOf locs."= /metrics" == [ "auth_request /__hive_authelia;" ]
+        && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
+        && lib.hasInfix "error_page 401 =302" locs."/".extraConfig
+        && lib.elem "auth_request /__hive_authelia;" (checksOf locs."/")
+        && checksOf locs."^~ /select/logsql/" == [ "auth_request /__swarm_logs_auth;" ];
+    }
+    {
+      # The `deny` that confines the log store's `/metrics` is scoped to that
+      # path, and comes after the collector's rule: every other path on the
+      # name still reaches `default_policy`, as it did with no rule at all.
+      name = "authelia confines the log store's /metrics to the collector and leaves its other paths alone";
+      ok =
+        let
+          h = autheliaOnly.services.hyperhive;
+          collector = "oauth2:client:${h.swarm.otel.clientId}";
+          rules =
+            autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules;
+          logsRules = lib.filter (r: r.domain == "logs.t.local") rules;
+          policy = subject: autheliaPolicy autheliaOnly subject "logs.t.local";
+        in
+        map (r: r.policy) logsRules == [
+          "one_factor"
+          "deny"
+        ]
+        && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) logsRules
+        && (lib.head logsRules).subject == [ collector ]
+        && policy collector "/metrics" == "one_factor"
+        && policy "user:someone" "/metrics" == "deny"
+        && policy "user:someone" "/metrics?x=1" == "deny"
+        && policy "user:someone" "/" == "one_factor"
+        && policy "user:someone" "/select/logsql/query" == "one_factor"
+        && policy "user:someone" "/insert/opentelemetry/v1/logs" == "one_factor";
+    }
+    {
+      # Single host: the store is scraped once, by its published name.
+      name = "a host running the log store and the collector scrapes it once, by name";
+      ok =
+        let
+          o = allLocal.services.hyperhive.swarm.otel;
+          jobs =
+            lib.filter (j: j.job_name == "victorialogs")
+              (otelSettings allLocal).receivers.prometheus.config.scrape_configs;
+        in
+        allLocal.services.hyperhive.deploy.victorialogs.enable
+        && allLocal.services.hyperhive.deploy.swarm-otel.enable
+        && !(o.scrapeTargets ? victorialogs)
+        && lib.length jobs == 1
+        && (lib.head jobs).static_configs == [ { targets = [ "logs.t.local" ]; } ];
+    }
     {
       # Read against the roster the fixture declares rather than against
       # names spelled here: an arm naming `h1` passes on a single-hive