swarm-otel: scrape nats through its published name

The queue's metrics were scraped over loopback from the exporter, declared
only where the queue and the collector share a host, so a collector on any
other host had no job for it. Its target is now declared on every host as
`https://<swarm.nats.domain>/metrics`, and the queue's host gets a gateway
vhost on that name serving `= /metrics` from the exporter behind the
collector's auth subrequest (`/__nats_metrics_authz`, the forge's shape).
Every other path there is 403; the monitoring endpoint is never proxied.
The rules derived from `publishedScrapeTargets` admit the collector's
client and deny every other subject. The loopback entry, and with it
`glue-otel-scrape-targets.nix`, is gone.

In self-signed mode the vhost serves the queue's own `swarm-nats` leaf
(issued for exactly that name, `server_flag` set), copied 0640
root:nginx by `swarm-nats-gateway-cert`; until the leaf exists that unit
writes a throwaway pair, so nginx never starts on a missing file. The
name stays out of `swarm.serviceDomains`, so a host whose only swarm
service is the queue fronts no services name and its gateway neither
waits for nor needs the swarm-services leaf: no services-issuer pair has
to be placed on it. With ACME or an operator cert the vhost uses the
gateway's own TLS, as every other name does.

The queue's host now enables the gateway (`mkDefault`), as every other
swarm service's host does.

Fixes #5205

nix/module-eval/swarm-otel-identity.nix

@@ -111,6 +111,7 @@ let
   grafanaMetricsUrl = "https://grafana.t.local/metrics";
   logsMetricsUrl = "https://logs.t.local/metrics";
   vmMetricsUrl = "https://metrics.t.local/metrics";
+  natsMetricsUrl = "https://nats.t.local/metrics";
 
   # The policy authelia applies to `subject` requesting `https://<domain><path>`,
   # where `path` is what authelia matches `resources` against: the cleaned path
@@ -378,8 +379,9 @@ let
           grafanaMetricsUrl
           logsMetricsUrl
           vmMetricsUrl
+          natsMetricsUrl
         ]
-        && lib.length targets == 10
+        && lib.length targets == 12
         && lib.all (t: policy collector t == "one_factor") targets
         && lib.all (t: policy "user:someone" t == "deny") targets
         && lib.all (t: policy "group:admins" t == "deny") targets;
@@ -427,7 +429,7 @@ let
           ];
           matched = row: builtins.match re row.path != null;
         in
-        lib.length metricsRules == 8
+        lib.length metricsRules == 10
         && lib.all (r: r.resources == [ re ]) metricsRules
         && re == "^/metrics([?].*)?$"
         && lib.all (row: matched row == (row.nginx == "= /metrics")) table;
@@ -621,6 +623,60 @@ let
         && lib.length jobs == 1
         && (lib.head jobs).static_configs == [ { targets = [ "grafana.t.local" ]; } ];
     }
+    {
+      name = "a collector on a host without the queue scrapes it by name";
+      ok =
+        let
+          s = otelSettings otelRemoteAuthelia;
+          job = lib.findFirst (j: j.job_name == "nats") null s.receivers.prometheus.config.scrape_configs;
+        in
+        !otelRemoteAuthelia.services.hyperhive.deploy.nats.enable
+        && job != null
+        && job.static_configs == [ { targets = [ "nats.t.local" ]; } ]
+        && job.metrics_path == "/metrics"
+        && job.oauth2.endpoint_params.audience == natsMetricsUrl;
+    }
+    {
+      # The `deny` that confines the queue's `/metrics` is scoped to that path,
+      # and comes after the collector's rule.
+      name = "authelia confines the queue's /metrics to the collector";
+      ok =
+        let
+          h = autheliaOnly.services.hyperhive;
+          collector = "oauth2:client:${h.swarm.otel.clientId}";
+          rules =
+            autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules;
+          natsRules = lib.filter (r: r.domain == "nats.t.local") rules;
+          policy = subject: autheliaPolicy autheliaOnly subject "nats.t.local";
+        in
+        map (r: r.policy) natsRules == [
+          "one_factor"
+          "deny"
+        ]
+        && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) natsRules
+        && (lib.head natsRules).subject == [ collector ]
+        && policy collector "/metrics" == "one_factor"
+        && policy "user:someone" "/metrics" == "deny"
+        && policy "user:someone" "/metrics?x=1" == "deny"
+        && policy "group:admins" "/metrics" == "deny";
+    }
+    {
+      # Single host: the queue is scraped once, by its published name.
+      name = "a host running the queue and the collector scrapes it once, by name";
+      ok =
+        let
+          o = allLocal.services.hyperhive.swarm.otel;
+          jobs =
+            lib.filter (j: j.job_name == "nats")
+              (otelSettings allLocal).receivers.prometheus.config.scrape_configs;
+        in
+        allLocal.services.hyperhive.deploy.nats.enable
+        && allLocal.services.hyperhive.deploy.swarm-otel.enable
+        && !(o.scrapeTargets ? nats)
+        && (o.publishedScrapeTargets.nats or null) == natsMetricsUrl
+        && lib.length jobs == 1
+        && (lib.head jobs).static_configs == [ { targets = [ "nats.t.local" ]; } ];
+    }
     {
       # Read against the roster the fixture declares rather than against
       # names spelled here: an arm naming `h1` passes on a single-hive