swarm-otel: scrape nats through its published name

The queue's metrics were scraped over loopback from the exporter, declared
only where the queue and the collector share a host, so a collector on any
other host had no job for it. Its target is now declared on every host as
`https://<swarm.nats.domain>/metrics`, and the queue's host gets a gateway
vhost on that name serving `= /metrics` from the exporter behind the
collector's auth subrequest (`/__nats_metrics_authz`, the forge's shape).
Every other path there is 403; the monitoring endpoint is never proxied.
The rules derived from `publishedScrapeTargets` admit the collector's
client and deny every other subject. The loopback entry, and with it
`glue-otel-scrape-targets.nix`, is gone.

In self-signed mode the vhost serves the queue's own `swarm-nats` leaf
(issued for exactly that name, `server_flag` set), copied 0640
root:nginx by `swarm-nats-gateway-cert`; until the leaf exists that unit
writes a throwaway pair, so nginx never starts on a missing file. The
name stays out of `swarm.serviceDomains`, so a host whose only swarm
service is the queue fronts no services name and its gateway neither
waits for nor needs the swarm-services leaf: no services-issuer pair has
to be placed on it. With ACME or an operator cert the vhost uses the
gateway's own TLS, as every other name does.

The queue's host now enables the gateway (`mkDefault`), as every other
swarm service's host does.

Fixes #5205

nix/host-modules/default.nix

@@ -44,7 +44,6 @@
     ./glue-nats-bao-identity.nix
     ./glue-nats-oidc-client.nix
     ./glue-nats-wireguard.nix
-    ./glue-otel-scrape-targets.nix
     ./glue-queue-agent-credential.nix
     ./glue-builder-oidc-client.nix
     ./glue-review-classifier-oidc-client.nix

nix/host-modules/glue-otel-scrape-targets.nix

@@ -1,37 +0,0 @@
-# Glue: declare the loopback scrape target of nats on the swarm's OTEL
-# collector, wherever it shares a host with it. Authelia and grafana are
-# `publishedScrapeTargets` jobs (./swarm-otel-service.nix); a loopback job of
-# the same name fails ./swarm-otel.nix's job-name assertion.
-#
-# ONE PAIRING — nats ← the collector. Deleting this leaves the collector not
-# scraping nats: no error, no metrics.
-#
-# The target is gated on the service AND the collector being enabled here. The
-# loopback address is only honest when both run on this host; a target nobody
-# can read asserts a collection that is not happening. Both default from
-# `deploy.allSwarmServices`, but `mkDefault` is an invitation to override, so
-# co-location is a property of the auto-deployed topology, not the modules.
-#
-# ⚠️ Silent when split: separate hosts are separate evaluations with no shared
-# context, so a host running nats and no collector declares nothing and
-# the collector's host never learns of it. No error, no warning — a
-# healthy exporter and an empty dashboard. Not guardable from here.
-#
-# ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart, and only
-# this one reads `scrapeTargets`.
-{
-  lib,
-  config,
-  ...
-}:
-let
-  hyperhiveCfg = config.services.hyperhive;
-  deployCfg = hyperhiveCfg.deploy;
-  swarmCfg = hyperhiveCfg.swarm;
-  withCollector = service: lib.mkIf (deployCfg.${service}.enable && deployCfg.swarm-otel.enable);
-in
-{
-  config.services.hyperhive.swarm.otel.scrapeTargets = {
-    nats = withCollector "nats" "127.0.0.1:${toString swarmCfg.nats.metricsPort}";
-  };
-}

nix/host-modules/swarm-nats-service.nix

@@ -40,8 +40,11 @@ in
         Every other hive resolves it through the operator's upstream DNS,
         which is where a multi-host swarm needs a record for it.
 
-        Not in {option}`services.hyperhive.swarm.serviceDomains`: that list
+        Not in {option}`services.hyperhive.swarm.serviceDomains`. The gateway
-        is the names a gateway vhost fronts, and nginx fronts nothing here.
+        on the queue's host serves `https://<domain>/metrics` to the swarm
+        collector, and in self-signed mode it serves the queue's own leaf
+        there rather than the swarm-services one, so that host never waits
+        for a services leaf (./swarm-nats.nix).
       '';
     };
 
@@ -89,7 +92,9 @@ in
       default = 7777;
       description = ''
         Port the Prometheus exporter serves NATS's metrics on, bound to
-        loopback for the swarm collector to scrape.
+        loopback. The gateway's `= /metrics` location on
+        {option}`services.hyperhive.swarm.nats.domain` proxies it, behind
+        the swarm collector's authelia check.
 
         ⚠️ This and {option}`monitorPort` are two more claims on a port
         space every swarm container shares — they run in this container but

nix/host-modules/swarm-nats.nix

@@ -286,6 +286,14 @@ let
   tlsKeyCredential = "tls-key";
   tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}";
 
+  # The gateway's copy of that leaf, for the `/metrics` vhost below: the key
+  # here is 0640 root:nginx, so nginx reads it while the queue's own stays
+  # 0600 root. Self-signed mode only, the mode `gateway.lib.tlsFor` hands the
+  # swarm-services leaf to; with ACME or an operator cert that issuer covers
+  # this name as it covers every other.
+  gatewayServesQueueLeaf = gatewayCfg.enable && gatewayCfg.useSelfSigned;
+  gatewayTlsDir = "/var/lib/swarm-nats-gateway-tls";
+
   # The responder's own store identity, for the callout xkey, agent queue
   # credentials and its OIDC client secret. Required: an assertion below refuses
   # a queue without it.
@@ -329,12 +337,12 @@ in
   # The swarm's message queue: one NATS server, reached by every hive at
   # `tls://<swarm.nats.domain>:<port>`.
   #
-  # ⚠️ There is deliberately NO gateway vhost here. NATS speaks its own TCP
+  # ⚠️ The gateway does NOT front the queue. NATS speaks its own TCP protocol
-  # protocol rather than HTTP, so nginx cannot front it the way it fronts the
+  # rather than HTTP, so the server terminates TLS itself on `port`. The
-  # forge, matrix and authelia, and the server terminates TLS itself. The
   # name is the half of the sibling pattern that applies, the way it is for
   # bao: `gateway.localNames` on this host, the operator's DNS everywhere
-  # else.
+  # else. The one gateway vhost on that name serves the exporter's
+  # `/metrics` to the swarm collector and nothing else.
 
   # What the queue IS to every hive, the ports it answers on and the client id
   # it is registered under, is `swarm.nats` in ./swarm-nats-service.nix. What
@@ -429,14 +437,124 @@ in
   };
 
   config = lib.mkIf deployCfg.nats.enable {
-    # Resolver only: NATS speaks its own protocol, so nginx fronts
+    # The auth responder introspects authelia by name, and the gateway serves
-    # nothing here — but the auth responder introspects authelia by name.
+    # the exporter's `/metrics` below.
+    services.hyperhive.gateway.enable = lib.mkDefault true;
     services.hyperhive.gateway.dns.enable = lib.mkDefault true;
 
     # The name every client dials, answered with the bridge IP on this host.
     # Other hives resolve it through the operator's DNS, as they do bao's.
     services.hyperhive.gateway.localNames = [ cfg.domain ];
 
+    # The exporter's metrics, for the swarm collector wherever it runs, and
+    # nothing else on this name: the monitoring endpoint (`/varz`, `/connz`)
+    # is never proxied, and every other path is 403.
+    #
+    # ⚠️ Not `gateway.lib.tlsFor`, and the name is not in
+    # `swarm.serviceDomains`. In self-signed mode this vhost serves the
+    # queue's own leaf (issued for exactly this name, `server_flag` set), so
+    # a host whose only swarm service is the queue never fronts a services
+    # name and its gateway does not wait for the swarm-services leaf.
+    #
+    # `forceSSL` for the reason ./swarm-victorialogs.nix gives: authelia
+    # answers a plain-http auth subrequest with 400.
+    services.nginx.virtualHosts."${cfg.domain}" =
+      (
+        if gatewayServesQueueLeaf then
+          {
+            sslCertificate = "${gatewayTlsDir}/cert.pem";
+            sslCertificateKey = "${gatewayTlsDir}/key.pem";
+          }
+        else
+          builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ]
+      )
+      // {
+        forceSSL = true;
+        listen = gatewayCfg.lib.listen;
+        extraConfig = gatewayCfg.lib.securityHeaders;
+        locations = {
+          "/".return = "403";
+
+          # The collector requests exactly this URL, declared as
+          # `publishedScrapeTargets.nats` in ./swarm-otel-service.nix; the
+          # two must agree, since that string is also the token's audience.
+          # Authelia's rules for it admit the collector's client and deny
+          # every other subject (`metricsRules` in ./swarm-authelia.nix).
+          "= /metrics" = {
+            proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics";
+            extraConfig = ''
+              auth_request /__nats_metrics_authz;
+            '';
+          };
+
+          # The subrequest, same shape as the forge's `= /__forge_metrics_authz`
+          # (./hive-forge/default.nix): no `error_page 401 =302`, so a scraper
+          # gets the 401 rather than a login page parsed as metrics.
+          "= /__nats_metrics_authz" = {
+            proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request";
+            # nixpkgs appends its OWN `Host $host` after extraConfig, which
+            # would override verifiedProxyTo's — see hive-gateway/vhost-lib.nix.
+            recommendedProxySettings = false;
+            extraConfig = ''
+              internal;
+              ${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain}
+              proxy_pass_request_body off;
+              proxy_set_header Content-Length "";
+              proxy_set_header X-Original-Method $request_method;
+              proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
+              proxy_set_header X-Forwarded-Proto $scheme;
+              proxy_set_header X-Forwarded-Host $http_host;
+              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            '';
+          };
+        };
+      };
+
+    # Puts a pair at the paths the vhost above names, before nginx starts:
+    # nginx refuses a config naming a missing cert file, and that takes the
+    # whole gateway down rather than this one name. The queue's leaf when it
+    # exists, else a throwaway self-signed pair until `swarm-bao-nats-tls`
+    # issues the leaf and starts this again. Never fails on a missing leaf.
+    systemd.services.swarm-nats-gateway-cert = lib.mkIf gatewayServesQueueLeaf {
+      description = "Copy the swarm queue's TLS leaf for the gateway's /metrics vhost";
+      wantedBy = [ "nginx.service" ];
+      before = [ "nginx.service" ];
+      path = [
+        pkgs.coreutils
+        pkgs.openssl
+        pkgs.systemd
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        UMask = "0077";
+        SyslogIdentifier = "swarm-nats-gateway-cert";
+      };
+      script = ''
+        set -eu
+        src=${lib.escapeShellArg tlsDir}
+        d=${lib.escapeShellArg gatewayTlsDir}
+        install -d -m 0755 "$d"
+        if [ -s "$src/cert.pem" ] && [ -s "$src/key.pem" ]; then
+          cp "$src/cert.pem" "$d/cert.pem.new"
+          cp "$src/key.pem" "$d/key.pem.new"
+        elif [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; then
+          exit 0
+        else
+          echo "no queue leaf yet — serving a throwaway self-signed certificate on ${cfg.domain} until swarm-bao-nats-tls issues one" >&2
+          openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \
+            -subj "/CN=${cfg.domain}" -keyout "$d/key.pem.new" -out "$d/cert.pem.new"
+        fi
+        chmod 0644 "$d/cert.pem.new"
+        chgrp nginx "$d/key.pem.new"
+        chmod 0640 "$d/key.pem.new"
+        mv -f "$d/key.pem.new" "$d/key.pem"
+        mv -f "$d/cert.pem.new" "$d/cert.pem"
+        if systemctl is-active --quiet nginx.service; then
+          systemctl reload nginx.service
+        fi
+      '';
+    };
+
     assertions = [
       {
         # The callout xkey and the responder's OIDC client secret are read
@@ -999,6 +1117,10 @@ in
         chmod 0644 "$d/cert.pem.new"
         mv -f "$d/key.pem.new" "$d/key.pem"
         mv -f "$d/cert.pem.new" "$d/cert.pem"
+        ${lib.optionalString gatewayServesQueueLeaf ''
+
+          # The gateway's `/metrics` vhost serves a copy of this leaf.
+          systemctl start swarm-nats-gateway-cert.service''}
 
         # Renewal, or the late-store retry. On a normal boot the container is
         # not up yet and starts the server with this leaf itself. A restart,

nix/host-modules/swarm-otel-service.nix

@@ -240,8 +240,8 @@ in
         declared on a host that does not serve it. An entry that also needs
         the collector on the same host (the loopback address is only honest
         then) is declared from a glue module instead, which reads both
-        switches: grafana and nats are declared by
+        switches. Every swarm service is a `publishedScrapeTargets` entry
-        `glue-otel-scrape-targets.nix`.
+        instead, which a collector on any host scrapes.
 
         ⚠️ **That constrains the TARGET, not the SCRAPER, and the difference
         is a silent gap.** Nothing here puts the collector on the same host.
@@ -364,5 +364,7 @@ in
     victorialogs = "https://${hyperhiveCfg.swarm.victorialogs.domain}/metrics";
     # The vhost location is in ./swarm-grafana.nix.
     grafana = "https://${hyperhiveCfg.swarm.grafana.domain}/metrics";
+    # The vhost location is in ./swarm-nats.nix.
+    nats = "https://${hyperhiveCfg.swarm.nats.domain}/metrics";
   };
 }

nix/host-modules/swarm.nix

@@ -93,6 +93,10 @@ let
   # services leaf over the hive one, so a gateway-published swarm name
   # missing here is served the hive certificate, which cannot cover a name
   # under a different apex.
+  #
+  # `swarm.nats.domain` is deliberately absent: its `/metrics` vhost names
+  # the queue's own leaf instead (./swarm-nats.nix), so a host running only
+  # the queue fronts no services name and never waits for that leaf.
   serviceDomains' = [
     swarmCfg.forge.domain
     swarmCfg.matrix.gatewayHost

nix/module-eval/forge-placement.nix

@@ -138,15 +138,16 @@ let
         && !(refusedOver bare "deploy.forgejo.sso.clientSecretFile");
     }
     {
-      # Authelia and grafana are published targets; ./swarm-otel-identity.nix
+      # All three are published targets; ./swarm-otel-identity.nix covers them.
-      # covers them.
+      name = "a host running the collector, authelia, grafana and nats scrapes none of them on loopback";
-      name = "a host running the collector, authelia, grafana and nats scrapes nats on loopback, authelia and grafana not";
       ok =
         let
           t = servicesHere.services.hyperhive.swarm.otel.scrapeTargets;
-          port = n: toString servicesHere.services.hyperhive.swarm.${n}.metricsPort;
         in
-        t.nats == "127.0.0.1:${port "nats"}" && !(t ? authelia) && !(t ? grafana);
+        servicesHere.services.hyperhive.deploy.nats.enable
+        && !(t ? authelia)
+        && !(t ? grafana)
+        && !(t ? nats);
     }
     {
       name = "hosting the swarm's shared services runs the forge";

nix/module-eval/nats-tls.nix

@@ -362,6 +362,74 @@ let
         failedAssertions remoteLeafNoUrl == [ ]
         && !(remoteLeafNoUrl.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL);
     }
+    {
+      # Only `/metrics`, from the exporter, behind the collector's subrequest:
+      # no `error_page` (a scraper would parse a login page as metrics), the
+      # monitoring endpoint never proxied, every other path refused. In
+      # self-signed mode the vhost names the queue's own leaf, as copied for
+      # nginx, not the swarm-services one.
+      name = "the queue's name serves only /metrics, behind the collector's auth subrequest, on the queue's own leaf";
+      ok =
+        let
+          h = allLocal.services.hyperhive;
+          v = allLocal.services.nginx.virtualHosts.${natsName};
+          locs = v.locations;
+          checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig);
+          authz = locs."= /__nats_metrics_authz";
+        in
+        lib.attrNames locs == [
+          "/"
+          "= /__nats_metrics_authz"
+          "= /metrics"
+        ]
+        && locs."/".return == "403"
+        && locs."/".proxyPass == null
+        && locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.nats.metricsPort}/metrics"
+        && checksOf locs."= /metrics" == [ "auth_request /__nats_metrics_authz;" ]
+        && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
+        && authz.proxyPass == "https://auth.t.local/api/authz/auth-request"
+        && lib.hasInfix "internal;" authz.extraConfig
+        && !(lib.any (
+          l:
+          lib.hasInfix ":${toString h.swarm.nats.monitorPort}" (
+            if l.proxyPass == null then "" else l.proxyPass
+          )
+        ) (lib.attrValues locs))
+        && v.forceSSL
+        && v.sslCertificate == "/var/lib/swarm-nats-gateway-tls/cert.pem"
+        && v.sslCertificateKey == "/var/lib/swarm-nats-gateway-tls/key.pem";
+    }
+    {
+      # A host whose only swarm service is the queue: its gateway serves the
+      # queue's name with the queue's leaf, fronts no swarm-services name, and
+      # so neither waits for nor requires the swarm-services leaf. The copy
+      # unit is only wanted by nginx, never required, and a rotated queue leaf
+      # re-runs it.
+      name = "a host running only the queue renders its gateway without requiring the swarm-services leaf";
+      ok =
+        let
+          h = queueNoStore.services.hyperhive;
+          s = queueNoStore.systemd.services;
+          copy = s.swarm-nats-gateway-cert;
+        in
+        h.deploy.nats.enable
+        && !h.deploy.hive-controller.enable
+        && h.gateway.enable
+        && h.gateway.useSelfSigned
+        &&
+          queueNoStore.services.nginx.virtualHosts.${natsName}.sslCertificate
+          == "/var/lib/swarm-nats-gateway-tls/cert.pem"
+        && h.swarm.localServiceDomains == [ ]
+        && !(lib.elem natsName h.swarm.serviceDomains)
+        && !(lib.elem "hive-gateway-self-signed-cert.service" (s.swarm-services-cert.requiredBy or [ ]))
+        && !(lib.hasInfix "serves swarm names only with it" s.hive-gateway-self-signed-cert.script)
+        && lib.elem "nginx.service" copy.wantedBy
+        && lib.elem "nginx.service" copy.before
+        && !(lib.elem "nginx.service" (copy.requiredBy or [ ]))
+        && lib.hasInfix "throwaway self-signed" copy.script
+        && lib.hasInfix "chgrp nginx" copy.script
+        && lib.hasInfix "systemctl start swarm-nats-gateway-cert.service" s.swarm-bao-nats-tls.script;
+    }
   ];
 in
 runGroup "nats-tls" cases

nix/module-eval/swarm-otel-identity.nix

@@ -111,6 +111,7 @@ let
   grafanaMetricsUrl = "https://grafana.t.local/metrics";
   logsMetricsUrl = "https://logs.t.local/metrics";
   vmMetricsUrl = "https://metrics.t.local/metrics";
+  natsMetricsUrl = "https://nats.t.local/metrics";
 
   # The policy authelia applies to `subject` requesting `https://<domain><path>`,
   # where `path` is what authelia matches `resources` against: the cleaned path
@@ -378,8 +379,9 @@ let
           grafanaMetricsUrl
           logsMetricsUrl
           vmMetricsUrl
+          natsMetricsUrl
         ]
-        && lib.length targets == 10
+        && lib.length targets == 12
         && lib.all (t: policy collector t == "one_factor") targets
         && lib.all (t: policy "user:someone" t == "deny") targets
         && lib.all (t: policy "group:admins" t == "deny") targets;
@@ -427,7 +429,7 @@ let
           ];
           matched = row: builtins.match re row.path != null;
         in
-        lib.length metricsRules == 8
+        lib.length metricsRules == 10
         && lib.all (r: r.resources == [ re ]) metricsRules
         && re == "^/metrics([?].*)?$"
         && lib.all (row: matched row == (row.nginx == "= /metrics")) table;
@@ -621,6 +623,60 @@ let
         && lib.length jobs == 1
         && (lib.head jobs).static_configs == [ { targets = [ "grafana.t.local" ]; } ];
     }
+    {
+      name = "a collector on a host without the queue scrapes it by name";
+      ok =
+        let
+          s = otelSettings otelRemoteAuthelia;
+          job = lib.findFirst (j: j.job_name == "nats") null s.receivers.prometheus.config.scrape_configs;
+        in
+        !otelRemoteAuthelia.services.hyperhive.deploy.nats.enable
+        && job != null
+        && job.static_configs == [ { targets = [ "nats.t.local" ]; } ]
+        && job.metrics_path == "/metrics"
+        && job.oauth2.endpoint_params.audience == natsMetricsUrl;
+    }
+    {
+      # The `deny` that confines the queue's `/metrics` is scoped to that path,
+      # and comes after the collector's rule.
+      name = "authelia confines the queue's /metrics to the collector";
+      ok =
+        let
+          h = autheliaOnly.services.hyperhive;
+          collector = "oauth2:client:${h.swarm.otel.clientId}";
+          rules =
+            autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules;
+          natsRules = lib.filter (r: r.domain == "nats.t.local") rules;
+          policy = subject: autheliaPolicy autheliaOnly subject "nats.t.local";
+        in
+        map (r: r.policy) natsRules == [
+          "one_factor"
+          "deny"
+        ]
+        && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) natsRules
+        && (lib.head natsRules).subject == [ collector ]
+        && policy collector "/metrics" == "one_factor"
+        && policy "user:someone" "/metrics" == "deny"
+        && policy "user:someone" "/metrics?x=1" == "deny"
+        && policy "group:admins" "/metrics" == "deny";
+    }
+    {
+      # Single host: the queue is scraped once, by its published name.
+      name = "a host running the queue and the collector scrapes it once, by name";
+      ok =
+        let
+          o = allLocal.services.hyperhive.swarm.otel;
+          jobs =
+            lib.filter (j: j.job_name == "nats")
+              (otelSettings allLocal).receivers.prometheus.config.scrape_configs;
+        in
+        allLocal.services.hyperhive.deploy.nats.enable
+        && allLocal.services.hyperhive.deploy.swarm-otel.enable
+        && !(o.scrapeTargets ? nats)
+        && (o.publishedScrapeTargets.nats or null) == natsMetricsUrl
+        && lib.length jobs == 1
+        && (lib.head jobs).static_configs == [ { targets = [ "nats.t.local" ]; } ];
+    }
     {
       # Read against the roster the fixture declares rather than against
       # names spelled here: an arm naming `h1` passes on a single-hive