fix(#3849): scrape the path openbao actually serves

The store's scrape target rendered as `host:port` alone, so the collector
requested `/metrics` — a path openbao does not serve on any listener. Every
scrape since the target landed has 404'd, which reads as a dead exporter
rather than a wrong address, and no bao sample has ever reached the store.

`scrapeTargets` values may now carry a path and query. Both are optional and
omitted when absent, so every existing target renders the config it rendered
before. The query cannot ride in `metrics_path`: prometheus percent-encodes
the `?`, so it has to become `params`.

Measured against openbao 2.6.2 and otelcol-contrib 0.151.0, the pinned
versions: `/metrics` 404s ("this listener only serves the metrics endpoint"),
`/v1/sys/metrics` answers JSON, and only `?format=prometheus` — or an Accept
header the collector happens to send today — returns exposition text. The
query param is the half that does not depend on content negotiation.

nix/host-modules/swarm-bao.nix

@@ -195,9 +195,9 @@ let
 
   # Metrics get their own listener rather than a flag on the one above, and
   # that follows from what a scraper can express: `swarm.otel.scrapeTargets`
-  # is `host:port` with no scheme and no credential, while the API listener is
+  # carries no scheme and no credential, while the API listener is TLS and —
-  # TLS and — once a client CA is set — demands a client certificate. The
+  # once a client CA is set — demands a client certificate. The collector
-  # collector cannot reach it at all.
+  # cannot reach it at all.
   #
   # `metrics_only` narrows this one to the metrics path (every other path 404s)
   # and the unauthenticated access is confined to loopback. **Deliberately
@@ -576,7 +576,11 @@ in
       ];
 
       services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere {
-        bao = "127.0.0.1:${toString baoDeploy.metricsPort}";
+        # Path and query, not just `host:port`: openbao serves no `/metrics`
+        # at all, and `/v1/sys/metrics` answers JSON unless the format is
+        # asked for. A scrape of the default path 404s, which reads as a
+        # dead exporter rather than a wrong address.
+        bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus";
       };
     })
 

nix/host-modules/swarm-otel.nix

@@ -91,6 +91,46 @@ let
   # error rather than a warning nobody reads.
   parsePublished = url: builtins.match "https://([^/]+)(/.*)" url;
 
+  # A loopback target may carry a path and a query as well as `host:port`,
+  # because not every exporter serves `/metrics`: openbao 404s there and
+  # answers on `/v1/sys/metrics?format=prometheus`. Split for the same reason
+  # the published one is — one declaration, three prometheus fields.
+  #
+  # ⚠️ The query may not ride along in `metrics_path`: prometheus percent-
+  # encodes the `?`, so the request goes to a path that does not exist. It has
+  # to become `params`, whose values are lists.
+  #
+  # `null` when the shape is wrong, which the assertion below reports by name.
+  parseScrape = t: builtins.match "([^/?]+)(/[^?]*)?(\\?(.+))?" t;
+
+  queryParams =
+    query:
+    lib.listToAttrs (
+      map (
+        kv:
+        let
+          parts = lib.splitString "=" kv;
+        in
+        lib.nameValuePair (lib.head parts) [ (lib.concatStringsSep "=" (lib.tail parts)) ]
+      ) (lib.splitString "&" query)
+    );
+
+  # Both optional fields are omitted rather than defaulted, so a plain
+  # `host:port` renders the config it rendered before this grammar existed.
+  loopbackScrapeConfig =
+    job: target:
+    let
+      parts = parseScrape target;
+      path = lib.elemAt parts 1;
+      query = lib.elemAt parts 3;
+    in
+    {
+      job_name = job;
+      static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ];
+    }
+    // lib.optionalAttrs (path != null) { metrics_path = path; }
+    // lib.optionalAttrs (query != null) { params = queryParams query; };
+
   # The client secret takes three names, and the reason is `DynamicUser`.
   #
   # Upstream's collector unit runs with `DynamicUser = true`, and the
@@ -332,6 +372,12 @@ in
         Prometheus exposition endpoints this collector scrapes, as
         `<job name> = "<host>:<port>"`.
 
+        A path and query may follow the port —
+        `"127.0.0.1:8202/v1/sys/metrics?format=prometheus"` — for an
+        exporter that does not serve `/metrics`. Both are optional and
+        omitted when absent, so a bare `host:port` is scraped exactly as
+        before.
+
         **A service declares its own entry, from its own module, under its
         own `enable`.** Do not assemble the list here: an entry then exists
         only where the service that named it runs, so a target is never
@@ -711,6 +757,23 @@ in
           token, and plain http would put it on the wire in clear text.
         '';
       }
+      {
+        # Same reason the published one is asserted: a value the grammar
+        # rejects surfaces as `attempt to call elemAt on null` from inside
+        # the renderer, naming neither the option nor the offending value.
+        assertion = lib.all (t: parseScrape t != null) (lib.attrValues cfg.scrapeTargets);
+        message = ''
+          services.hyperhive.swarm.otel.scrapeTargets has ${
+            lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") (
+              lib.filter (kv: parseScrape kv.value == null) (lib.attrsToList cfg.scrapeTargets)
+            )
+          }, which is not of the form <host>:<port>[/<path>][?<k>=<v>].
+
+          A trailing `?` with nothing after it is the usual cause. This
+          option is loopback-and-unauthenticated by contract, so it takes
+          no scheme and no credential.
+        '';
+      }
       {
         # A job name used by BOTH scrape options. Within one option this
         # cannot happen — the module system refuses two definitions of the
@@ -927,10 +990,7 @@ in
                 # and deploys perfectly while adding no data.
                 // lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) {
                   prometheus.config.scrape_configs =
-                    lib.mapAttrsToList (job: target: {
+                    lib.mapAttrsToList loopbackScrapeConfig cfg.scrapeTargets
-                      job_name = job;
-                      static_configs = [ { targets = [ target ]; } ];
-                    }) cfg.scrapeTargets
                     # ⚠️ `++`, so the two kinds of target land in ONE list —
                     # which is exactly why a job name may not appear in both
                     # options. A list concatenation does not resolve a

nix/module-eval.nix

@@ -118,6 +118,10 @@ let
   baoWithCollector = hive {
     deploy.bao.enable = true;
     deploy.swarm-otel.enable = true;
+    # A second job declared as bare `host:port`, so the pair of cases below
+    # reads one rendered scrape list: the store's entry carries a path, this
+    # one carries none.
+    swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
   };
   baoNoCollector = hive {
     deploy.bao.enable = true;
@@ -133,6 +137,14 @@ let
   # look there rather than at the host's service set.
   baoUnits = machine: machine.containers.swarm-bao.config.systemd.services;
 
+  # The scrape list prometheus is handed, not the option a service declared:
+  # the address, the path and the query are one string on the way in and three
+  # fields on the way out, and only the second shape is what gets requested.
+  scrapeJob =
+    machine: job:
+    lib.findFirst (c: c.job_name == job) null
+      machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs;
+
   # A priority collision is a property of the *option*, not
   # of the merged value's interior — nix throws the moment the value is
   # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
@@ -385,6 +397,28 @@ let
         in
         s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
     }
+    {
+      # openbao serves no `/metrics` at all, so a scrape of the default path
+      # 404s: the store looks like a dead exporter, and every panel built on
+      # it renders empty rather than erroring.
+      name = "the store's scrape asks for the path openbao serves";
+      ok =
+        let
+          j = scrapeJob baoWithCollector "bao";
+        in
+        (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ];
+    }
+    {
+      # Presence control for the case above: both fields are omitted rather
+      # than defaulted, so a target declared as bare `host:port` renders what
+      # it rendered before the path grammar existed.
+      name = "a target with no path renders neither metrics_path nor params";
+      ok =
+        let
+          j = scrapeJob baoWithCollector "plain";
+        in
+        j != null && !(j ? metrics_path) && !(j ? params);
+    }
     {
       # Absence arm. Unauthenticated by design, so it must not exist where
       # nothing reads it.