vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fix(#3849): scrape the path openbao actually serves

Browse source 
The store's scrape target rendered as `host:port` alone, so the collector
requested `/metrics` — a path openbao does not serve on any listener. Every
scrape since the target landed has 404'd, which reads as a dead exporter
rather than a wrong address, and no bao sample has ever reached the store.

`scrapeTargets` values may now carry a path and query. Both are optional and
omitted when absent, so every existing target renders the config it rendered
before. The query cannot ride in `metrics_path`: prometheus percent-encodes
the `?`, so it has to become `params`.

Measured against openbao 2.6.2 and otelcol-contrib 0.151.0, the pinned
versions: `/metrics` 404s ("this listener only serves the metrics endpoint"),
`/v1/sys/metrics` answers JSON, and only `?format=prometheus` — or an Accept
header the collector happens to send today — returns exposition text. The
query param is the half that does not depend on content negotiation.
This commit is contained in:
atlas 2026-09-01 09:43:27 +02:00 • committed by mara
commit 7eb5c92d15
3 changed files with 106 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

12
nix/host-modules/swarm-bao.nix
View file

@ -195,9 +195,9 @@ let
# Metrics get their own listener rather than a flag on the one above, and
# that follows from what a scraper can express: `swarm.otel.scrapeTargets`
# is `host:port` with no scheme and no credential, while the API listener is
# TLS and — once a client CA is set — demands a client certificate. The
# collector cannot reach it at all.
# carries no scheme and no credential, while the API listener is TLS and —
# once a client CA is set — demands a client certificate. The collector
# cannot reach it at all.
#
# `metrics_only` narrows this one to the metrics path (every other path 404s)
# and the unauthenticated access is confined to loopback. **Deliberately
@ -576,7 +576,11 @@ in
];
services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere {
bao = "127.0.0.1:${toString baoDeploy.metricsPort}";
# Path and query, not just `host:port`: openbao serves no `/metrics`
# at all, and `/v1/sys/metrics` answers JSON unless the format is
# asked for. A scrape of the default path 404s, which reads as a
# dead exporter rather than a wrong address.
bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus";
};
})

68
nix/host-modules/swarm-otel.nix
View file

@ -91,6 +91,46 @@ let
# error rather than a warning nobody reads.
parsePublished = url: builtins.match "https://([^/]+)(/.*)" url;
# A loopback target may carry a path and a query as well as `host:port`,
# because not every exporter serves `/metrics`: openbao 404s there and
# answers on `/v1/sys/metrics?format=prometheus`. Split for the same reason
# the published one is — one declaration, three prometheus fields.
#
# ⚠️ The query may not ride along in `metrics_path`: prometheus percent-
# encodes the `?`, so the request goes to a path that does not exist. It has
# to become `params`, whose values are lists.
#
# `null` when the shape is wrong, which the assertion below reports by name.
parseScrape = t: builtins.match "([^/?]+)(/[^?]*)?(\\?(.+))?" t;
queryParams =
query:
lib.listToAttrs (
map (
kv:
let
parts = lib.splitString "=" kv;
in
lib.nameValuePair (lib.head parts) [ (lib.concatStringsSep "=" (lib.tail parts)) ]
) (lib.splitString "&" query)
);
# Both optional fields are omitted rather than defaulted, so a plain
# `host:port` renders the config it rendered before this grammar existed.
loopbackScrapeConfig =
job: target:
let
parts = parseScrape target;
path = lib.elemAt parts 1;
query = lib.elemAt parts 3;
in
{
job_name = job;
static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ];
}
// lib.optionalAttrs (path != null) { metrics_path = path; }
// lib.optionalAttrs (query != null) { params = queryParams query; };
# The client secret takes three names, and the reason is `DynamicUser`.
#
# Upstream's collector unit runs with `DynamicUser = true`, and the
@ -332,6 +372,12 @@ in
Prometheus exposition endpoints this collector scrapes, as
`<job name> = "<host>:<port>"`.
A path and query may follow the port —
`"127.0.0.1:8202/v1/sys/metrics?format=prometheus"` — for an
exporter that does not serve `/metrics`. Both are optional and
omitted when absent, so a bare `host:port` is scraped exactly as
before.
**A service declares its own entry, from its own module, under its
own `enable`.** Do not assemble the list here: an entry then exists
only where the service that named it runs, so a target is never
@ -711,6 +757,23 @@ in
token, and plain http would put it on the wire in clear text.
'';
}
{
# Same reason the published one is asserted: a value the grammar
# rejects surfaces as `attempt to call elemAt on null` from inside
# the renderer, naming neither the option nor the offending value.
assertion = lib.all (t: parseScrape t != null) (lib.attrValues cfg.scrapeTargets);
message = ''
services.hyperhive.swarm.otel.scrapeTargets has ${
lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") (
lib.filter (kv: parseScrape kv.value == null) (lib.attrsToList cfg.scrapeTargets)
)
}, which is not of the form <host>:<port>[/<path>][?<k>=<v>].
A trailing `?` with nothing after it is the usual cause. This
option is loopback-and-unauthenticated by contract, so it takes
no scheme and no credential.
'';
}
{
# A job name used by BOTH scrape options. Within one option this
# cannot happen — the module system refuses two definitions of the
@ -927,10 +990,7 @@ in
# and deploys perfectly while adding no data.
// lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) {
prometheus.config.scrape_configs =
lib.mapAttrsToList (job: target: {
job_name = job;
static_configs = [ { targets = [ target ]; } ];
}) cfg.scrapeTargets
lib.mapAttrsToList loopbackScrapeConfig cfg.scrapeTargets
# ⚠️ `++`, so the two kinds of target land in ONE list —
# which is exactly why a job name may not appear in both
# options. A list concatenation does not resolve a

34
nix/module-eval.nix
View file

@ -118,6 +118,10 @@ let
baoWithCollector = hive {
deploy.bao.enable = true;
deploy.swarm-otel.enable = true;
# A second job declared as bare `host:port`, so the pair of cases below
# reads one rendered scrape list: the store's entry carries a path, this
# one carries none.
swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
};
baoNoCollector = hive {
deploy.bao.enable = true;
@ -133,6 +137,14 @@ let
# look there rather than at the host's service set.
baoUnits = machine: machine.containers.swarm-bao.config.systemd.services;
# The scrape list prometheus is handed, not the option a service declared:
# the address, the path and the query are one string on the way in and three
# fields on the way out, and only the second shape is what gets requested.
scrapeJob =
machine: job:
lib.findFirst (c: c.job_name == job) null
machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs;
# A priority collision is a property of the *option*, not
# of the merged value's interior — nix throws the moment the value is
# demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
@ -385,6 +397,28 @@ let
in
s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
}
{
# openbao serves no `/metrics` at all, so a scrape of the default path
# 404s: the store looks like a dead exporter, and every panel built on
# it renders empty rather than erroring.
name = "the store's scrape asks for the path openbao serves";
ok =
let
j = scrapeJob baoWithCollector "bao";
in
(j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ];
}
{
# Presence control for the case above: both fields are omitted rather
# than defaulted, so a target declared as bare `host:port` renders what
# it rendered before the path grammar existed.
name = "a target with no path renders neither metrics_path nor params";
ok =
let
j = scrapeJob baoWithCollector "plain";
in
j != null && !(j ? metrics_path) && !(j ? params);
}
{
# Absence arm. Unauthenticated by design, so it must not exist where
# nothing reads it.