From 7eb5c92d15d114f0c55662cad1e49806d0cda809 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 1 Sep 2026 09:43:27 +0200 Subject: [PATCH 1/2] fix(#3849): scrape the path openbao actually serves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The store's scrape target rendered as `host:port` alone, so the collector requested `/metrics` — a path openbao does not serve on any listener. Every scrape since the target landed has 404'd, which reads as a dead exporter rather than a wrong address, and no bao sample has ever reached the store. `scrapeTargets` values may now carry a path and query. Both are optional and omitted when absent, so every existing target renders the config it rendered before. The query cannot ride in `metrics_path`: prometheus percent-encodes the `?`, so it has to become `params`. Measured against openbao 2.6.2 and otelcol-contrib 0.151.0, the pinned versions: `/metrics` 404s ("this listener only serves the metrics endpoint"), `/v1/sys/metrics` answers JSON, and only `?format=prometheus` — or an Accept header the collector happens to send today — returns exposition text. The query param is the half that does not depend on content negotiation. --- nix/host-modules/swarm-bao.nix | 12 ++++-- nix/host-modules/swarm-otel.nix | 68 +++++++++++++++++++++++++++++++-- nix/module-eval.nix | 34 +++++++++++++++++ 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index f5e632c9..b43ab84e 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -195,9 +195,9 @@ let # Metrics get their own listener rather than a flag on the one above, and # that follows from what a scraper can express: `swarm.otel.scrapeTargets` - # is `host:port` with no scheme and no credential, while the API listener is - # TLS and — once a client CA is set — demands a client certificate. The - # collector cannot reach it at all. + # carries no scheme and no credential, while the API listener is TLS and — + # once a client CA is set — demands a client certificate. The collector + # cannot reach it at all. # # `metrics_only` narrows this one to the metrics path (every other path 404s) # and the unauthenticated access is confined to loopback. **Deliberately @@ -576,7 +576,11 @@ in ]; services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere { - bao = "127.0.0.1:${toString baoDeploy.metricsPort}"; + # Path and query, not just `host:port`: openbao serves no `/metrics` + # at all, and `/v1/sys/metrics` answers JSON unless the format is + # asked for. A scrape of the default path 404s, which reads as a + # dead exporter rather than a wrong address. + bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus"; }; }) diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index e4e40e17..2f5cdbea 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -91,6 +91,46 @@ let # error rather than a warning nobody reads. parsePublished = url: builtins.match "https://([^/]+)(/.*)" url; + # A loopback target may carry a path and a query as well as `host:port`, + # because not every exporter serves `/metrics`: openbao 404s there and + # answers on `/v1/sys/metrics?format=prometheus`. Split for the same reason + # the published one is — one declaration, three prometheus fields. + # + # ⚠️ The query may not ride along in `metrics_path`: prometheus percent- + # encodes the `?`, so the request goes to a path that does not exist. It has + # to become `params`, whose values are lists. + # + # `null` when the shape is wrong, which the assertion below reports by name. + parseScrape = t: builtins.match "([^/?]+)(/[^?]*)?(\\?(.+))?" t; + + queryParams = + query: + lib.listToAttrs ( + map ( + kv: + let + parts = lib.splitString "=" kv; + in + lib.nameValuePair (lib.head parts) [ (lib.concatStringsSep "=" (lib.tail parts)) ] + ) (lib.splitString "&" query) + ); + + # Both optional fields are omitted rather than defaulted, so a plain + # `host:port` renders the config it rendered before this grammar existed. + loopbackScrapeConfig = + job: target: + let + parts = parseScrape target; + path = lib.elemAt parts 1; + query = lib.elemAt parts 3; + in + { + job_name = job; + static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ]; + } + // lib.optionalAttrs (path != null) { metrics_path = path; } + // lib.optionalAttrs (query != null) { params = queryParams query; }; + # The client secret takes three names, and the reason is `DynamicUser`. # # Upstream's collector unit runs with `DynamicUser = true`, and the @@ -332,6 +372,12 @@ in Prometheus exposition endpoints this collector scrapes, as ` = ":"`. + A path and query may follow the port — + `"127.0.0.1:8202/v1/sys/metrics?format=prometheus"` — for an + exporter that does not serve `/metrics`. Both are optional and + omitted when absent, so a bare `host:port` is scraped exactly as + before. + **A service declares its own entry, from its own module, under its own `enable`.** Do not assemble the list here: an entry then exists only where the service that named it runs, so a target is never @@ -711,6 +757,23 @@ in token, and plain http would put it on the wire in clear text. ''; } + { + # Same reason the published one is asserted: a value the grammar + # rejects surfaces as `attempt to call elemAt on null` from inside + # the renderer, naming neither the option nor the offending value. + assertion = lib.all (t: parseScrape t != null) (lib.attrValues cfg.scrapeTargets); + message = '' + services.hyperhive.swarm.otel.scrapeTargets has ${ + lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") ( + lib.filter (kv: parseScrape kv.value == null) (lib.attrsToList cfg.scrapeTargets) + ) + }, which is not of the form :[/][?=]. + + A trailing `?` with nothing after it is the usual cause. This + option is loopback-and-unauthenticated by contract, so it takes + no scheme and no credential. + ''; + } { # A job name used by BOTH scrape options. Within one option this # cannot happen — the module system refuses two definitions of the @@ -927,10 +990,7 @@ in # and deploys perfectly while adding no data. // lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) { prometheus.config.scrape_configs = - lib.mapAttrsToList (job: target: { - job_name = job; - static_configs = [ { targets = [ target ]; } ]; - }) cfg.scrapeTargets + lib.mapAttrsToList loopbackScrapeConfig cfg.scrapeTargets # ⚠️ `++`, so the two kinds of target land in ONE list — # which is exactly why a job name may not appear in both # options. A list concatenation does not resolve a diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 4028a376..b3cb38ab 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -118,6 +118,10 @@ let baoWithCollector = hive { deploy.bao.enable = true; deploy.swarm-otel.enable = true; + # A second job declared as bare `host:port`, so the pair of cases below + # reads one rendered scrape list: the store's entry carries a path, this + # one carries none. + swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; }; baoNoCollector = hive { deploy.bao.enable = true; @@ -133,6 +137,14 @@ let # look there rather than at the host's service set. baoUnits = machine: machine.containers.swarm-bao.config.systemd.services; + # The scrape list prometheus is handed, not the option a service declared: + # the address, the path and the query are one string on the way in and three + # fields on the way out, and only the second shape is what gets requested. + scrapeJob = + machine: job: + lib.findFirst (c: c.job_name == job) null + machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs; + # A priority collision is a property of the *option*, not # of the merged value's interior — nix throws the moment the value is # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is @@ -385,6 +397,28 @@ let in s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s"; } + { + # openbao serves no `/metrics` at all, so a scrape of the default path + # 404s: the store looks like a dead exporter, and every panel built on + # it renders empty rather than erroring. + name = "the store's scrape asks for the path openbao serves"; + ok = + let + j = scrapeJob baoWithCollector "bao"; + in + (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ]; + } + { + # Presence control for the case above: both fields are omitted rather + # than defaulted, so a target declared as bare `host:port` renders what + # it rendered before the path grammar existed. + name = "a target with no path renders neither metrics_path nor params"; + ok = + let + j = scrapeJob baoWithCollector "plain"; + in + j != null && !(j ? metrics_path) && !(j ? params); + } { # Absence arm. Unauthenticated by design, so it must not exist where # nothing reads it. From a3612f5168be4c6945d8677f5f5d3575deae59bd Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 1 Sep 2026 09:43:34 +0200 Subject: [PATCH 2/2] feat(#3849): a grafana board for the swarm's secret store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine panels against the metric names openbao actually emits — the prefix is `vault_`, the fork kept upstream's names, and a board built on `openbao_` would render perfectly and show nothing. The top row works on a SEALED store, which is what a hive has until an operator runs `operator init`: seal state, uptime, goroutines and a request count that reads 0 rather than going blank. Everything below is empty until then, and each panel's description says so rather than leaving an operator to wonder which of the two it is looking at. The seal panel reduces with `max()`. Once unsealed there are two `vault_core_unsealed` series and the `cluster=""` one stays 0 forever, so a bare selector reads SEALED on a healthy store — invisible while sealed, which is why the metric names were sampled in both states. --- nix/host-modules/swarm-grafana.nix | 1 + .../swarm-grafana/dashboards/bao.json | 569 ++++++++++++++++++ 2 files changed, 570 insertions(+) create mode 100644 nix/host-modules/swarm-grafana/dashboards/bao.json diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 4c4b159b..26c0d8ae 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -61,6 +61,7 @@ let [ "agents.json" "authelia.json" + "bao.json" "claude-usage.json" "forge.json" "logstore.json" diff --git a/nix/host-modules/swarm-grafana/dashboards/bao.json b/nix/host-modules/swarm-grafana/dashboards/bao.json new file mode 100644 index 00000000..931f5308 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/bao.json @@ -0,0 +1,569 @@ +{ + "title": "hyperhive · openbao", + "uid": "hyperhive-bao", + "description": "The swarm's secret store. Two things to know before editing: the metric prefix is vault_, not openbao_ — the fork kept upstream's names, and a panel built on the name you would guess renders perfectly and shows nothing. And a SEALED store emits almost nothing: only the top row works before an operator has run `bao operator init`, everything below it is empty until then and that is the expected reading, not a broken query. Targets are labelled service.name, NOT job; there is no job label in this store. Counters are counts over the dashboard range rather than per-second rates, because a store this quiet rates to exactly zero.", + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "hyperhive", + "swarm", + "bao", + "secrets" + ], + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "templating": { + "list": [] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Seal state", + "description": "SEALED means the store is up but refusing every read — the state it boots into, and the state it returns to on restart unless auto-unseal is configured. The max() is load-bearing: once unsealed there are two vault_core_unsealed series and the cluster=\"\" one stays 0 forever, so a bare selector reads SEALED on a healthy store.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max(vault_core_unsealed{\"service.name\"=\"bao\"})", + "instant": true, + "legendFormat": "seal" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "SEALED", + "color": "red", + "index": 0 + }, + "1": { + "text": "unsealed", + "color": "green", + "index": 1 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "background", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 2, + "type": "stat", + "title": "Uptime", + "description": "Time since the openbao process started. Read it beside the seal state: a restart re-seals the store, so a short uptime and a SEALED panel are one event rather than two problems.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "time() - process_start_time_seconds{\"service.name\"=\"bao\"}", + "instant": true, + "legendFormat": "uptime" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Goroutines", + "description": "A Go runtime metric, so it is present even while sealed — which makes it the panel that distinguishes a store with nothing to say from a scrape that is not arriving at all. No data here means the collector is not reaching the metrics listener.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "go_goroutines{\"service.name\"=\"bao\"}", + "instant": true, + "legendFormat": "goroutines" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Requests (range)", + "description": "vault_core_handle_request_count over the dashboard range — every API call the store served, unseal attempts included. The summary's _count series, not the summary itself. Zero on a sealed store, and zero is the honest answer there rather than an absent panel.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(vault_core_handle_request_count{\"service.name\"=\"bao\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "requests" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 10, + "type": "stat", + "title": "Seal operations (range)", + "description": "Encrypt and decrypt calls against the seal wrapper — on this swarm a TPM through PKCS#11, so each one is a round trip to hardware. These are counters with no _total suffix. A climbing decrypt count with a flat request count means something is re-reading the barrier rather than serving clients.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(vault_seal_encrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "encrypt" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(vault_seal_decrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "decrypt" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value_and_name", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 11, + "type": "stat", + "title": "Raft peers", + "description": "Voting members of the store's raft cluster. This swarm runs a single node, so 1 is the expected value and anything else means a peer was added — or that this node no longer sees itself as part of the cluster it stores data for.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vault_raft_peers{\"service.name\"=\"bao\"}", + "instant": true, + "legendFormat": "peers" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Request latency", + "description": "Quantiles openbao computes itself — a summary, so the store picked the buckets and a histogram_quantile() is neither needed nor possible here. Units are MILLISECONDS: vault's timers report ms, and reading them as seconds understates the store by three orders of magnitude.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.99\"}", + "legendFormat": "p99" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.5\"}", + "legendFormat": "p50" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 21, + "type": "timeseries", + "title": "Leases", + "description": "Outstanding leases, and the irrevocable ones beside them. The second line should sit at zero: an irrevocable lease is one the store gave up trying to revoke, so a credential somewhere outlives what issued it and no amount of waiting will clean it up.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vault_expire_num_leases{\"service.name\"=\"bao\"}", + "legendFormat": "leases" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vault_expire_num_irrevocable_leases{\"service.name\"=\"bao\"}", + "legendFormat": "irrevocable" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 30, + "type": "timeseries", + "title": "Heap in use", + "description": "go_memstats_heap_inuse_bytes. Present while sealed, so it is a live line even on a store that is serving nothing — a sawtooth is normal Go behaviour, a monotonic climb on an idle store is not.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "go_memstats_heap_inuse_bytes{\"service.name\"=\"bao\"}", + "legendFormat": "heap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +}