diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index f5e632c9..b43ab84e 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -195,9 +195,9 @@ let # Metrics get their own listener rather than a flag on the one above, and # that follows from what a scraper can express: `swarm.otel.scrapeTargets` - # is `host:port` with no scheme and no credential, while the API listener is - # TLS and — once a client CA is set — demands a client certificate. The - # collector cannot reach it at all. + # carries no scheme and no credential, while the API listener is TLS and — + # once a client CA is set — demands a client certificate. The collector + # cannot reach it at all. # # `metrics_only` narrows this one to the metrics path (every other path 404s) # and the unauthenticated access is confined to loopback. **Deliberately @@ -576,7 +576,11 @@ in ]; services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere { - bao = "127.0.0.1:${toString baoDeploy.metricsPort}"; + # Path and query, not just `host:port`: openbao serves no `/metrics` + # at all, and `/v1/sys/metrics` answers JSON unless the format is + # asked for. A scrape of the default path 404s, which reads as a + # dead exporter rather than a wrong address. + bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus"; }; }) diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 4c4b159b..26c0d8ae 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -61,6 +61,7 @@ let [ "agents.json" "authelia.json" + "bao.json" "claude-usage.json" "forge.json" "logstore.json" diff --git a/nix/host-modules/swarm-grafana/dashboards/bao.json b/nix/host-modules/swarm-grafana/dashboards/bao.json new file mode 100644 index 00000000..931f5308 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/bao.json @@ -0,0 +1,569 @@ +{ + "title": "hyperhive · openbao", + "uid": "hyperhive-bao", + "description": "The swarm's secret store. Two things to know before editing: the metric prefix is vault_, not openbao_ — the fork kept upstream's names, and a panel built on the name you would guess renders perfectly and shows nothing. And a SEALED store emits almost nothing: only the top row works before an operator has run `bao operator init`, everything below it is empty until then and that is the expected reading, not a broken query. Targets are labelled service.name, NOT job; there is no job label in this store. Counters are counts over the dashboard range rather than per-second rates, because a store this quiet rates to exactly zero.", + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "hyperhive", + "swarm", + "bao", + "secrets" + ], + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "templating": { + "list": [] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Seal state", + "description": "SEALED means the store is up but refusing every read — the state it boots into, and the state it returns to on restart unless auto-unseal is configured. The max() is load-bearing: once unsealed there are two vault_core_unsealed series and the cluster=\"\" one stays 0 forever, so a bare selector reads SEALED on a healthy store.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max(vault_core_unsealed{\"service.name\"=\"bao\"})", + "instant": true, + "legendFormat": "seal" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "SEALED", + "color": "red", + "index": 0 + }, + "1": { + "text": "unsealed", + "color": "green", + "index": 1 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "background", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 2, + "type": "stat", + "title": "Uptime", + "description": "Time since the openbao process started. Read it beside the seal state: a restart re-seals the store, so a short uptime and a SEALED panel are one event rather than two problems.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "time() - process_start_time_seconds{\"service.name\"=\"bao\"}", + "instant": true, + "legendFormat": "uptime" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Goroutines", + "description": "A Go runtime metric, so it is present even while sealed — which makes it the panel that distinguishes a store with nothing to say from a scrape that is not arriving at all. No data here means the collector is not reaching the metrics listener.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "go_goroutines{\"service.name\"=\"bao\"}", + "instant": true, + "legendFormat": "goroutines" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Requests (range)", + "description": "vault_core_handle_request_count over the dashboard range — every API call the store served, unseal attempts included. The summary's _count series, not the summary itself. Zero on a sealed store, and zero is the honest answer there rather than an absent panel.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(vault_core_handle_request_count{\"service.name\"=\"bao\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "requests" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 10, + "type": "stat", + "title": "Seal operations (range)", + "description": "Encrypt and decrypt calls against the seal wrapper — on this swarm a TPM through PKCS#11, so each one is a round trip to hardware. These are counters with no _total suffix. A climbing decrypt count with a flat request count means something is re-reading the barrier rather than serving clients.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(vault_seal_encrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "encrypt" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(vault_seal_decrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "decrypt" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value_and_name", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 11, + "type": "stat", + "title": "Raft peers", + "description": "Voting members of the store's raft cluster. This swarm runs a single node, so 1 is the expected value and anything else means a peer was added — or that this node no longer sees itself as part of the cluster it stores data for.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vault_raft_peers{\"service.name\"=\"bao\"}", + "instant": true, + "legendFormat": "peers" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Request latency", + "description": "Quantiles openbao computes itself — a summary, so the store picked the buckets and a histogram_quantile() is neither needed nor possible here. Units are MILLISECONDS: vault's timers report ms, and reading them as seconds understates the store by three orders of magnitude.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.99\"}", + "legendFormat": "p99" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.5\"}", + "legendFormat": "p50" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 21, + "type": "timeseries", + "title": "Leases", + "description": "Outstanding leases, and the irrevocable ones beside them. The second line should sit at zero: an irrevocable lease is one the store gave up trying to revoke, so a credential somewhere outlives what issued it and no amount of waiting will clean it up.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vault_expire_num_leases{\"service.name\"=\"bao\"}", + "legendFormat": "leases" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "vault_expire_num_irrevocable_leases{\"service.name\"=\"bao\"}", + "legendFormat": "irrevocable" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 30, + "type": "timeseries", + "title": "Heap in use", + "description": "go_memstats_heap_inuse_bytes. Present while sealed, so it is a live line even on a store that is serving nothing — a sawtooth is normal Go behaviour, a monotonic climb on an idle store is not.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "go_memstats_heap_inuse_bytes{\"service.name\"=\"bao\"}", + "legendFormat": "heap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +} diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index e4e40e17..2f5cdbea 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -91,6 +91,46 @@ let # error rather than a warning nobody reads. parsePublished = url: builtins.match "https://([^/]+)(/.*)" url; + # A loopback target may carry a path and a query as well as `host:port`, + # because not every exporter serves `/metrics`: openbao 404s there and + # answers on `/v1/sys/metrics?format=prometheus`. Split for the same reason + # the published one is — one declaration, three prometheus fields. + # + # ⚠️ The query may not ride along in `metrics_path`: prometheus percent- + # encodes the `?`, so the request goes to a path that does not exist. It has + # to become `params`, whose values are lists. + # + # `null` when the shape is wrong, which the assertion below reports by name. + parseScrape = t: builtins.match "([^/?]+)(/[^?]*)?(\\?(.+))?" t; + + queryParams = + query: + lib.listToAttrs ( + map ( + kv: + let + parts = lib.splitString "=" kv; + in + lib.nameValuePair (lib.head parts) [ (lib.concatStringsSep "=" (lib.tail parts)) ] + ) (lib.splitString "&" query) + ); + + # Both optional fields are omitted rather than defaulted, so a plain + # `host:port` renders the config it rendered before this grammar existed. + loopbackScrapeConfig = + job: target: + let + parts = parseScrape target; + path = lib.elemAt parts 1; + query = lib.elemAt parts 3; + in + { + job_name = job; + static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ]; + } + // lib.optionalAttrs (path != null) { metrics_path = path; } + // lib.optionalAttrs (query != null) { params = queryParams query; }; + # The client secret takes three names, and the reason is `DynamicUser`. # # Upstream's collector unit runs with `DynamicUser = true`, and the @@ -332,6 +372,12 @@ in Prometheus exposition endpoints this collector scrapes, as ` = ":"`. + A path and query may follow the port — + `"127.0.0.1:8202/v1/sys/metrics?format=prometheus"` — for an + exporter that does not serve `/metrics`. Both are optional and + omitted when absent, so a bare `host:port` is scraped exactly as + before. + **A service declares its own entry, from its own module, under its own `enable`.** Do not assemble the list here: an entry then exists only where the service that named it runs, so a target is never @@ -711,6 +757,23 @@ in token, and plain http would put it on the wire in clear text. ''; } + { + # Same reason the published one is asserted: a value the grammar + # rejects surfaces as `attempt to call elemAt on null` from inside + # the renderer, naming neither the option nor the offending value. + assertion = lib.all (t: parseScrape t != null) (lib.attrValues cfg.scrapeTargets); + message = '' + services.hyperhive.swarm.otel.scrapeTargets has ${ + lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") ( + lib.filter (kv: parseScrape kv.value == null) (lib.attrsToList cfg.scrapeTargets) + ) + }, which is not of the form :[/][?=]. + + A trailing `?` with nothing after it is the usual cause. This + option is loopback-and-unauthenticated by contract, so it takes + no scheme and no credential. + ''; + } { # A job name used by BOTH scrape options. Within one option this # cannot happen — the module system refuses two definitions of the @@ -927,10 +990,7 @@ in # and deploys perfectly while adding no data. // lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) { prometheus.config.scrape_configs = - lib.mapAttrsToList (job: target: { - job_name = job; - static_configs = [ { targets = [ target ]; } ]; - }) cfg.scrapeTargets + lib.mapAttrsToList loopbackScrapeConfig cfg.scrapeTargets # ⚠️ `++`, so the two kinds of target land in ONE list — # which is exactly why a job name may not appear in both # options. A list concatenation does not resolve a diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 4028a376..b3cb38ab 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -118,6 +118,10 @@ let baoWithCollector = hive { deploy.bao.enable = true; deploy.swarm-otel.enable = true; + # A second job declared as bare `host:port`, so the pair of cases below + # reads one rendered scrape list: the store's entry carries a path, this + # one carries none. + swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; }; baoNoCollector = hive { deploy.bao.enable = true; @@ -133,6 +137,14 @@ let # look there rather than at the host's service set. baoUnits = machine: machine.containers.swarm-bao.config.systemd.services; + # The scrape list prometheus is handed, not the option a service declared: + # the address, the path and the query are one string on the way in and three + # fields on the way out, and only the second shape is what gets requested. + scrapeJob = + machine: job: + lib.findFirst (c: c.job_name == job) null + machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs; + # A priority collision is a property of the *option*, not # of the merged value's interior — nix throws the moment the value is # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is @@ -385,6 +397,28 @@ let in s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s"; } + { + # openbao serves no `/metrics` at all, so a scrape of the default path + # 404s: the store looks like a dead exporter, and every panel built on + # it renders empty rather than erroring. + name = "the store's scrape asks for the path openbao serves"; + ok = + let + j = scrapeJob baoWithCollector "bao"; + in + (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ]; + } + { + # Presence control for the case above: both fields are omitted rather + # than defaulted, so a target declared as bare `host:port` renders what + # it rendered before the path grammar existed. + name = "a target with no path renders neither metrics_path nor params"; + ok = + let + j = scrapeJob baoWithCollector "plain"; + in + j != null && !(j ? metrics_path) && !(j ? params); + } { # Absence arm. Unauthenticated by design, so it must not exist where # nothing reads it.