diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index b43ab84e..f5e632c9 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -195,9 +195,9 @@ let # Metrics get their own listener rather than a flag on the one above, and # that follows from what a scraper can express: `swarm.otel.scrapeTargets` - # carries no scheme and no credential, while the API listener is TLS and — - # once a client CA is set — demands a client certificate. The collector - # cannot reach it at all. + # is `host:port` with no scheme and no credential, while the API listener is + # TLS and — once a client CA is set — demands a client certificate. The + # collector cannot reach it at all. # # `metrics_only` narrows this one to the metrics path (every other path 404s) # and the unauthenticated access is confined to loopback. **Deliberately @@ -576,11 +576,7 @@ in ]; services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere { - # Path and query, not just `host:port`: openbao serves no `/metrics` - # at all, and `/v1/sys/metrics` answers JSON unless the format is - # asked for. A scrape of the default path 404s, which reads as a - # dead exporter rather than a wrong address. - bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus"; + bao = "127.0.0.1:${toString baoDeploy.metricsPort}"; }; }) diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 26c0d8ae..4c4b159b 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -61,7 +61,6 @@ let [ "agents.json" "authelia.json" - "bao.json" "claude-usage.json" "forge.json" "logstore.json" diff --git a/nix/host-modules/swarm-grafana/dashboards/bao.json b/nix/host-modules/swarm-grafana/dashboards/bao.json deleted file mode 100644 index 931f5308..00000000 --- a/nix/host-modules/swarm-grafana/dashboards/bao.json +++ /dev/null @@ -1,569 +0,0 @@ -{ - "title": "hyperhive · openbao", - "uid": "hyperhive-bao", - "description": "The swarm's secret store. Two things to know before editing: the metric prefix is vault_, not openbao_ — the fork kept upstream's names, and a panel built on the name you would guess renders perfectly and shows nothing. And a SEALED store emits almost nothing: only the top row works before an operator has run `bao operator init`, everything below it is empty until then and that is the expected reading, not a broken query. Targets are labelled service.name, NOT job; there is no job label in this store. Counters are counts over the dashboard range rather than per-second rates, because a store this quiet rates to exactly zero.", - "editable": true, - "refresh": "1m", - "schemaVersion": 39, - "tags": [ - "hyperhive", - "swarm", - "bao", - "secrets" - ], - "time": { - "from": "now-24h", - "to": "now" - }, - "timezone": "utc", - "version": 1, - "templating": { - "list": [] - }, - "panels": [ - { - "id": 1, - "type": "stat", - "title": "Seal state", - "description": "SEALED means the store is up but refusing every read — the state it boots into, and the state it returns to on restart unless auto-unseal is configured. The max() is load-bearing: once unsealed there are two vault_core_unsealed series and the cluster=\"\" one stays 0 forever, so a bare selector reads SEALED on a healthy store.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "max(vault_core_unsealed{\"service.name\"=\"bao\"})", - "instant": true, - "legendFormat": "seal" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "SEALED", - "color": "red", - "index": 0 - }, - "1": { - "text": "unsealed", - "color": "green", - "index": 1 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "background", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - } - }, - { - "id": 2, - "type": "stat", - "title": "Uptime", - "description": "Time since the openbao process started. Read it beside the seal state: a restart re-seals the store, so a short uptime and a SEALED panel are one event rather than two problems.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "time() - process_start_time_seconds{\"service.name\"=\"bao\"}", - "instant": true, - "legendFormat": "uptime" - } - ], - "fieldConfig": { - "defaults": { - "unit": "s", - "decimals": 0, - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - } - }, - { - "id": 3, - "type": "stat", - "title": "Goroutines", - "description": "A Go runtime metric, so it is present even while sealed — which makes it the panel that distinguishes a store with nothing to say from a scrape that is not arriving at all. No data here means the collector is not reaching the metrics listener.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "go_goroutines{\"service.name\"=\"bao\"}", - "instant": true, - "legendFormat": "goroutines" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - } - }, - { - "id": 4, - "type": "stat", - "title": "Requests (range)", - "description": "vault_core_handle_request_count over the dashboard range — every API call the store served, unseal attempts included. The summary's _count series, not the summary itself. Zero on a sealed store, and zero is the honest answer there rather than an absent panel.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase(vault_core_handle_request_count{\"service.name\"=\"bao\"}[$__range])) or vector(0)", - "instant": true, - "legendFormat": "requests" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - } - }, - { - "id": 10, - "type": "stat", - "title": "Seal operations (range)", - "description": "Encrypt and decrypt calls against the seal wrapper — on this swarm a TPM through PKCS#11, so each one is a round trip to hardware. These are counters with no _total suffix. A climbing decrypt count with a flat request count means something is re-reading the barrier rather than serving clients.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 4 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase(vault_seal_encrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)", - "instant": true, - "legendFormat": "encrypt" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "sum(increase(vault_seal_decrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)", - "instant": true, - "legendFormat": "decrypt" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value_and_name", - "justifyMode": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - } - }, - { - "id": 11, - "type": "stat", - "title": "Raft peers", - "description": "Voting members of the store's raft cluster. This swarm runs a single node, so 1 is the expected value and anything else means a peer was added — or that this node no longer sees itself as part of the cluster it stores data for.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 4 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "vault_raft_peers{\"service.name\"=\"bao\"}", - "instant": true, - "legendFormat": "peers" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "fixed", - "fixedColor": "text" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "graphMode": "none", - "colorMode": "value", - "textMode": "value", - "justifyMode": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - } - }, - { - "id": 20, - "type": "timeseries", - "title": "Request latency", - "description": "Quantiles openbao computes itself — a summary, so the store picked the buckets and a histogram_quantile() is neither needed nor possible here. Units are MILLISECONDS: vault's timers report ms, and reading them as seconds understates the store by three orders of magnitude.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.99\"}", - "legendFormat": "p99" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.5\"}", - "legendFormat": "p50" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "decimals": 1, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 0, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 21, - "type": "timeseries", - "title": "Leases", - "description": "Outstanding leases, and the irrevocable ones beside them. The second line should sit at zero: an irrevocable lease is one the store gave up trying to revoke, so a credential somewhere outlives what issued it and no amount of waiting will clean it up.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "vault_expire_num_leases{\"service.name\"=\"bao\"}", - "legendFormat": "leases" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "vault_expire_num_irrevocable_leases{\"service.name\"=\"bao\"}", - "legendFormat": "irrevocable" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 30, - "type": "timeseries", - "title": "Heap in use", - "description": "go_memstats_heap_inuse_bytes. Present while sealed, so it is a live line even on a store that is serving nothing — a sawtooth is normal Go behaviour, a monotonic climb on an idle store is not.", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 16 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "@datasourceUid@" - }, - "expr": "go_memstats_heap_inuse_bytes{\"service.name\"=\"bao\"}", - "legendFormat": "heap" - } - ], - "fieldConfig": { - "defaults": { - "unit": "bytes", - "decimals": 1, - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false, - "axisSoftMin": 0 - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - } - ] -} diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 2f5cdbea..e4e40e17 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -91,46 +91,6 @@ let # error rather than a warning nobody reads. parsePublished = url: builtins.match "https://([^/]+)(/.*)" url; - # A loopback target may carry a path and a query as well as `host:port`, - # because not every exporter serves `/metrics`: openbao 404s there and - # answers on `/v1/sys/metrics?format=prometheus`. Split for the same reason - # the published one is — one declaration, three prometheus fields. - # - # ⚠️ The query may not ride along in `metrics_path`: prometheus percent- - # encodes the `?`, so the request goes to a path that does not exist. It has - # to become `params`, whose values are lists. - # - # `null` when the shape is wrong, which the assertion below reports by name. - parseScrape = t: builtins.match "([^/?]+)(/[^?]*)?(\\?(.+))?" t; - - queryParams = - query: - lib.listToAttrs ( - map ( - kv: - let - parts = lib.splitString "=" kv; - in - lib.nameValuePair (lib.head parts) [ (lib.concatStringsSep "=" (lib.tail parts)) ] - ) (lib.splitString "&" query) - ); - - # Both optional fields are omitted rather than defaulted, so a plain - # `host:port` renders the config it rendered before this grammar existed. - loopbackScrapeConfig = - job: target: - let - parts = parseScrape target; - path = lib.elemAt parts 1; - query = lib.elemAt parts 3; - in - { - job_name = job; - static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ]; - } - // lib.optionalAttrs (path != null) { metrics_path = path; } - // lib.optionalAttrs (query != null) { params = queryParams query; }; - # The client secret takes three names, and the reason is `DynamicUser`. # # Upstream's collector unit runs with `DynamicUser = true`, and the @@ -372,12 +332,6 @@ in Prometheus exposition endpoints this collector scrapes, as ` = ":"`. - A path and query may follow the port — - `"127.0.0.1:8202/v1/sys/metrics?format=prometheus"` — for an - exporter that does not serve `/metrics`. Both are optional and - omitted when absent, so a bare `host:port` is scraped exactly as - before. - **A service declares its own entry, from its own module, under its own `enable`.** Do not assemble the list here: an entry then exists only where the service that named it runs, so a target is never @@ -757,23 +711,6 @@ in token, and plain http would put it on the wire in clear text. ''; } - { - # Same reason the published one is asserted: a value the grammar - # rejects surfaces as `attempt to call elemAt on null` from inside - # the renderer, naming neither the option nor the offending value. - assertion = lib.all (t: parseScrape t != null) (lib.attrValues cfg.scrapeTargets); - message = '' - services.hyperhive.swarm.otel.scrapeTargets has ${ - lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") ( - lib.filter (kv: parseScrape kv.value == null) (lib.attrsToList cfg.scrapeTargets) - ) - }, which is not of the form :[/][?=]. - - A trailing `?` with nothing after it is the usual cause. This - option is loopback-and-unauthenticated by contract, so it takes - no scheme and no credential. - ''; - } { # A job name used by BOTH scrape options. Within one option this # cannot happen — the module system refuses two definitions of the @@ -990,7 +927,10 @@ in # and deploys perfectly while adding no data. // lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) { prometheus.config.scrape_configs = - lib.mapAttrsToList loopbackScrapeConfig cfg.scrapeTargets + lib.mapAttrsToList (job: target: { + job_name = job; + static_configs = [ { targets = [ target ]; } ]; + }) cfg.scrapeTargets # ⚠️ `++`, so the two kinds of target land in ONE list — # which is exactly why a job name may not appear in both # options. A list concatenation does not resolve a diff --git a/nix/module-eval.nix b/nix/module-eval.nix index b3cb38ab..4028a376 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -118,10 +118,6 @@ let baoWithCollector = hive { deploy.bao.enable = true; deploy.swarm-otel.enable = true; - # A second job declared as bare `host:port`, so the pair of cases below - # reads one rendered scrape list: the store's entry carries a path, this - # one carries none. - swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; }; baoNoCollector = hive { deploy.bao.enable = true; @@ -137,14 +133,6 @@ let # look there rather than at the host's service set. baoUnits = machine: machine.containers.swarm-bao.config.systemd.services; - # The scrape list prometheus is handed, not the option a service declared: - # the address, the path and the query are one string on the way in and three - # fields on the way out, and only the second shape is what gets requested. - scrapeJob = - machine: job: - lib.findFirst (c: c.job_name == job) null - machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs; - # A priority collision is a property of the *option*, not # of the merged value's interior — nix throws the moment the value is # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is @@ -397,28 +385,6 @@ let in s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s"; } - { - # openbao serves no `/metrics` at all, so a scrape of the default path - # 404s: the store looks like a dead exporter, and every panel built on - # it renders empty rather than erroring. - name = "the store's scrape asks for the path openbao serves"; - ok = - let - j = scrapeJob baoWithCollector "bao"; - in - (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ]; - } - { - # Presence control for the case above: both fields are omitted rather - # than defaulted, so a target declared as bare `host:port` renders what - # it rendered before the path grammar existed. - name = "a target with no path renders neither metrics_path nor params"; - ok = - let - j = scrapeJob baoWithCollector "plain"; - in - j != null && !(j ? metrics_path) && !(j ? params); - } { # Absence arm. Unauthenticated by design, so it must not exist where # nothing reads it.