vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

Compare commits

base: vinzenz:fe81dcaf59b3aa9c8334ffc67fd06a51c5297cd7
Branches Tags
vinzenz:main
...
compare: vinzenz:a3612f5168be4c6945d8677f5f5d3575deae59bd
Branches Tags
vinzenz:main
Author SHA1 Message Date
atlas
a3612f5168 feat(#3849): a grafana board for the swarm's secret store 
Nine panels against the metric names openbao actually emits — the prefix is
`vault_`, the fork kept upstream's names, and a board built on `openbao_`
would render perfectly and show nothing.

The top row works on a SEALED store, which is what a hive has until an
operator runs `operator init`: seal state, uptime, goroutines and a request
count that reads 0 rather than going blank. Everything below is empty until
then, and each panel's description says so rather than leaving an operator to
wonder which of the two it is looking at.

The seal panel reduces with `max()`. Once unsealed there are two
`vault_core_unsealed` series and the `cluster=""` one stays 0 forever, so a
bare selector reads SEALED on a healthy store — invisible while sealed, which
is why the metric names were sampled in both states.
 2026-09-01 11:10:30 +02:00
atlas
7eb5c92d15 fix(#3849): scrape the path openbao actually serves 
The store's scrape target rendered as `host:port` alone, so the collector
requested `/metrics` — a path openbao does not serve on any listener. Every
scrape since the target landed has 404'd, which reads as a dead exporter
rather than a wrong address, and no bao sample has ever reached the store.

`scrapeTargets` values may now carry a path and query. Both are optional and
omitted when absent, so every existing target renders the config it rendered
before. The query cannot ride in `metrics_path`: prometheus percent-encodes
the `?`, so it has to become `params`.

Measured against openbao 2.6.2 and otelcol-contrib 0.151.0, the pinned
versions: `/metrics` 404s ("this listener only serves the metrics endpoint"),
`/v1/sys/metrics` answers JSON, and only `?format=prometheus` — or an Accept
header the collector happens to send today — returns exposition text. The
query param is the half that does not depend on content negotiation.
 2026-09-01 11:10:30 +02:00
5 changed files with 676 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

12
nix/host-modules/swarm-bao.nix
View file

@ -195,9 +195,9 @@ let
# Metrics get their own listener rather than a flag on the one above, and
# that follows from what a scraper can express: `swarm.otel.scrapeTargets`
# is `host:port` with no scheme and no credential, while the API listener is
# TLS and — once a client CA is set — demands a client certificate. The
# collector cannot reach it at all.
# carries no scheme and no credential, while the API listener is TLS and —
# once a client CA is set — demands a client certificate. The collector
# cannot reach it at all.
#
# `metrics_only` narrows this one to the metrics path (every other path 404s)
# and the unauthenticated access is confined to loopback. **Deliberately
@ -576,7 +576,11 @@ in
];
services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere {
bao = "127.0.0.1:${toString baoDeploy.metricsPort}";
# Path and query, not just `host:port`: openbao serves no `/metrics`
# at all, and `/v1/sys/metrics` answers JSON unless the format is
# asked for. A scrape of the default path 404s, which reads as a
# dead exporter rather than a wrong address.
bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus";
};
})

1
nix/host-modules/swarm-grafana.nix
View file

@ -61,6 +61,7 @@ let
[
"agents.json"
"authelia.json"
"bao.json"
"claude-usage.json"
"forge.json"
"logstore.json"

569
nix/host-modules/swarm-grafana/dashboards/bao.json Normal file
View file

@ -0,0 +1,569 @@
{
"title": "hyperhive · openbao",
"uid": "hyperhive-bao",
"description": "The swarm's secret store. Two things to know before editing: the metric prefix is vault_, not openbao_ — the fork kept upstream's names, and a panel built on the name you would guess renders perfectly and shows nothing. And a SEALED store emits almost nothing: only the top row works before an operator has run `bao operator init`, everything below it is empty until then and that is the expected reading, not a broken query. Targets are labelled service.name, NOT job; there is no job label in this store. Counters are counts over the dashboard range rather than per-second rates, because a store this quiet rates to exactly zero.",
"editable": true,
"refresh": "1m",
"schemaVersion": 39,
"tags": [
"hyperhive",
"swarm",
"bao",
"secrets"
],
"time": {
"from": "now-24h",
"to": "now"
},
"timezone": "utc",
"version": 1,
"templating": {
"list": []
},
"panels": [
{
"id": 1,
"type": "stat",
"title": "Seal state",
"description": "SEALED means the store is up but refusing every read — the state it boots into, and the state it returns to on restart unless auto-unseal is configured. The max() is load-bearing: once unsealed there are two vault_core_unsealed series and the cluster=\"\" one stays 0 forever, so a bare selector reads SEALED on a healthy store.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max(vault_core_unsealed{\"service.name\"=\"bao\"})",
"instant": true,
"legendFormat": "seal"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "thresholds"
},
"mappings": [
{
"type": "value",
"options": {
"0": {
"text": "SEALED",
"color": "red",
"index": 0
},
"1": {
"text": "unsealed",
"color": "green",
"index": 1
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "background",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 2,
"type": "stat",
"title": "Uptime",
"description": "Time since the openbao process started. Read it beside the seal state: a restart re-seals the store, so a short uptime and a SEALED panel are one event rather than two problems.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - process_start_time_seconds{\"service.name\"=\"bao\"}",
"instant": true,
"legendFormat": "uptime"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "text"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 3,
"type": "stat",
"title": "Goroutines",
"description": "A Go runtime metric, so it is present even while sealed — which makes it the panel that distinguishes a store with nothing to say from a scrape that is not arriving at all. No data here means the collector is not reaching the metrics listener.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "go_goroutines{\"service.name\"=\"bao\"}",
"instant": true,
"legendFormat": "goroutines"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "text"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 4,
"type": "stat",
"title": "Requests (range)",
"description": "vault_core_handle_request_count over the dashboard range — every API call the store served, unseal attempts included. The summary's _count series, not the summary itself. Zero on a sealed store, and zero is the honest answer there rather than an absent panel.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase(vault_core_handle_request_count{\"service.name\"=\"bao\"}[$__range])) or vector(0)",
"instant": true,
"legendFormat": "requests"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "text"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 10,
"type": "stat",
"title": "Seal operations (range)",
"description": "Encrypt and decrypt calls against the seal wrapper — on this swarm a TPM through PKCS#11, so each one is a round trip to hardware. These are counters with no _total suffix. A climbing decrypt count with a flat request count means something is re-reading the barrier rather than serving clients.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 12,
"x": 0,
"y": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase(vault_seal_encrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)",
"instant": true,
"legendFormat": "encrypt"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "sum(increase(vault_seal_decrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)",
"instant": true,
"legendFormat": "decrypt"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value_and_name",
"justifyMode": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 11,
"type": "stat",
"title": "Raft peers",
"description": "Voting members of the store's raft cluster. This swarm runs a single node, so 1 is the expected value and anything else means a peer was added — or that this node no longer sees itself as part of the cluster it stores data for.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 4,
"w": 12,
"x": 12,
"y": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "vault_raft_peers{\"service.name\"=\"bao\"}",
"instant": true,
"legendFormat": "peers"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "text"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
}
},
{
"id": 20,
"type": "timeseries",
"title": "Request latency",
"description": "Quantiles openbao computes itself — a summary, so the store picked the buckets and a histogram_quantile() is neither needed nor possible here. Units are MILLISECONDS: vault's timers report ms, and reading them as seconds understates the store by three orders of magnitude.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.99\"}",
"legendFormat": "p99"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.5\"}",
"legendFormat": "p50"
}
],
"fieldConfig": {
"defaults": {
"unit": "ms",
"decimals": 1,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 0,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 21,
"type": "timeseries",
"title": "Leases",
"description": "Outstanding leases, and the irrevocable ones beside them. The second line should sit at zero: an irrevocable lease is one the store gave up trying to revoke, so a credential somewhere outlives what issued it and no amount of waiting will clean it up.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "vault_expire_num_leases{\"service.name\"=\"bao\"}",
"legendFormat": "leases"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "vault_expire_num_irrevocable_leases{\"service.name\"=\"bao\"}",
"legendFormat": "irrevocable"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 30,
"type": "timeseries",
"title": "Heap in use",
"description": "go_memstats_heap_inuse_bytes. Present while sealed, so it is a live line even on a store that is serving nothing — a sawtooth is normal Go behaviour, a monotonic climb on an idle store is not.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 16
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "go_memstats_heap_inuse_bytes{\"service.name\"=\"bao\"}",
"legendFormat": "heap"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"decimals": 1,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
}
]
}

68
nix/host-modules/swarm-otel.nix
View file

@ -91,6 +91,46 @@ let
# error rather than a warning nobody reads.
parsePublished = url: builtins.match "https://([^/]+)(/.*)" url;
# A loopback target may carry a path and a query as well as `host:port`,
# because not every exporter serves `/metrics`: openbao 404s there and
# answers on `/v1/sys/metrics?format=prometheus`. Split for the same reason
# the published one is — one declaration, three prometheus fields.
#
# ⚠️ The query may not ride along in `metrics_path`: prometheus percent-
# encodes the `?`, so the request goes to a path that does not exist. It has
# to become `params`, whose values are lists.
#
# `null` when the shape is wrong, which the assertion below reports by name.
parseScrape = t: builtins.match "([^/?]+)(/[^?]*)?(\\?(.+))?" t;
queryParams =
query:
lib.listToAttrs (
map (
kv:
let
parts = lib.splitString "=" kv;
in
lib.nameValuePair (lib.head parts) [ (lib.concatStringsSep "=" (lib.tail parts)) ]
) (lib.splitString "&" query)
);
# Both optional fields are omitted rather than defaulted, so a plain
# `host:port` renders the config it rendered before this grammar existed.
loopbackScrapeConfig =
job: target:
let
parts = parseScrape target;
path = lib.elemAt parts 1;
query = lib.elemAt parts 3;
in
{
job_name = job;
static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ];
}
// lib.optionalAttrs (path != null) { metrics_path = path; }
// lib.optionalAttrs (query != null) { params = queryParams query; };
# The client secret takes three names, and the reason is `DynamicUser`.
#
# Upstream's collector unit runs with `DynamicUser = true`, and the
@ -332,6 +372,12 @@ in
Prometheus exposition endpoints this collector scrapes, as
`<job name> = "<host>:<port>"`.
A path and query may follow the port —
`"127.0.0.1:8202/v1/sys/metrics?format=prometheus"` — for an
exporter that does not serve `/metrics`. Both are optional and
omitted when absent, so a bare `host:port` is scraped exactly as
before.
**A service declares its own entry, from its own module, under its
own `enable`.** Do not assemble the list here: an entry then exists
only where the service that named it runs, so a target is never
@ -711,6 +757,23 @@ in
token, and plain http would put it on the wire in clear text.
'';
}
{
# Same reason the published one is asserted: a value the grammar
# rejects surfaces as `attempt to call elemAt on null` from inside
# the renderer, naming neither the option nor the offending value.
assertion = lib.all (t: parseScrape t != null) (lib.attrValues cfg.scrapeTargets);
message = ''
services.hyperhive.swarm.otel.scrapeTargets has ${
lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") (
lib.filter (kv: parseScrape kv.value == null) (lib.attrsToList cfg.scrapeTargets)
)
}, which is not of the form <host>:<port>[/<path>][?<k>=<v>].
A trailing `?` with nothing after it is the usual cause. This
option is loopback-and-unauthenticated by contract, so it takes
no scheme and no credential.
'';
}
{
# A job name used by BOTH scrape options. Within one option this
# cannot happen — the module system refuses two definitions of the
@ -927,10 +990,7 @@ in
# and deploys perfectly while adding no data.
// lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) {
prometheus.config.scrape_configs =
lib.mapAttrsToList (job: target: {
job_name = job;
static_configs = [ { targets = [ target ]; } ];
}) cfg.scrapeTargets
lib.mapAttrsToList loopbackScrapeConfig cfg.scrapeTargets
# ⚠️ `++`, so the two kinds of target land in ONE list —
# which is exactly why a job name may not appear in both
# options. A list concatenation does not resolve a

34
nix/module-eval.nix
View file

@ -118,6 +118,10 @@ let
baoWithCollector = hive {
deploy.bao.enable = true;
deploy.swarm-otel.enable = true;
# A second job declared as bare `host:port`, so the pair of cases below
# reads one rendered scrape list: the store's entry carries a path, this
# one carries none.
swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
};
baoNoCollector = hive {
deploy.bao.enable = true;
@ -133,6 +137,14 @@ let
# look there rather than at the host's service set.
baoUnits = machine: machine.containers.swarm-bao.config.systemd.services;
# The scrape list prometheus is handed, not the option a service declared:
# the address, the path and the query are one string on the way in and three
# fields on the way out, and only the second shape is what gets requested.
scrapeJob =
machine: job:
lib.findFirst (c: c.job_name == job) null
machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs;
# A priority collision is a property of the *option*, not
# of the merged value's interior — nix throws the moment the value is
# demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
@ -385,6 +397,28 @@ let
in
s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
}
{
# openbao serves no `/metrics` at all, so a scrape of the default path
# 404s: the store looks like a dead exporter, and every panel built on
# it renders empty rather than erroring.
name = "the store's scrape asks for the path openbao serves";
ok =
let
j = scrapeJob baoWithCollector "bao";
in
(j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ];
}
{
# Presence control for the case above: both fields are omitted rather
# than defaulted, so a target declared as bare `host:port` renders what
# it rendered before the path grammar existed.
name = "a target with no path renders neither metrics_path nor params";
ok =
let
j = scrapeJob baoWithCollector "plain";
in
j != null && !(j ? metrics_path) && !(j ? params);
}
{
# Absence arm. Unauthenticated by design, so it must not exist where
# nothing reads it.