a3612f5168
Nine panels against the metric names openbao actually emits — the prefix is `vault_`, the fork kept upstream's names, and a board built on `openbao_` would render perfectly and show nothing. The top row works on a SEALED store, which is what a hive has until an operator runs `operator init`: seal state, uptime, goroutines and a request count that reads 0 rather than going blank. Everything below is empty until then, and each panel's description says so rather than leaving an operator to wonder which of the two it is looking at. The seal panel reduces with `max()`. Once unsealed there are two `vault_core_unsealed` series and the `cluster=""` one stays 0 forever, so a bare selector reads SEALED on a healthy store — invisible while sealed, which is why the metric names were sampled in both states.
569 lines
15 KiB
JSON
569 lines
15 KiB
JSON
{
|
|
"title": "hyperhive · openbao",
|
|
"uid": "hyperhive-bao",
|
|
"description": "The swarm's secret store. Two things to know before editing: the metric prefix is vault_, not openbao_ — the fork kept upstream's names, and a panel built on the name you would guess renders perfectly and shows nothing. And a SEALED store emits almost nothing: only the top row works before an operator has run `bao operator init`, everything below it is empty until then and that is the expected reading, not a broken query. Targets are labelled service.name, NOT job; there is no job label in this store. Counters are counts over the dashboard range rather than per-second rates, because a store this quiet rates to exactly zero.",
|
|
"editable": true,
|
|
"refresh": "1m",
|
|
"schemaVersion": 39,
|
|
"tags": [
|
|
"hyperhive",
|
|
"swarm",
|
|
"bao",
|
|
"secrets"
|
|
],
|
|
"time": {
|
|
"from": "now-24h",
|
|
"to": "now"
|
|
},
|
|
"timezone": "utc",
|
|
"version": 1,
|
|
"templating": {
|
|
"list": []
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "stat",
|
|
"title": "Seal state",
|
|
"description": "SEALED means the store is up but refusing every read — the state it boots into, and the state it returns to on restart unless auto-unseal is configured. The max() is load-bearing: once unsealed there are two vault_core_unsealed series and the cluster=\"\" one stays 0 forever, so a bare selector reads SEALED on a healthy store.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "max(vault_core_unsealed{\"service.name\"=\"bao\"})",
|
|
"instant": true,
|
|
"legendFormat": "seal"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [
|
|
{
|
|
"type": "value",
|
|
"options": {
|
|
"0": {
|
|
"text": "SEALED",
|
|
"color": "red",
|
|
"index": 0
|
|
},
|
|
"1": {
|
|
"text": "unsealed",
|
|
"color": "green",
|
|
"index": 1
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "background",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "Uptime",
|
|
"description": "Time since the openbao process started. Read it beside the seal state: a restart re-seals the store, so a short uptime and a SEALED panel are one event rather than two problems.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "time() - process_start_time_seconds{\"service.name\"=\"bao\"}",
|
|
"instant": true,
|
|
"legendFormat": "uptime"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
},
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "Goroutines",
|
|
"description": "A Go runtime metric, so it is present even while sealed — which makes it the panel that distinguishes a store with nothing to say from a scrape that is not arriving at all. No data here means the collector is not reaching the metrics listener.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "go_goroutines{\"service.name\"=\"bao\"}",
|
|
"instant": true,
|
|
"legendFormat": "goroutines"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
},
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "Requests (range)",
|
|
"description": "vault_core_handle_request_count over the dashboard range — every API call the store served, unseal attempts included. The summary's _count series, not the summary itself. Zero on a sealed store, and zero is the honest answer there rather than an absent panel.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum(increase(vault_core_handle_request_count{\"service.name\"=\"bao\"}[$__range])) or vector(0)",
|
|
"instant": true,
|
|
"legendFormat": "requests"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
},
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 10,
|
|
"type": "stat",
|
|
"title": "Seal operations (range)",
|
|
"description": "Encrypt and decrypt calls against the seal wrapper — on this swarm a TPM through PKCS#11, so each one is a round trip to hardware. These are counters with no _total suffix. A climbing decrypt count with a flat request count means something is re-reading the barrier rather than serving clients.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 4
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum(increase(vault_seal_encrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)",
|
|
"instant": true,
|
|
"legendFormat": "encrypt"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "sum(increase(vault_seal_decrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)",
|
|
"instant": true,
|
|
"legendFormat": "decrypt"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value_and_name",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "stat",
|
|
"title": "Raft peers",
|
|
"description": "Voting members of the store's raft cluster. This swarm runs a single node, so 1 is the expected value and anything else means a peer was added — or that this node no longer sees itself as part of the cluster it stores data for.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 4
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "vault_raft_peers{\"service.name\"=\"bao\"}",
|
|
"instant": true,
|
|
"legendFormat": "peers"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
},
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"graphMode": "none",
|
|
"colorMode": "value",
|
|
"textMode": "value",
|
|
"justifyMode": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 20,
|
|
"type": "timeseries",
|
|
"title": "Request latency",
|
|
"description": "Quantiles openbao computes itself — a summary, so the store picked the buckets and a histogram_quantile() is neither needed nor possible here. Units are MILLISECONDS: vault's timers report ms, and reading them as seconds understates the store by three orders of magnitude.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.99\"}",
|
|
"legendFormat": "p99"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.5\"}",
|
|
"legendFormat": "p50"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"decimals": 1,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 0,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"axisSoftMin": 0
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true,
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 21,
|
|
"type": "timeseries",
|
|
"title": "Leases",
|
|
"description": "Outstanding leases, and the irrevocable ones beside them. The second line should sit at zero: an irrevocable lease is one the store gave up trying to revoke, so a credential somewhere outlives what issued it and no amount of waiting will clean it up.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 8
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "vault_expire_num_leases{\"service.name\"=\"bao\"}",
|
|
"legendFormat": "leases"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "vault_expire_num_irrevocable_leases{\"service.name\"=\"bao\"}",
|
|
"legendFormat": "irrevocable"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"axisSoftMin": 0
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true,
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 30,
|
|
"type": "timeseries",
|
|
"title": "Heap in use",
|
|
"description": "go_memstats_heap_inuse_bytes. Present while sealed, so it is a live line even on a store that is serving nothing — a sawtooth is normal Go behaviour, a monotonic climb on an idle store is not.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "@datasourceUid@"
|
|
},
|
|
"expr": "go_memstats_heap_inuse_bytes{\"service.name\"=\"bao\"}",
|
|
"legendFormat": "heap"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"decimals": 1,
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"axisSoftMin": 0
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true,
|
|
"calcs": []
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|