From ae2ecab9f2749eca5d1603989adf7525e528d9d8 Mon Sep 17 00:00:00 2001 From: iris Date: Wed, 28 Oct 2026 18:01:10 +0100 Subject: [PATCH 1/3] swarm-grafana: the bao dashboard reads openbao's journal The bao board was nine metrics panels, so a gap in log delivery looked like a quiet bao. Three panels read the log store for openbao.service: lines delivered over time, WARN and ERROR lines over time, and the recent WARN and ERROR lines themselves. A module-eval case pins them by query. --- .../swarm-grafana/dashboards/bao.json | 151 ++++++++++++++++++ nix/module-eval/grafana.nix | 19 +++ 2 files changed, 170 insertions(+) diff --git a/nix/host-modules/swarm-grafana/dashboards/bao.json b/nix/host-modules/swarm-grafana/dashboards/bao.json index cc3af46dc..c466739d4 100644 --- a/nix/host-modules/swarm-grafana/dashboards/bao.json +++ b/nix/host-modules/swarm-grafana/dashboards/bao.json @@ -547,6 +547,157 @@ "sort": "desc" } } + }, + { + "id": 40, + "type": "timeseries", + "title": "Log lines delivered", + "description": "Lines from openbao.service that reached the log store.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"openbao.service\" | stats count() as lines", + "queryType": "statsRange", + "legendFormat": "lines" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 60, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 41, + "type": "timeseries", + "title": "Warnings and errors", + "description": "WARN and ERROR lines from openbao.service.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"openbao.service\" _msg:~\"\\\\[(WARN|ERROR)\\\\]\" | stats count() as lines", + "queryType": "statsRange", + "legendFormat": "lines" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 60, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 42, + "type": "logs", + "title": "Recent warnings and errors", + "description": "The WARN and ERROR lines themselves, newest first.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"openbao.service\" _msg:~\"\\\\[(WARN|ERROR)\\\\]\"", + "queryType": "instant" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + } } ] } diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 0db99b6a6..b87ae67a4 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -691,6 +691,25 @@ let && builtins.length selectors == 12 && lib.all (e: lib.hasInfix "\"service.name\"=\"${stamped.value}\"" e) selectors; } + { + # The bao board's metrics panels can show a gap in the metrics, never a + # gap in the logs: "bao is quiet" and "bao's logs are not arriving" look + # the same there. These panels read the journal itself. Asserted on the + # query, as for the logs board, because a panel that keeps its title and + # loses its expression renders an empty graph. + name = "the bao dashboard reads openbao's journal from the log store"; + ok = + let + board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/bao.json); + logPanels = lib.filter ( + p: (p.datasource.type or "") == "victoriametrics-logs-datasource" + ) board.panels; + exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) logPanels; + in + builtins.length logPanels == 3 + && lib.all (e: lib.hasInfix "_SYSTEMD_UNIT:\"openbao.service\"" e) exprs + && lib.any (p: p.type == "logs") logPanels; + } { # A module declares its vhost under `gateway.vhosts` and the gateway adds # what it owns: the listen set, the cert for the name and the security From 2472e3868dce17db4683c97e31d41e2ded19e5d9 Mon Sep 17 00:00:00 2001 From: iris Date: Wed, 28 Oct 2026 18:05:41 +0100 Subject: [PATCH 2/3] docs: say what the bao dashboard's log panels show The classifier asked for the docs to move with the three new panels. bao.md gets a short 'Watching the store' section, including that a flat zero in the delivered-lines panel means delivery is broken, not that bao is quiet. --- docs/swarm/bao.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/swarm/bao.md b/docs/swarm/bao.md index e5b41f13f..cf0a33758 100644 --- a/docs/swarm/bao.md +++ b/docs/swarm/bao.md @@ -97,6 +97,16 @@ constrain what a policy says. What bounds it: responds while every read times out — the failure looks like a stall, not like a store that was never initialised. +## Watching the store + +The `hyperhive · openbao` dashboard in Grafana shows seal state, uptime, +request latency, leases and heap from the store's own metrics. Three more +panels read its journal: the lines delivered, the `[WARN]` and `[ERROR]` lines +over time, and the recent ones. A flat zero in **Log lines delivered** while +the store is up means its logs aren't arriving, not that it's quiet. The +journal gives these lines no severity, so the panels match on the level the +store writes into the message. + ## TLS The store serves TLS, and on a host that deploys it you need do nothing: From 698183c16aa78d34d2cfba433d0c88f2a4ab5c49 Mon Sep 17 00:00:00 2001 From: iris Date: Wed, 28 Oct 2026 18:06:58 +0100 Subject: [PATCH 3/3] docs: drop the bao dashboard section again; a dashboard needs no page Reverts the previous commit: the panels describe themselves, and the operator ruled that this change needs no docs page. --- docs/swarm/bao.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/swarm/bao.md b/docs/swarm/bao.md index cf0a33758..e5b41f13f 100644 --- a/docs/swarm/bao.md +++ b/docs/swarm/bao.md @@ -97,16 +97,6 @@ constrain what a policy says. What bounds it: responds while every read times out — the failure looks like a stall, not like a store that was never initialised. -## Watching the store - -The `hyperhive · openbao` dashboard in Grafana shows seal state, uptime, -request latency, leases and heap from the store's own metrics. Three more -panels read its journal: the lines delivered, the `[WARN]` and `[ERROR]` lines -over time, and the recent ones. A flat zero in **Log lines delivered** while -the store is up means its logs aren't arriving, not that it's quiet. The -journal gives these lines no severity, so the panels match on the level the -store writes into the message. - ## TLS The store serves TLS, and on a host that deploys it you need do nothing: