From ae2ecab9f2749eca5d1603989adf7525e528d9d8 Mon Sep 17 00:00:00 2001
From: iris <iris@hyperhive.local>
Date: Wed, 28 Oct 2026 18:01:10 +0100
Subject: [PATCH] swarm-grafana: the bao dashboard reads openbao's journal

The bao board was nine metrics panels, so a gap in log delivery looked like
a quiet bao. Three panels read the log store for openbao.service: lines
delivered over time, WARN and ERROR lines over time, and the recent WARN and
ERROR lines themselves. A module-eval case pins them by query.
---
 .../swarm-grafana/dashboards/bao.json         | 151 ++++++++++++++++++
 nix/module-eval/grafana.nix                   |  19 +++
 2 files changed, 170 insertions(+)

diff --git a/nix/host-modules/swarm-grafana/dashboards/bao.json b/nix/host-modules/swarm-grafana/dashboards/bao.json
index cc3af46d..c466739d 100644
--- a/nix/host-modules/swarm-grafana/dashboards/bao.json
+++ b/nix/host-modules/swarm-grafana/dashboards/bao.json
@@ -547,6 +547,157 @@
           "sort": "desc"
         }
       }
+    },
+    {
+      "id": 40,
+      "type": "timeseries",
+      "title": "Log lines delivered",
+      "description": "Lines from openbao.service that reached the log store.",
+      "datasource": {
+        "type": "victoriametrics-logs-datasource",
+        "uid": "@logsDatasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 22
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "victoriametrics-logs-datasource",
+            "uid": "@logsDatasourceUid@"
+          },
+          "expr": "_SYSTEMD_UNIT:\"openbao.service\" | stats count() as lines",
+          "queryType": "statsRange",
+          "legendFormat": "lines"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "bars",
+            "lineWidth": 1,
+            "fillOpacity": 60,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 41,
+      "type": "timeseries",
+      "title": "Warnings and errors",
+      "description": "WARN and ERROR lines from openbao.service.",
+      "datasource": {
+        "type": "victoriametrics-logs-datasource",
+        "uid": "@logsDatasourceUid@"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 22
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "victoriametrics-logs-datasource",
+            "uid": "@logsDatasourceUid@"
+          },
+          "expr": "_SYSTEMD_UNIT:\"openbao.service\" _msg:~\"\\\\[(WARN|ERROR)\\\\]\" | stats count() as lines",
+          "queryType": "statsRange",
+          "legendFormat": "lines"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "bars",
+            "lineWidth": 1,
+            "fillOpacity": 60,
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": 0
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false,
+          "calcs": []
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 42,
+      "type": "logs",
+      "title": "Recent warnings and errors",
+      "description": "The WARN and ERROR lines themselves, newest first.",
+      "datasource": {
+        "type": "victoriametrics-logs-datasource",
+        "uid": "@logsDatasourceUid@"
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 30
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {
+            "type": "victoriametrics-logs-datasource",
+            "uid": "@logsDatasourceUid@"
+          },
+          "expr": "_SYSTEMD_UNIT:\"openbao.service\" _msg:~\"\\\\[(WARN|ERROR)\\\\]\"",
+          "queryType": "instant"
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "wrapLogMessage": true,
+        "prettifyLogMessage": false,
+        "enableLogDetails": true,
+        "sortOrder": "Descending",
+        "dedupStrategy": "none"
+      }
     }
   ]
 }
diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix
index 0db99b6a..b87ae67a 100644
--- a/nix/module-eval/grafana.nix
+++ b/nix/module-eval/grafana.nix
@@ -691,6 +691,25 @@ let
         && builtins.length selectors == 12
         && lib.all (e: lib.hasInfix "\"service.name\"=\"${stamped.value}\"" e) selectors;
     }
+    {
+      # The bao board's metrics panels can show a gap in the metrics, never a
+      # gap in the logs: "bao is quiet" and "bao's logs are not arriving" look
+      # the same there. These panels read the journal itself. Asserted on the
+      # query, as for the logs board, because a panel that keeps its title and
+      # loses its expression renders an empty graph.
+      name = "the bao dashboard reads openbao's journal from the log store";
+      ok =
+        let
+          board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/bao.json);
+          logPanels = lib.filter (
+            p: (p.datasource.type or "") == "victoriametrics-logs-datasource"
+          ) board.panels;
+          exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) logPanels;
+        in
+        builtins.length logPanels == 3
+        && lib.all (e: lib.hasInfix "_SYSTEMD_UNIT:\"openbao.service\"" e) exprs
+        && lib.any (p: p.type == "logs") logPanels;
+    }
     {
       # A module declares its vhost under `gateway.vhosts` and the gateway adds
       # what it owns: the listen set, the cert for the name and the security