swarm-grafana: the bao dashboard reads openbao's journal
The bao board was nine metrics panels, so a gap in log delivery looked like a quiet bao. Three panels read the log store for openbao.service: lines delivered over time, WARN and ERROR lines over time, and the recent WARN and ERROR lines themselves. A module-eval case pins them by query.
This commit is contained in:
iris
mara
parent
4d5a65a185
commit
ae2ecab9f2
2 changed files with 170 additions and 0 deletions
|
|
@ -547,6 +547,157 @@
|
||||||
"sort": "desc"
|
"sort": "desc"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 40,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Log lines delivered",
|
||||||
|
"description": "Lines from openbao.service that reached the log store.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 22
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"openbao.service\" | stats count() as lines",
|
||||||
|
"queryType": "statsRange",
|
||||||
|
"legendFormat": "lines"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "bars",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 60,
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"axisSoftMin": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": false,
|
||||||
|
"calcs": []
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 41,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Warnings and errors",
|
||||||
|
"description": "WARN and ERROR lines from openbao.service.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 22
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"openbao.service\" _msg:~\"\\\\[(WARN|ERROR)\\\\]\" | stats count() as lines",
|
||||||
|
"queryType": "statsRange",
|
||||||
|
"legendFormat": "lines"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "bars",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 60,
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"axisSoftMin": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": false,
|
||||||
|
"calcs": []
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 42,
|
||||||
|
"type": "logs",
|
||||||
|
"title": "Recent warnings and errors",
|
||||||
|
"description": "The WARN and ERROR lines themselves, newest first.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 10,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 30
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"openbao.service\" _msg:~\"\\\\[(WARN|ERROR)\\\\]\"",
|
||||||
|
"queryType": "instant"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"showTime": true,
|
||||||
|
"wrapLogMessage": true,
|
||||||
|
"prettifyLogMessage": false,
|
||||||
|
"enableLogDetails": true,
|
||||||
|
"sortOrder": "Descending",
|
||||||
|
"dedupStrategy": "none"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -691,6 +691,25 @@ let
|
||||||
&& builtins.length selectors == 12
|
&& builtins.length selectors == 12
|
||||||
&& lib.all (e: lib.hasInfix "\"service.name\"=\"${stamped.value}\"" e) selectors;
|
&& lib.all (e: lib.hasInfix "\"service.name\"=\"${stamped.value}\"" e) selectors;
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# The bao board's metrics panels can show a gap in the metrics, never a
|
||||||
|
# gap in the logs: "bao is quiet" and "bao's logs are not arriving" look
|
||||||
|
# the same there. These panels read the journal itself. Asserted on the
|
||||||
|
# query, as for the logs board, because a panel that keeps its title and
|
||||||
|
# loses its expression renders an empty graph.
|
||||||
|
name = "the bao dashboard reads openbao's journal from the log store";
|
||||||
|
ok =
|
||||||
|
let
|
||||||
|
board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/bao.json);
|
||||||
|
logPanels = lib.filter (
|
||||||
|
p: (p.datasource.type or "") == "victoriametrics-logs-datasource"
|
||||||
|
) board.panels;
|
||||||
|
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) logPanels;
|
||||||
|
in
|
||||||
|
builtins.length logPanels == 3
|
||||||
|
&& lib.all (e: lib.hasInfix "_SYSTEMD_UNIT:\"openbao.service\"" e) exprs
|
||||||
|
&& lib.any (p: p.type == "logs") logPanels;
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# A module declares its vhost under `gateway.vhosts` and the gateway adds
|
# A module declares its vhost under `gateway.vhosts` and the gateway adds
|
||||||
# what it owns: the listen set, the cert for the name and the security
|
# what it owns: the listen set, the cert for the name and the security
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue