diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 640d8f32..c85b8f9c 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -53,6 +53,7 @@ let }) [ "agents.json" + "authelia.json" "claude-usage.json" ] ); diff --git a/nix/host-modules/swarm-grafana/dashboards/authelia.json b/nix/host-modules/swarm-grafana/dashboards/authelia.json new file mode 100644 index 00000000..aa2a0b09 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/authelia.json @@ -0,0 +1,633 @@ +{ + "title": "hyperhive · authelia", + "uid": "hyperhive-authelia", + "description": "The swarm's identity provider. Split out of the swarm-services board because it was most of it — every other service either has no exporter yet or is still failing to scrape. Two things to know before editing: counters here are COUNTS over the dashboard range, not per-second rates, because this IdP has served a few hundred requests in its whole lifetime and rate() over any sane window is exactly zero — which draws a flat line and is indistinguishable from a broken query. And targets are labelled service.name, NOT job; there is no job label in this store.", + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "hyperhive", + "swarm", + "authelia", + "auth" + ], + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "templating": { + "list": [] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Uptime", + "description": "Time since the authelia process started. A reset here explains a counter reset below — worth checking before reading a drop as a change in behaviour. Note authelia is restarted by swarmctl on identity changes, so short uptimes are normal rather than alarming.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "time() - process_start_time_seconds{\"service.name\"=\"authelia\"}", + "instant": true, + "legendFormat": "uptime" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 2, + "type": "stat", + "title": "Authentications (range)", + "description": "authelia_authn_duration_count over the dashboard's own time range — a login, not a per-request authorization check. Deliberately range-scoped rather than a fixed window: at this volume an hourly figure reads 0 nearly always, and a zero that is always zero teaches nobody anything.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(authelia_authn_duration_count[$__range])) or vector(0)", + "instant": true, + "legendFormat": "authn" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Failed authentications (range)", + "description": "The success=\"false\" arm of the same counter. Red on any non-zero value — a failed login against the swarm's identity provider is worth a look even as a single event, because nothing here should be guessing passwords.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(authelia_authn_duration_count{success=\"false\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "failed" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "x-zero-is-healthy": true + }, + { + "id": 4, + "type": "stat", + "title": "Authorization decisions (range)", + "description": "authelia_authz — the forward-auth verdict the gateway asks for on every proxied request. This is the busy one: it fires per request, where authn fires per login, which is why the two counts differ by orders of magnitude and should.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum(increase(authelia_authz[$__range])) or vector(0)", + "instant": true, + "legendFormat": "authz" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 10, + "type": "bargauge", + "title": "Requests by status code (range)", + "description": "Counts over the dashboard range, not a per-second rate — see the board description for why. A 401 here is normal traffic (an unauthenticated request being told to log in); a 5xx is not.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (code) (increase(authelia_request[$__range]))", + "legendFormat": "{{code}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 11, + "type": "bargauge", + "title": "Authorization verdicts by code (range)", + "description": "The gateway's forward-auth outcomes. A rising 403/401 count against a flat authn count means something is being REFUSED, not that someone is failing to log in — that distinction is the reason this panel sits beside the requests one rather than being folded into it.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "sum by (code) (increase(authelia_authz[$__range]))", + "legendFormat": "{{code}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Request latency (p95)", + "description": "histogram_quantile over authelia_request_duration_bucket, using increase() over the dashboard range rather than rate() over a short window. Measured: at this traffic level a 1m rate window contains no requests, every bucket is zero, and the quantile is NaN — the panel renders EMPTY, not zero. Note the sum by (le): dropping every other label before the quantile is required, or buckets from different codes get mixed and the answer is silently wrong rather than absent.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "histogram_quantile(0.95, sum by (le) (increase(authelia_request_duration_bucket[$__range])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 3, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 21, + "type": "timeseries", + "title": "OIDC request latency (p95)", + "description": "The same quantile over authelia's separate openid_connect duration family. Worth its own panel rather than being merged into the one beside it: every machine-to-machine credential in this swarm is minted through the OIDC endpoints, so this is the latency that agents and services actually feel, while the general one is dominated by browser traffic.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "histogram_quantile(0.95, sum by (le) (increase(authelia_request_duration_openid_connect_bucket[$__range])))", + "legendFormat": "p95 oidc" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 3, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 30, + "type": "timeseries", + "title": "Resident memory", + "description": "process_resident_memory_bytes for the authelia process. Included because authelia holds its user store in memory and reloads it on file change (file.watch, #3403) — a step here after an identity is created is expected; a monotonic climb is not.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 20 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "process_resident_memory_bytes{\"service.name\"=\"authelia\"}", + "legendFormat": "rss" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 31, + "type": "timeseries", + "title": "Goroutines", + "description": "The cheapest leak detector Go gives you. Flat is healthy; a staircase that never comes down usually means a handler is blocking on something that never returns — which for an IdP is most often an outbound call to a service that is down.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 20 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "go_goroutines{\"service.name\"=\"authelia\"}", + "legendFormat": "goroutines" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 32, + "type": "timeseries", + "title": "Open file descriptors", + "description": "process_open_fds. Paired with goroutines because the two rise together when connections are being held rather than closed, and an fd exhaustion is one of the few ways this process fails hard rather than returning an error.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 20 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "process_open_fds{\"service.name\"=\"authelia\"}", + "legendFormat": "fds" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +}