add the egress log dashboard
This commit is contained in:
atlas
parent
fd2861c097
commit
700279026f
3 changed files with 327 additions and 3 deletions
|
|
@ -275,6 +275,7 @@ let
|
|||
"claude-usage.json"
|
||||
"collector.json"
|
||||
"controller.json"
|
||||
"egress.json"
|
||||
"forge.json"
|
||||
"gateway.json"
|
||||
"logstore.json"
|
||||
|
|
|
|||
299
nix/host-modules/swarm-grafana/dashboards/egress.json
Normal file
299
nix/host-modules/swarm-grafana/dashboards/egress.json
Normal file
|
|
@ -0,0 +1,299 @@
|
|||
{
|
||||
"title": "hyperhive · egress log mode",
|
||||
"uid": "hyperhive-egress",
|
||||
"description": "What agents in log mode reached that restricted mode would refuse.",
|
||||
"editable": true,
|
||||
"refresh": "1m",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["hyperhive", "swarm", "egress", "squid"],
|
||||
"time": {
|
||||
"from": "now-24h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "utc",
|
||||
"version": 1,
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "agent",
|
||||
"label": "Agent",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"query": {
|
||||
"refId": "agent",
|
||||
"type": "fieldValue",
|
||||
"field": "agent",
|
||||
"query": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user=<agent> method=<method> host=<dest> port=<port> result=<result>\"",
|
||||
"limit": 200
|
||||
},
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"sort": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Would-deny requests",
|
||||
"description": "Requests let through that restricted mode would have refused, in the range.",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"expr": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user=<agent> method=<method> host=<dest> port=<port> result=<result>\" | agent:$agent | stats count() as requests",
|
||||
"queryType": "stats"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Agents logging",
|
||||
"description": "Agents with at least one would-deny request in the range.",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"expr": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user=<agent> method=<method> host=<dest> port=<port> result=<result>\" | agent:$agent | stats by (agent) count() as requests | stats count() as agents",
|
||||
"queryType": "stats"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Proxied requests · all agents (control)",
|
||||
"description": "Every squid request line in the range; zero means the log feed is down.",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"gridPos": { "h": 5, "w": 12, "x": 12, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"expr": "_SYSTEMD_UNIT:\"squid.service\" \"TCP_\" | stats count() as requests",
|
||||
"queryType": "stats"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "table",
|
||||
"title": "Hosts by agent",
|
||||
"description": "Each host an agent reached that restricted mode would refuse, with its request count.",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 5 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"expr": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user=<agent> method=<method> host=<dest> port=<port> result=<result>\" | agent:$agent | stats by (agent, dest, port) count() as requests | sort by (requests) desc",
|
||||
"queryType": "stats"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": { "Time": true },
|
||||
"renameByName": {
|
||||
"agent": "Agent",
|
||||
"dest": "Host",
|
||||
"port": "Port",
|
||||
"requests": "Requests"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "short", "decimals": 0 },
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [{ "displayName": "Requests", "desc": true }]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Would-deny requests over time",
|
||||
"description": "Would-deny requests per agent over the range.",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"expr": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user=<agent> method=<method> host=<dest> port=<port> result=<result>\" | agent:$agent | stats by (agent) count() as requests",
|
||||
"queryType": "statsRange",
|
||||
"legendFormat": "{{agent}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 60,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "logs",
|
||||
"title": "Recent would-deny requests",
|
||||
"description": "The would-deny lines themselves, newest first.",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 25 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "@logsDatasourceUid@"
|
||||
},
|
||||
"expr": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user=<agent> method=<method> host=<dest> port=<port> result=<result>\" | agent:$agent",
|
||||
"queryType": "instant"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": false,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -452,6 +452,27 @@ let
|
|||
]
|
||||
&& lib.all (m: lib.hasInfix "\"${m}\"" recorded) queried;
|
||||
}
|
||||
{
|
||||
# The egress board ships, and the `extract` pattern its queries use is the
|
||||
# field layout of the `would_deny` line squid logs: a changed `logformat`
|
||||
# is a board of empty panels.
|
||||
name = "the egress dashboard ships, and reads the field layout of squid's would_deny log line";
|
||||
ok =
|
||||
let
|
||||
provider = lib.head grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.dashboards.settings.providers;
|
||||
shipped = lib.attrNames provider.options.path.entries;
|
||||
board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/egress.json);
|
||||
exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels;
|
||||
pattern = "would_deny user=<agent> method=<method> host=<dest> port=<port> result=<result>";
|
||||
agentVar = lib.findFirst (v: v.name == "agent") null board.templating.list;
|
||||
squid = builtins.readFile ../host-modules/hive-squid.nix;
|
||||
in
|
||||
lib.elem "egress.json" shipped
|
||||
&& lib.length (lib.filter (e: lib.hasInfix pattern e) exprs) == 5
|
||||
&& agentVar != null
|
||||
&& lib.hasInfix pattern agentVar.query.query
|
||||
&& lib.hasInfix "logformat would_deny would_deny user=%[un method=%rm host=%>rd port=%>rP result=" squid;
|
||||
}
|
||||
{
|
||||
# The standing check on the collector's PRIORITY mapping — whether a line
|
||||
# ever reaches the log store carrying no severity. It is shipped config
|
||||
|
|
|
|||
Loading…
Reference in a new issue