From 700279026f30a8dfd4a4b771b7b9b1ba4177ca35 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 30 Oct 2026 02:22:02 +0100 Subject: [PATCH] add the egress log dashboard --- docs/networking/network.md | 5 +- docs/swarm/services.md | 1 + docs/swarm/ui.md | 3 +- nix/host-modules/swarm-grafana.nix | 1 + .../swarm-grafana/dashboards/egress.json | 299 ++++++++++++++++++ nix/module-eval/grafana.nix | 21 ++ 6 files changed, 327 insertions(+), 3 deletions(-) create mode 100644 nix/host-modules/swarm-grafana/dashboards/egress.json diff --git a/docs/networking/network.md b/docs/networking/network.md index 33f1bce7..94bbd912 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -301,8 +301,9 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. rule lets through gets a second log line, `would_deny user= method= host= port=

result=`, from the `would_denied` `access_log` in `hive-squid.nix`; the usual request line keeps - its format. Only the operator changes the mode: nothing moves an agent to - `restricted` after a time or a count. The self-check's users aren't agents, + its format. The `egress` dashboard in Grafana lists the hosts per agent. Only + the operator changes the mode: nothing moves an agent to `restricted` after a + time or a count. The self-check's users aren't agents, so `deny-canary.invalid` stays a 403. - **Base list.** `services.hyperhive.deploy.hive-controller.squid.baseAllowedDomains` is open to every diff --git a/docs/swarm/services.md b/docs/swarm/services.md index e5595b59..96626af3 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -203,6 +203,7 @@ kept. - `agents`: per-agent resource usage and Claude Code activity. - `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue. - `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents. +- `egress`: the requests that agents in `log` mode made to hosts restricted mode would refuse, by agent and host. Filter by agent. - `controller`: the swarm controller's job graph by state, and the push and commit rates it sees on the forge. - `review-classifier`: review requests by outcome, the verdict mix, results per check, retried asks, and run duration p50 and p95. - `forge`: what the forge holds, and the forge process's CPU, memory and descriptors. diff --git a/docs/swarm/ui.md b/docs/swarm/ui.md index a0156f89..b5386b95 100644 --- a/docs/swarm/ui.md +++ b/docs/swarm/ui.md @@ -592,7 +592,8 @@ modes (**allow unrestricted**, **log**, **restrict**) set the mode with a `PUT` without the list resets a stored list to the default. An agent with no stored list sends the flags alone. `log` sends `unrestricted_internet: false` with `internet_log: true`; the other two leave `internet_log` out, which ends -logging. +logging. The **egress** dashboard in Grafana shows the hosts a `log` agent +reached. The hive's egress proxy applies the flag to each agent's proxy credential (see [the egress proxy](../networking/network.md#egress-proxy-squid)). An agent sends its credential from its next start after it has fetched it, so a flag change applies diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 49853a4e..ee1b864f 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -275,6 +275,7 @@ let "claude-usage.json" "collector.json" "controller.json" + "egress.json" "forge.json" "gateway.json" "logstore.json" diff --git a/nix/host-modules/swarm-grafana/dashboards/egress.json b/nix/host-modules/swarm-grafana/dashboards/egress.json new file mode 100644 index 00000000..72501fd4 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/egress.json @@ -0,0 +1,299 @@ +{ + "title": "hyperhive · egress log mode", + "uid": "hyperhive-egress", + "description": "What agents in log mode reached that restricted mode would refuse.", + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "tags": ["hyperhive", "swarm", "egress", "squid"], + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "templating": { + "list": [ + { + "name": "agent", + "label": "Agent", + "type": "query", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "query": { + "refId": "agent", + "type": "fieldValue", + "field": "agent", + "query": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user= method= host= port= result=\"", + "limit": 200 + }, + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Would-deny requests", + "description": "Requests let through that restricted mode would have refused, in the range.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user= method= host= port= result=\" | agent:$agent | stats count() as requests", + "queryType": "stats" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 2, + "type": "stat", + "title": "Agents logging", + "description": "Agents with at least one would-deny request in the range.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user= method= host= port= result=\" | agent:$agent | stats by (agent) count() as requests | stats count() as agents", + "queryType": "stats" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Proxied requests · all agents (control)", + "description": "Every squid request line in the range; zero means the log feed is down.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { "h": 5, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"squid.service\" \"TCP_\" | stats count() as requests", + "queryType": "stats" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 4, + "type": "table", + "title": "Hosts by agent", + "description": "Each host an agent reached that restricted mode would refuse, with its request count.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { "h": 12, "w": 24, "x": 0, "y": 5 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user= method= host= port= result=\" | agent:$agent | stats by (agent, dest, port) count() as requests | sort by (requests) desc", + "queryType": "stats" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { "Time": true }, + "renameByName": { + "agent": "Agent", + "dest": "Host", + "port": "Port", + "requests": "Requests" + } + } + } + ], + "fieldConfig": { + "defaults": { "unit": "short", "decimals": 0 }, + "overrides": [] + }, + "options": { + "showHeader": true, + "sortBy": [{ "displayName": "Requests", "desc": true }] + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Would-deny requests over time", + "description": "Would-deny requests per agent over the range.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user= method= host= port= result=\" | agent:$agent | stats by (agent) count() as requests", + "queryType": "statsRange", + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 60, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + }, + { + "id": 6, + "type": "logs", + "title": "Recent would-deny requests", + "description": "The would-deny lines themselves, newest first.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 25 }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:\"squid.service\" \"would_deny user=\" | extract \"would_deny user= method= host= port= result=\" | agent:$agent", + "queryType": "instant" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + } + } + ] +} diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index b87ae67a..f05def3d 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -452,6 +452,27 @@ let ] && lib.all (m: lib.hasInfix "\"${m}\"" recorded) queried; } + { + # The egress board ships, and the `extract` pattern its queries use is the + # field layout of the `would_deny` line squid logs: a changed `logformat` + # is a board of empty panels. + name = "the egress dashboard ships, and reads the field layout of squid's would_deny log line"; + ok = + let + provider = lib.head grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.dashboards.settings.providers; + shipped = lib.attrNames provider.options.path.entries; + board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/egress.json); + exprs = lib.concatMap (p: map (t: t.expr or "") (p.targets or [ ])) board.panels; + pattern = "would_deny user= method= host= port= result="; + agentVar = lib.findFirst (v: v.name == "agent") null board.templating.list; + squid = builtins.readFile ../host-modules/hive-squid.nix; + in + lib.elem "egress.json" shipped + && lib.length (lib.filter (e: lib.hasInfix pattern e) exprs) == 5 + && agentVar != null + && lib.hasInfix pattern agentVar.query.query + && lib.hasInfix "logformat would_deny would_deny user=%[un method=%rm host=%>rd port=%>rP result=" squid; + } { # The standing check on the collector's PRIORITY mapping — whether a line # ever reaches the log store carrying no severity. It is shipped config