diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 9f9b3df7..d0b67762 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -204,6 +204,7 @@ kept. - `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue. - `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents. - `egress`: what agents send through the egress proxy, in three tabs. **overview** has the totals (requests, agents active, refused, credential challenges, data transferred), requests over time by status, and the busiest agents and destinations. **refused** lists the hosts the proxy refused for each agent. **would-deny** lists what agents in `log` mode reached that restricted mode would refuse. The host lists are tables, one row per host (and agent), with a count column you can sort. Filter by agent. +- `builds`: the container builds hive-c0re ran on every hive, in two tabs. **builds** has the count, the failures and the longest, builds over time by outcome, and a table of each build with its id, agent, kind, outcome and runtime. **output** shows every line of the build picked in the **Build** filter, with its start and finish lines. It reads the lines hive-c0re logs for each build, so a hive-c0re that doesn't log them leaves it empty. Filter by agent. - `controller`: the swarm controller's job graph by state, and the push and commit rates it sees on the forge. - `review-classifier`: review requests by outcome, the verdict mix, results per check, retried asks, and run duration p50 and p95. - `forge`: what the forge holds, and the forge process's CPU, memory and descriptors. diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index d5a7c7e4..7e167aac 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -271,6 +271,7 @@ let "agents.json" "authelia.json" "bao.json" + "builds.json" "ci.json" "claude-usage.json" "collector.json" diff --git a/nix/host-modules/swarm-grafana/dashboards/builds.json b/nix/host-modules/swarm-grafana/dashboards/builds.json new file mode 100644 index 00000000..d50a6ff3 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/builds.json @@ -0,0 +1,684 @@ +{ + "apiVersion": "dashboard.grafana.app/v2beta1", + "kind": "Dashboard", + "metadata": { + "name": "hyperhive-builds" + }, + "spec": { + "title": "hyperhive ยท builds", + "description": "Container builds on every hive: what ran, how it ended, how long it took, and its output.", + "editable": true, + "cursorSync": "Off", + "liveNow": false, + "preload": false, + "tags": ["hyperhive", "swarm", "builds"], + "links": [], + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "datasource": { + "name": "-- Grafana --" + }, + "spec": {} + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "agent", + "label": "Agent", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "victoriametrics-logs-datasource", + "version": "v0", + "datasource": { + "name": "@logsDatasourceUid@" + }, + "spec": { + "refId": "agent", + "type": "fieldValue", + "field": "F_AGENT", + "query": "_SYSTEMD_UNIT:\"hive-c0re.service\" TARGET:\"nixos-container\" F_EVENT:finish", + "limit": 200 + } + }, + "regex": "", + "options": [], + "allowCustomValue": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": "alphabeticalAsc", + "multi": true, + "includeAll": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "build", + "label": "Build", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "victoriametrics-logs-datasource", + "version": "v0", + "datasource": { + "name": "@logsDatasourceUid@" + }, + "spec": { + "refId": "build", + "type": "fieldValue", + "field": "F_BUILD_ID", + "query": "_SYSTEMD_UNIT:\"hive-c0re.service\" TARGET:\"nixos-container\" F_EVENT:start", + "limit": 200 + } + }, + "regex": "", + "options": [], + "allowCustomValue": true, + "current": {}, + "sort": "numericalDesc", + "multi": false, + "includeAll": false + } + } + ], + "timeSettings": { + "timezone": "utc", + "from": "now-24h", + "to": "now", + "autoRefresh": "1m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "Builds", + "description": "Container builds that finished in the range; zero means the log feed is down.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "victoriametrics-logs-datasource", + "version": "v0", + "datasource": { + "name": "@logsDatasourceUid@" + }, + "spec": { + "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" TARGET:\"nixos-container\" F_EVENT:finish | filter F_AGENT:$agent | stats count() as builds", + "queryType": "stats" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + } + } + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Failed builds", + "description": "Builds that ended in failure; open the output tab to read one.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "victoriametrics-logs-datasource", + "version": "v0", + "datasource": { + "name": "@logsDatasourceUid@" + }, + "spec": { + "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" TARGET:\"nixos-container\" F_EVENT:finish | filter F_AGENT:$agent | filter F_STATUS:fail | stats count() as failed", + "queryType": "stats" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + } + } + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Longest build", + "description": "The longest build in the range, in seconds.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "victoriametrics-logs-datasource", + "version": "v0", + "datasource": { + "name": "@logsDatasourceUid@" + }, + "spec": { + "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" TARGET:\"nixos-container\" F_EVENT:finish | filter F_AGENT:$agent | stats max(F_RUNTIME_SECS) as seconds", + "queryType": "stats" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + } + } + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Builds over time by status", + "description": "Finished builds per outcome over the range.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "victoriametrics-logs-datasource", + "version": "v0", + "datasource": { + "name": "@logsDatasourceUid@" + }, + "spec": { + "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" TARGET:\"nixos-container\" F_EVENT:finish | filter F_AGENT:$agent | stats by (F_STATUS) count() as builds", + "queryType": "statsRange", + "legendFormat": "{{F_STATUS}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 60, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + } + } + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Recent builds", + "description": "Each build in the range: its id, agent, kind, outcome and runtime. Pick the id on the output tab.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "victoriametrics-logs-datasource", + "version": "v0", + "datasource": { + "name": "@logsDatasourceUid@" + }, + "spec": { + "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" TARGET:\"nixos-container\" F_EVENT:finish | filter F_AGENT:$agent | stats by (F_BUILD_ID, F_AGENT, F_KIND, F_STATUS) max(F_RUNTIME_SECS) as runtime | sort by (F_BUILD_ID) desc | limit 100", + "queryType": "stats" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [ + { + "kind": "joinByLabels", + "spec": { + "id": "joinByLabels", + "options": { + "join": ["F_BUILD_ID", "F_AGENT", "F_KIND", "F_STATUS"], + "value": "__name__" + } + } + }, + { + "kind": "organize", + "spec": { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "F_BUILD_ID": "Build", + "F_AGENT": "Agent", + "F_KIND": "Kind", + "F_STATUS": "Status", + "runtime": "Runtime (s)" + } + } + } + } + ], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "", + "spec": { + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Build", + "desc": true + } + ] + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0 + }, + "overrides": [] + } + } + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Build output", + "description": "Every line of the build picked above, oldest first, with its start and finish lines.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "victoriametrics-logs-datasource", + "version": "v0", + "datasource": { + "name": "@logsDatasourceUid@" + }, + "spec": { + "expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" TARGET:\"nixos-container\" F_BUILD_ID:$build | sort by (_time)", + "queryType": "instant" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "logs", + "version": "", + "spec": { + "options": { + "showTime": true, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Ascending", + "dedupStrategy": "none" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + } + } + } + }, + "layout": { + "kind": "TabsLayout", + "spec": { + "tabs": [ + { + "kind": "TabsLayoutTab", + "spec": { + "title": "builds", + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 0, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 0, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 4, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 12, + "width": 24, + "height": 14, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + } + ] + } + } + } + }, + { + "kind": "TabsLayoutTab", + "spec": { + "title": "output", + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 26, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 3c947a12..60613d18 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -478,6 +478,52 @@ let ] && (lib.head xf).spec.options.byField == "agent"; } + { + # The builds board reads the fields hive-c0re puts on every container-build + # log line: the build id, agent and kind on each, the event on the start and + # finish lines, and the status and runtime on the finish line. A renamed + # field in hive-c0re is a board of empty panels, so each field the board + # filters or groups by must still be set there. + name = "the builds dashboard ships, and reads the fields hive-c0re logs on a container build"; + ok = + let + provider = lib.head grafanaWithAuthelia.containers.swarm-grafana.config.services.grafana.provision.dashboards.settings.providers; + shipped = lib.attrNames provider.options.path.entries; + board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/builds.json); + exprs = map (e: (lib.head e.spec.data.spec.queries).spec.query.spec.expr) ( + lib.attrValues board.spec.elements + ); + tabs = map (t: t.spec.title) board.spec.layout.spec.tabs; + logged = builtins.readFile ../../hive-c0re/src/lifecycle/mod.rs; + tables = lib.filter (e: e.spec.vizConfig.group == "table") (lib.attrValues board.spec.elements); + in + lib.elem "builds.json" shipped + && + tabs == [ + "builds" + "output" + ] + && lib.all (e: lib.hasInfix "TARGET:\"nixos-container\"" e) exprs + && lib.hasInfix "event = \"finish\"" logged + && lib.hasInfix "event = \"start\"" logged + # each field as the logging macro spells it (`status` is a bare local) + && lib.all (f: lib.hasInfix f logged) [ + "build_id = " + "agent = " + "kind = " + "status," + "runtime_secs = " + ] + && lib.hasInfix "target: \"nixos-container\"" logged + # a grouped stats table is one frame per group: joined by labels, one row per group + && lib.all ( + e: + map (t: t.spec.id) e.spec.data.spec.transformations == [ + "joinByLabels" + "organize" + ] + ) tables; + } { # The egress board ships, and its queries read the two squid log lines as # squid writes them: the would_deny line's field layout, and the default