document budgeted-agent swap and oom metrics; add oom kills panel

nix/host-modules/swarm-grafana/dashboards/agents.json

@@ -3874,6 +3874,85 @@
             }
           }
         }
+      },
+      "panel-85": {
+        "kind": "Panel",
+        "spec": {
+          "id": 85,
+          "title": "OOM kills by agent",
+          "description": "Processes the kernel killed for memory in each agent container, over the selected range.",
+          "links": [],
+          "data": {
+            "kind": "QueryGroup",
+            "spec": {
+              "queries": [
+                {
+                  "kind": "PanelQuery",
+                  "spec": {
+                    "query": {
+                      "kind": "DataQuery",
+                      "group": "prometheus",
+                      "version": "v0",
+                      "datasource": {
+                        "name": "@datasourceUid@"
+                      },
+                      "spec": {
+                        "expr": "sort_desc(sum by (agent) (increase({__name__=\"hyperhive.container.memory.oom_kills\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}[$__range])))",
+                        "instant": true,
+                        "legendFormat": "{{agent}}"
+                      }
+                    },
+                    "refId": "A",
+                    "hidden": false
+                  }
+                }
+              ],
+              "transformations": [],
+              "queryOptions": {}
+            }
+          },
+          "vizConfig": {
+            "kind": "VizConfig",
+            "group": "bargauge",
+            "version": "",
+            "spec": {
+              "options": {
+                "displayMode": "basic",
+                "orientation": "horizontal",
+                "reduceOptions": {
+                  "calcs": ["lastNotNull"],
+                  "fields": "",
+                  "values": false
+                },
+                "showUnfilled": true,
+                "valueMode": "text"
+              },
+              "fieldConfig": {
+                "defaults": {
+                  "unit": "short",
+                  "decimals": 0,
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      {
+                        "value": null,
+                        "color": "green"
+                      },
+                      {
+                        "value": 1,
+                        "color": "red"
+                      }
+                    ]
+                  },
+                  "color": {
+                    "mode": "thresholds"
+                  }
+                },
+                "overrides": []
+              }
+            }
+          }
+        }
       }
     },
     "layout": {
@@ -4472,6 +4551,19 @@
                           "name": "panel-84"
                         }
                       }
+                    },
+                    {
+                      "kind": "GridLayoutItem",
+                      "spec": {
+                        "x": 0,
+                        "y": 54,
+                        "width": 24,
+                        "height": 7,
+                        "element": {
+                          "kind": "ElementReference",
+                          "name": "panel-85"
+                        }
+                      }
                     }
                   ]
                 }

nix/module-eval/grafana.nix

@@ -547,6 +547,25 @@ let
         && lib.any (e: lib.hasInfix "swarm_queue.publish_skipped" e) exprs
         && lib.hasInfix "hyperhive.container.cpu.percent" connected;
     }
+    {
+      # The agents board's health tab shows OOM kills per agent from the counter
+      # hive-c0re exports. A renamed instrument is an empty panel, so the name
+      # must still be spelled in the exporter and read by a panel on that tab.
+      name = "the agents dashboard's health tab reads the OOM-kill counter hive-c0re exports";
+      ok =
+        let
+          board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/agents.json);
+          tab = lib.findFirst (t: t.spec.title == "health") null board.spec.layout.spec.tabs;
+          names = map (i: i.spec.element.name) tab.spec.layout.spec.items;
+          exprs = lib.concatMap (
+            n: map (q: q.spec.query.spec.expr) board.spec.elements.${n}.spec.data.spec.queries
+          ) names;
+          exported = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs;
+        in
+        tab != null
+        && lib.hasInfix "\"hyperhive.container.memory.oom_kills\"" exported
+        && lib.any (e: lib.hasInfix "hyperhive.container.memory.oom_kills" e) exprs;
+    }
     {
       # The agents board's turns tab reads the turn, tool, bash command, reminder
       # and session metrics the harness records, grouped by the labels it sets on them. A renamed instrument or