From c014b48c22e79db3fa4a679651d58cc1af198400 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 11 Nov 2026 01:58:30 +0100 Subject: [PATCH] document budgeted-agent swap and oom metrics; add oom kills panel --- .../4-implementation/scheduler/coordinator.md | 15 ++- .../scheduler/observability.md | 24 ++--- .../swarm-grafana/dashboards/agents.json | 92 +++++++++++++++++++ nix/module-eval/grafana.nix | 19 ++++ 4 files changed, 138 insertions(+), 12 deletions(-) diff --git a/docs/4-implementation/scheduler/coordinator.md b/docs/4-implementation/scheduler/coordinator.md index dc6cd195e..d21b67e3a 100644 --- a/docs/4-implementation/scheduler/coordinator.md +++ b/docs/4-implementation/scheduler/coordinator.md @@ -446,7 +446,13 @@ changed cap reaches them at the next container start without a rebuild. `hyperhive-memory-cap-check.service` logs a warning naming the path in the container's journal. The container's `MemoryMax=` still applies. -When the container reaches its cap, the kernel kills in this order: +A container with a finite `MemoryMax=` also gets `MemorySwapMax=0` in the same +drop-in, so a budgeted agent never swaps: paging it out to host swap would +move the pressure onto the host. `MemoryMax=infinity` leaves swap alone. Like +the rest of the drop-in, it reaches a running agent at its next spawn or +rebuild. + +When the container reaches its cap, the kernel kills inside it, in this order: 1. a subagent run (`oom_score_adj` +1000); 2. a bash task (0); @@ -461,6 +467,13 @@ daemon itself sits in `daemon/` beside them. A daemon that can't set these up logs one warning at start and runs everything in its own cgroup; subagent runs still get +1000. +The container stays up through these kills. Each one counts in the +`hyperhive.container.memory.oom_kills` metric, shown as "OOM kills by agent" +on the health tab of the agents dashboard. `hyperhive.container.memory.swap.usage` +is the memory the container has swapped out: nonzero only for an agent with +no budget, or one whose container started before its drop-in carried +`MemorySwapMax=0`. +
Implementation: swarm caps swarm-controller keeps the caps in the `agent-limits` KV bucket, one key per diff --git a/docs/4-implementation/scheduler/observability.md b/docs/4-implementation/scheduler/observability.md index a483fa8d8..929c0160b 100644 --- a/docs/4-implementation/scheduler/observability.md +++ b/docs/4-implementation/scheduler/observability.md @@ -344,17 +344,19 @@ CI runner, the same gauges also cover its `swarm-ci-runner` container, with filters those series out and the host dashboard's top-10 panels group by `container.name`. Its storage usage isn't measured, so that series is absent. -| Metric | Unit | Kind | Source | -| --------------------------------------- | ---- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds | -| `container.memory.usage` | `By` | gauge | `memory.current` | -| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) | -| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) | -| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + the root template (the live ephemeral layer isn't counted), measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) | -| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) | -| `hyperhive.agent.unrestricted_internet` | `1` | gauge | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable | -| `hyperhive.agent.closure_pullable` | `1` | gauge | `1` when this hive can fetch the agent's latest closure from the swarm's binary cache and verify its signature, else `0` with a `reason` label (`no-record`, `not-cached`, `no-credentials`, `fetch-failed`, `bad-signature`; see [the builder docs](../swarm/builder.md#what-hives-do-with-the-cache)). Omitted before the first probe pass, every 5 minutes | -| `hyperhive.agent.needs_login` | `1` | gauge | `1` when a running agent needs a claude login (the dashboard's needs-login flag), `0` when it has a login. Omitted for a stopped agent and before the first container scan | +| Metric | Unit | Kind | Source | +| --------------------------------------- | ----------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds | +| `container.memory.usage` | `By` | gauge | `memory.current` | +| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) | +| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) | +| `hyperhive.container.memory.oom_kills` | `{process}` | counter | `memory.events` `oom_kill`: processes the kernel killed for memory in the container and below it (custom; omitted if unavailable) | +| `hyperhive.container.memory.swap.usage` | `By` | gauge | `memory.swap.current` (custom; omitted if unavailable) | +| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + the root template (the live ephemeral layer isn't counted), measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) | +| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) | +| `hyperhive.agent.unrestricted_internet` | `1` | gauge | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable | +| `hyperhive.agent.closure_pullable` | `1` | gauge | `1` when this hive can fetch the agent's latest closure from the swarm's binary cache and verify its signature, else `0` with a `reason` label (`no-record`, `not-cached`, `no-credentials`, `fetch-failed`, `bad-signature`; see [the builder docs](../swarm/builder.md#what-hives-do-with-the-cache)). Omitted before the first probe pass, every 5 minutes | +| `hyperhive.agent.needs_login` | `1` | gauge | `1` when a running agent needs a claude login (the dashboard's needs-login flag), `0` when it has a login. Omitted for a stopped agent and before the first container scan | The `hyperhive.`-prefixed metrics have no semconv equivalent (memory limit + peak, on-disk footprint, and an instantaneous cpu percent kept diff --git a/nix/host-modules/swarm-grafana/dashboards/agents.json b/nix/host-modules/swarm-grafana/dashboards/agents.json index b9c22b65b..24abe4312 100644 --- a/nix/host-modules/swarm-grafana/dashboards/agents.json +++ b/nix/host-modules/swarm-grafana/dashboards/agents.json @@ -3874,6 +3874,85 @@ } } } + }, + "panel-85": { + "kind": "Panel", + "spec": { + "id": 85, + "title": "OOM kills by agent", + "description": "Processes the kernel killed for memory in each agent container, over the selected range.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@datasourceUid@" + }, + "spec": { + "expr": "sort_desc(sum by (agent) (increase({__name__=\"hyperhive.container.memory.oom_kills\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}[$__range])))", + "instant": true, + "legendFormat": "{{agent}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "bargauge", + "version": "", + "spec": { + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "text" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } } }, "layout": { @@ -4472,6 +4551,19 @@ "name": "panel-84" } } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 54, + "width": 24, + "height": 7, + "element": { + "kind": "ElementReference", + "name": "panel-85" + } + } } ] } diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 28be2514f..36f71ceaa 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -547,6 +547,25 @@ let && lib.any (e: lib.hasInfix "swarm_queue.publish_skipped" e) exprs && lib.hasInfix "hyperhive.container.cpu.percent" connected; } + { + # The agents board's health tab shows OOM kills per agent from the counter + # hive-c0re exports. A renamed instrument is an empty panel, so the name + # must still be spelled in the exporter and read by a panel on that tab. + name = "the agents dashboard's health tab reads the OOM-kill counter hive-c0re exports"; + ok = + let + board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/agents.json); + tab = lib.findFirst (t: t.spec.title == "health") null board.spec.layout.spec.tabs; + names = map (i: i.spec.element.name) tab.spec.layout.spec.items; + exprs = lib.concatMap ( + n: map (q: q.spec.query.spec.expr) board.spec.elements.${n}.spec.data.spec.queries + ) names; + exported = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs; + in + tab != null + && lib.hasInfix "\"hyperhive.container.memory.oom_kills\"" exported + && lib.any (e: lib.hasInfix "hyperhive.container.memory.oom_kills" e) exprs; + } { # The agents board's turns tab reads the turn, tool, bash command, reminder # and session metrics the harness records, grouped by the labels it sets on them. A renamed instrument or