diff --git a/docs/4-implementation/scheduler/observability.md b/docs/4-implementation/scheduler/observability.md index b4bc3e3fb..8bc3b25cf 100644 --- a/docs/4-implementation/scheduler/observability.md +++ b/docs/4-implementation/scheduler/observability.md @@ -344,19 +344,20 @@ CI runner, the same gauges also cover its `swarm-ci-runner` container, with filters those series out and the host dashboard's top-10 panels group by `container.name`. Its storage usage isn't measured, so that series is absent. -| Metric | Unit | Kind | Source | -| --------------------------------------- | ----------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds | -| `container.memory.usage` | `By` | gauge | `memory.current` | -| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) | -| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) | -| `hyperhive.container.memory.oom_kills` | `{process}` | counter | `memory.events` `oom_kill`: processes the kernel killed for memory in the container and below it (custom; omitted if unavailable) | -| `hyperhive.container.memory.swap.usage` | `By` | gauge | `memory.swap.current` (custom; omitted if unavailable) | -| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + the root template (the live ephemeral layer isn't counted), measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) | -| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) | -| `hyperhive.agent.unrestricted_internet` | `1` | gauge | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable | -| `hyperhive.agent.closure_pullable` | `1` | gauge | `1` when this hive can fetch the agent's latest closure from the swarm's binary cache and verify its signature, else `0` with a `reason` label (`no-record`, `not-cached`, `no-credentials`, `fetch-failed`, `bad-signature`; see [the builder docs](../swarm/builder.md#what-hives-do-with-the-cache)). Omitted before the first probe pass, every 5 minutes | -| `hyperhive.agent.needs_login` | `1` | gauge | `1` when a running agent needs a claude login (the dashboard's needs-login flag), `0` when it has a login. Omitted for a stopped agent and before the first container scan | +| Metric | Unit | Kind | Source | +| ------------------------------------------- | ----------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds | +| `container.memory.usage` | `By` | gauge | `memory.current` | +| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) | +| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) | +| `hyperhive.container.memory.oom_kills` | `{process}` | counter | `memory.events` `oom_kill`: processes the kernel killed for memory in the container and below it (custom; omitted if unavailable) | +| `hyperhive.container.memory.pressure.stall` | `us` | counter | `memory.pressure` `total` in microseconds, split by a `kind` attribute (`some`, `full`); a rate is the stalled share of time (custom; omitted without PSI) | +| `hyperhive.container.memory.swap.usage` | `By` | gauge | `memory.swap.current` (custom; omitted if unavailable) | +| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + the root template (the live ephemeral layer isn't counted), measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) | +| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) | +| `hyperhive.agent.unrestricted_internet` | `1` | gauge | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable | +| `hyperhive.agent.closure_pullable` | `1` | gauge | `1` when this hive can fetch the agent's latest closure from the swarm's binary cache and verify its signature, else `0` with a `reason` label (`no-record`, `not-cached`, `no-credentials`, `fetch-failed`, `bad-signature`; see [the builder docs](../swarm/builder.md#what-hives-do-with-the-cache)). Omitted before the first probe pass, every 5 minutes | +| `hyperhive.agent.needs_login` | `1` | gauge | `1` when a running agent needs a claude login (the dashboard's needs-login flag), `0` when it has a login. Omitted for a stopped agent and before the first container scan | The `hyperhive.`-prefixed metrics have no semconv equivalent (memory limit + peak, on-disk footprint, and an instantaneous cpu percent kept @@ -524,7 +525,7 @@ whole-swarm spend figure. hive-c0re measures everything above **per agent**, tagged with the hive it -runs in. It measures these five per **hive** instead, and they carry no +runs in. It measures these six per **hive** instead, and they carry no `agent` label — so a hive that hosts no agents still reports, and "this hive is quiet" is distinguishable from "this hive is gone." Select them with @@ -532,13 +533,14 @@ distinguishable from "this hive is gone." Select them with -| Metric | Unit | Kind | Meaning | -| -------------------------------------- | ---- | ----- | -------------------------------------------------------------------------------------------------------------------- | -| `process.uptime` | `s` | gauge | seconds since this hive's `hive-c0re` started exporting; a restart reads as a drop to ~0 | -| `hyperhive.hive.degraded` | `1` | gauge | `1` while the hive reports itself unhealthy — the same verdict `/health/ready` gives and the swarm status view shows | -| `hyperhive.hive.warnings` | `1` | gauge | how many warnings are currently raised, split by a `level` attribute (`warn`, `crit`) | -| `hyperhive.hive.warning` | `1` | gauge | `1` per raised warning, with `kind` and `level` attributes; a cleared warning stops reporting | -| `hyperhive.hive.nix_store.utilization` | `1` | gauge | used fraction (0–1) of the filesystem that holds the nix store; no sample when the probe fails | +| Metric | Unit | Kind | Meaning | +| -------------------------------------- | ---- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `process.uptime` | `s` | gauge | seconds since this hive's `hive-c0re` started exporting; a restart reads as a drop to ~0 | +| `hyperhive.hive.degraded` | `1` | gauge | `1` while the hive reports itself unhealthy — the same verdict `/health/ready` gives and the swarm status view shows | +| `hyperhive.hive.warnings` | `1` | gauge | how many warnings are currently raised, split by a `level` attribute (`warn`, `crit`) | +| `hyperhive.hive.warning` | `1` | gauge | `1` per raised warning, with `kind` and `level` attributes; a cleared warning stops reporting | +| `hyperhive.hive.nix_store.utilization` | `1` | gauge | used fraction (0–1) of the filesystem that holds the nix store; no sample when the probe fails | +| `hyperhive.hive.pressure.stall` | `us` | counter | host-wide `/proc/pressure/{cpu,memory,io}` `total` in microseconds, split by `resource` (`cpu`, `memory`, `io`) and `kind` (`some`, `full`); a rate is the stalled share of time; no sample without PSI | hive-c0re reports both levels of `hyperhive.hive.warnings` every cycle, `0` included, so a healthy hive is visible as zeros rather than as missing diff --git a/nix/host-modules/swarm-grafana/dashboards/host.json b/nix/host-modules/swarm-grafana/dashboards/host.json index 6c1c7b099..f63dfc3c2 100644 --- a/nix/host-modules/swarm-grafana/dashboards/host.json +++ b/nix/host-modules/swarm-grafana/dashboards/host.json @@ -998,6 +998,174 @@ "sort": "desc" } } + }, + { + "id": 17, + "type": "timeseries", + "title": "Pressure stall: some", + "description": "Share of time at least one task waited on CPU, memory or IO, averaged over hosts.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "avg by (resource) (rate({__name__=\"hyperhive.hive.pressure.stall\", kind=\"some\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])) / 1000000", + "legendFormat": "{{resource}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 18, + "type": "timeseries", + "title": "Pressure stall: full", + "description": "Share of time all non-idle tasks waited on memory or IO at once, averaged over hosts.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "avg by (resource) (rate({__name__=\"hyperhive.hive.pressure.stall\", kind=\"full\", resource!=\"cpu\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])) / 1000000", + "legendFormat": "{{resource}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 19, + "type": "timeseries", + "title": "Top 10 containers by memory stall", + "description": "Agent containers stalled longest on memory, as a share of time.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "topk(10, sum by (\"container.name\") (rate({__name__=\"hyperhive.container.memory.pressure.stall\", kind=\"some\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])) / 1000000)", + "legendFormat": "{{container.name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } } ] }