host dashboard: pressure stall panels, docs for the new series
This commit is contained in:
atlas
parent
9769cec843
commit
5b319e0600
2 changed files with 191 additions and 21 deletions
|
|
@ -344,19 +344,20 @@ CI runner, the same gauges also cover its `swarm-ci-runner` container, with
|
||||||
filters those series out and the host dashboard's top-10 panels group by
|
filters those series out and the host dashboard's top-10 panels group by
|
||||||
`container.name`. Its storage usage isn't measured, so that series is absent.
|
`container.name`. Its storage usage isn't measured, so that series is absent.
|
||||||
|
|
||||||
| Metric | Unit | Kind | Source |
|
| Metric | Unit | Kind | Source |
|
||||||
| --------------------------------------- | ----------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| ------------------------------------------- | ----------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds |
|
| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds |
|
||||||
| `container.memory.usage` | `By` | gauge | `memory.current` |
|
| `container.memory.usage` | `By` | gauge | `memory.current` |
|
||||||
| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) |
|
| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) |
|
||||||
| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) |
|
| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) |
|
||||||
| `hyperhive.container.memory.oom_kills` | `{process}` | counter | `memory.events` `oom_kill`: processes the kernel killed for memory in the container and below it (custom; omitted if unavailable) |
|
| `hyperhive.container.memory.oom_kills` | `{process}` | counter | `memory.events` `oom_kill`: processes the kernel killed for memory in the container and below it (custom; omitted if unavailable) |
|
||||||
| `hyperhive.container.memory.swap.usage` | `By` | gauge | `memory.swap.current` (custom; omitted if unavailable) |
|
| `hyperhive.container.memory.pressure.stall` | `us` | counter | `memory.pressure` `total` in microseconds, split by a `kind` attribute (`some`, `full`); a rate is the stalled share of time (custom; omitted without PSI) |
|
||||||
| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + the root template (the live ephemeral layer isn't counted), measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) |
|
| `hyperhive.container.memory.swap.usage` | `By` | gauge | `memory.swap.current` (custom; omitted if unavailable) |
|
||||||
| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) |
|
| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + the root template (the live ephemeral layer isn't counted), measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) |
|
||||||
| `hyperhive.agent.unrestricted_internet` | `1` | gauge | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable |
|
| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) |
|
||||||
| `hyperhive.agent.closure_pullable` | `1` | gauge | `1` when this hive can fetch the agent's latest closure from the swarm's binary cache and verify its signature, else `0` with a `reason` label (`no-record`, `not-cached`, `no-credentials`, `fetch-failed`, `bad-signature`; see [the builder docs](../swarm/builder.md#what-hives-do-with-the-cache)). Omitted before the first probe pass, every 5 minutes |
|
| `hyperhive.agent.unrestricted_internet` | `1` | gauge | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable |
|
||||||
| `hyperhive.agent.needs_login` | `1` | gauge | `1` when a running agent needs a claude login (the dashboard's needs-login flag), `0` when it has a login. Omitted for a stopped agent and before the first container scan |
|
| `hyperhive.agent.closure_pullable` | `1` | gauge | `1` when this hive can fetch the agent's latest closure from the swarm's binary cache and verify its signature, else `0` with a `reason` label (`no-record`, `not-cached`, `no-credentials`, `fetch-failed`, `bad-signature`; see [the builder docs](../swarm/builder.md#what-hives-do-with-the-cache)). Omitted before the first probe pass, every 5 minutes |
|
||||||
|
| `hyperhive.agent.needs_login` | `1` | gauge | `1` when a running agent needs a claude login (the dashboard's needs-login flag), `0` when it has a login. Omitted for a stopped agent and before the first container scan |
|
||||||
|
|
||||||
The `hyperhive.`-prefixed metrics have no semconv equivalent (memory
|
The `hyperhive.`-prefixed metrics have no semconv equivalent (memory
|
||||||
limit + peak, on-disk footprint, and an instantaneous cpu percent kept
|
limit + peak, on-disk footprint, and an instantaneous cpu percent kept
|
||||||
|
|
@ -524,7 +525,7 @@ whole-swarm spend figure.
|
||||||
<!-- vale write-good.Passive = NO -->
|
<!-- vale write-good.Passive = NO -->
|
||||||
|
|
||||||
hive-c0re measures everything above **per agent**, tagged with the hive it
|
hive-c0re measures everything above **per agent**, tagged with the hive it
|
||||||
runs in. It measures these five per **hive** instead, and they carry no
|
runs in. It measures these six per **hive** instead, and they carry no
|
||||||
`agent` label — so a hive
|
`agent` label — so a hive
|
||||||
that hosts no agents still reports, and "this hive is quiet" is
|
that hosts no agents still reports, and "this hive is quiet" is
|
||||||
distinguishable from "this hive is gone." Select them with
|
distinguishable from "this hive is gone." Select them with
|
||||||
|
|
@ -532,13 +533,14 @@ distinguishable from "this hive is gone." Select them with
|
||||||
|
|
||||||
<!-- vale write-good.Passive = YES -->
|
<!-- vale write-good.Passive = YES -->
|
||||||
|
|
||||||
| Metric | Unit | Kind | Meaning |
|
| Metric | Unit | Kind | Meaning |
|
||||||
| -------------------------------------- | ---- | ----- | -------------------------------------------------------------------------------------------------------------------- |
|
| -------------------------------------- | ---- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `process.uptime` | `s` | gauge | seconds since this hive's `hive-c0re` started exporting; a restart reads as a drop to ~0 |
|
| `process.uptime` | `s` | gauge | seconds since this hive's `hive-c0re` started exporting; a restart reads as a drop to ~0 |
|
||||||
| `hyperhive.hive.degraded` | `1` | gauge | `1` while the hive reports itself unhealthy — the same verdict `/health/ready` gives and the swarm status view shows |
|
| `hyperhive.hive.degraded` | `1` | gauge | `1` while the hive reports itself unhealthy — the same verdict `/health/ready` gives and the swarm status view shows |
|
||||||
| `hyperhive.hive.warnings` | `1` | gauge | how many warnings are currently raised, split by a `level` attribute (`warn`, `crit`) |
|
| `hyperhive.hive.warnings` | `1` | gauge | how many warnings are currently raised, split by a `level` attribute (`warn`, `crit`) |
|
||||||
| `hyperhive.hive.warning` | `1` | gauge | `1` per raised warning, with `kind` and `level` attributes; a cleared warning stops reporting |
|
| `hyperhive.hive.warning` | `1` | gauge | `1` per raised warning, with `kind` and `level` attributes; a cleared warning stops reporting |
|
||||||
| `hyperhive.hive.nix_store.utilization` | `1` | gauge | used fraction (0–1) of the filesystem that holds the nix store; no sample when the probe fails |
|
| `hyperhive.hive.nix_store.utilization` | `1` | gauge | used fraction (0–1) of the filesystem that holds the nix store; no sample when the probe fails |
|
||||||
|
| `hyperhive.hive.pressure.stall` | `us` | counter | host-wide `/proc/pressure/{cpu,memory,io}` `total` in microseconds, split by `resource` (`cpu`, `memory`, `io`) and `kind` (`some`, `full`); a rate is the stalled share of time; no sample without PSI |
|
||||||
|
|
||||||
hive-c0re reports both levels of `hyperhive.hive.warnings` every cycle, `0`
|
hive-c0re reports both levels of `hyperhive.hive.warnings` every cycle, `0`
|
||||||
included, so a healthy hive is visible as zeros rather than as missing
|
included, so a healthy hive is visible as zeros rather than as missing
|
||||||
|
|
|
||||||
|
|
@ -998,6 +998,174 @@
|
||||||
"sort": "desc"
|
"sort": "desc"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 17,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Pressure stall: some",
|
||||||
|
"description": "Share of time at least one task waited on CPU, memory or IO, averaged over hosts.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 48
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "avg by (resource) (rate({__name__=\"hyperhive.hive.pressure.stall\", kind=\"some\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])) / 1000000",
|
||||||
|
"legendFormat": "{{resource}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percentunit",
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "line",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 10,
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"axisSoftMin": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true,
|
||||||
|
"calcs": []
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 18,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Pressure stall: full",
|
||||||
|
"description": "Share of time all non-idle tasks waited on memory or IO at once, averaged over hosts.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 48
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "avg by (resource) (rate({__name__=\"hyperhive.hive.pressure.stall\", kind=\"full\", resource!=\"cpu\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])) / 1000000",
|
||||||
|
"legendFormat": "{{resource}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percentunit",
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "line",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 10,
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"axisSoftMin": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true,
|
||||||
|
"calcs": []
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 19,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Top 10 containers by memory stall",
|
||||||
|
"description": "Agent containers stalled longest on memory, as a share of time.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 56
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "topk(10, sum by (\"container.name\") (rate({__name__=\"hyperhive.container.memory.pressure.stall\", kind=\"some\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])) / 1000000)",
|
||||||
|
"legendFormat": "{{container.name}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percentunit",
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "line",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 10,
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"axisSoftMin": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true,
|
||||||
|
"calcs": []
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue