From 1c3b37e46ad207dc357aa370cecafdaa061717eb Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 9 Nov 2026 10:56:50 +0100 Subject: [PATCH] page pending_logins from hh-hive-warning for hives without the needs-login gauge --- docs/4-implementation/scheduler/observability.md | 16 ++++++++-------- nix/host-modules/swarm-grafana.nix | 9 +++++---- nix/module-eval/grafana.nix | 15 +++++++++------ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/docs/4-implementation/scheduler/observability.md b/docs/4-implementation/scheduler/observability.md index e17998812..ea3cf88f5 100644 --- a/docs/4-implementation/scheduler/observability.md +++ b/docs/4-implementation/scheduler/observability.md @@ -553,16 +553,16 @@ failing sweep such as `swarm_status_publish`, `pending_logins`), and the message The swarm Grafana (`nix/host-modules/swarm-grafana.nix`) alerts on these: -| Alert | Fires when | -| ----------------------- | -------------------------------------------------------------------------------------------------------- | -| `hh-hive-degraded` | `hyperhive.hive.degraded` is `1` for 2 minutes | -| `hh-hive-warning` | any `hyperhive.hive.warning` series other than `disk_pressure` and `pending_logins` exists for 2 minutes | -| `hh-agent-needs-login` | `hyperhive.agent.needs_login` is `1` for an agent for 5 minutes, one alert per agent | -| `hh-host-disk-high` | `hyperhive.hive.nix_store.utilization` stays above 0.85 for 10 minutes | -| `hh-host-disk-critical` | `hyperhive.hive.nix_store.utilization` stays above 0.95 for 2 minutes | +| Alert | Fires when | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `hh-hive-degraded` | `hyperhive.hive.degraded` is `1` for 2 minutes | +| `hh-hive-warning` | any `hyperhive.hive.warning` series other than `disk_pressure` exists (`pending_logins` only for a hive that doesn't export `hyperhive.agent.needs_login`) for 2 minutes | +| `hh-agent-needs-login` | `hyperhive.agent.needs_login` is `1` for an agent for 5 minutes, one alert per agent | +| `hh-host-disk-high` | `hyperhive.hive.nix_store.utilization` stays above 0.85 for 10 minutes | +| `hh-host-disk-critical` | `hyperhive.hive.nix_store.utilization` stays above 0.95 for 2 minutes | The disk alerts replace the old dashboard banner for the nix store, and -`hh-hive-warning` replaces it for every other warning. `pending_logins` has its own per-agent alert, `hh-agent-needs-login`, so a login never pages twice. Disk pressure has its +`hh-hive-warning` replaces it for every other warning. On a hive that exports `hyperhive.agent.needs_login`, the per-agent alert `hh-agent-needs-login` covers `pending_logins`, so a login never pages twice; a hive on an older hive-c0re keeps the `hh-hive-warning` page. Disk pressure has its own pair so the thresholds sit on the metric rather than on the registry entry. diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 7146ec7d1..a6a18b283 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -1720,13 +1720,14 @@ in # pressure is left out: it has its own thresholds below, on # the filesystem metric rather than the registry entry. # Sweeps raise their warning only after several misses, so - # `for` is short. `pending_logins` is left out: it has no - # `agent` label, and `hh-agent-needs-login` below pages - # once per agent instead. + # `for` is short. `pending_logins` is dropped for a hive that exports + # `hyperhive.agent.needs_login`, where `hh-agent-needs-login` + # below pages once per agent; a hive on an older hive-c0re + # has no such series, so it keeps paging here. (metricThresholdRule { uid = "hh-hive-warning"; title = "hive warning raised"; - expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!~"disk_pressure|pending_logins"})''; + expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!~"disk_pressure|pending_logins"} or ({__name__="hyperhive.hive.warning", kind="pending_logins"} unless on (hive) group by (hive) ({__name__="hyperhive.agent.needs_login"})))''; threshold = 0; for = "2m"; summary = "hive {{ $labels.hive }} has a {{ $labels.level }} warning raised: {{ $labels.kind }}; the message is the `hive warning raised` line in the hive-c0re log"; diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 2c0ca843b..ce4a19c0d 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -942,13 +942,14 @@ let } { # The needs-login alert is one instance per (hive, agent), fires after - # 5 minutes on the gauge hive-c0re exports, and `hh-hive-warning` leaves - # `pending_logins` to it so a login never pages twice. The disk rule is + # 5 minutes on the gauge hive-c0re exports, and `hh-hive-warning` drops + # `pending_logins` only for hives that export the gauge, so a login never + # pages twice and a hive on an older hive-c0re still pages. The disk rule is # the presence control: it goes through the same lookup. # 🔬 Invert-proof: change `for`, `max by (hive, agent)` or the metric on - # `hh-agent-needs-login`, drop `pending_logins` from `hh-hive-warning`, - # or rename the gauge in `otel_metrics.rs`. - name = "the needs-login alert is per agent, fires after 5 minutes and the hive warning rule skips pending_logins"; + # `hh-agent-needs-login`, exclude `pending_logins` from `hh-hive-warning` for every hive + # instead of per hive (drop the `unless on (hive)` clause), or rename the gauge in `otel_metrics.rs`. + name = "the needs-login alert is per agent, fires after 5 minutes and the hive warning rule skips pending_logins only for hives with the gauge"; ok = let groups = @@ -965,7 +966,9 @@ let && lib.hasInfix ''{__name__="hyperhive.agent.needs_login"}'' (expr "hh-agent-needs-login") && lib.hasInfix ''"hyperhive.agent.needs_login"'' source && lib.hasInfix "max by (hive, agent)" (expr "hh-agent-needs-login") - && lib.hasInfix "pending_logins" (expr "hh-hive-warning") + && lib.hasInfix ''{__name__="hyperhive.hive.warning", kind="pending_logins"} unless on (hive) group by (hive) ({__name__="hyperhive.agent.needs_login"})'' ( + expr "hh-hive-warning" + ) && lib.hasInfix "disk_pressure" (expr "hh-hive-warning"); } {