From 9043be4a61bca13dde91ba65386e6b3417200efb Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 9 Nov 2026 10:35:16 +0100 Subject: [PATCH] export hyperhive.agent.needs_login and alert per agent --- .../scheduler/observability.md | 16 ++++--- hive-c0re/src/coordinator.rs | 1 + hive-c0re/src/stats/host_stats.rs | 48 +++++++++++++++++++ hive-c0re/src/stats/otel_metrics.rs | 15 ++++++ nix/host-modules/swarm-grafana.nix | 21 ++++++-- nix/module-eval/grafana.nix | 28 +++++++++++ 6 files changed, 117 insertions(+), 12 deletions(-) diff --git a/docs/4-implementation/scheduler/observability.md b/docs/4-implementation/scheduler/observability.md index 9377b994a..00ca137c2 100644 --- a/docs/4-implementation/scheduler/observability.md +++ b/docs/4-implementation/scheduler/observability.md @@ -354,6 +354,7 @@ filters those series out and the host dashboard's top-10 panels group by | `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) | | `hyperhive.agent.unrestricted_internet` | `1` | gauge | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable | | `hyperhive.agent.closure_pullable` | `1` | gauge | `1` when this hive can fetch the agent's latest closure from the swarm's binary cache and verify its signature, else `0` with a `reason` label (`no-record`, `not-cached`, `no-credentials`, `fetch-failed`, `bad-signature`; see [the builder docs](../swarm/builder.md#what-hives-do-with-the-cache)). Omitted before the first probe pass, every 5 minutes | +| `hyperhive.agent.needs_login` | `1` | gauge | `1` when a running agent needs a claude login (the dashboard's needs-login flag), `0` when it is logged in. Omitted for a stopped agent and before the first container scan | The `hyperhive.`-prefixed metrics have no semconv equivalent (memory limit + peak, on-disk footprint, and an instantaneous cpu percent kept @@ -552,15 +553,16 @@ failing sweep such as `swarm_status_publish`, `pending_logins`), and the message The swarm Grafana (`nix/host-modules/swarm-grafana.nix`) alerts on these: -| Alert | Fires when | -| ----------------------- | ----------------------------------------------------------------------------------- | -| `hh-hive-degraded` | `hyperhive.hive.degraded` is `1` for 2 minutes | -| `hh-hive-warning` | any `hyperhive.hive.warning` series other than `disk_pressure` exists for 2 minutes | -| `hh-host-disk-high` | `hyperhive.hive.nix_store.utilization` stays above 0.85 for 10 minutes | -| `hh-host-disk-critical` | `hyperhive.hive.nix_store.utilization` stays above 0.95 for 2 minutes | +| Alert | Fires when | +| ----------------------- | -------------------------------------------------------------------------------------------------------- | +| `hh-hive-degraded` | `hyperhive.hive.degraded` is `1` for 2 minutes | +| `hh-hive-warning` | any `hyperhive.hive.warning` series other than `disk_pressure` and `pending_logins` exists for 2 minutes | +| `hh-agent-needs-login` | `hyperhive.agent.needs_login` is `1` for an agent for 5 minutes, one alert per agent | +| `hh-host-disk-high` | `hyperhive.hive.nix_store.utilization` stays above 0.85 for 10 minutes | +| `hh-host-disk-critical` | `hyperhive.hive.nix_store.utilization` stays above 0.95 for 2 minutes | The disk alerts replace the old dashboard banner for the nix store, and -`hh-hive-warning` replaces it for every other warning. Disk pressure has its +`hh-hive-warning` replaces it for every other warning. `pending_logins` has its own per-agent alert, `hh-agent-needs-login`, so a login never pages twice. Disk pressure has its own pair so the thresholds sit on the metric rather than on the registry entry. diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index 8445931f4..cc60cc0bd 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -496,6 +496,7 @@ impl Coordinator { } }; crate::host_stats::refresh_pending_logins(&fresh); + crate::host_stats::record_login_states(&fresh); let mut last = self.last_containers.lock().await; let mut changed_or_new = Vec::new(); let mut removed = Vec::new(); diff --git a/hive-c0re/src/stats/host_stats.rs b/hive-c0re/src/stats/host_stats.rs index 44c172957..bd02e9d53 100644 --- a/hive-c0re/src/stats/host_stats.rs +++ b/hive-c0re/src/stats/host_stats.rs @@ -113,6 +113,40 @@ pub fn refresh_pending_logins(containers: &[ContainerView]) { } } +/// Per-agent login state of the last container snapshot; see +/// [`record_login_states`]. +static LOGIN_STATES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + +/// `(agent, 1 if it needs a login else 0)` for every running agent. A stopped +/// agent has no entry: `needs_login` is already running-gated in +/// `container_view::build_all`, so its `false` would read as "logged in". +#[must_use] +pub fn login_readings(containers: &[ContainerView]) -> Vec<(String, u64)> { + containers + .iter() + .filter(|c| c.running) + .map(|c| (c.name.clone(), u64::from(c.needs_login))) + .collect() +} + +/// Hold the login state of the current container snapshot for the +/// `hyperhive.agent.needs_login` gauge. Driven by the container rescan, next +/// to [`refresh_pending_logins`]. +pub fn record_login_states(containers: &[ContainerView]) { + *LOGIN_STATES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = login_readings(containers); +} + +/// The held readings of [`record_login_states`]; empty before the first rescan. +#[must_use] +pub fn held_login_states() -> Vec<(String, u64)> { + LOGIN_STATES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() +} + /// Agent-state warnings derived from the live container snapshot: agents /// that need a claude login. Takes the container list from its caller /// because that is where the snapshot lives. @@ -307,6 +341,20 @@ mod tests { assert!(current().is_none(), "no agent needing a login clears it"); } + #[test] + fn login_readings_are_per_running_agent_by_name() { + let mut stopped = cv("zed", false); + stopped.running = false; + let mut machine = cv("amy", true); + machine.container = "h-amy".to_owned(); + let readings = login_readings(&[machine, cv("bob", false), stopped]); + assert_eq!( + readings, + vec![("amy".to_owned(), 1), ("bob".to_owned(), 0)], + "agent names, 1 for needs-login, 0 for logged in, nothing for stopped" + ); + } + /// The gauge value is a fraction, so a percent leaking through would put /// every alert threshold 100x off. #[test] diff --git a/hive-c0re/src/stats/otel_metrics.rs b/hive-c0re/src/stats/otel_metrics.rs index 804bd7d9f..a9dadd1ba 100644 --- a/hive-c0re/src/stats/otel_metrics.rs +++ b/hive-c0re/src/stats/otel_metrics.rs @@ -259,6 +259,7 @@ fn register_instruments(provider: &SdkMeterProvider, snapshot: Arc Option { diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 67677dbf9..7146ec7d1 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -1720,18 +1720,29 @@ in # pressure is left out: it has its own thresholds below, on # the filesystem metric rather than the registry entry. # Sweeps raise their warning only after several misses, so - # `for` is short. `pending_logins` pages too, on purpose: - # an agent that waits more than 2m for its claude login - # needs a human to act, which is what the banner it - # replaces was for. + # `for` is short. `pending_logins` is left out: it has no + # `agent` label, and `hh-agent-needs-login` below pages + # once per agent instead. (metricThresholdRule { uid = "hh-hive-warning"; title = "hive warning raised"; - expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!="disk_pressure"})''; + expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!~"disk_pressure|pending_logins"})''; threshold = 0; for = "2m"; summary = "hive {{ $labels.hive }} has a {{ $labels.level }} warning raised: {{ $labels.kind }}; the message is the `hive warning raised` line in the hive-c0re log"; }) + # `hyperhive.agent.needs_login` is `1` for a running agent + # without a usable claude login; stopped agents report + # nothing. One alert instance per `(hive, agent)`; `for` + # lets a login in progress finish first. + (metricThresholdRule { + uid = "hh-agent-needs-login"; + title = "agent needs a claude login"; + expr = ''max by (hive, agent) ({__name__="hyperhive.agent.needs_login"}) == 1''; + threshold = 0; + for = "5m"; + summary = "agent {{ $labels.agent }} on hive {{ $labels.hive }} has needed a claude login for 5 minutes; open its page in the dashboard to complete it"; + }) # `hyperhive.hive.nix_store.utilization` is the used fraction of the # filesystem holding the nix store, the one that failed CI # with ENOSPC before the GC guardrails. diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 83fe5d0ca..2c0ca843b 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -940,6 +940,34 @@ let && lib.hasInfix "/ 18000" expr && lib.hasInfix "max by (account, window)" expr; } + { + # The needs-login alert is one instance per (hive, agent), fires after + # 5 minutes on the gauge hive-c0re exports, and `hh-hive-warning` leaves + # `pending_logins` to it so a login never pages twice. The disk rule is + # the presence control: it goes through the same lookup. + # 🔬 Invert-proof: change `for`, `max by (hive, agent)` or the metric on + # `hh-agent-needs-login`, drop `pending_logins` from `hh-hive-warning`, + # or rename the gauge in `otel_metrics.rs`. + name = "the needs-login alert is per agent, fires after 5 minutes and the hive warning rule skips pending_logins"; + ok = + let + groups = + grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups; + rules = lib.concatMap (g: g.rules) groups; + source = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs; + rule = uid: lib.findFirst (r: r.uid == uid) null rules; + login = rule "hh-agent-needs-login"; + expr = uid: (lib.head (rule uid).data).model.expr; + in + login != null + && login.for == "5m" + && rule "hh-host-disk-high" != null + && lib.hasInfix ''{__name__="hyperhive.agent.needs_login"}'' (expr "hh-agent-needs-login") + && lib.hasInfix ''"hyperhive.agent.needs_login"'' source + && lib.hasInfix "max by (hive, agent)" (expr "hh-agent-needs-login") + && lib.hasInfix "pending_logins" (expr "hh-hive-warning") + && lib.hasInfix "disk_pressure" (expr "hh-hive-warning"); + } { # The held-deploys alert fires on any agent held for 30 minutes and # selects the gauge name `held_deploys.rs` spells in Rust. The disk rule