export hyperhive.agent.needs_login and alert per agent

docs/4-implementation/scheduler/observability.md

@@ -354,6 +354,7 @@ filters those series out and the host dashboard's top-10 panels group by
 | `hyperhive.container.cpu.percent`       | `%`  | gauge   | host-normalised percent (custom — no `rate()` needed)                                                                                                                                                                                                                                                                                                         |
 | `hyperhive.agent.unrestricted_internet` | `1`  | gauge   | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable                                                                                                                                                                   |
 | `hyperhive.agent.closure_pullable`      | `1`  | gauge   | `1` when this hive can fetch the agent's latest closure from the swarm's binary cache and verify its signature, else `0` with a `reason` label (`no-record`, `not-cached`, `no-credentials`, `fetch-failed`, `bad-signature`; see [the builder docs](../swarm/builder.md#what-hives-do-with-the-cache)). Omitted before the first probe pass, every 5 minutes |
+| `hyperhive.agent.needs_login`           | `1`  | gauge   | `1` when a running agent needs a claude login (the dashboard's needs-login flag), `0` when it is logged in. Omitted for a stopped agent and before the first container scan                                                                                                                                                                                   |
 
 The `hyperhive.`-prefixed metrics have no semconv equivalent (memory
 limit + peak, on-disk footprint, and an instantaneous cpu percent kept
@@ -552,15 +553,16 @@ failing sweep such as `swarm_status_publish`, `pending_logins`), and the message
 
 The swarm Grafana (`nix/host-modules/swarm-grafana.nix`) alerts on these:
 
-| Alert                   | Fires when                                                                          |
+| Alert                   | Fires when                                                                                               |
-| ----------------------- | ----------------------------------------------------------------------------------- |
+| ----------------------- | -------------------------------------------------------------------------------------------------------- |
-| `hh-hive-degraded`      | `hyperhive.hive.degraded` is `1` for 2 minutes                                      |
+| `hh-hive-degraded`      | `hyperhive.hive.degraded` is `1` for 2 minutes                                                           |
-| `hh-hive-warning`       | any `hyperhive.hive.warning` series other than `disk_pressure` exists for 2 minutes |
+| `hh-hive-warning`       | any `hyperhive.hive.warning` series other than `disk_pressure` and `pending_logins` exists for 2 minutes |
-| `hh-host-disk-high`     | `hyperhive.hive.nix_store.utilization` stays above 0.85 for 10 minutes              |
+| `hh-agent-needs-login`  | `hyperhive.agent.needs_login` is `1` for an agent for 5 minutes, one alert per agent                     |
-| `hh-host-disk-critical` | `hyperhive.hive.nix_store.utilization` stays above 0.95 for 2 minutes               |
+| `hh-host-disk-high`     | `hyperhive.hive.nix_store.utilization` stays above 0.85 for 10 minutes                                   |
+| `hh-host-disk-critical` | `hyperhive.hive.nix_store.utilization` stays above 0.95 for 2 minutes                                    |
 
 The disk alerts replace the old dashboard banner for the nix store, and
-`hh-hive-warning` replaces it for every other warning. Disk pressure has its
+`hh-hive-warning` replaces it for every other warning. `pending_logins` has its own per-agent alert, `hh-agent-needs-login`, so a login never pages twice. Disk pressure has its
 own pair so the thresholds sit on the metric rather than on the registry
 entry.
 

hive-c0re/src/coordinator.rs

@@ -496,6 +496,7 @@ impl Coordinator {
             }
         };
         crate::host_stats::refresh_pending_logins(&fresh);
+        crate::host_stats::record_login_states(&fresh);
         let mut last = self.last_containers.lock().await;
         let mut changed_or_new = Vec::new();
         let mut removed = Vec::new();

hive-c0re/src/stats/host_stats.rs

@@ -113,6 +113,40 @@ pub fn refresh_pending_logins(containers: &[ContainerView]) {
     }
 }
 
+/// Per-agent login state of the last container snapshot; see
+/// [`record_login_states`].
+static LOGIN_STATES: std::sync::Mutex<Vec<(String, u64)>> = std::sync::Mutex::new(Vec::new());
+
+/// `(agent, 1 if it needs a login else 0)` for every running agent. A stopped
+/// agent has no entry: `needs_login` is already running-gated in
+/// `container_view::build_all`, so its `false` would read as "logged in".
+#[must_use]
+pub fn login_readings(containers: &[ContainerView]) -> Vec<(String, u64)> {
+    containers
+        .iter()
+        .filter(|c| c.running)
+        .map(|c| (c.name.clone(), u64::from(c.needs_login)))
+        .collect()
+}
+
+/// Hold the login state of the current container snapshot for the
+/// `hyperhive.agent.needs_login` gauge. Driven by the container rescan, next
+/// to [`refresh_pending_logins`].
+pub fn record_login_states(containers: &[ContainerView]) {
+    *LOGIN_STATES
+        .lock()
+        .unwrap_or_else(std::sync::PoisonError::into_inner) = login_readings(containers);
+}
+
+/// The held readings of [`record_login_states`]; empty before the first rescan.
+#[must_use]
+pub fn held_login_states() -> Vec<(String, u64)> {
+    LOGIN_STATES
+        .lock()
+        .unwrap_or_else(std::sync::PoisonError::into_inner)
+        .clone()
+}
+
 /// Agent-state warnings derived from the live container snapshot: agents
 /// that need a claude login. Takes the container list from its caller
 /// because that is where the snapshot lives.
@@ -307,6 +341,20 @@ mod tests {
         assert!(current().is_none(), "no agent needing a login clears it");
     }
 
+    #[test]
+    fn login_readings_are_per_running_agent_by_name() {
+        let mut stopped = cv("zed", false);
+        stopped.running = false;
+        let mut machine = cv("amy", true);
+        machine.container = "h-amy".to_owned();
+        let readings = login_readings(&[machine, cv("bob", false), stopped]);
+        assert_eq!(
+            readings,
+            vec![("amy".to_owned(), 1), ("bob".to_owned(), 0)],
+            "agent names, 1 for needs-login, 0 for logged in, nothing for stopped"
+        );
+    }
+
     /// The gauge value is a fraction, so a percent leaking through would put
     /// every alert threshold 100x off.
     #[test]

hive-c0re/src/stats/otel_metrics.rs

@@ -259,6 +259,7 @@ fn register_instruments(provider: &SdkMeterProvider, snapshot: Arc<Mutex<Vec<Con
         .build();
 
     register_closure_pullable(&meter);
+    register_needs_login(&meter);
     register_hive_instruments(&meter);
 }
 
@@ -281,6 +282,20 @@ fn register_closure_pullable(meter: &opentelemetry::metrics::Meter) {
         .build();
 }
 
+/// 1 when a running agent needs a claude login, else 0, from the container
+/// snapshot the dashboard shows. A stopped agent reports nothing.
+fn register_needs_login(meter: &opentelemetry::metrics::Meter) {
+    meter
+        .u64_observable_gauge("hyperhive.agent.needs_login")
+        .with_unit("1")
+        .with_callback(|obs| {
+            for (agent, value) in crate::host_stats::held_login_states() {
+                obs.observe(value, &[KeyValue::new("agent", agent)]);
+            }
+        })
+        .build();
+}
+
 /// The gauge value for a sampled container: `None` for one that belongs to no
 /// agent (the CI runner), else [`unrestricted_value`] for its agent.
 fn unrestricted_reading(stored: &permissions::Stored, c: &ContainerResource) -> Option<u64> {

nix/host-modules/swarm-grafana.nix

@@ -1720,18 +1720,29 @@ in
                     # pressure is left out: it has its own thresholds below, on
                     # the filesystem metric rather than the registry entry.
                     # Sweeps raise their warning only after several misses, so
-                    # `for` is short. `pending_logins` pages too, on purpose:
+                    # `for` is short. `pending_logins` is left out: it has no
-                    # an agent that waits more than 2m for its claude login
+                    # `agent` label, and `hh-agent-needs-login` below pages
-                    # needs a human to act, which is what the banner it
+                    # once per agent instead.
-                    # replaces was for.
                     (metricThresholdRule {
                       uid = "hh-hive-warning";
                       title = "hive warning raised";
-                      expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!="disk_pressure"})'';
+                      expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!~"disk_pressure|pending_logins"})'';
                       threshold = 0;
                       for = "2m";
                       summary = "hive {{ $labels.hive }} has a {{ $labels.level }} warning raised: {{ $labels.kind }}; the message is the `hive warning raised` line in the hive-c0re log";
                     })
+                    # `hyperhive.agent.needs_login` is `1` for a running agent
+                    # without a usable claude login; stopped agents report
+                    # nothing. One alert instance per `(hive, agent)`; `for`
+                    # lets a login in progress finish first.
+                    (metricThresholdRule {
+                      uid = "hh-agent-needs-login";
+                      title = "agent needs a claude login";
+                      expr = ''max by (hive, agent) ({__name__="hyperhive.agent.needs_login"}) == 1'';
+                      threshold = 0;
+                      for = "5m";
+                      summary = "agent {{ $labels.agent }} on hive {{ $labels.hive }} has needed a claude login for 5 minutes; open its page in the dashboard to complete it";
+                    })
                     # `hyperhive.hive.nix_store.utilization` is the used fraction of the
                     # filesystem holding the nix store, the one that failed CI
                     # with ENOSPC before the GC guardrails.

nix/module-eval/grafana.nix

@@ -940,6 +940,34 @@ let
         && lib.hasInfix "/ 18000" expr
         && lib.hasInfix "max by (account, window)" expr;
     }
+    {
+      # The needs-login alert is one instance per (hive, agent), fires after
+      # 5 minutes on the gauge hive-c0re exports, and `hh-hive-warning` leaves
+      # `pending_logins` to it so a login never pages twice. The disk rule is
+      # the presence control: it goes through the same lookup.
+      # 🔬 Invert-proof: change `for`, `max by (hive, agent)` or the metric on
+      # `hh-agent-needs-login`, drop `pending_logins` from `hh-hive-warning`,
+      # or rename the gauge in `otel_metrics.rs`.
+      name = "the needs-login alert is per agent, fires after 5 minutes and the hive warning rule skips pending_logins";
+      ok =
+        let
+          groups =
+            grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
+          rules = lib.concatMap (g: g.rules) groups;
+          source = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs;
+          rule = uid: lib.findFirst (r: r.uid == uid) null rules;
+          login = rule "hh-agent-needs-login";
+          expr = uid: (lib.head (rule uid).data).model.expr;
+        in
+        login != null
+        && login.for == "5m"
+        && rule "hh-host-disk-high" != null
+        && lib.hasInfix ''{__name__="hyperhive.agent.needs_login"}'' (expr "hh-agent-needs-login")
+        && lib.hasInfix ''"hyperhive.agent.needs_login"'' source
+        && lib.hasInfix "max by (hive, agent)" (expr "hh-agent-needs-login")
+        && lib.hasInfix "pending_logins" (expr "hh-hive-warning")
+        && lib.hasInfix "disk_pressure" (expr "hh-hive-warning");
+    }
     {
       # The held-deploys alert fires on any agent held for 30 minutes and
       # selects the gauge name `held_deploys.rs` spells in Rust. The disk rule