hive-c0re: reconcile declared agents periodically, with backoff

hive-c0re/src/stats/otel_metrics.rs

@@ -27,6 +27,7 @@
 //! `agent` attribute**, and that absence is the point: it is what makes them
 //! selectable as `{hive!="", agent=""}`.
 
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, Mutex, OnceLock};
 use std::time::{Duration, Instant};
 
@@ -58,6 +59,14 @@ static PROVIDER: OnceLock<SdkMeterProvider> = OnceLock::new();
 /// doc says exporter-start rather than claiming a precision it does not have.
 static STARTED: OnceLock<Instant> = OnceLock::new();
 
+/// Restarts the periodic reconcile tick has queued since this process started.
+static RECONCILE_RESTARTS: AtomicU64 = AtomicU64::new(0);
+
+/// Count one restart queued by the periodic reconcile tick.
+pub fn count_reconcile_restart() {
+    RECONCILE_RESTARTS.fetch_add(1, Ordering::Relaxed);
+}
+
 /// Spawn the container-resource OTEL exporter if OTEL is configured
 /// (`OTEL_EXPORTER_OTLP_ENDPOINT` non-empty). No-op otherwise. Call once at
 /// startup.
@@ -497,6 +506,17 @@ fn register_hive_instruments(meter: &opentelemetry::metrics::Meter) {
         })
         .build();
 
+    // Restarts the periodic reconcile tick queued for an agent that was wanted
+    // up and found down. Hive-scoped: the agent is in the log line, and a
+    // per-agent label would grow a series for every agent that ever failed.
+    meter
+        .u64_observable_counter("hyperhive.hive.reconcile_tick.restarts")
+        .with_unit("{restart}")
+        .with_callback(|obs| {
+            obs.observe(RECONCILE_RESTARTS.load(Ordering::Relaxed), HIVE_ATTRS);
+        })
+        .build();
+
     // Host-wide stall totals from `/proc/pressure`, by `resource` (`cpu`,
     // `memory`, `io`) and `kind` (`some`, `full`). Counters in microseconds,
     // so a rate over them is the stalled fraction of wall time. No sample on a