review classifier: export request, verdict, check and run metrics, with a dashboard

The controller counts review requests, verdicts and per-check results and times each run; the classifier service counts retried asks. A provisioned Grafana dashboard charts them.

swarm-review-classifier/src/cost.rs

@@ -7,6 +7,10 @@
 //! `claude` exports for itself on the claude runtime, so the cache hit rate is
 //! `cacheRead` over the three input types on either runtime.
 //!
+//! The counter `hyperhive.review_classifier.ask.retries` counts the asks
+//! repeated because an answer did not validate, with the run's `runtime`
+//! (`api_key` or `acp`); it is exported on either runtime.
+//!
 //! The amount is computed here from the prices the run carries and the
 //! tokens the agent reports, so it does not depend on how, or whether, the
 //! agent prices a custom provider.
@@ -23,7 +27,7 @@ use opentelemetry::KeyValue;
 use opentelemetry::metrics::{Counter, MeterProvider as _};
 use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig};
 use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
-use swarm_wire::review_classifier::Prices;
+use swarm_wire::review_classifier::{Prices, Runtime};
 
 /// The currency every price is in.
 pub const CURRENCY: &str = "USD";
@@ -68,7 +72,6 @@ pub struct Otel;
 
 impl CostSink for Otel {
     fn record(&self, cost: &Cost) {
-        static COUNTERS: OnceLock<Option<(SdkMeterProvider, Counters)>> = OnceLock::new();
         let Some((_, counters)) = COUNTERS.get_or_init(build) else {
             return;
         };
@@ -76,10 +79,21 @@ impl CostSink for Otel {
     }
 }
 
-/// The cost counter, and the token counter beside it.
+/// Counts one repeated ask on `runtime`'s model.
+pub fn ask_retried(runtime: &Runtime) {
+    let Some((_, counters)) = COUNTERS.get_or_init(build) else {
+        return;
+    };
+    add_retry(counters, runtime);
+}
+
+static COUNTERS: OnceLock<Option<(SdkMeterProvider, Counters)>> = OnceLock::new();
+
+/// The cost counter, the token counter beside it, and the retry counter.
 struct Counters {
     cost: Counter<f64>,
     tokens: Counter<u64>,
+    retries: Counter<u64>,
 }
 
 /// The counters on `provider`.
@@ -92,9 +106,21 @@ fn counters(provider: &SdkMeterProvider) -> Counters {
         tokens: meter
             .u64_counter("hyperhive.review_classifier.token.usage")
             .build(),
+        retries: meter
+            .u64_counter("hyperhive.review_classifier.ask.retries")
+            .build(),
     }
 }
 
+/// Adds one retry to `counters`, labelled with `runtime`.
+fn add_retry(counters: &Counters, runtime: &Runtime) {
+    let name = match runtime {
+        Runtime::ApiKey => "api_key",
+        Runtime::Acp(_) => "acp",
+    };
+    counters.retries.add(1, &[KeyValue::new("runtime", name)]);
+}
+
 /// Adds `cost` to `counters` with the run's labels.
 fn add(counters: &Counters, cost: &Cost) {
     let labels = [
@@ -293,4 +319,34 @@ mod tests {
             ]
         );
     }
+
+    #[test]
+    fn a_retry_adds_one_under_the_counter_name_and_the_runtime() {
+        let exporter = InMemoryMetricExporter::default();
+        let reader = PeriodicReader::builder(exporter.clone()).build();
+        let provider = SdkMeterProvider::builder().with_reader(reader).build();
+        let counters = counters(&provider);
+        add_retry(&counters, &Runtime::ApiKey);
+        add_retry(&counters, &Runtime::ApiKey);
+        provider.force_flush().expect("flush");
+
+        let metrics = exporter.get_finished_metrics().expect("metrics");
+        let metric = metrics
+            .iter()
+            .flat_map(ResourceMetrics::scope_metrics)
+            .flat_map(ScopeMetrics::metrics)
+            .find(|m| m.name() == "hyperhive.review_classifier.ask.retries")
+            .expect("the retry counter was exported");
+        let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
+            panic!("not a u64 sum: {:?}", metric.data());
+        };
+        let points: Vec<_> = sum.data_points().collect();
+        assert_eq!(points.len(), 1);
+        assert_eq!(points[0].value(), 2);
+        let labels: Vec<_> = points[0]
+            .attributes()
+            .map(|kv| (kv.key.to_string(), kv.value.to_string()))
+            .collect();
+        assert_eq!(labels, [("runtime".to_owned(), "api_key".to_owned())]);
+    }
 }