review classifier: export request, verdict, check and run metrics, with a dashboard

The controller counts review requests, verdicts and per-check results and times each run; the classifier service counts retried asks. A provisioned Grafana dashboard charts them.

swarm-review-classifier/src/cost.rs

@@ -7,6 +7,10 @@
 //! `claude` exports for itself on the claude runtime, so the cache hit rate is
 //! `cacheRead` over the three input types on either runtime.
 //!
+//! The counter `hyperhive.review_classifier.ask.retries` counts the asks
+//! repeated because an answer did not validate, with the run's `runtime`
+//! (`api_key` or `acp`); it is exported on either runtime.
+//!
 //! The amount is computed here from the prices the run carries and the
 //! tokens the agent reports, so it does not depend on how, or whether, the
 //! agent prices a custom provider.
@@ -23,7 +27,7 @@ use opentelemetry::KeyValue;
 use opentelemetry::metrics::{Counter, MeterProvider as _};
 use opentelemetry_otlp::{MetricExporter, Protocol, WithExportConfig};
 use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
-use swarm_wire::review_classifier::Prices;
+use swarm_wire::review_classifier::{Prices, Runtime};
 
 /// The currency every price is in.
 pub const CURRENCY: &str = "USD";
@@ -68,7 +72,6 @@ pub struct Otel;
 
 impl CostSink for Otel {
     fn record(&self, cost: &Cost) {
-        static COUNTERS: OnceLock<Option<(SdkMeterProvider, Counters)>> = OnceLock::new();
         let Some((_, counters)) = COUNTERS.get_or_init(build) else {
             return;
         };
@@ -76,10 +79,21 @@ impl CostSink for Otel {
     }
 }
 
-/// The cost counter, and the token counter beside it.
+/// Counts one repeated ask on `runtime`'s model.
+pub fn ask_retried(runtime: &Runtime) {
+    let Some((_, counters)) = COUNTERS.get_or_init(build) else {
+        return;
+    };
+    add_retry(counters, runtime);
+}
+
+static COUNTERS: OnceLock<Option<(SdkMeterProvider, Counters)>> = OnceLock::new();
+
+/// The cost counter, the token counter beside it, and the retry counter.
 struct Counters {
     cost: Counter<f64>,
     tokens: Counter<u64>,
+    retries: Counter<u64>,
 }
 
 /// The counters on `provider`.
@@ -92,9 +106,21 @@ fn counters(provider: &SdkMeterProvider) -> Counters {
         tokens: meter
             .u64_counter("hyperhive.review_classifier.token.usage")
             .build(),
+        retries: meter
+            .u64_counter("hyperhive.review_classifier.ask.retries")
+            .build(),
     }
 }
 
+/// Adds one retry to `counters`, labelled with `runtime`.
+fn add_retry(counters: &Counters, runtime: &Runtime) {
+    let name = match runtime {
+        Runtime::ApiKey => "api_key",
+        Runtime::Acp(_) => "acp",
+    };
+    counters.retries.add(1, &[KeyValue::new("runtime", name)]);
+}
+
 /// Adds `cost` to `counters` with the run's labels.
 fn add(counters: &Counters, cost: &Cost) {
     let labels = [
@@ -293,4 +319,34 @@ mod tests {
             ]
         );
     }
+
+    #[test]
+    fn a_retry_adds_one_under_the_counter_name_and_the_runtime() {
+        let exporter = InMemoryMetricExporter::default();
+        let reader = PeriodicReader::builder(exporter.clone()).build();
+        let provider = SdkMeterProvider::builder().with_reader(reader).build();
+        let counters = counters(&provider);
+        add_retry(&counters, &Runtime::ApiKey);
+        add_retry(&counters, &Runtime::ApiKey);
+        provider.force_flush().expect("flush");
+
+        let metrics = exporter.get_finished_metrics().expect("metrics");
+        let metric = metrics
+            .iter()
+            .flat_map(ResourceMetrics::scope_metrics)
+            .flat_map(ScopeMetrics::metrics)
+            .find(|m| m.name() == "hyperhive.review_classifier.ask.retries")
+            .expect("the retry counter was exported");
+        let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
+            panic!("not a u64 sum: {:?}", metric.data());
+        };
+        let points: Vec<_> = sum.data_points().collect();
+        assert_eq!(points.len(), 1);
+        assert_eq!(points[0].value(), 2);
+        let labels: Vec<_> = points[0]
+            .attributes()
+            .map(|kv| (kv.key.to_string(), kv.value.to_string()))
+            .collect();
+        assert_eq!(labels, [("runtime".to_owned(), "api_key".to_owned())]);
+    }
 }

swarm-review-classifier/src/run.rs

@@ -22,7 +22,7 @@ use swarm_wire::agent_inference::Effort;
 use swarm_wire::review_classifier::{CLASSIFICATION_DEADLINE, Reply, Row, Runtime, TryOut};
 
 use crate::prompt::Prompt;
-use crate::{aggregate, answer, prompt};
+use crate::{aggregate, answer, cost, prompt};
 
 /// How long `claude` may print nothing before it is stopped.
 const IDLE_TIMEOUT: Duration = Duration::from_mins(2);
@@ -69,6 +69,7 @@ pub async fn answer(keys: &impl KeySource, model: &impl Model, run: &TryOut) ->
             run,
             CLASSIFICATION_DEADLINE,
             RETRY_FLOOR,
+            || cost::ask_retried(&run.runtime),
         )
         .await;
         rows.push(aggregate::row(c, outcome));
@@ -96,7 +97,7 @@ const RETRY_FLOOR: Duration = Duration::from_secs(20);
 /// answer that does not validate is asked for once more, with a reminder to
 /// reply with the JSON object alone, and only the time the first ask left of
 /// `budget` to do it in; with less than `floor` left, or after a failed ask,
-/// it is not asked again.
+/// it is not asked again. `on_retry` runs just before the second ask.
 async fn classify(
     model: &impl Model,
     prompt: &Prompt<'_>,
@@ -104,6 +105,7 @@ async fn classify(
     run: &TryOut,
     budget: Duration,
     floor: Duration,
+    on_retry: impl FnOnce(),
 ) -> Result<answer::Answer, String> {
     let started = Instant::now();
     let first = model.ask(prompt, key, run, budget).await?;
@@ -115,6 +117,7 @@ async fn classify(
     if left < floor {
         return Err(unparsed);
     }
+    on_retry();
     let again = model
         .ask(&prompt.with_json_reminder(), key, run, left)
         .await?;
@@ -643,9 +646,27 @@ mod tests {
     const FLOOR: Duration = Duration::from_millis(100);
 
     async fn classify_slow(model: &Slow) -> Result<answer::Answer, String> {
+        classify_counting(model).await.0
+    }
+
+    /// The result, and how often `on_retry` ran.
+    async fn classify_counting(model: &Slow) -> (Result<answer::Answer, String>, usize) {
         let run = run(&["c"]);
         let prompt = Prompt::of("q");
-        classify(model, &prompt, "k", &run, BUDGET, FLOOR).await
+        let retries = std::cell::Cell::new(0);
+        let result = classify(model, &prompt, "k", &run, BUDGET, FLOOR, || {
+            retries.set(retries.get() + 1);
+        })
+        .await;
+        (result, retries.get())
+    }
+
+    #[tokio::test]
+    async fn a_retry_is_counted_and_a_skipped_one_is_not() {
+        let (_, retries) = classify_counting(&Slow::new(Duration::from_millis(200))).await;
+        assert_eq!(retries, 1);
+        let (_, retries) = classify_counting(&Slow::new(BUDGET)).await;
+        assert_eq!(retries, 0, "no second ask, so no retry");
     }
 
     #[tokio::test]