review classifier: export request, verdict, check and run metrics, with a dashboard

The controller counts review requests, verdicts and per-check results and times each run; the classifier service counts retried asks. A provisioned Grafana dashboard charts them.

swarm-review-classifier/src/run.rs

@@ -22,7 +22,7 @@ use swarm_wire::agent_inference::Effort;
 use swarm_wire::review_classifier::{CLASSIFICATION_DEADLINE, Reply, Row, Runtime, TryOut};
 
 use crate::prompt::Prompt;
-use crate::{aggregate, answer, prompt};
+use crate::{aggregate, answer, cost, prompt};
 
 /// How long `claude` may print nothing before it is stopped.
 const IDLE_TIMEOUT: Duration = Duration::from_mins(2);
@@ -69,6 +69,7 @@ pub async fn answer(keys: &impl KeySource, model: &impl Model, run: &TryOut) ->
             run,
             CLASSIFICATION_DEADLINE,
             RETRY_FLOOR,
+            || cost::ask_retried(&run.runtime),
         )
         .await;
         rows.push(aggregate::row(c, outcome));
@@ -96,7 +97,7 @@ const RETRY_FLOOR: Duration = Duration::from_secs(20);
 /// answer that does not validate is asked for once more, with a reminder to
 /// reply with the JSON object alone, and only the time the first ask left of
 /// `budget` to do it in; with less than `floor` left, or after a failed ask,
-/// it is not asked again.
+/// it is not asked again. `on_retry` runs just before the second ask.
 async fn classify(
     model: &impl Model,
     prompt: &Prompt<'_>,
@@ -104,6 +105,7 @@ async fn classify(
     run: &TryOut,
     budget: Duration,
     floor: Duration,
+    on_retry: impl FnOnce(),
 ) -> Result<answer::Answer, String> {
     let started = Instant::now();
     let first = model.ask(prompt, key, run, budget).await?;
@@ -115,6 +117,7 @@ async fn classify(
     if left < floor {
         return Err(unparsed);
     }
+    on_retry();
     let again = model
         .ask(&prompt.with_json_reminder(), key, run, left)
         .await?;
@@ -643,9 +646,27 @@ mod tests {
     const FLOOR: Duration = Duration::from_millis(100);
 
     async fn classify_slow(model: &Slow) -> Result<answer::Answer, String> {
+        classify_counting(model).await.0
+    }
+
+    /// The result, and how often `on_retry` ran.
+    async fn classify_counting(model: &Slow) -> (Result<answer::Answer, String>, usize) {
         let run = run(&["c"]);
         let prompt = Prompt::of("q");
-        classify(model, &prompt, "k", &run, BUDGET, FLOOR).await
+        let retries = std::cell::Cell::new(0);
+        let result = classify(model, &prompt, "k", &run, BUDGET, FLOOR, || {
+            retries.set(retries.get() + 1);
+        })
+        .await;
+        (result, retries.get())
+    }
+
+    #[tokio::test]
+    async fn a_retry_is_counted_and_a_skipped_one_is_not() {
+        let (_, retries) = classify_counting(&Slow::new(Duration::from_millis(200))).await;
+        assert_eq!(retries, 1);
+        let (_, retries) = classify_counting(&Slow::new(BUDGET)).await;
+        assert_eq!(retries, 0, "no second ask, so no retry");
     }
 
     #[tokio::test]