review classifier: export request, verdict, check and run metrics, with a dashboard

The controller counts review requests, verdicts and per-check results and times each run; the classifier service counts retried asks. A provisioned Grafana dashboard charts them.

swarm-controller/src/review_classifier/metrics.rs

@@ -0,0 +1,283 @@
+//! OTEL export of what review requests and review runs come to: the counters
+//! `hyperhive.review_classifier.requests` (`outcome`), `.verdicts` (`outcome`)
+//! and `.checks` (`check`, `result`), and the histogram
+//! `hyperhive.review_classifier.run.duration` in seconds (`outcome`). The
+//! classifier service exports the retries beside its cost counters, see
+//! `swarm-review-classifier`'s `cost` module.
+//!
+//! Same exporter as [`crate::vcs_metrics`], built on the first record; with no
+//! `OTEL_EXPORTER_OTLP_ENDPOINT` every record does nothing.
+
+use std::sync::{Arc, OnceLock};
+
+use opentelemetry::KeyValue;
+use opentelemetry::metrics::{Counter, Histogram, MeterProvider as _};
+use opentelemetry_sdk::metrics::SdkMeterProvider;
+use swarm_wire::review_classifier::{Row, Verdict as RowVerdict};
+
+use super::runs::RunOutcome;
+
+/// What became of a review request.
+#[derive(Debug, Clone, Copy)]
+pub(super) enum Request {
+    /// Stored in the request queue.
+    Queued,
+    /// The queue did not store it; a no-verdict comment was posted.
+    QueueFailed,
+    /// Taken from the queue but unreadable, and dropped.
+    Unparsable,
+    /// Taken while the PR's review was going; put back in the queue.
+    Deferred,
+    /// Taken, and a fresh run began.
+    Started,
+    /// Taken for a head already reviewed; its verdict was posted again.
+    Reposted,
+}
+
+impl Request {
+    const fn label(self) -> &'static str {
+        match self {
+            Self::Queued => "queued",
+            Self::QueueFailed => "queue_failed",
+            Self::Unparsable => "unparsable",
+            Self::Deferred => "deferred",
+            Self::Started => "started",
+            Self::Reposted => "reposted",
+        }
+    }
+}
+
+/// The review event a run chose.
+#[derive(Debug, Clone, Copy)]
+pub(super) enum Choice {
+    Approve,
+    RequestChanges,
+    NoVerdict,
+}
+
+impl Choice {
+    const fn label(self) -> &'static str {
+        match self {
+            Self::Approve => "approve",
+            Self::RequestChanges => "request_changes",
+            Self::NoVerdict => "no_verdict",
+        }
+    }
+}
+
+/// A run's `outcome` label: the verdict names, plus `post_failed`.
+const fn run_label(outcome: RunOutcome) -> &'static str {
+    match outcome {
+        RunOutcome::Approved => "approve",
+        RunOutcome::ChangesRequested => "request_changes",
+        RunOutcome::NoVerdict => "no_verdict",
+        RunOutcome::PostFailed => "post_failed",
+        RunOutcome::QueueFull => "queue_full",
+        RunOutcome::Done => "done",
+        RunOutcome::Failed => "failed",
+    }
+}
+
+/// A row's `result` label.
+const fn check_result(row: &Row) -> &'static str {
+    match (row.error.is_some(), row.verdict) {
+        (true, _) => "no_answer",
+        (false, RowVerdict::Match) => "match",
+        (false, RowVerdict::NoMatch) => "no_match",
+        (false, RowVerdict::NotApplicable) => "unsure",
+    }
+}
+
+/// Seconds. A run is one model call per classification plus the forge reads
+/// and the post.
+const DURATION_BOUNDARIES: [f64; 12] = [
+    1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 60.0, 90.0, 120.0, 180.0, 300.0, 600.0,
+];
+
+struct Instruments {
+    /// Exports only while it lives.
+    _provider: SdkMeterProvider,
+    requests: Counter<u64>,
+    verdicts: Counter<u64>,
+    checks: Counter<u64>,
+    duration: Histogram<f64>,
+}
+
+impl Instruments {
+    fn new(provider: &SdkMeterProvider) -> Self {
+        let meter = provider.meter("hyperhive.review_classifier");
+        Self {
+            _provider: provider.clone(),
+            requests: meter
+                .u64_counter("hyperhive.review_classifier.requests")
+                .build(),
+            verdicts: meter
+                .u64_counter("hyperhive.review_classifier.verdicts")
+                .build(),
+            checks: meter
+                .u64_counter("hyperhive.review_classifier.checks")
+                .build(),
+            duration: meter
+                .f64_histogram("hyperhive.review_classifier.run.duration")
+                .with_unit("s")
+                .with_boundaries(DURATION_BOUNDARIES.to_vec())
+                .build(),
+        }
+    }
+}
+
+/// The recorder: a handle on the instruments, or on nothing where no
+/// collector is configured.
+#[derive(Clone)]
+pub(crate) struct Metrics(Option<Arc<Instruments>>);
+
+static GLOBAL: OnceLock<Option<Arc<Instruments>>> = OnceLock::new();
+
+impl Metrics {
+    /// The process's recorder, building its exporter on the first call.
+    pub(crate) fn global() -> Self {
+        Self(GLOBAL.get_or_init(|| build().map(Arc::new)).clone())
+    }
+
+    /// Records nothing.
+    #[cfg(test)]
+    pub(super) const fn off() -> Self {
+        Self(None)
+    }
+
+    /// Instruments on `provider`.
+    #[cfg(test)]
+    pub(super) fn on(provider: &SdkMeterProvider) -> Self {
+        Self(Some(Arc::new(Instruments::new(provider))))
+    }
+
+    pub(super) fn request(&self, request: Request) {
+        if let Some(inst) = &self.0 {
+            inst.requests
+                .add(1, &[KeyValue::new("outcome", request.label())]);
+        }
+    }
+
+    pub(super) fn verdict(&self, choice: Choice) {
+        if let Some(inst) = &self.0 {
+            inst.verdicts
+                .add(1, &[KeyValue::new("outcome", choice.label())]);
+        }
+    }
+
+    /// One count per classification the run answered or failed to.
+    pub(super) fn checks(&self, rows: &[Row]) {
+        let Some(inst) = &self.0 else { return };
+        for row in rows {
+            inst.checks.add(
+                1,
+                &[
+                    KeyValue::new("check", row.name.clone()),
+                    KeyValue::new("result", check_result(row)),
+                ],
+            );
+        }
+    }
+
+    pub(super) fn run_duration(&self, outcome: RunOutcome, seconds: f64) {
+        if let Some(inst) = &self.0 {
+            inst.duration
+                .record(seconds, &[KeyValue::new("outcome", run_label(outcome))]);
+        }
+    }
+}
+
+fn build() -> Option<Instruments> {
+    let endpoint = crate::vcs_metrics::endpoint()?;
+    let interval = crate::vcs_metrics::EXPORT_INTERVAL;
+    match crate::vcs_metrics::build_provider(interval) {
+        Ok(provider) => {
+            tracing::info!(%endpoint, ?interval, "otel review-classifier metrics: exporter enabled");
+            Some(Instruments::new(&provider))
+        }
+        Err(e) => {
+            tracing::warn!(error = ?e, "otel review-classifier metrics: exporter init failed");
+            None
+        }
+    }
+}
+
+#[cfg(test)]
+pub(super) mod testing {
+    use std::collections::BTreeMap;
+
+    use opentelemetry_sdk::metrics::data::{
+        AggregatedMetrics, MetricData, ResourceMetrics, ScopeMetrics,
+    };
+    use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader};
+
+    use super::{Metrics, SdkMeterProvider};
+
+    /// Instruments on a provider that exports to memory.
+    pub(in crate::review_classifier) struct Recorder {
+        pub(in crate::review_classifier) metrics: Metrics,
+        provider: SdkMeterProvider,
+        exporter: InMemoryMetricExporter,
+    }
+
+    impl Recorder {
+        pub(in crate::review_classifier) fn new() -> Self {
+            let exporter = InMemoryMetricExporter::default();
+            let reader = PeriodicReader::builder(exporter.clone()).build();
+            let provider = SdkMeterProvider::builder().with_reader(reader).build();
+            Self {
+                metrics: Metrics::on(&provider),
+                provider,
+                exporter,
+            }
+        }
+
+        /// `read` of the newest export of the metric `name`.
+        fn export<R>(&self, name: &str, read: impl FnOnce(&AggregatedMetrics) -> R) -> Option<R> {
+            self.provider.force_flush().expect("flush");
+            let all = self.exporter.get_finished_metrics().expect("metrics");
+            all.last()
+                .into_iter()
+                .flat_map(ResourceMetrics::scope_metrics)
+                .flat_map(ScopeMetrics::metrics)
+                .find(|m| m.name() == name)
+                .map(|m| read(m.data()))
+        }
+
+        /// The counter `name`'s value per label set, each set as
+        /// `key=value,key=value` in key order. Empty when nothing was added.
+        pub(in crate::review_classifier) fn counter(&self, name: &str) -> BTreeMap<String, u64> {
+            self.export(name, |data| {
+                let AggregatedMetrics::U64(MetricData::Sum(sum)) = data else {
+                    panic!("not a u64 sum: {data:?}");
+                };
+                sum.data_points()
+                    .map(|p| (labels(p.attributes()), p.value()))
+                    .collect()
+            })
+            .unwrap_or_default()
+        }
+
+        /// The histogram `name`'s sample count per label set, as
+        /// [`Self::counter`] writes them.
+        pub(in crate::review_classifier) fn histogram(&self, name: &str) -> BTreeMap<String, u64> {
+            self.export(name, |data| {
+                let AggregatedMetrics::F64(MetricData::Histogram(h)) = data else {
+                    panic!("not an f64 histogram: {data:?}");
+                };
+                h.data_points()
+                    .map(|p| (labels(p.attributes()), p.count()))
+                    .collect()
+            })
+            .unwrap_or_default()
+        }
+    }
+
+    fn labels<'a>(attributes: impl Iterator<Item = &'a opentelemetry::KeyValue>) -> String {
+        let mut all: Vec<String> = attributes
+            .map(|kv| format!("{}={}", kv.key, kv.value))
+            .collect();
+        all.sort();
+        all.join(",")
+    }
+}