hyperhive/swarm-controller/src/review_classifier/metrics.rs

//! OTEL export of what review requests and review runs come to: the counters
//! `hyperhive.review_classifier.requests` (`outcome`), `.verdicts` (`outcome`)
//! and `.checks` (`check`, `result`), and the histogram
//! `hyperhive.review_classifier.run.duration` in seconds (`outcome`). The
//! classifier service exports the retries beside its cost counters, see
//! `swarm-review-classifier`'s `cost` module.
//!
//! Same exporter as [`crate::vcs_metrics`], built on the first record; with no
//! `OTEL_EXPORTER_OTLP_ENDPOINT` every record does nothing.

use std::sync::{Arc, OnceLock};

use opentelemetry::KeyValue;
use opentelemetry::metrics::{Counter, Histogram, MeterProvider as _};
use opentelemetry_sdk::metrics::SdkMeterProvider;
use swarm_wire::review_classifier::{Row, Verdict as RowVerdict};

use super::runs::RunOutcome;

/// What became of a review request.
#[derive(Debug, Clone, Copy)]
pub(super) enum Request {
    /// Stored in the request queue.
    Queued,
    /// The queue did not store it; a no-verdict comment was posted.
    QueueFailed,
    /// Taken from the queue but unreadable, and dropped.
    Unparsable,
    /// Taken while the PR's review was going; put back in the queue.
    Deferred,
    /// Taken, and a fresh run began.
    Started,
    /// Taken for a head already reviewed; its verdict was posted again.
    Reposted,
}

impl Request {
    const fn label(self) -> &'static str {
        match self {
            Self::Queued => "queued",
            Self::QueueFailed => "queue_failed",
            Self::Unparsable => "unparsable",
            Self::Deferred => "deferred",
            Self::Started => "started",
            Self::Reposted => "reposted",
        }
    }
}

/// The review event a run chose.
#[derive(Debug, Clone, Copy)]
pub(super) enum Choice {
    Approve,
    RequestChanges,
    NoVerdict,
}

impl Choice {
    const fn label(self) -> &'static str {
        match self {
            Self::Approve => "approve",
            Self::RequestChanges => "request_changes",
            Self::NoVerdict => "no_verdict",
        }
    }
}

/// A run's `outcome` label: the verdict names, plus `post_failed`.
const fn run_label(outcome: RunOutcome) -> &'static str {
    match outcome {
        RunOutcome::Approved => "approve",
        RunOutcome::ChangesRequested => "request_changes",
        RunOutcome::NoVerdict => "no_verdict",
        RunOutcome::PostFailed => "post_failed",
        RunOutcome::QueueFull => "queue_full",
        RunOutcome::Done => "done",
        RunOutcome::Failed => "failed",
    }
}

/// A row's `result` label.
const fn check_result(row: &Row) -> &'static str {
    match (row.error.is_some(), row.verdict) {
        (true, _) => "no_answer",
        (false, RowVerdict::Match) => "match",
        (false, RowVerdict::NoMatch) => "no_match",
        (false, RowVerdict::NotApplicable) => "unsure",
    }
}

/// Seconds. A run is one model call per classification plus the forge reads
/// and the post.
const DURATION_BOUNDARIES: [f64; 12] = [
    1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 60.0, 90.0, 120.0, 180.0, 300.0, 600.0,
];

struct Instruments {
    /// Exports only while it lives.
    _provider: SdkMeterProvider,
    requests: Counter<u64>,
    verdicts: Counter<u64>,
    checks: Counter<u64>,
    duration: Histogram<f64>,
}

impl Instruments {
    fn new(provider: &SdkMeterProvider) -> Self {
        let meter = provider.meter("hyperhive.review_classifier");
        Self {
            _provider: provider.clone(),
            requests: meter
                .u64_counter("hyperhive.review_classifier.requests")
                .build(),
            verdicts: meter
                .u64_counter("hyperhive.review_classifier.verdicts")
                .build(),
            checks: meter
                .u64_counter("hyperhive.review_classifier.checks")
                .build(),
            duration: meter
                .f64_histogram("hyperhive.review_classifier.run.duration")
                .with_unit("s")
                .with_boundaries(DURATION_BOUNDARIES.to_vec())
                .build(),
        }
    }
}

/// The recorder: a handle on the instruments, or on nothing where no
/// collector is configured.
#[derive(Clone)]
pub(crate) struct Metrics(Option<Arc<Instruments>>);

static GLOBAL: OnceLock<Option<Arc<Instruments>>> = OnceLock::new();

impl Metrics {
    /// The process's recorder, building its exporter on the first call.
    pub(crate) fn global() -> Self {
        Self(GLOBAL.get_or_init(|| build().map(Arc::new)).clone())
    }

    /// Records nothing.
    #[cfg(test)]
    pub(super) const fn off() -> Self {
        Self(None)
    }

    /// Instruments on `provider`.
    #[cfg(test)]
    pub(super) fn on(provider: &SdkMeterProvider) -> Self {
        Self(Some(Arc::new(Instruments::new(provider))))
    }

    pub(super) fn request(&self, request: Request) {
        if let Some(inst) = &self.0 {
            inst.requests
                .add(1, &[KeyValue::new("outcome", request.label())]);
        }
    }

    pub(super) fn verdict(&self, choice: Choice) {
        if let Some(inst) = &self.0 {
            inst.verdicts
                .add(1, &[KeyValue::new("outcome", choice.label())]);
        }
    }

    /// One count per classification the run answered or failed to.
    pub(super) fn checks(&self, rows: &[Row]) {
        let Some(inst) = &self.0 else { return };
        for row in rows {
            inst.checks.add(
                1,
                &[
                    KeyValue::new("check", row.name.clone()),
                    KeyValue::new("result", check_result(row)),
                ],
            );
        }
    }

    pub(super) fn run_duration(&self, outcome: RunOutcome, seconds: f64) {
        if let Some(inst) = &self.0 {
            inst.duration
                .record(seconds, &[KeyValue::new("outcome", run_label(outcome))]);
        }
    }
}

fn build() -> Option<Instruments> {
    let endpoint = crate::vcs_metrics::endpoint()?;
    let interval = crate::vcs_metrics::EXPORT_INTERVAL;
    match crate::vcs_metrics::build_provider(interval) {
        Ok(provider) => {
            tracing::info!(%endpoint, ?interval, "otel review-classifier metrics: exporter enabled");
            Some(Instruments::new(&provider))
        }
        Err(e) => {
            tracing::warn!(error = ?e, "otel review-classifier metrics: exporter init failed");
            None
        }
    }
}

#[cfg(test)]
pub(super) mod testing {
    use std::collections::BTreeMap;

    use opentelemetry_sdk::metrics::data::{
        AggregatedMetrics, MetricData, ResourceMetrics, ScopeMetrics,
    };
    use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader};

    use super::{Metrics, SdkMeterProvider};

    /// Instruments on a provider that exports to memory.
    pub(in crate::review_classifier) struct Recorder {
        pub(in crate::review_classifier) metrics: Metrics,
        provider: SdkMeterProvider,
        exporter: InMemoryMetricExporter,
    }

    impl Recorder {
        pub(in crate::review_classifier) fn new() -> Self {
            let exporter = InMemoryMetricExporter::default();
            let reader = PeriodicReader::builder(exporter.clone()).build();
            let provider = SdkMeterProvider::builder().with_reader(reader).build();
            Self {
                metrics: Metrics::on(&provider),
                provider,
                exporter,
            }
        }

        /// `read` of the newest export of the metric `name`.
        fn export<R>(&self, name: &str, read: impl FnOnce(&AggregatedMetrics) -> R) -> Option<R> {
            self.provider.force_flush().expect("flush");
            let all = self.exporter.get_finished_metrics().expect("metrics");
            all.last()
                .into_iter()
                .flat_map(ResourceMetrics::scope_metrics)
                .flat_map(ScopeMetrics::metrics)
                .find(|m| m.name() == name)
                .map(|m| read(m.data()))
        }

        /// The counter `name`'s value per label set, each set as
        /// `key=value,key=value` in key order. Empty when nothing was added.
        pub(in crate::review_classifier) fn counter(&self, name: &str) -> BTreeMap<String, u64> {
            self.export(name, |data| {
                let AggregatedMetrics::U64(MetricData::Sum(sum)) = data else {
                    panic!("not a u64 sum: {data:?}");
                };
                sum.data_points()
                    .map(|p| (labels(p.attributes()), p.value()))
                    .collect()
            })
            .unwrap_or_default()
        }

        /// The histogram `name`'s sample count per label set, as
        /// [`Self::counter`] writes them.
        pub(in crate::review_classifier) fn histogram(&self, name: &str) -> BTreeMap<String, u64> {
            self.export(name, |data| {
                let AggregatedMetrics::F64(MetricData::Histogram(h)) = data else {
                    panic!("not an f64 histogram: {data:?}");
                };
                h.data_points()
                    .map(|p| (labels(p.attributes()), p.count()))
                    .collect()
            })
            .unwrap_or_default()
        }
    }

    fn labels<'a>(attributes: impl Iterator<Item = &'a opentelemetry::KeyValue>) -> String {
        let mut all: Vec<String> = attributes
            .map(|kv| format!("{}={}", kv.key, kv.value))
            .collect();
        all.sort();
        all.join(",")
    }
}