//! OTEL export of what review requests and review runs come to: the counters //! `hyperhive.review_classifier.requests` (`outcome`), `.verdicts` (`outcome`) //! and `.checks` (`check`, `result`), and the histogram //! `hyperhive.review_classifier.run.duration` in seconds (`outcome`). The //! classifier service exports the retries beside its cost counters, see //! `swarm-review-classifier`'s `cost` module. //! //! Same exporter as [`crate::vcs_metrics`], built on the first record; with no //! `OTEL_EXPORTER_OTLP_ENDPOINT` every record does nothing. use std::sync::{Arc, OnceLock}; use opentelemetry::KeyValue; use opentelemetry::metrics::{Counter, Histogram, MeterProvider as _}; use opentelemetry_sdk::metrics::SdkMeterProvider; use swarm_wire::review_classifier::{Row, Verdict as RowVerdict}; use super::runs::RunOutcome; /// What became of a review request. #[derive(Debug, Clone, Copy)] pub(super) enum Request { /// Stored in the request queue. Queued, /// The queue did not store it; a no-verdict comment was posted. QueueFailed, /// Taken from the queue but unreadable, and dropped. Unparsable, /// Taken while the PR's review was going; put back in the queue. Deferred, /// Taken, and a fresh run began. Started, /// Taken for a head already reviewed; its verdict was posted again. Reposted, } impl Request { const fn label(self) -> &'static str { match self { Self::Queued => "queued", Self::QueueFailed => "queue_failed", Self::Unparsable => "unparsable", Self::Deferred => "deferred", Self::Started => "started", Self::Reposted => "reposted", } } } /// The review event a run chose. #[derive(Debug, Clone, Copy)] pub(super) enum Choice { Approve, RequestChanges, NoVerdict, } impl Choice { const fn label(self) -> &'static str { match self { Self::Approve => "approve", Self::RequestChanges => "request_changes", Self::NoVerdict => "no_verdict", } } } /// A run's `outcome` label: the verdict names, plus `post_failed`. const fn run_label(outcome: RunOutcome) -> &'static str { match outcome { RunOutcome::Approved => "approve", RunOutcome::ChangesRequested => "request_changes", RunOutcome::NoVerdict => "no_verdict", RunOutcome::PostFailed => "post_failed", RunOutcome::QueueFull => "queue_full", RunOutcome::Done => "done", RunOutcome::Failed => "failed", } } /// A row's `result` label. const fn check_result(row: &Row) -> &'static str { match (row.error.is_some(), row.verdict) { (true, _) => "no_answer", (false, RowVerdict::Match) => "match", (false, RowVerdict::NoMatch) => "no_match", (false, RowVerdict::NotApplicable) => "unsure", } } /// Seconds. A run is one model call per classification plus the forge reads /// and the post. const DURATION_BOUNDARIES: [f64; 12] = [ 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 60.0, 90.0, 120.0, 180.0, 300.0, 600.0, ]; struct Instruments { /// Exports only while it lives. _provider: SdkMeterProvider, requests: Counter, verdicts: Counter, checks: Counter, duration: Histogram, } impl Instruments { fn new(provider: &SdkMeterProvider) -> Self { let meter = provider.meter("hyperhive.review_classifier"); Self { _provider: provider.clone(), requests: meter .u64_counter("hyperhive.review_classifier.requests") .build(), verdicts: meter .u64_counter("hyperhive.review_classifier.verdicts") .build(), checks: meter .u64_counter("hyperhive.review_classifier.checks") .build(), duration: meter .f64_histogram("hyperhive.review_classifier.run.duration") .with_unit("s") .with_boundaries(DURATION_BOUNDARIES.to_vec()) .build(), } } } /// The recorder: a handle on the instruments, or on nothing where no /// collector is configured. #[derive(Clone)] pub(crate) struct Metrics(Option>); static GLOBAL: OnceLock>> = OnceLock::new(); impl Metrics { /// The process's recorder, building its exporter on the first call. pub(crate) fn global() -> Self { Self(GLOBAL.get_or_init(|| build().map(Arc::new)).clone()) } /// Records nothing. #[cfg(test)] pub(super) const fn off() -> Self { Self(None) } /// Instruments on `provider`. #[cfg(test)] pub(super) fn on(provider: &SdkMeterProvider) -> Self { Self(Some(Arc::new(Instruments::new(provider)))) } pub(super) fn request(&self, request: Request) { if let Some(inst) = &self.0 { inst.requests .add(1, &[KeyValue::new("outcome", request.label())]); } } pub(super) fn verdict(&self, choice: Choice) { if let Some(inst) = &self.0 { inst.verdicts .add(1, &[KeyValue::new("outcome", choice.label())]); } } /// One count per classification the run answered or failed to. pub(super) fn checks(&self, rows: &[Row]) { let Some(inst) = &self.0 else { return }; for row in rows { inst.checks.add( 1, &[ KeyValue::new("check", row.name.clone()), KeyValue::new("result", check_result(row)), ], ); } } pub(super) fn run_duration(&self, outcome: RunOutcome, seconds: f64) { if let Some(inst) = &self.0 { inst.duration .record(seconds, &[KeyValue::new("outcome", run_label(outcome))]); } } } fn build() -> Option { let endpoint = crate::vcs_metrics::endpoint()?; let interval = crate::vcs_metrics::EXPORT_INTERVAL; match crate::vcs_metrics::build_provider(interval) { Ok(provider) => { tracing::info!(%endpoint, ?interval, "otel review-classifier metrics: exporter enabled"); Some(Instruments::new(&provider)) } Err(e) => { tracing::warn!(error = ?e, "otel review-classifier metrics: exporter init failed"); None } } } #[cfg(test)] pub(super) mod testing { use std::collections::BTreeMap; use opentelemetry_sdk::metrics::data::{ AggregatedMetrics, MetricData, ResourceMetrics, ScopeMetrics, }; use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader}; use super::{Metrics, SdkMeterProvider}; /// Instruments on a provider that exports to memory. pub(in crate::review_classifier) struct Recorder { pub(in crate::review_classifier) metrics: Metrics, provider: SdkMeterProvider, exporter: InMemoryMetricExporter, } impl Recorder { pub(in crate::review_classifier) fn new() -> Self { let exporter = InMemoryMetricExporter::default(); let reader = PeriodicReader::builder(exporter.clone()).build(); let provider = SdkMeterProvider::builder().with_reader(reader).build(); Self { metrics: Metrics::on(&provider), provider, exporter, } } /// `read` of the newest export of the metric `name`. fn export(&self, name: &str, read: impl FnOnce(&AggregatedMetrics) -> R) -> Option { self.provider.force_flush().expect("flush"); let all = self.exporter.get_finished_metrics().expect("metrics"); all.last() .into_iter() .flat_map(ResourceMetrics::scope_metrics) .flat_map(ScopeMetrics::metrics) .find(|m| m.name() == name) .map(|m| read(m.data())) } /// The counter `name`'s value per label set, each set as /// `key=value,key=value` in key order. Empty when nothing was added. pub(in crate::review_classifier) fn counter(&self, name: &str) -> BTreeMap { self.export(name, |data| { let AggregatedMetrics::U64(MetricData::Sum(sum)) = data else { panic!("not a u64 sum: {data:?}"); }; sum.data_points() .map(|p| (labels(p.attributes()), p.value())) .collect() }) .unwrap_or_default() } /// The histogram `name`'s sample count per label set, as /// [`Self::counter`] writes them. pub(in crate::review_classifier) fn histogram(&self, name: &str) -> BTreeMap { self.export(name, |data| { let AggregatedMetrics::F64(MetricData::Histogram(h)) = data else { panic!("not an f64 histogram: {data:?}"); }; h.data_points() .map(|p| (labels(p.attributes()), p.count())) .collect() }) .unwrap_or_default() } } fn labels<'a>(attributes: impl Iterator) -> String { let mut all: Vec = attributes .map(|kv| format!("{}={}", kv.key, kv.value)) .collect(); all.sort(); all.join(",") } }