push ci run metrics to victoriametrics, with a grafana board

A swarm-ci-metrics host unit on a 5-minute timer lists every forge
repo's Actions runs as the new read-only ci-metrics account and pushes
each finished run's duration and queue wait over OTLP, stamped at the
run's stop time. The first pass backfills from run 1. swarm-controller
mints the account's token into bao and keeps a ci-metrics team with
read on the Actions unit in every organisation.

Closes #5376

swarm-ci-metrics/src/sample.rs

@@ -0,0 +1,274 @@
+//! One finished forge Actions run to the samples it becomes, or the reason it
+//! becomes none.
+//!
+//! The span is `stopped − started`. The run's `duration` field is not used: it
+//! is in nanoseconds, absent on some finished runs, and set on running ones.
+
+use forgejo_api::structs::ActionRun;
+use time::OffsetDateTime;
+
+/// The run's wall time, `stopped − started`, in seconds.
+pub const DURATION_METRIC: &str = "hyperhive.ci.run.duration";
+
+/// The time the run waited for a runner, `started − created`, in seconds.
+pub const QUEUE_WAIT_METRIC: &str = "hyperhive.ci.run.queue_wait";
+
+/// How long a run that has not finished keeps the cursor from passing it.
+/// Past this the run is counted as [`Skip::Abandoned`] and not looked at
+/// again, so one run the forge never closes cannot make every pass re-read
+/// the whole history above it.
+pub const ABANDON_AFTER: time::Duration = time::Duration::days(1);
+
+/// The label set both samples of a run carry.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Labels {
+    /// `owner/name`, as `hyperhive.vcs.*` spells it.
+    pub repo: String,
+    /// The run number. A rerun of one commit is a run of its own, so with
+    /// this label every run is a series of its own.
+    pub run: String,
+    /// The workflow file, e.g. `ci.yml`.
+    pub workflow: String,
+    /// The PR number for a PR run, otherwise the ref the run ran on.
+    pub pr: String,
+    /// The first 12 characters of the commit.
+    pub sha: String,
+    /// `success` or `failure`.
+    pub outcome: String,
+    /// The webhook event, e.g. `pull_request`.
+    pub event: String,
+}
+
+/// What one finished run is pushed as.
+#[derive(Debug, Clone, PartialEq)]
+pub struct RunSamples {
+    pub labels: Labels,
+    /// `stopped`, the timestamp of both samples.
+    pub at_unix_nanos: i128,
+    pub duration_s: f64,
+    /// `None` when the run has no `created`, or one after `started`.
+    pub queue_wait_s: Option<f64>,
+}
+
+/// Why a run is pushed as nothing.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Skip {
+    /// Cancelled: its span is not a run time.
+    Cancelled,
+    /// The forge skipped the run; nothing ran.
+    NotRun,
+    /// `started` absent or at the epoch, which the forge writes for a run
+    /// cancelled before it started.
+    NoStart,
+    /// `stopped` absent, or not after `started`.
+    BadSpan,
+    /// Not finished after [`ABANDON_AFTER`].
+    Abandoned,
+}
+
+/// What [`classify`] makes of one run.
+#[derive(Debug, Clone, PartialEq)]
+pub enum Verdict {
+    Push(Box<RunSamples>),
+    Skip(Skip),
+    /// Not finished yet: the cursor stays at or below it.
+    Pending,
+}
+
+/// The `pr` label: the number of a `#N` ref, else the ref as the forge
+/// prints it (`main`, a branch, a tag).
+pub fn pr_label(prettyref: &str) -> String {
+    match prettyref.strip_prefix('#') {
+        Some(n) if !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit()) => n.to_owned(),
+        _ => prettyref.to_owned(),
+    }
+}
+
+/// What `run` of `repo` becomes, judged at `now`.
+pub fn classify(repo: &str, run: &ActionRun, now: OffsetDateTime) -> Verdict {
+    let outcome = match run.status.as_deref() {
+        Some(s @ ("success" | "failure")) => s,
+        Some("cancelled") => return Verdict::Skip(Skip::Cancelled),
+        Some("skipped") => return Verdict::Skip(Skip::NotRun),
+        _ => {
+            return match run.created {
+                Some(created) if now - created > ABANDON_AFTER => Verdict::Skip(Skip::Abandoned),
+                _ => Verdict::Pending,
+            };
+        }
+    };
+    let Some(started) = run.started.filter(|s| s.unix_timestamp() > 0) else {
+        return Verdict::Skip(Skip::NoStart);
+    };
+    let Some(stopped) = run.stopped.filter(|s| *s > started) else {
+        return Verdict::Skip(Skip::BadSpan);
+    };
+    let queue_wait_s = run
+        .created
+        .filter(|c| *c <= started)
+        .map(|c| (started - c).as_seconds_f64());
+    let sha = run.commit_sha.as_deref().unwrap_or_default();
+    Verdict::Push(Box::new(RunSamples {
+        labels: Labels {
+            repo: repo.to_owned(),
+            run: run.index_in_repo.map(|n| n.to_string()).unwrap_or_default(),
+            workflow: run.workflow_id.clone().unwrap_or_default(),
+            pr: pr_label(run.prettyref.as_deref().unwrap_or_default()),
+            sha: sha.chars().take(12).collect(),
+            outcome: outcome.to_owned(),
+            event: run.event.clone().unwrap_or_default(),
+        },
+        at_unix_nanos: stopped.unix_timestamp_nanos(),
+        duration_s: (stopped - started).as_seconds_f64(),
+        queue_wait_s,
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    pub(crate) fn run(v: serde_json::Value) -> ActionRun {
+        let mut v = v;
+        let obj = v.as_object_mut().expect("an object");
+        for key in ["created", "started", "stopped", "updated", "html_url"] {
+            obj.entry(key).or_insert(serde_json::Value::Null);
+        }
+        serde_json::from_value(v).expect("an ActionRun")
+    }
+
+    fn now() -> OffsetDateTime {
+        OffsetDateTime::parse(
+            "2026-10-16T18:00:00+02:00",
+            &time::format_description::well_known::Rfc3339,
+        )
+        .expect("a timestamp")
+    }
+
+    /// A finished PR run, shaped as the forge returns one.
+    fn finished(status: &str) -> serde_json::Value {
+        json!({
+            "index_in_repo": 5638,
+            "status": status,
+            "event": "pull_request",
+            "prettyref": "#7",
+            "workflow_id": "ci.yml",
+            "commit_sha": "d0ab2edd05bcce7a35ea15bbc2ef5a366fc93d18",
+            "created": "2026-10-16T16:50:38+02:00",
+            "started": "2026-10-16T16:55:17+02:00",
+            "stopped": "2026-10-16T17:13:30+02:00",
+            "duration": 1_093_000_000_000_i64,
+        })
+    }
+
+    #[test]
+    fn a_finished_run_is_its_span_and_its_wait_stamped_at_stopped() {
+        let Verdict::Push(s) = classify("hyperhive/hyperhive", &run(finished("success")), now())
+        else {
+            panic!("a finished run is pushed");
+        };
+        assert_eq!(
+            s.labels,
+            Labels {
+                repo: "hyperhive/hyperhive".to_owned(),
+                run: "5638".to_owned(),
+                workflow: "ci.yml".to_owned(),
+                pr: "7".to_owned(),
+                sha: "d0ab2edd05bc".to_owned(),
+                outcome: "success".to_owned(),
+                event: "pull_request".to_owned(),
+            }
+        );
+        // 16:55:17 → 17:13:30, not the `duration` field's 1093 s.
+        assert!((s.duration_s - 1093.0).abs() < f64::EPSILON);
+        // 16:50:38 → 16:55:17.
+        assert_eq!(s.queue_wait_s, Some(279.0));
+        // 2026-10-16T15:13:30Z.
+        assert_eq!(s.at_unix_nanos, 1_792_163_610_000_000_000);
+    }
+
+    #[test]
+    fn the_span_comes_from_the_timestamps_not_the_duration_field() {
+        let mut v = finished("failure");
+        v["duration"] = json!(5);
+        let Verdict::Push(s) = classify("o/r", &run(v), now()) else {
+            panic!("a failed run is pushed");
+        };
+        assert!((s.duration_s - 1093.0).abs() < f64::EPSILON);
+        assert_eq!(s.labels.outcome, "failure");
+    }
+
+    #[test]
+    fn cancelled_skipped_and_unstarted_runs_are_skips() {
+        assert_eq!(
+            classify("o/r", &run(finished("cancelled")), now()),
+            Verdict::Skip(Skip::Cancelled)
+        );
+        assert_eq!(
+            classify("o/r", &run(finished("skipped")), now()),
+            Verdict::Skip(Skip::NotRun)
+        );
+        let mut epoch = finished("failure");
+        epoch["started"] = json!("1970-01-01T01:00:00+01:00");
+        assert_eq!(
+            classify("o/r", &run(epoch), now()),
+            Verdict::Skip(Skip::NoStart)
+        );
+        let mut absent = finished("success");
+        absent["started"] = serde_json::Value::Null;
+        assert_eq!(
+            classify("o/r", &run(absent), now()),
+            Verdict::Skip(Skip::NoStart)
+        );
+    }
+
+    #[test]
+    fn a_span_that_does_not_move_forward_is_a_skip() {
+        let mut equal = finished("success");
+        equal["stopped"] = equal["started"].clone();
+        assert_eq!(
+            classify("o/r", &run(equal), now()),
+            Verdict::Skip(Skip::BadSpan)
+        );
+        let mut absent = finished("success");
+        absent["stopped"] = serde_json::Value::Null;
+        assert_eq!(
+            classify("o/r", &run(absent), now()),
+            Verdict::Skip(Skip::BadSpan)
+        );
+    }
+
+    #[test]
+    fn an_unfinished_run_waits_until_it_is_abandoned() {
+        let mut running = finished("running");
+        running["created"] = json!("2026-10-16T17:30:00+02:00");
+        assert_eq!(classify("o/r", &run(running), now()), Verdict::Pending);
+        let mut stale = finished("waiting");
+        stale["created"] = json!("2026-10-15T17:30:00+02:00");
+        assert_eq!(
+            classify("o/r", &run(stale), now()),
+            Verdict::Skip(Skip::Abandoned)
+        );
+    }
+
+    #[test]
+    fn a_created_after_started_drops_the_wait_not_the_run() {
+        let mut v = finished("success");
+        v["created"] = json!("2026-10-16T17:00:00+02:00");
+        let Verdict::Push(s) = classify("o/r", &run(v), now()) else {
+            panic!("the span is still good");
+        };
+        assert_eq!(s.queue_wait_s, None);
+    }
+
+    #[test]
+    fn the_pr_label_is_the_number_or_the_ref() {
+        assert_eq!(pr_label("#7"), "7");
+        assert_eq!(pr_label("main"), "main");
+        assert_eq!(pr_label("2100-hive-metric"), "2100-hive-metric");
+        assert_eq!(pr_label("#"), "#");
+        assert_eq!(pr_label("#12x"), "#12x");
+        assert_eq!(pr_label(""), "");
+    }
+}