b072fbf4d6
A swarm-ci-metrics host unit on a 5-minute timer lists every forge repo's Actions runs as the new read-only ci-metrics account and pushes each finished run's duration and queue wait over OTLP, stamped at the run's stop time. The first pass backfills from run 1. swarm-controller mints the account's token into bao and keeps a ci-metrics team with read on the Actions unit in every organisation. Closes #5376
274 lines
9.5 KiB
Rust
274 lines
9.5 KiB
Rust
//! One finished forge Actions run to the samples it becomes, or the reason it
|
||
//! becomes none.
|
||
//!
|
||
//! The span is `stopped − started`. The run's `duration` field is not used: it
|
||
//! is in nanoseconds, absent on some finished runs, and set on running ones.
|
||
|
||
use forgejo_api::structs::ActionRun;
|
||
use time::OffsetDateTime;
|
||
|
||
/// The run's wall time, `stopped − started`, in seconds.
|
||
pub const DURATION_METRIC: &str = "hyperhive.ci.run.duration";
|
||
|
||
/// The time the run waited for a runner, `started − created`, in seconds.
|
||
pub const QUEUE_WAIT_METRIC: &str = "hyperhive.ci.run.queue_wait";
|
||
|
||
/// How long a run that has not finished keeps the cursor from passing it.
|
||
/// Past this the run is counted as [`Skip::Abandoned`] and not looked at
|
||
/// again, so one run the forge never closes cannot make every pass re-read
|
||
/// the whole history above it.
|
||
pub const ABANDON_AFTER: time::Duration = time::Duration::days(1);
|
||
|
||
/// The label set both samples of a run carry.
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub struct Labels {
|
||
/// `owner/name`, as `hyperhive.vcs.*` spells it.
|
||
pub repo: String,
|
||
/// The run number. A rerun of one commit is a run of its own, so with
|
||
/// this label every run is a series of its own.
|
||
pub run: String,
|
||
/// The workflow file, e.g. `ci.yml`.
|
||
pub workflow: String,
|
||
/// The PR number for a PR run, otherwise the ref the run ran on.
|
||
pub pr: String,
|
||
/// The first 12 characters of the commit.
|
||
pub sha: String,
|
||
/// `success` or `failure`.
|
||
pub outcome: String,
|
||
/// The webhook event, e.g. `pull_request`.
|
||
pub event: String,
|
||
}
|
||
|
||
/// What one finished run is pushed as.
|
||
#[derive(Debug, Clone, PartialEq)]
|
||
pub struct RunSamples {
|
||
pub labels: Labels,
|
||
/// `stopped`, the timestamp of both samples.
|
||
pub at_unix_nanos: i128,
|
||
pub duration_s: f64,
|
||
/// `None` when the run has no `created`, or one after `started`.
|
||
pub queue_wait_s: Option<f64>,
|
||
}
|
||
|
||
/// Why a run is pushed as nothing.
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||
pub enum Skip {
|
||
/// Cancelled: its span is not a run time.
|
||
Cancelled,
|
||
/// The forge skipped the run; nothing ran.
|
||
NotRun,
|
||
/// `started` absent or at the epoch, which the forge writes for a run
|
||
/// cancelled before it started.
|
||
NoStart,
|
||
/// `stopped` absent, or not after `started`.
|
||
BadSpan,
|
||
/// Not finished after [`ABANDON_AFTER`].
|
||
Abandoned,
|
||
}
|
||
|
||
/// What [`classify`] makes of one run.
|
||
#[derive(Debug, Clone, PartialEq)]
|
||
pub enum Verdict {
|
||
Push(Box<RunSamples>),
|
||
Skip(Skip),
|
||
/// Not finished yet: the cursor stays at or below it.
|
||
Pending,
|
||
}
|
||
|
||
/// The `pr` label: the number of a `#N` ref, else the ref as the forge
|
||
/// prints it (`main`, a branch, a tag).
|
||
pub fn pr_label(prettyref: &str) -> String {
|
||
match prettyref.strip_prefix('#') {
|
||
Some(n) if !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit()) => n.to_owned(),
|
||
_ => prettyref.to_owned(),
|
||
}
|
||
}
|
||
|
||
/// What `run` of `repo` becomes, judged at `now`.
|
||
pub fn classify(repo: &str, run: &ActionRun, now: OffsetDateTime) -> Verdict {
|
||
let outcome = match run.status.as_deref() {
|
||
Some(s @ ("success" | "failure")) => s,
|
||
Some("cancelled") => return Verdict::Skip(Skip::Cancelled),
|
||
Some("skipped") => return Verdict::Skip(Skip::NotRun),
|
||
_ => {
|
||
return match run.created {
|
||
Some(created) if now - created > ABANDON_AFTER => Verdict::Skip(Skip::Abandoned),
|
||
_ => Verdict::Pending,
|
||
};
|
||
}
|
||
};
|
||
let Some(started) = run.started.filter(|s| s.unix_timestamp() > 0) else {
|
||
return Verdict::Skip(Skip::NoStart);
|
||
};
|
||
let Some(stopped) = run.stopped.filter(|s| *s > started) else {
|
||
return Verdict::Skip(Skip::BadSpan);
|
||
};
|
||
let queue_wait_s = run
|
||
.created
|
||
.filter(|c| *c <= started)
|
||
.map(|c| (started - c).as_seconds_f64());
|
||
let sha = run.commit_sha.as_deref().unwrap_or_default();
|
||
Verdict::Push(Box::new(RunSamples {
|
||
labels: Labels {
|
||
repo: repo.to_owned(),
|
||
run: run.index_in_repo.map(|n| n.to_string()).unwrap_or_default(),
|
||
workflow: run.workflow_id.clone().unwrap_or_default(),
|
||
pr: pr_label(run.prettyref.as_deref().unwrap_or_default()),
|
||
sha: sha.chars().take(12).collect(),
|
||
outcome: outcome.to_owned(),
|
||
event: run.event.clone().unwrap_or_default(),
|
||
},
|
||
at_unix_nanos: stopped.unix_timestamp_nanos(),
|
||
duration_s: (stopped - started).as_seconds_f64(),
|
||
queue_wait_s,
|
||
}))
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use serde_json::json;
|
||
|
||
pub(crate) fn run(v: serde_json::Value) -> ActionRun {
|
||
let mut v = v;
|
||
let obj = v.as_object_mut().expect("an object");
|
||
for key in ["created", "started", "stopped", "updated", "html_url"] {
|
||
obj.entry(key).or_insert(serde_json::Value::Null);
|
||
}
|
||
serde_json::from_value(v).expect("an ActionRun")
|
||
}
|
||
|
||
fn now() -> OffsetDateTime {
|
||
OffsetDateTime::parse(
|
||
"2026-10-16T18:00:00+02:00",
|
||
&time::format_description::well_known::Rfc3339,
|
||
)
|
||
.expect("a timestamp")
|
||
}
|
||
|
||
/// A finished PR run, shaped as the forge returns one.
|
||
fn finished(status: &str) -> serde_json::Value {
|
||
json!({
|
||
"index_in_repo": 5638,
|
||
"status": status,
|
||
"event": "pull_request",
|
||
"prettyref": "#7",
|
||
"workflow_id": "ci.yml",
|
||
"commit_sha": "d0ab2edd05bcce7a35ea15bbc2ef5a366fc93d18",
|
||
"created": "2026-10-16T16:50:38+02:00",
|
||
"started": "2026-10-16T16:55:17+02:00",
|
||
"stopped": "2026-10-16T17:13:30+02:00",
|
||
"duration": 1_093_000_000_000_i64,
|
||
})
|
||
}
|
||
|
||
#[test]
|
||
fn a_finished_run_is_its_span_and_its_wait_stamped_at_stopped() {
|
||
let Verdict::Push(s) = classify("hyperhive/hyperhive", &run(finished("success")), now())
|
||
else {
|
||
panic!("a finished run is pushed");
|
||
};
|
||
assert_eq!(
|
||
s.labels,
|
||
Labels {
|
||
repo: "hyperhive/hyperhive".to_owned(),
|
||
run: "5638".to_owned(),
|
||
workflow: "ci.yml".to_owned(),
|
||
pr: "7".to_owned(),
|
||
sha: "d0ab2edd05bc".to_owned(),
|
||
outcome: "success".to_owned(),
|
||
event: "pull_request".to_owned(),
|
||
}
|
||
);
|
||
// 16:55:17 → 17:13:30, not the `duration` field's 1093 s.
|
||
assert!((s.duration_s - 1093.0).abs() < f64::EPSILON);
|
||
// 16:50:38 → 16:55:17.
|
||
assert_eq!(s.queue_wait_s, Some(279.0));
|
||
// 2026-10-16T15:13:30Z.
|
||
assert_eq!(s.at_unix_nanos, 1_792_163_610_000_000_000);
|
||
}
|
||
|
||
#[test]
|
||
fn the_span_comes_from_the_timestamps_not_the_duration_field() {
|
||
let mut v = finished("failure");
|
||
v["duration"] = json!(5);
|
||
let Verdict::Push(s) = classify("o/r", &run(v), now()) else {
|
||
panic!("a failed run is pushed");
|
||
};
|
||
assert!((s.duration_s - 1093.0).abs() < f64::EPSILON);
|
||
assert_eq!(s.labels.outcome, "failure");
|
||
}
|
||
|
||
#[test]
|
||
fn cancelled_skipped_and_unstarted_runs_are_skips() {
|
||
assert_eq!(
|
||
classify("o/r", &run(finished("cancelled")), now()),
|
||
Verdict::Skip(Skip::Cancelled)
|
||
);
|
||
assert_eq!(
|
||
classify("o/r", &run(finished("skipped")), now()),
|
||
Verdict::Skip(Skip::NotRun)
|
||
);
|
||
let mut epoch = finished("failure");
|
||
epoch["started"] = json!("1970-01-01T01:00:00+01:00");
|
||
assert_eq!(
|
||
classify("o/r", &run(epoch), now()),
|
||
Verdict::Skip(Skip::NoStart)
|
||
);
|
||
let mut absent = finished("success");
|
||
absent["started"] = serde_json::Value::Null;
|
||
assert_eq!(
|
||
classify("o/r", &run(absent), now()),
|
||
Verdict::Skip(Skip::NoStart)
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn a_span_that_does_not_move_forward_is_a_skip() {
|
||
let mut equal = finished("success");
|
||
equal["stopped"] = equal["started"].clone();
|
||
assert_eq!(
|
||
classify("o/r", &run(equal), now()),
|
||
Verdict::Skip(Skip::BadSpan)
|
||
);
|
||
let mut absent = finished("success");
|
||
absent["stopped"] = serde_json::Value::Null;
|
||
assert_eq!(
|
||
classify("o/r", &run(absent), now()),
|
||
Verdict::Skip(Skip::BadSpan)
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn an_unfinished_run_waits_until_it_is_abandoned() {
|
||
let mut running = finished("running");
|
||
running["created"] = json!("2026-10-16T17:30:00+02:00");
|
||
assert_eq!(classify("o/r", &run(running), now()), Verdict::Pending);
|
||
let mut stale = finished("waiting");
|
||
stale["created"] = json!("2026-10-15T17:30:00+02:00");
|
||
assert_eq!(
|
||
classify("o/r", &run(stale), now()),
|
||
Verdict::Skip(Skip::Abandoned)
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn a_created_after_started_drops_the_wait_not_the_run() {
|
||
let mut v = finished("success");
|
||
v["created"] = json!("2026-10-16T17:00:00+02:00");
|
||
let Verdict::Push(s) = classify("o/r", &run(v), now()) else {
|
||
panic!("the span is still good");
|
||
};
|
||
assert_eq!(s.queue_wait_s, None);
|
||
}
|
||
|
||
#[test]
|
||
fn the_pr_label_is_the_number_or_the_ref() {
|
||
assert_eq!(pr_label("#7"), "7");
|
||
assert_eq!(pr_label("main"), "main");
|
||
assert_eq!(pr_label("2100-hive-metric"), "2100-hive-metric");
|
||
assert_eq!(pr_label("#"), "#");
|
||
assert_eq!(pr_label("#12x"), "#12x");
|
||
assert_eq!(pr_label(""), "");
|
||
}
|
||
}
|