vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

push ci run metrics to victoriametrics, with a grafana board

Browse source 
A swarm-ci-metrics host unit on a 5-minute timer lists every forge
repo's Actions runs as the new read-only ci-metrics account and pushes
each finished run's duration and queue wait over OTLP, stamped at the
run's stop time. The first pass backfills from run 1. swarm-controller
mints the account's token into bao and keeps a ci-metrics team with
read on the Actions unit in every organisation.

Closes #5376
This commit is contained in:
atlas 2026-10-16 18:23:59 +02:00 • committed by mara
commit b072fbf4d6
36 changed files with 2849 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

258
swarm-ci-metrics/src/pass.rs Normal file
View file

@ -0,0 +1,258 @@
//! One repo's pass: which of the runs read from the forge to push, and the
//! order of pushes and cursor writes that keeps a crash from losing a run.
use std::collections::BTreeMap;
use anyhow::Result;
use forgejo_api::structs::ActionRun;
use time::OffsetDateTime;
use crate::cursor::State;
use crate::sample::{RunSamples, Skip, Verdict, classify};
/// Runs per OTLP request: two points each, well under any body limit on the
/// way to the collector.
pub const CHUNK: usize = 100;
/// What one pass does with a repo's runs.
#[derive(Debug, Default)]
pub struct Plan {
/// Finished runs not yet pushed, oldest first, by run number.
pub push: Vec<(u64, RunSamples)>,
/// Finished runs that become no sample, by run number.
pub skip: Vec<(u64, Skip)>,
/// The lowest run number still going.
pub lowest_pending: Option<u64>,
/// The highest run number read.
pub newest_seen: Option<u64>,
}
/// Sort `runs` of `repo` against its cursor in `state`. A run listed twice
/// (the list shifts while it is paged) counts once; a run with no number is
/// left out.
pub fn plan(state: &State, repo: &str, runs: &[ActionRun], now: OffsetDateTime) -> Plan {
let cursor = state.repos.get(repo).cloned().unwrap_or_default();
let by_number: BTreeMap<u64, &ActionRun> = runs
.iter()
.filter_map(|r| {
let n = u64::try_from(r.index_in_repo?).ok()?;
Some((n, r))
})
.collect();
let mut plan = Plan {
newest_seen: by_number.keys().next_back().copied(),
..Plan::default()
};
for (&n, run) in &by_number {
if cursor.is_handled(n) {
continue;
}
match classify(repo, run, now) {
Verdict::Push(s) => plan.push.push((n, *s)),
Verdict::Skip(why) => plan.skip.push((n, why)),
Verdict::Pending => {
plan.lowest_pending.get_or_insert(n);
}
}
}
plan
}
/// Push `plan` for `repo` in chunks of [`CHUNK`] runs, marking each chunk in
/// `state` and saving it only after `push` accepted that chunk; then move the
/// cursor and save once more. The skips are marked with the first save.
///
/// A failed push returns before anything it carried is marked. A crash after
/// a push and before its save leaves those runs unmarked, so they go again.
pub async fn apply(
state: &mut State,
repo: &str,
plan: Plan,
mut push: impl AsyncFnMut(&[RunSamples]) -> Result<()>,
mut save: impl FnMut(&State) -> Result<()>,
) -> Result<()> {
state.repo(repo).mark(plan.skip.iter().map(|(n, _)| *n));
for chunk in plan.push.chunks(CHUNK) {
let samples: Vec<RunSamples> = chunk.iter().map(|(_, s)| s.clone()).collect();
push(&samples).await?;
state.repo(repo).mark(chunk.iter().map(|(n, _)| *n));
save(state)?;
}
state
.repo(repo)
.advance(plan.lowest_pending, plan.newest_seen);
save(state)
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn now() -> OffsetDateTime {
OffsetDateTime::parse(
"2026-10-16T18:00:00+02:00",
&time::format_description::well_known::Rfc3339,
)
.expect("a timestamp")
}
fn run(n: i64, status: &str) -> ActionRun {
let mut v = json!({
"index_in_repo": n,
"status": status,
"event": "pull_request",
"prettyref": format!("#{n}"),
"workflow_id": "ci.yml",
"commit_sha": format!("{n:040}"),
"created": "2026-10-16T16:50:38+02:00",
"started": "2026-10-16T16:55:17+02:00",
"stopped": "2026-10-16T17:13:30+02:00",
});
let obj = v.as_object_mut().expect("an object");
for key in ["updated", "html_url"] {
obj.insert(key.to_owned(), serde_json::Value::Null);
}
serde_json::from_value(v).expect("an ActionRun")
}
/// `count` finished runs numbered from 1, newest first as the forge lists.
fn history(count: i64) -> Vec<ActionRun> {
(1..=count).rev().map(|n| run(n, "success")).collect()
}
fn pushed_numbers(samples: &[RunSamples]) -> Vec<String> {
samples.iter().map(|s| s.labels.pr.clone()).collect()
}
#[tokio::test]
async fn the_first_pass_pushes_every_finished_run_and_the_next_pushes_none() {
let mut state = State::default();
let runs = history(250);
let p = plan(&state, "o/r", &runs, now());
assert_eq!(p.push.len(), 250);
let mut requests = 0;
apply(
&mut state,
"o/r",
p,
async |_| {
requests += 1;
Ok(())
},
|_| Ok(()),
)
.await
.expect("apply");
assert_eq!(requests, 3);
assert_eq!(state.repos["o/r"].low, 251);
let again = plan(&state, "o/r", &runs, now());
assert!(again.push.is_empty() && again.skip.is_empty());
}
#[tokio::test]
async fn a_failed_push_marks_nothing_it_carried_and_the_next_pass_resumes() {
let mut state = State::default();
let runs = history(250);
let mut saved = None;
let mut calls = 0;
let p = plan(&state, "o/r", &runs, now());
let err = apply(
&mut state,
"o/r",
p,
async |_| {
calls += 1;
if calls == 2 {
anyhow::bail!("the collector answered 503");
}
Ok(())
},
|s| {
saved = Some(s.clone());
Ok(())
},
)
.await;
assert!(err.is_err());
// What survives the crash is the last save: the first chunk.
let state = saved.expect("the first chunk was saved");
assert_eq!(state.repos["o/r"].handled.len(), CHUNK);
assert_eq!(state.repos["o/r"].low, 0);
let resumed = plan(&state, "o/r", &runs, now());
let numbers: Vec<u64> = resumed.push.iter().map(|(n, _)| *n).collect();
assert_eq!(numbers, (101..=250).collect::<Vec<_>>());
}
#[tokio::test]
async fn a_crash_between_a_push_and_its_save_pushes_those_runs_again() {
let mut state = State::default();
let runs = history(3);
let mut first = Vec::new();
let p = plan(&state, "o/r", &runs, now());
let err = apply(
&mut state,
"o/r",
p,
async |s| {
first.extend(pushed_numbers(s));
Ok(())
},
|_| Err(anyhow::anyhow!("disk full")),
)
.await;
assert!(err.is_err());
// The file on disk is still the empty state.
let on_disk = State::default();
let again = plan(&on_disk, "o/r", &runs, now());
let second: Vec<String> = again
.push
.iter()
.map(|(_, s)| s.labels.pr.clone())
.collect();
assert_eq!(first, second);
}
#[tokio::test]
async fn a_run_finishing_after_a_newer_one_is_still_pushed() {
let mut state = State::default();
let mut runs = vec![run(3, "success"), run(2, "running"), run(1, "success")];
let mut pushed = Vec::new();
let p = plan(&state, "o/r", &runs, now());
apply(
&mut state,
"o/r",
p,
async |s| {
pushed.extend(pushed_numbers(s));
Ok(())
},
|_| Ok(()),
)
.await
.expect("apply");
assert_eq!(pushed, ["1", "3"]);
assert_eq!(state.repos["o/r"].low, 2);
runs[1] = run(2, "failure");
let next = plan(&state, "o/r", &runs, now());
let numbers: Vec<u64> = next.push.iter().map(|(n, _)| *n).collect();
assert_eq!(numbers, [2]);
}
#[test]
fn skips_are_counted_not_pushed_and_a_relisted_run_counts_once() {
let state = State::default();
let runs = vec![
run(3, "cancelled"),
run(2, "success"),
run(2, "success"),
run(1, "skipped"),
];
let p = plan(&state, "o/r", &runs, now());
assert_eq!(p.push.len(), 1);
assert_eq!(p.skip, [(1, Skip::NotRun), (3, Skip::Cancelled)]);
assert_eq!(p.newest_seen, Some(3));
}
}