vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

push ci run metrics to victoriametrics, with a grafana board

Browse source 
A swarm-ci-metrics host unit on a 5-minute timer lists every forge
repo's Actions runs as the new read-only ci-metrics account and pushes
each finished run's duration and queue wait over OTLP, stamped at the
run's stop time. The first pass backfills from run 1. swarm-controller
mints the account's token into bao and keeps a ci-metrics team with
read on the Actions unit in every organisation.

Closes #5376
This commit is contained in:
atlas 2026-10-16 18:23:59 +02:00 • committed by mara
commit b072fbf4d6
36 changed files with 2849 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

62
swarm-controller/src/main.rs
View file

@ -163,6 +163,11 @@ enum SwarmNodeKind {
/// the `hives` team, and holds a live token in the swarm store, where the
/// builder reads it to fetch config repos. See `forge::builder_account`.
MintBuilderForgeToken,
/// Make sure the CI metrics poller's read-only forge account exists, is in
/// its team in every organisation, and holds a live token in the swarm
/// store, where the poller reads it to list CI runs. See
/// `forge::ci_metrics_account`.
MintCiMetricsForgeToken,
/// Build `agent`'s config at `rev`, the head of its config PR `pr`, on
/// the swarm builder, and write the result as a commit status on `rev`.
/// See `builder`.
@ -212,6 +217,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
SwarmNodeKind::MintHiveForgeToken { .. } => "mint_hive_forge_token".to_owned(),
SwarmNodeKind::MintClassifierForgeToken => "mint_classifier_forge_token".to_owned(),
SwarmNodeKind::MintBuilderForgeToken => "mint_builder_forge_token".to_owned(),
SwarmNodeKind::MintCiMetricsForgeToken => "mint_ci_metrics_forge_token".to_owned(),
SwarmNodeKind::BuildAgent { .. } => "build_agent".to_owned(),
SwarmNodeKind::RenewAgentQueueCredential { .. } => {
"renew_agent_queue_credential".to_owned()
@ -243,9 +249,9 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
}
SwarmNodeKind::MintHiveSenderToken { hive }
| SwarmNodeKind::MintHiveForgeToken { hive } => serde_json::json!({ "hive": hive }),
SwarmNodeKind::MintClassifierForgeToken | SwarmNodeKind::MintBuilderForgeToken => {
serde_json::json!({})
}
SwarmNodeKind::MintClassifierForgeToken
| SwarmNodeKind::MintBuilderForgeToken
| SwarmNodeKind::MintCiMetricsForgeToken => serde_json::json!({}),
SwarmNodeKind::BuildAgent { agent, rev, pr } => {
serde_json::json!({ "agent": agent, "rev": rev, "pr": pr })
}
@ -413,6 +419,7 @@ async fn run_swarm_node(
SwarmNodeKind::MintHiveForgeToken { hive } => mint_hive_forge_token(forge, &hive).await,
SwarmNodeKind::MintClassifierForgeToken => mint_classifier_forge_token(forge).await,
SwarmNodeKind::MintBuilderForgeToken => mint_builder_forge_token(forge).await,
SwarmNodeKind::MintCiMetricsForgeToken => mint_ci_metrics_forge_token(forge).await,
SwarmNodeKind::BuildAgent { agent, rev, pr } => {
build_agent(&deps, forge, agent, rev, pr).await
}
@ -606,6 +613,30 @@ async fn mint_builder_forge_token(
}
}
/// The `MintCiMetricsForgeToken` arm, lifted out for the same reason.
async fn mint_ci_metrics_forge_token(
forge: Option<Arc<forge::Client>>,
) -> hive_jobq::scheduler::Outcome {
use hive_jobq::scheduler::Outcome;
let Some(client) = forge else {
return Outcome::Failed(
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
not in the swarm secret store), so no ci metrics forge token can be minted"
.to_owned(),
);
};
match forge_node(
"mint_ci_metrics_forge_token",
client.ensure_ci_metrics_forge_token(),
)
.await
{
Ok(()) => Outcome::Done,
Err(e) => Outcome::Failed(format!("{e:#}")),
}
}
/// The `MintClassifierForgeToken` arm, lifted out for the same reason.
async fn mint_classifier_forge_token(
forge: Option<Arc<forge::Client>>,
@ -2760,6 +2791,20 @@ fn queue_builder_forge_token_mint(
.map_err(|e| anyhow::anyhow!("{e}"))
}
/// Insert one `MintCiMetricsForgeToken` job and return its node's id.
/// `forge::ci_metrics_account::spawn`'s periodic pass is the one caller.
fn queue_ci_metrics_forge_token_mint(
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
) -> Result<Vec<hive_jobq::NodeId>> {
sched
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert_job(None, |b| {
vec![b.node(SwarmNodeKind::MintCiMetricsForgeToken).guid()]
})
.map_err(|e| anyhow::anyhow!("{e}"))
}
/// The homeserver agents' own matrix accounts are minted on. Unset is this
/// controller creating agents with no matrix account, which the mint node
/// reports by name rather than `main` refusing to start.
@ -2835,6 +2880,12 @@ fn spawn_forge_workers(
}
});
let sched = Arc::clone(jobq);
forge::ci_metrics_account::spawn(Arc::clone(&client), move || {
if let Err(e) = queue_ci_metrics_forge_token_mint(&sched) {
tracing::warn!(error = %format!("{e:#}"), "ci metrics forge token: queueing failed");
}
});
let sched = Arc::clone(jobq);
forge::docs_grant::spawn(Arc::clone(&client), move |agents| {
if let Err(e) = queue_docs_grants(&sched, agents) {
tracing::warn!(error = %format!("{e:#}"), "docs grants: queueing failed");
@ -4725,6 +4776,7 @@ mod tests {
for name in [
crate::forge::builder_account::LOGIN,
crate::forge::classifier_account::LOGIN,
crate::forge::ci_metrics_account::LOGIN,
] {
let mut warnings = Vec::new();
let broken = super::broken_name_rules(name, Some(&raw), &mut warnings);
@ -4959,6 +5011,10 @@ mod tests {
"mint_builder_forge_token",
SwarmNodeKind::MintBuilderForgeToken,
),
(
"mint_ci_metrics_forge_token",
SwarmNodeKind::MintCiMetricsForgeToken,
),
];
for (label, kind) in nodes {
let mut sched = hive_jobq::scheduler::Scheduler::new(