push ci run metrics to victoriametrics, with a grafana board

A swarm-ci-metrics host unit on a 5-minute timer lists every forge
repo's Actions runs as the new read-only ci-metrics account and pushes
each finished run's duration and queue wait over OTLP, stamped at the
run's stop time. The first pass backfills from run 1. swarm-controller
mints the account's token into bao and keeps a ci-metrics team with
read on the Actions unit in every organisation.

Closes #5376

swarm-controller/src/forge/ci_metrics_account.rs

@@ -0,0 +1,476 @@
+//! The CI metrics poller's read-only forge account, [`LOGIN`]: created here
+//! with the admin API, put in a [`TEAM`] in every organisation that reads the
+//! Actions unit of each of its repos (including ones created later), and
+//! given one token, stored at
+//! `swarm_secret_client::ci_metrics::forge_token_path`, where the poller reads
+//! it under its own store identity to list CI runs.
+//!
+//! A public repo's runs need no grant. A private repo owned by a user gets
+//! none: the forge grants a user's repo only to a collaborator, and a
+//! collaborator reads every unit, code included.
+//!
+//! The token decision is [`super::agent_token`]'s, under this module's name
+//! and scopes, so a stored token the forge still lists with the same last
+//! eight characters and scopes is kept, and a pass whose reads fail queues
+//! nothing.
+
+use std::sync::Arc;
+
+use anyhow::{Context, Result, bail};
+use forgejo_api::structs::{
+    CreateTeamOption, CreateTeamOptionPermission, EditTeamOption, EditTeamOptionPermission,
+    RepoSearchQuery, Team, TeamPermission,
+};
+use forgejo_api::{Auth, Forgejo};
+use swarm_secret_client::ci_metrics::forge_token_path;
+use swarm_secret_client::forge;
+
+use super::agent_token::{Decision, Observed, classify_named, is_not_found};
+use super::hive_token::TeamSeen;
+use super::{Client, folds_into_success};
+
+/// The poller's forge login.
+pub const LOGIN: &str = "ci-metrics";
+
+/// The forge's name for the account's one token.
+pub const TOKEN_NAME: &str = "swarm-ci-metrics";
+
+/// What the token may do: read repositories, which the runs list is filed
+/// under. Which repositories is the account's business: public ones, and
+/// those its [`TEAM`]s reach.
+pub const TOKEN_SCOPES: &str = "read:repository";
+
+/// The team in each organisation the account is the one member of.
+const TEAM: &str = "ci-metrics";
+
+const TEAM_DESCRIPTION: &str = "hyperhive CI metrics — read on every repo's Actions runs";
+
+/// The one unit the team reads.
+const TEAM_UNITS: [&str; 1] = ["repo.actions"];
+
+/// Organisations per page of the admin listing.
+const ORG_PAGE: u32 = 50;
+
+/// How often [`spawn`] re-checks the account, its teams and its token.
+const RECONCILE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(5);
+
+/// The email this module creates the account with, and the one it requires
+/// before acting on an account named [`LOGIN`]. Under its own domain, so it
+/// equals no agent's or hive's account email.
+const EMAIL: &str = "ci-metrics@services.hyperhive.local";
+
+/// Whether `t` has every setting [`Client::ensure_ci_metrics_team`] gives it.
+fn team_matches(t: &Team) -> bool {
+    t.units
+        .as_deref()
+        .is_some_and(|units| units.iter().map(String::as_str).eq(TEAM_UNITS))
+        && t.permission == Some(TeamPermission::Read)
+        && t.includes_all_repositories == Some(true)
+        && t.can_create_org_repo == Some(false)
+        && t.description.as_deref() == Some(TEAM_DESCRIPTION)
+}
+
+/// Whether a pass that found `seen`, and `granted` for whether every
+/// organisation's [`TEAM`] is as the job leaves it with the account in it,
+/// queues a `MintCiMetricsForgeToken` job. [`Observed::Unknown`] never does,
+/// so a forge or store outage can't rotate the token, and neither does a
+/// [`Decision::Conflict`], which the job would refuse. A kept token queues
+/// only when some grant is known to be missing; that job finds the token
+/// current and does not rotate it.
+pub fn needs_job(seen: Observed, granted: Option<bool>) -> bool {
+    match seen {
+        Observed::NoForgeUser | Observed::Decided(Decision::Mint(_)) => true,
+        Observed::Decided(Decision::Keep) => granted == Some(false),
+        Observed::Unknown | Observed::Decided(Decision::Conflict) => false,
+    }
+}
+
+impl Client {
+    /// Make sure [`LOGIN`] exists, is in [`TEAM`] in every organisation, and
+    /// holds a live [`TOKEN_NAME`] token in the store, minting one when it
+    /// does not. The whole job of the `MintCiMetricsForgeToken` node.
+    ///
+    /// # Errors
+    /// When the store or the forge refuses a step, when an account named
+    /// [`LOGIN`] exists with another email, or when the read-back does not
+    /// show the new token.
+    pub async fn ensure_ci_metrics_forge_token(&self) -> Result<()> {
+        let path = forge_token_path()?;
+        self.ensure_ci_metrics_user().await?;
+        for org in self.all_orgs().await? {
+            let team = self.ensure_ci_metrics_team(&org).await?;
+            self.api
+                .call(|api| api.org_add_team_member(team, LOGIN).into_future())
+                .await
+                .with_context(|| format!("adding {LOGIN} to {org}/{TEAM}"))?;
+        }
+
+        let store = crate::store::connect()
+            .await
+            .context("logging in to the swarm secret store")?;
+        let (stored, listed) = self.read_ci_metrics_token(&store, &path).await?;
+        let reason = match classify_named(TOKEN_NAME, TOKEN_SCOPES, stored.as_ref(), &listed) {
+            Decision::Keep => {
+                tracing::debug!("ci metrics forge token is current; left as it is");
+                return Ok(());
+            }
+            Decision::Conflict => bail!(
+                "the forge lists more than one {TOKEN_NAME:?} token for {LOGIN:?}; delete them \
+                 by id and re-run"
+            ),
+            Decision::Mint(reason) => reason,
+        };
+        let credential = self
+            .replace_token(&store, LOGIN, TOKEN_NAME, TOKEN_SCOPES, &listed, &path)
+            .await?;
+
+        Forgejo::new(Auth::Token(&credential.value), self.url.clone())
+            .context("building a forge client with the new token")?
+            .repo_search(RepoSearchQuery::default())
+            .await
+            .with_context(|| format!("using {LOGIN}'s new forge token"))?;
+        let (stored, listed) = self.read_ci_metrics_token(&store, &path).await?;
+        let now = classify_named(TOKEN_NAME, TOKEN_SCOPES, stored.as_ref(), &listed);
+        if now != Decision::Keep {
+            bail!("{LOGIN}'s new forge token does not read back as current: {now:?}");
+        }
+        tracing::info!(?reason, %path, "ci metrics forge token minted and stored");
+        Ok(())
+    }
+
+    /// The stored token at `path` and every token the forge lists for
+    /// [`LOGIN`], which must exist.
+    async fn read_ci_metrics_token(
+        &self,
+        store: &swarm_secret_client::SecretStore,
+        path: &str,
+    ) -> Result<(
+        Option<forge::Credential>,
+        Vec<forgejo_api::structs::AccessToken>,
+    )> {
+        let stored = store
+            .read_optional(path)
+            .await
+            .with_context(|| format!("reading {path}"))?;
+        let Some(listed) = self.list_agent_tokens(LOGIN).await? else {
+            bail!("the forge has no user {LOGIN:?} after this job created it");
+        };
+        Ok((stored, listed))
+    }
+
+    /// Create [`LOGIN`] if there is none, refuse one with another email, and
+    /// lock it out of creating repos.
+    async fn ensure_ci_metrics_user(&self) -> Result<()> {
+        self.create_user(LOGIN, EMAIL).await?;
+        let user = self
+            .api
+            .call(|api| api.user_get(LOGIN).into_future())
+            .await
+            .with_context(|| format!("read forge user {LOGIN}"))?;
+        if user.email.as_deref() != Some(EMAIL) {
+            bail!(
+                "forge account `{LOGIN}` exists but is not the CI metrics poller's (email {:?}, \
+                 expected `{EMAIL}`); refusing to give it read on every organisation's runs",
+                user.email
+            );
+        }
+        self.api
+            .call(|api| {
+                api.admin_edit_user(LOGIN, super::repo_creation_lockdown(LOGIN))
+                    .into_future()
+            })
+            .await
+            .with_context(|| format!("disable repo creation for forge user {LOGIN}"))?;
+        Ok(())
+    }
+
+    /// Every organisation on the forge, by name.
+    async fn all_orgs(&self) -> Result<Vec<String>> {
+        let mut names = Vec::new();
+        let mut page = 1;
+        loop {
+            let (_, orgs) = self
+                .api
+                .call(|api| {
+                    api.admin_get_all_orgs()
+                        .page(page)
+                        .page_size(ORG_PAGE)
+                        .into_future()
+                })
+                .await
+                .context("listing the forge's organisations")?;
+            let last = orgs.len() < ORG_PAGE as usize;
+            names.extend(orgs.into_iter().filter_map(|o| o.username.or(o.name)));
+            if last {
+                return Ok(names);
+            }
+            page += 1;
+        }
+    }
+
+    /// The id of `org`'s [`TEAM`], created or put back to its settings.
+    async fn ensure_ci_metrics_team(&self, org: &str) -> Result<i64> {
+        match self.observe_ci_metrics_team(org).await {
+            TeamSeen::Matches(id) => Ok(id),
+            TeamSeen::Drifted(id) => {
+                self.api
+                    .call(|api| {
+                        api.org_edit_team(
+                            id,
+                            EditTeamOption {
+                                can_create_org_repo: Some(false),
+                                description: Some(TEAM_DESCRIPTION.to_owned()),
+                                includes_all_repositories: Some(true),
+                                name: TEAM.to_owned(),
+                                permission: Some(EditTeamOptionPermission::Read),
+                                units: Some(TEAM_UNITS.map(str::to_owned).to_vec()),
+                                units_map: None,
+                            },
+                        )
+                        .into_future()
+                    })
+                    .await
+                    .with_context(|| format!("reconcile {org}/{TEAM} team settings"))?;
+                tracing::info!("swarm forge: reconciled {org}/{TEAM} team settings");
+                Ok(id)
+            }
+            TeamSeen::Absent => self.create_ci_metrics_team(org).await,
+            TeamSeen::Unknown => bail!("could not read the teams of {org}"),
+        }
+    }
+
+    async fn create_ci_metrics_team(&self, org: &str) -> Result<i64> {
+        let team = CreateTeamOption {
+            can_create_org_repo: Some(false),
+            description: Some(TEAM_DESCRIPTION.to_owned()),
+            includes_all_repositories: Some(true),
+            name: TEAM.to_owned(),
+            permission: Some(CreateTeamOptionPermission::Read),
+            units: Some(TEAM_UNITS.map(str::to_owned).to_vec()),
+            units_map: None,
+        };
+        match self
+            .api
+            .call(|api| api.org_create_team(org, team.clone()).into_future())
+            .await
+        {
+            Ok(created) => {
+                tracing::info!("swarm forge: created {org}/{TEAM} team");
+                created
+                    .id
+                    .with_context(|| format!("the forge created {org}/{TEAM} but returned no id"))
+            }
+            // A duplicate is a 409 or a 422; either counts once a re-read
+            // finds the team, which also supplies its id.
+            Err(e) => match self.observe_ci_metrics_team(org).await {
+                TeamSeen::Matches(id) | TeamSeen::Drifted(id) if folds_into_success(&e, true) => {
+                    Ok(id)
+                }
+                _ => Err(e).with_context(|| format!("create team {org}/{TEAM}")),
+            },
+        }
+    }
+
+    async fn observe_ci_metrics_team(&self, org: &str) -> TeamSeen {
+        match self
+            .api
+            .call(|api| api.org_list_teams(org).into_future())
+            .await
+        {
+            Ok((_, teams)) => teams
+                .into_iter()
+                .find(|t| t.name.as_deref() == Some(TEAM))
+                .map_or(TeamSeen::Absent, |t| match t.id {
+                    Some(id) if team_matches(&t) => TeamSeen::Matches(id),
+                    Some(id) => TeamSeen::Drifted(id),
+                    None => TeamSeen::Unknown,
+                }),
+            Err(e) => {
+                tracing::warn!(error = %e, "ci metrics forge token: reading the teams of {org} failed");
+                TeamSeen::Unknown
+            }
+        }
+    }
+
+    /// Whether every organisation's [`TEAM`] is as the job leaves it, with
+    /// the account in it; `None` when a read failed.
+    async fn observe_ci_metrics_grants(&self) -> Option<bool> {
+        let orgs = match self.all_orgs().await {
+            Ok(orgs) => orgs,
+            Err(e) => {
+                tracing::warn!(error = %format!("{e:#}"), "ci metrics forge token: organisation read failed");
+                return None;
+            }
+        };
+        for org in orgs {
+            let id = match self.observe_ci_metrics_team(&org).await {
+                TeamSeen::Matches(id) => id,
+                TeamSeen::Drifted(_) | TeamSeen::Absent => return Some(false),
+                TeamSeen::Unknown => return None,
+            };
+            match self
+                .api
+                .call(|api| api.org_list_team_member(id, LOGIN).into_future())
+                .await
+            {
+                Ok(_) => {}
+                Err(e) if is_not_found(&e) => return Some(false),
+                Err(e) => {
+                    tracing::warn!(error = %e, "ci metrics forge token: team membership read failed");
+                    return None;
+                }
+            }
+        }
+        Some(true)
+    }
+
+    /// What the store and the forge say about the account's token, and
+    /// whether its grants are all in place.
+    async fn observe_ci_metrics(&self) -> Result<(Observed, Option<bool>)> {
+        let path = forge_token_path()?;
+        let store = crate::store::connect()
+            .await
+            .context("logging in to the swarm secret store")?;
+        let stored = match store.read_optional::<forge::Credential>(&path).await {
+            Ok(stored) => stored,
+            Err(e) => {
+                tracing::warn!(error = %e, "ci metrics forge token: store read failed");
+                return Ok((Observed::Unknown, None));
+            }
+        };
+        let seen = match self.list_agent_tokens(LOGIN).await {
+            Ok(Some(listed)) => Observed::Decided(classify_named(
+                TOKEN_NAME,
+                TOKEN_SCOPES,
+                stored.as_ref(),
+                &listed,
+            )),
+            Ok(None) => Observed::NoForgeUser,
+            Err(e) => {
+                tracing::warn!(error = %format!("{e:#}"), "ci metrics forge token: forge read failed");
+                Observed::Unknown
+            }
+        };
+        Ok((seen, self.observe_ci_metrics_grants().await))
+    }
+}
+
+/// Check the account now and every [`RECONCILE_INTERVAL`] after, and call
+/// `enqueue`, which inserts a `MintCiMetricsForgeToken` node, when it needs a
+/// job. A pass that fails is logged and retried on the next tick.
+pub fn spawn(client: Arc<Client>, enqueue: impl Fn() + Send + 'static) {
+    tokio::spawn(async move {
+        let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
+        loop {
+            ticker.tick().await;
+            let Some(outcome) =
+                super::within_forge_deadline("ci metrics forge token", client.observe_ci_metrics())
+                    .await
+            else {
+                continue;
+            };
+            match outcome {
+                Ok((seen, granted)) if needs_job(seen, granted) => {
+                    tracing::info!(?seen, ?granted, "ci metrics forge token: queueing a job");
+                    enqueue();
+                }
+                Ok((seen, granted)) => {
+                    tracing::debug!(?seen, ?granted, "ci metrics forge token: no job");
+                }
+                Err(e) => tracing::warn!(
+                    error = %format!("{e:#}"),
+                    retry_in_s = RECONCILE_INTERVAL.as_secs(),
+                    "ci metrics forge token: pass failed; retrying next tick"
+                ),
+            }
+        }
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::agent_token::MintReason;
+    use super::*;
+
+    #[test]
+    fn the_token_reads_repositories_and_nothing_else() {
+        assert_eq!(TOKEN_SCOPES, "read:repository");
+        assert_ne!(TOKEN_NAME, super::super::agent_token::AGENT_TOKEN_NAME);
+        assert_ne!(TOKEN_NAME, super::super::hive_token::HIVE_TOKEN_NAME);
+        assert_ne!(TOKEN_NAME, super::super::classifier_account::TOKEN_NAME);
+        assert_ne!(TOKEN_NAME, super::super::builder_account::TOKEN_NAME);
+    }
+
+    #[test]
+    fn a_mint_a_missing_account_or_a_missing_grant_queues() {
+        let mint = Observed::Decided(Decision::Mint(MintReason::NotStored));
+        let keep = Observed::Decided(Decision::Keep);
+        assert!(needs_job(Observed::NoForgeUser, None));
+        assert!(needs_job(mint, Some(true)));
+        assert!(needs_job(keep, Some(false)));
+        assert!(!needs_job(keep, Some(true)));
+        assert!(!needs_job(keep, None));
+        assert!(!needs_job(
+            Observed::Decided(Decision::Conflict),
+            Some(false)
+        ));
+        assert!(!needs_job(Observed::Unknown, Some(false)));
+    }
+
+    fn team(units: &[&str], permission: TeamPermission) -> Team {
+        Team {
+            can_create_org_repo: Some(false),
+            description: Some(TEAM_DESCRIPTION.to_owned()),
+            id: Some(7),
+            includes_all_repositories: Some(true),
+            name: Some(TEAM.to_owned()),
+            organization: None,
+            permission: Some(permission),
+            units: Some(units.iter().map(|u| (*u).to_owned()).collect()),
+            units_map: None,
+        }
+    }
+
+    #[test]
+    fn the_team_reads_actions_and_nothing_else() {
+        assert!(team_matches(&team(&["repo.actions"], TeamPermission::Read)));
+        assert!(!team_matches(&team(
+            &["repo.actions", "repo.code"],
+            TeamPermission::Read
+        )));
+        assert!(!team_matches(&team(
+            &["repo.actions"],
+            TeamPermission::Write
+        )));
+        let mut some_repos = team(&["repo.actions"], TeamPermission::Read);
+        some_repos.includes_all_repositories = Some(false);
+        assert!(!team_matches(&some_repos));
+    }
+
+    #[test]
+    fn the_accounts_email_is_no_agents_or_hives() {
+        assert_ne!(EMAIL, super::super::agent_email(LOGIN));
+        assert_ne!(EMAIL, super::super::legacy_agent_email(LOGIN));
+        assert!(!LOGIN.starts_with(hive_types::RESERVED_AGENT_PREFIX));
+    }
+
+    /// An existing account named like the poller with an agent's email is
+    /// refused before the lockdown, so it never joins a team.
+    #[tokio::test]
+    async fn the_job_refuses_an_account_that_is_not_the_pollers() {
+        use super::super::tests::{lockdown_patch, stub_forge};
+
+        let (client, seen) = stub_forge(
+            (409, r#"{"message":"user already exists"}"#),
+            (200, r#"{"email":"ci-metrics@hyperhive.local"}"#),
+            (200, "{}"),
+        )
+        .await;
+        let err = client
+            .ensure_ci_metrics_user()
+            .await
+            .expect_err("an agent's account is not the poller's");
+        assert!(format!("{err:#}").contains("not the CI metrics poller's"));
+        assert!(lockdown_patch(&seen, LOGIN).is_none());
+    }
+}