//! The CI metrics poller's read-only forge account, [`LOGIN`]: created here //! with the admin API, put in a [`TEAM`] in every organisation that reads the //! Actions unit of each of its repos (including ones created later), and //! given one token, stored at //! `swarm_secret_client::ci_metrics::forge_token_path`, where the poller reads //! it under its own store identity to list CI runs. //! //! A public repo's runs need no grant. A private repo owned by a user gets //! none: the forge grants a user's repo only to a collaborator, and a //! collaborator reads every unit, code included. //! //! The token decision is [`super::agent_token`]'s, under this module's name //! and scopes, so a stored token the forge still lists with the same last //! eight characters and scopes is kept, and a pass whose reads fail queues //! nothing. use std::sync::Arc; use anyhow::{Context, Result, bail}; use forge_client::async_client; use forgejo_api::structs::{ CreateTeamOption, CreateTeamOptionPermission, EditTeamOption, EditTeamOptionPermission, RepoSearchQuery, Team, TeamPermission, }; use swarm_secret_client::ci_metrics::forge_token_path; use swarm_secret_client::forge; use super::agent_token::{Decision, Observed, classify_named, is_not_found}; use super::hive_token::TeamSeen; use super::{Client, folds_into_success}; /// The poller's forge login. pub const LOGIN: &str = "ci-metrics"; /// The forge's name for the account's one token. pub const TOKEN_NAME: &str = "swarm-ci-metrics"; /// What the token may do: read repositories, which the runs list is filed /// under. Which repositories is the account's business: public ones, and /// those its [`TEAM`]s reach. pub const TOKEN_SCOPES: &str = "read:repository"; /// The team in each organisation the account is the one member of. const TEAM: &str = "ci-metrics"; const TEAM_DESCRIPTION: &str = "hyperhive CI metrics — read on every repo's Actions runs"; /// The one unit the team reads. const TEAM_UNITS: [&str; 1] = ["repo.actions"]; /// Organisations per page of the admin listing. const ORG_PAGE: u32 = 50; /// How often [`spawn`] re-checks the account, its teams and its token. const RECONCILE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(5); /// The email this module creates the account with, and the one it requires /// before acting on an account named [`LOGIN`]. Under its own domain, so it /// equals no agent's or hive's account email. const EMAIL: &str = "ci-metrics@services.hyperhive.local"; /// Whether `t` has every setting [`Client::ensure_ci_metrics_team`] gives it. fn team_matches(t: &Team) -> bool { t.units .as_deref() .is_some_and(|units| units.iter().map(String::as_str).eq(TEAM_UNITS)) && t.permission == Some(TeamPermission::Read) && t.includes_all_repositories == Some(true) && t.can_create_org_repo == Some(false) && t.description.as_deref() == Some(TEAM_DESCRIPTION) } /// Whether a pass that found `seen`, and `granted` for whether every /// organisation's [`TEAM`] is as the job leaves it with the account in it, /// queues a `MintCiMetricsForgeToken` job. [`Observed::Unknown`] never does, /// so a forge or store outage can't rotate the token, and neither does a /// [`Decision::Conflict`], which the job would refuse. A kept token queues /// only when some grant is known to be missing; that job finds the token /// current and does not rotate it. pub fn needs_job(seen: Observed, granted: Option) -> bool { match seen { Observed::NoForgeUser | Observed::Decided(Decision::Mint(_)) => true, Observed::Decided(Decision::Keep) => granted == Some(false), Observed::Unknown | Observed::Decided(Decision::Conflict) => false, } } impl Client { /// Make sure [`LOGIN`] exists, is in [`TEAM`] in every organisation, and /// holds a live [`TOKEN_NAME`] token in the store, minting one when it /// does not. The whole job of the `MintCiMetricsForgeToken` node. /// /// # Errors /// When the store or the forge refuses a step, when an account named /// [`LOGIN`] exists with another email, or when the read-back does not /// show the new token. pub async fn ensure_ci_metrics_forge_token(&self) -> Result<()> { let path = forge_token_path()?; self.ensure_ci_metrics_user().await?; for org in self.all_orgs().await? { let team = self.ensure_ci_metrics_team(&org).await?; self.api .call(|api| api.org_add_team_member(team, LOGIN).into_future()) .await .with_context(|| format!("adding {LOGIN} to {org}/{TEAM}"))?; } let store = crate::store::connect() .await .context("logging in to the swarm secret store")?; let (stored, listed) = self.read_ci_metrics_token(&store, &path).await?; let reason = match classify_named(TOKEN_NAME, TOKEN_SCOPES, stored.as_ref(), &listed) { Decision::Keep => { tracing::debug!("ci metrics forge token is current; left as it is"); return Ok(()); } Decision::Conflict => bail!( "the forge lists more than one {TOKEN_NAME:?} token for {LOGIN:?}; delete them \ by id and re-run" ), Decision::Mint(reason) => reason, }; let credential = self .replace_token(&store, LOGIN, TOKEN_NAME, TOKEN_SCOPES, &listed, &path) .await?; async_client(self.url.clone(), &credential.value) .context("building a forge client with the new token")? .repo_search(RepoSearchQuery::default()) .await .with_context(|| format!("using {LOGIN}'s new forge token"))?; let (stored, listed) = self.read_ci_metrics_token(&store, &path).await?; let now = classify_named(TOKEN_NAME, TOKEN_SCOPES, stored.as_ref(), &listed); if now != Decision::Keep { bail!("{LOGIN}'s new forge token does not read back as current: {now:?}"); } tracing::info!(?reason, %path, "ci metrics forge token minted and stored"); Ok(()) } /// The stored token at `path` and every token the forge lists for /// [`LOGIN`], which must exist. async fn read_ci_metrics_token( &self, store: &swarm_secret_client::SecretStore, path: &str, ) -> Result<( Option, Vec, )> { let stored = store .read_optional(path) .await .with_context(|| format!("reading {path}"))?; let Some(listed) = self.list_agent_tokens(LOGIN).await? else { bail!("the forge has no user {LOGIN:?} after this job created it"); }; Ok((stored, listed)) } /// Create [`LOGIN`] if there is none, refuse one with another email, and /// lock it out of creating repos. async fn ensure_ci_metrics_user(&self) -> Result<()> { self.create_user(LOGIN, EMAIL).await?; let user = self .api .call(|api| api.user_get(LOGIN).into_future()) .await .with_context(|| format!("read forge user {LOGIN}"))?; if user.email.as_deref() != Some(EMAIL) { bail!( "forge account `{LOGIN}` exists but is not the CI metrics poller's (email {:?}, \ expected `{EMAIL}`); refusing to give it read on every organisation's runs", user.email ); } self.api .call(|api| { api.admin_edit_user(LOGIN, super::repo_creation_lockdown(LOGIN)) .into_future() }) .await .with_context(|| format!("disable repo creation for forge user {LOGIN}"))?; Ok(()) } /// Every organisation on the forge, by name. async fn all_orgs(&self) -> Result> { let mut names = Vec::new(); let mut page = 1; loop { let (_, orgs) = self .api .call(|api| { api.admin_get_all_orgs() .page(page) .page_size(ORG_PAGE) .into_future() }) .await .context("listing the forge's organisations")?; let last = orgs.len() < ORG_PAGE as usize; names.extend(orgs.into_iter().filter_map(|o| o.username.or(o.name))); if last { return Ok(names); } page += 1; } } /// The id of `org`'s [`TEAM`], created or put back to its settings. async fn ensure_ci_metrics_team(&self, org: &str) -> Result { match self.observe_ci_metrics_team(org).await { TeamSeen::Matches(id) => Ok(id), TeamSeen::Drifted(id) => { self.api .call(|api| { api.org_edit_team( id, EditTeamOption { can_create_org_repo: Some(false), description: Some(TEAM_DESCRIPTION.to_owned()), includes_all_repositories: Some(true), name: TEAM.to_owned(), permission: Some(EditTeamOptionPermission::Read), units: Some(TEAM_UNITS.map(str::to_owned).to_vec()), units_map: None, }, ) .into_future() }) .await .with_context(|| format!("reconcile {org}/{TEAM} team settings"))?; tracing::info!("swarm forge: reconciled {org}/{TEAM} team settings"); Ok(id) } TeamSeen::Absent => self.create_ci_metrics_team(org).await, TeamSeen::Unknown => bail!("could not read the teams of {org}"), } } async fn create_ci_metrics_team(&self, org: &str) -> Result { let team = CreateTeamOption { can_create_org_repo: Some(false), description: Some(TEAM_DESCRIPTION.to_owned()), includes_all_repositories: Some(true), name: TEAM.to_owned(), permission: Some(CreateTeamOptionPermission::Read), units: Some(TEAM_UNITS.map(str::to_owned).to_vec()), units_map: None, }; match self .api .call(|api| api.org_create_team(org, team.clone()).into_future()) .await { Ok(created) => { tracing::info!("swarm forge: created {org}/{TEAM} team"); created .id .with_context(|| format!("the forge created {org}/{TEAM} but returned no id")) } // A duplicate is a 409 or a 422; either counts once a re-read // finds the team, which also supplies its id. Err(e) => match self.observe_ci_metrics_team(org).await { TeamSeen::Matches(id) | TeamSeen::Drifted(id) if folds_into_success(&e, true) => { Ok(id) } _ => Err(e).with_context(|| format!("create team {org}/{TEAM}")), }, } } async fn observe_ci_metrics_team(&self, org: &str) -> TeamSeen { match self .api .call(|api| api.org_list_teams(org).into_future()) .await { Ok((_, teams)) => teams .into_iter() .find(|t| t.name.as_deref() == Some(TEAM)) .map_or(TeamSeen::Absent, |t| match t.id { Some(id) if team_matches(&t) => TeamSeen::Matches(id), Some(id) => TeamSeen::Drifted(id), None => TeamSeen::Unknown, }), Err(e) => { tracing::warn!(error = %e, "ci metrics forge token: reading the teams of {org} failed"); TeamSeen::Unknown } } } /// Whether every organisation's [`TEAM`] is as the job leaves it, with /// the account in it; `None` when a read failed. async fn observe_ci_metrics_grants(&self) -> Option { let orgs = match self.all_orgs().await { Ok(orgs) => orgs, Err(e) => { tracing::warn!(error = %format!("{e:#}"), "ci metrics forge token: organisation read failed"); return None; } }; for org in orgs { let id = match self.observe_ci_metrics_team(&org).await { TeamSeen::Matches(id) => id, TeamSeen::Drifted(_) | TeamSeen::Absent => return Some(false), TeamSeen::Unknown => return None, }; match self .api .call(|api| api.org_list_team_member(id, LOGIN).into_future()) .await { Ok(_) => {} Err(e) if is_not_found(&e) => return Some(false), Err(e) => { tracing::warn!(error = %e, "ci metrics forge token: team membership read failed"); return None; } } } Some(true) } /// What the store and the forge say about the account's token, and /// whether its grants are all in place. async fn observe_ci_metrics(&self) -> Result<(Observed, Option)> { let path = forge_token_path()?; let store = crate::store::connect() .await .context("logging in to the swarm secret store")?; let stored = match store.read_optional::(&path).await { Ok(stored) => stored, Err(e) => { tracing::warn!(error = %e, "ci metrics forge token: store read failed"); return Ok((Observed::Unknown, None)); } }; let seen = match self.list_agent_tokens(LOGIN).await { Ok(Some(listed)) => Observed::Decided(classify_named( TOKEN_NAME, TOKEN_SCOPES, stored.as_ref(), &listed, )), Ok(None) => Observed::NoForgeUser, Err(e) => { tracing::warn!(error = %format!("{e:#}"), "ci metrics forge token: forge read failed"); Observed::Unknown } }; Ok((seen, self.observe_ci_metrics_grants().await)) } } /// Check the account now and every [`RECONCILE_INTERVAL`] after, and call /// `enqueue`, which inserts a `MintCiMetricsForgeToken` node, when it needs a /// job. A pass that fails is logged and retried on the next tick. pub fn spawn(client: Arc, enqueue: impl Fn() + Send + 'static) { tokio::spawn(async move { let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); loop { ticker.tick().await; let Some(outcome) = super::within_forge_deadline("ci metrics forge token", client.observe_ci_metrics()) .await else { continue; }; match outcome { Ok((seen, granted)) if needs_job(seen, granted) => { tracing::info!(?seen, ?granted, "ci metrics forge token: queueing a job"); enqueue(); } Ok((seen, granted)) => { tracing::debug!(?seen, ?granted, "ci metrics forge token: no job"); } Err(e) => tracing::warn!( error = %format!("{e:#}"), retry_in_s = RECONCILE_INTERVAL.as_secs(), "ci metrics forge token: pass failed; retrying next tick" ), } } }); } #[cfg(test)] mod tests { use super::super::agent_token::MintReason; use super::*; #[test] fn the_token_reads_repositories_and_nothing_else() { assert_eq!(TOKEN_SCOPES, "read:repository"); assert_ne!(TOKEN_NAME, super::super::agent_token::AGENT_TOKEN_NAME); assert_ne!(TOKEN_NAME, super::super::hive_token::HIVE_TOKEN_NAME); assert_ne!(TOKEN_NAME, super::super::classifier_account::TOKEN_NAME); assert_ne!(TOKEN_NAME, super::super::builder_account::TOKEN_NAME); } #[test] fn a_mint_a_missing_account_or_a_missing_grant_queues() { let mint = Observed::Decided(Decision::Mint(MintReason::NotStored)); let keep = Observed::Decided(Decision::Keep); assert!(needs_job(Observed::NoForgeUser, None)); assert!(needs_job(mint, Some(true))); assert!(needs_job(keep, Some(false))); assert!(!needs_job(keep, Some(true))); assert!(!needs_job(keep, None)); assert!(!needs_job( Observed::Decided(Decision::Conflict), Some(false) )); assert!(!needs_job(Observed::Unknown, Some(false))); } fn team(units: &[&str], permission: TeamPermission) -> Team { Team { can_create_org_repo: Some(false), description: Some(TEAM_DESCRIPTION.to_owned()), id: Some(7), includes_all_repositories: Some(true), name: Some(TEAM.to_owned()), organization: None, permission: Some(permission), units: Some(units.iter().map(|u| (*u).to_owned()).collect()), units_map: None, } } #[test] fn the_team_reads_actions_and_nothing_else() { assert!(team_matches(&team(&["repo.actions"], TeamPermission::Read))); assert!(!team_matches(&team( &["repo.actions", "repo.code"], TeamPermission::Read ))); assert!(!team_matches(&team( &["repo.actions"], TeamPermission::Write ))); let mut some_repos = team(&["repo.actions"], TeamPermission::Read); some_repos.includes_all_repositories = Some(false); assert!(!team_matches(&some_repos)); } #[test] fn the_accounts_email_is_no_agents_or_hives() { assert_ne!(EMAIL, super::super::agent_email(LOGIN)); assert_ne!(EMAIL, super::super::legacy_agent_email(LOGIN)); assert!(!LOGIN.starts_with(hive_types::RESERVED_AGENT_PREFIX)); } /// An existing account named like the poller with an agent's email is /// refused before the lockdown, so it never joins a team. #[tokio::test] async fn the_job_refuses_an_account_that_is_not_the_pollers() { use super::super::tests::{lockdown_patch, stub_forge}; let (client, seen) = stub_forge( (409, r#"{"message":"user already exists"}"#), (200, r#"{"email":"ci-metrics@hyperhive.local"}"#), (200, "{}"), ) .await; let err = client .ensure_ci_metrics_user() .await .expect_err("an agent's account is not the poller's"); assert!(format!("{err:#}").contains("not the CI metrics poller's")); assert!(lockdown_patch(&seen, LOGIN).is_none()); } /// An existing account is locked out of repo creation and of /// organizations on every pass. #[tokio::test] async fn the_job_locks_the_poller_out_of_repos_and_organizations() { use super::super::tests::{lockdown_patch, stub_forge}; let (client, seen) = stub_forge( (409, r#"{"message":"user already exists"}"#), (200, r#"{"email":"ci-metrics@services.hyperhive.local"}"#), (200, "{}"), ) .await; client .ensure_ci_metrics_user() .await .expect("the poller's own account"); let body = lockdown_patch(&seen, LOGIN).expect("no lockdown PATCH sent"); assert_eq!(body["max_repo_creation"], 0); assert_eq!(body["allow_create_organization"], false); } // The forge side, against a stand-in forge that answers by path. use super::super::tests::{Seen as Requests, routed_forge, sent, writes}; use serde_json::{Value, json}; use std::sync::atomic::{AtomicUsize, Ordering}; fn team_json(id: i64, permission: &str, units: &[&str]) -> Value { json!({ "id": id, "name": TEAM, "description": TEAM_DESCRIPTION, "permission": permission, "includes_all_repositories": true, "can_create_org_repo": false, "units": units, }) } /// A forge listing `orgs` and giving each the teams in `teams`, every /// request not named by `rest` a 201 for a POST and 200 otherwise. async fn forge_of( orgs: &[&str], teams: Vec<(&'static str, u16, Value)>, rest: Vec<(String, u16, Value)>, ) -> (Client, Requests) { let orgs: Vec = orgs.iter().map(|o| json!({"username": o})).collect(); let teams: Vec<(String, u16, Value)> = teams .into_iter() .map(|(k, s, b)| (k.to_owned(), s, b)) .collect(); routed_forge(move |method, path, _| { let key = format!("{method} {path}"); if key == "GET /api/v1/admin/orgs" { return (200, json!(orgs).to_string()); } if let Some((_, s, b)) = teams.iter().chain(rest.iter()).find(|(k, _, _)| *k == key) { return (*s, b.to_string()); } match method { "GET" => (404, r#"{"message":"no"}"#.to_owned()), "POST" => (201, "{}".to_owned()), _ => (200, "{}".to_owned()), } }) .await } #[tokio::test] async fn the_account_is_created_and_locked_out_of_creating_repos() { let (client, seen) = forge_of( &[], vec![], vec![at( "GET /api/v1/users/ci-metrics", 200, json!({"email": EMAIL}), )], ) .await; client.ensure_ci_metrics_user().await.unwrap(); assert_eq!( writes(&seen), [ "POST /api/v1/admin/users".to_owned(), "PATCH /api/v1/admin/users/ci-metrics".to_owned() ] ); let lock = sent(&seen, "PATCH", "/api/v1/admin/users/ci-metrics").unwrap(); assert_eq!(lock["max_repo_creation"], 0); assert_eq!(lock["allow_create_organization"], false); assert_eq!(lock["login_name"], LOGIN); } #[tokio::test] async fn a_refused_create_stops_before_the_account_is_read_or_locked() { let (client, seen) = forge_of( &[], vec![], vec![ at("POST /api/v1/admin/users", 500, json!({"message": "x"})), at("GET /api/v1/users/ci-metrics", 200, json!({"email": EMAIL})), ], ) .await; client.ensure_ci_metrics_user().await.expect_err("refused"); assert_eq!(writes(&seen), ["POST /api/v1/admin/users"]); } #[tokio::test] async fn a_team_that_matches_is_left_and_one_with_the_wrong_grant_is_put_back() { let path = "GET /api/v1/orgs/o/teams"; let (client, seen) = forge_of( &[], vec![(path, 200, json!([team_json(3, "read", &TEAM_UNITS)]))], vec![], ) .await; assert_eq!(client.ensure_ci_metrics_team("o").await.unwrap(), 3); assert!(writes(&seen).is_empty()); let wide = team_json(3, "write", &["repo.actions", "repo.code"]); let (client, seen) = forge_of(&[], vec![(path, 200, json!([wide]))], vec![]).await; assert_eq!(client.ensure_ci_metrics_team("o").await.unwrap(), 3); assert_eq!(writes(&seen), ["PATCH /api/v1/teams/3"]); let edit = sent(&seen, "PATCH", "/api/v1/teams/3").unwrap(); assert_eq!(edit["permission"], "read"); assert_eq!(edit["units"], json!(TEAM_UNITS)); assert_eq!(edit["includes_all_repositories"], true); assert_eq!(edit["can_create_org_repo"], false); } #[tokio::test] async fn a_missing_team_is_created_read_only_on_actions_and_a_missing_id_fails() { let list = ("GET /api/v1/orgs/o/teams", 200, json!([])); let (client, seen) = forge_of( &[], vec![list.clone()], vec![at("POST /api/v1/orgs/o/teams", 201, json!({"id": 11}))], ) .await; assert_eq!(client.ensure_ci_metrics_team("o").await.unwrap(), 11); let team = sent(&seen, "POST", "/api/v1/orgs/o/teams").unwrap(); assert_eq!(team["name"], TEAM); assert_eq!(team["permission"], "read"); assert_eq!(team["units"], json!(TEAM_UNITS)); assert_eq!(team["includes_all_repositories"], true); assert_eq!(team["can_create_org_repo"], false); let (client, _) = forge_of(&[], vec![list], vec![]).await; client .ensure_ci_metrics_team("o") .await .expect_err("created, but no id came back"); } #[tokio::test] async fn an_unreadable_team_list_is_an_error_and_never_a_create() { let (client, seen) = forge_of( &[], vec![("GET /api/v1/orgs/o/teams", 500, json!({"message": "x"}))], vec![], ) .await; client .ensure_ci_metrics_team("o") .await .expect_err("unknown"); assert!(writes(&seen).is_empty()); } #[tokio::test] async fn a_duplicate_create_is_found_by_a_re_read_and_a_refusal_with_no_team_fails() { let reads = Arc::new(AtomicUsize::new(0)); let counted = Arc::clone(&reads); let (client, _) = routed_forge(move |method, path, _| match (method, path) { ("GET", "/api/v1/orgs/o/teams") if counted.fetch_add(1, Ordering::SeqCst) == 0 => { (200, "[]".to_owned()) } ("GET", "/api/v1/orgs/o/teams") => { (200, json!([team_json(4, "read", &TEAM_UNITS)]).to_string()) } ("POST", _) => (409, r#"{"message":"exists"}"#.to_owned()), _ => (200, "{}".to_owned()), }) .await; assert_eq!(client.ensure_ci_metrics_team("o").await.unwrap(), 4); let (client, _) = forge_of( &[], vec![("GET /api/v1/orgs/o/teams", 200, json!([]))], vec![at( "POST /api/v1/orgs/o/teams", 409, json!({"message": "x"}), )], ) .await; client .ensure_ci_metrics_team("o") .await .expect_err("not found again"); } #[tokio::test] async fn every_page_of_organisations_is_read() { let asked = Arc::new(AtomicUsize::new(0)); let counted = Arc::clone(&asked); let (client, _) = routed_forge(move |_, path, _| { assert_eq!(path, "/api/v1/admin/orgs"); let page = counted.fetch_add(1, Ordering::SeqCst); let n = if page == 0 { ORG_PAGE as usize } else { 1 }; let orgs: Vec = (0..n) .map(|i| json!({"username": format!("org-{page}-{i}")})) .collect(); (200, json!(orgs).to_string()) }) .await; let names = client.all_orgs().await.unwrap(); assert_eq!(names.len(), ORG_PAGE as usize + 1); assert_eq!(asked.load(Ordering::SeqCst), 2); assert_eq!(names.last().unwrap(), "org-1-0"); } fn at(key: &str, status: u16, body: Value) -> (String, u16, Value) { (key.to_owned(), status, body) } fn member(id: i64) -> (String, u16, Value) { at( &format!("GET /api/v1/teams/{id}/members/ci-metrics"), 200, json!({}), ) } #[tokio::test] async fn the_grants_are_complete_only_when_every_org_has_the_team_with_the_account_in_it() { let teams = |ok: bool| { vec![ ( "GET /api/v1/orgs/a/teams", 200, json!([team_json(1, "read", &TEAM_UNITS)]), ), ( "GET /api/v1/orgs/b/teams", 200, if ok { json!([team_json(2, "read", &TEAM_UNITS)]) } else { json!([]) }, ), ] }; let members = vec![member(1), member(2)]; let (client, _) = forge_of(&["a", "b"], teams(true), members.clone()).await; assert_eq!(client.observe_ci_metrics_grants().await, Some(true)); // A team missing in one org. let (client, _) = forge_of(&["a", "b"], teams(false), members.clone()).await; assert_eq!(client.observe_ci_metrics_grants().await, Some(false)); // The account missing from a team (404), against a failed read (500). let (client, _) = forge_of(&["a", "b"], teams(true), vec![member(1)]).await; assert_eq!(client.observe_ci_metrics_grants().await, Some(false)); let (client, _) = forge_of( &["a", "b"], teams(true), vec![ member(1), at( "GET /api/v1/teams/2/members/ci-metrics", 500, json!({"message": "x"}), ), ], ) .await; assert_eq!(client.observe_ci_metrics_grants().await, None); // An unreadable team list, and an unreadable org list. let mut unreadable = teams(true); unreadable[1] = ("GET /api/v1/orgs/b/teams", 500, json!({"message": "x"})); let (client, _) = forge_of(&["a", "b"], unreadable, members).await; assert_eq!(client.observe_ci_metrics_grants().await, None); let (client, _) = routed_forge(|_, _, _| (500, r#"{"message":"x"}"#.to_owned())).await; assert_eq!(client.observe_ci_metrics_grants().await, None); } }