vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-controller: re-read the forge token from the store when the forge rejects it

Browse source 
The controller read its forge token from the swarm secret store once, when
its client slot filled, and kept sending it until the daemon restarted. A
token the forge re-minted or revoked left every forge-backed path failing
with 401 until someone restarted the controller, and a restart drops the
runtime-only job graph.

Every forge request now goes through `forge::api::Api::call`. On a 401 it
asks a `StoredSecret` (the queue clients' debounced, single-flight re-read
with backoff) for a token to try instead, rebuilds the forgejo client with
it, and sends the request once more. A second 401, or a store still holding
the refused token, surfaces the error as before. Only 401 counts: unlike an
IdP, the forge answers 400 to a malformed request, and a re-read started by
one would put a real 401 that follows inside the backoff.

forgejo-api bakes the token into its own reqwest client's default headers
and takes no client or middleware from outside, so there is no single
request function to hook. Each `self.api.<endpoint>(..)` site becomes
`self.api.call(|api| api.<endpoint>(..).into_future())`; option structs a
site moved into the call are cloned, since the closure can run twice. The
`OnceLock<Arc<Client>>` slot and its `get()` callers are unchanged.

Refs #5017
This commit is contained in:
atlas 2026-10-13 21:38:35 +02:00 • committed by mara
commit 6ba29c4d33
11 changed files with 638 additions and 192 deletions
Download patch file Download diff file Expand all files Collapse all files

140
swarm-controller/src/forge/objects.rs
View file

@ -642,7 +642,11 @@ impl Client {
async fn observe(&self, desired: &Desired) -> Observed {
let mut observed = Observed::default();
for org in &desired.orgs {
let org_seen = seen(self.api.org_get(org).await, &format!("org {org}"), drop);
let org_seen = seen(
self.api.call(|api| api.org_get(org).into_future()).await,
&format!("org {org}"),
drop,
);
observed.orgs.insert(org.clone(), org_seen);
}
for org in desired.team_orgs() {
@ -650,7 +654,9 @@ impl Client {
continue;
}
let teams = seen(
self.api.org_list_teams(org).await,
self.api
.call(|api| api.org_list_teams(org).into_future())
.await,
&format!("teams of {org}"),
|(_headers, teams)| teams,
);
@ -685,7 +691,9 @@ impl Client {
continue;
}
let repo = seen(
self.api.repo_get(&owner, &name).await,
self.api
.call(|api| api.repo_get(&owner, &name).into_future())
.await,
&format!("repo {owner}/{name}"),
|r| RepoState {
private: r.private.unwrap_or(false),
@ -704,7 +712,11 @@ impl Client {
/// Read every config repo's `main` rule into `observed.config_rules`.
async fn observe_config_rules(&self, observed: &mut Observed) {
let repos = match self.api.org_list_repos(CONFIG_ORG).all().await {
let repos = match self
.api
.call(|api| Box::pin(api.org_list_repos(CONFIG_ORG).all()))
.await
{
Ok(repos) => repos,
Err(e) => {
tracing::warn!(error = %e, "swarm forge objects: listing {CONFIG_ORG} repos failed; skipping their branch rules this pass");
@ -715,7 +727,10 @@ impl Client {
for name in repos.into_iter().filter_map(|r| r.name) {
let rule = seen(
self.api
.repo_get_branch_protection(CONFIG_ORG, &name, "main")
.call(|api| {
api.repo_get_branch_protection(CONFIG_ORG, &name, "main")
.into_future()
})
.await,
&format!("branch protection of {CONFIG_ORG}/{name}"),
|rule| config_rule_matches(&rule),
@ -759,7 +774,15 @@ impl Client {
Action::ReconcileTeam { org, id } => self.reconcile_operators_team(org, *id).await,
Action::ConvergeConfigRule { repo } => {
self.api
.repo_edit_branch_protection(CONFIG_ORG, repo, "main", config_rule_edit())
.call(|api| {
api.repo_edit_branch_protection(
CONFIG_ORG,
repo,
"main",
config_rule_edit(),
)
.into_future()
})
.await
.with_context(|| format!("converge {CONFIG_ORG}/{repo} main branch rule"))?;
tracing::info!(%repo, "swarm forge objects: config repo merge gate converged");
@ -774,7 +797,7 @@ impl Client {
let mut edit = sparse_edit_repo_option();
edit.private = Some(false);
self.api
.repo_edit(owner, name, edit)
.call(|api| api.repo_edit(owner, name, edit.clone()).into_future())
.await
.with_context(|| format!("edit {owner}/{name} (set public)"))?;
tracing::info!(%owner, %name, "swarm forge objects: repo set to public");
@ -790,7 +813,7 @@ impl Client {
let mut edit = sparse_edit_repo_option();
edit.mirror_interval = Some(MIRROR_INTERVAL.to_owned());
self.api
.repo_edit(owner, repo, edit)
.call(|api| api.repo_edit(owner, repo, edit.clone()).into_future())
.await
.with_context(|| format!("set mirror_interval on {owner}/{repo}"))?;
tracing::info!(%owner, %repo, interval = MIRROR_INTERVAL, "swarm forge objects: pull-mirror interval updated");
@ -816,14 +839,22 @@ impl Client {
visibility: None,
website: None,
};
match self.api.org_create(option).await {
match self
.api
.call(|api| api.org_create(option.clone()).into_future())
.await
{
Ok(_) => {
tracing::info!(%org, "swarm forge objects: created org");
Ok(())
}
Err(e) => {
let existing =
is_ambiguous_validation_failure(&e) && self.api.org_get(org).await.is_ok();
let existing = is_ambiguous_validation_failure(&e)
&& self
.api
.call(|api| api.org_get(org).into_future())
.await
.is_ok();
if folds_into_success(&e, existing) {
Ok(())
} else {
@ -853,7 +884,11 @@ impl Client {
units: Some(OPERATORS_TEAM_UNITS.map(str::to_owned).to_vec()),
units_map: None,
};
match self.api.org_create_team(org, team).await {
match self
.api
.call(|api| api.org_create_team(org, team.clone()).into_future())
.await
{
Ok(_) => {
tracing::info!(%org, "swarm forge objects: created {OPERATORS_TEAM} team");
Ok(())
@ -863,11 +898,15 @@ impl Client {
// listing; the next pass reconciles its settings if they differ.
Err(e) => {
let existing = is_ambiguous_validation_failure(&e)
&& self.api.org_list_teams(org).await.is_ok_and(|(_, teams)| {
teams
.iter()
.any(|t| t.name.as_deref() == Some(OPERATORS_TEAM))
});
&& self
.api
.call(|api| api.org_list_teams(org).into_future())
.await
.is_ok_and(|(_, teams)| {
teams
.iter()
.any(|t| t.name.as_deref() == Some(OPERATORS_TEAM))
});
if folds_into_success(&e, existing) {
Ok(())
} else {
@ -891,7 +930,7 @@ impl Client {
units_map: None,
};
self.api
.org_edit_team(id, edit)
.call(|api| api.org_edit_team(id, edit.clone()).into_future())
.await
.with_context(|| format!("reconcile {org}/{OPERATORS_TEAM} team settings"))?;
tracing::info!(%org, "swarm forge objects: reconciled {OPERATORS_TEAM} team settings");
@ -902,7 +941,10 @@ impl Client {
async fn create_shared_repo(&self, owner: &str, name: &str, private: bool) -> Result<()> {
match self
.api
.create_org_repo(owner, Self::repo_option(name, private))
.call(|api| {
api.create_org_repo(owner, Self::repo_option(name, private))
.into_future()
})
.await
{
Ok(_) => {
@ -911,7 +953,11 @@ impl Client {
}
Err(e) => {
let existing = is_ambiguous_validation_failure(&e)
&& self.api.repo_get(owner, name).await.is_ok();
&& self
.api
.call(|api| api.repo_get(owner, name).into_future())
.await
.is_ok();
if folds_into_success(&e, existing) {
Ok(())
} else {
@ -933,21 +979,24 @@ impl Client {
sha: None,
}];
self.api
.repo_change_files(
owner,
name,
ChangeFilesOptions {
author: None,
branch: None,
committer: None,
dates: None,
files,
force_overwrite_new_branch: None,
message: Some("init: seed README".to_owned()),
new_branch: None,
signoff: None,
},
)
.call(|api| {
api.repo_change_files(
owner,
name,
ChangeFilesOptions {
author: None,
branch: None,
committer: None,
dates: None,
files: files.clone(),
force_overwrite_new_branch: None,
message: Some("init: seed README".to_owned()),
new_branch: None,
signoff: None,
},
)
.into_future()
})
.await
.with_context(|| format!("seed README.md in {owner}/{name}"))?;
tracing::info!(%owner, %name, "swarm forge objects: seeded README.md");
@ -984,7 +1033,11 @@ impl Client {
uid: None,
wiki: None,
};
match self.api.repo_migrate(opts).await {
match self
.api
.call(|api| api.repo_migrate(opts.clone()).into_future())
.await
{
Ok(_) => {
tracing::info!(%owner, %repo, %upstream, interval = MIRROR_INTERVAL, "swarm forge objects: created pull-mirror");
Ok(())
@ -1006,12 +1059,15 @@ impl Client {
.await
.with_context(|| format!("read {CONFIG_ORG} avatar PNG from {}", png.display()))?;
self.api
.org_update_avatar(
CONFIG_ORG,
UpdateUserAvatarOption {
image: Some(base64::engine::general_purpose::STANDARD.encode(&bytes)),
},
)
.call(|api| {
api.org_update_avatar(
CONFIG_ORG,
UpdateUserAvatarOption {
image: Some(base64::engine::general_purpose::STANDARD.encode(&bytes)),
},
)
.into_future()
})
.await
.with_context(|| format!("set {CONFIG_ORG} avatar"))?;
if let Err(e) = std::fs::write(marker, "") {