swarm-controller gains forge access when the token arrives, without a restart
A controller that started before the forge's host had published its token ran without forge access until someone restarted it: on a split-host first deploy that is a manual step. Now the forge client lives in a slot that is filled once (`Late<T>`, an `Arc<OnceLock<T>>`): - `forge::enable` tries the store once at start. On failure it logs once and re-reads in the background with backoff (30s doubling to a 5-minute cap, no log per retry), fills the slot and logs once when the token is there. - The forge passes (#5014's per-hive token mint among them), the config-PR poll and the webhook registration start from that moment, through `start_forge`'s callback. - Job nodes and HTTP handlers read the slot when they run, so they answer "no forge" before and work after. A token replaced in the store after the slot is filled still reaches the controller with its next restart. The round-1 change that shared `queue_identity::fetch_with_retry` is reverted; nothing else uses it. Refs #4998
This commit is contained in:
atlas
parent
121c4678aa
commit
c45f7a5330
11 changed files with 243 additions and 135 deletions
|
|
@ -234,6 +234,10 @@ impl hive_jobq_wire::WireResource for SwarmResourceKind {
|
|||
}
|
||||
}
|
||||
|
||||
/// A dependency this daemon may only gain after it starts serving: empty
|
||||
/// until then, filled once. Readers take what is there at the moment they ask.
|
||||
type Late<T> = Arc<std::sync::OnceLock<T>>;
|
||||
|
||||
/// Everything a claimed node's executor arm might need to reach outside
|
||||
/// this process — bundled into one `Clone` struct rather than growing
|
||||
/// `run_swarm_node`'s parameter list per node kind (three forge-shaped
|
||||
|
|
@ -245,7 +249,9 @@ impl hive_jobq_wire::WireResource for SwarmResourceKind {
|
|||
#[derive(Clone)]
|
||||
struct WorkerDeps {
|
||||
auth: Option<std::sync::Arc<auth::AuthBridge>>,
|
||||
forge: Option<std::sync::Arc<forge::Client>>,
|
||||
/// Empty until the forge token has been read (`forge::enable`), so a node
|
||||
/// run before then fails as "no forge access", and one run after succeeds.
|
||||
forge: Late<Arc<forge::Client>>,
|
||||
/// The swarm queue connection, for nodes whose effect is a published
|
||||
/// event rather than an API call. A handle rather than the status
|
||||
/// reader it is cloned from: a node that announces a deploy has no
|
||||
|
|
@ -288,6 +294,7 @@ async fn run_swarm_node(
|
|||
) {
|
||||
use hive_jobq::scheduler::Outcome;
|
||||
|
||||
let forge = deps.forge.get().cloned();
|
||||
let outcome = match kind {
|
||||
SwarmNodeKind::CreateIdentity { agent } => match deps.auth {
|
||||
None => {
|
||||
|
|
@ -311,7 +318,7 @@ async fn run_swarm_node(
|
|||
Err(e) => Outcome::Failed(format!("{e:#}")),
|
||||
},
|
||||
},
|
||||
SwarmNodeKind::CreateRepo { agent } => match deps.forge {
|
||||
SwarmNodeKind::CreateRepo { agent } => match forge {
|
||||
None => Outcome::Failed(
|
||||
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
||||
not in the swarm secret store)"
|
||||
|
|
@ -325,7 +332,7 @@ async fn run_swarm_node(
|
|||
Err(e) => Outcome::Failed(format!("{e:#}")),
|
||||
},
|
||||
},
|
||||
SwarmNodeKind::CreateForgeUser { agent } => match deps.forge {
|
||||
SwarmNodeKind::CreateForgeUser { agent } => match forge {
|
||||
None => Outcome::Failed(
|
||||
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
||||
not in the swarm secret store)"
|
||||
|
|
@ -336,7 +343,7 @@ async fn run_swarm_node(
|
|||
Err(e) => Outcome::Failed(format!("{e:#}")),
|
||||
},
|
||||
},
|
||||
SwarmNodeKind::AddRepoMember { agent } => match deps.forge {
|
||||
SwarmNodeKind::AddRepoMember { agent } => match forge {
|
||||
None => Outcome::Failed(
|
||||
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
||||
not in the swarm secret store)"
|
||||
|
|
@ -347,8 +354,8 @@ async fn run_swarm_node(
|
|||
Err(e) => Outcome::Failed(format!("{e:#}")),
|
||||
},
|
||||
},
|
||||
SwarmNodeKind::GrantDocsRead { agent } => grant_docs_read(deps.forge, &agent).await,
|
||||
SwarmNodeKind::InitAgentConfigRepo { agent } => match deps.forge {
|
||||
SwarmNodeKind::GrantDocsRead { agent } => grant_docs_read(forge, &agent).await,
|
||||
SwarmNodeKind::InitAgentConfigRepo { agent } => match forge {
|
||||
None => Outcome::Failed(
|
||||
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
||||
not in the swarm secret store)"
|
||||
|
|
@ -360,16 +367,14 @@ async fn run_swarm_node(
|
|||
},
|
||||
},
|
||||
SwarmNodeKind::MintAgentIdentity { agent } => mint_identity(&agent).await,
|
||||
SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(deps.forge, &agent).await,
|
||||
SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(forge, &agent).await,
|
||||
SwarmNodeKind::MintAgentMatrixAccount { agent } => {
|
||||
mint_matrix_account(deps.matrix_homeserver.as_deref(), &agent).await
|
||||
}
|
||||
SwarmNodeKind::MintHiveSenderToken { hive } => {
|
||||
mint_hive_sender(deps.matrix_homeserver.as_deref(), &hive).await
|
||||
}
|
||||
SwarmNodeKind::MintHiveForgeToken { hive } => {
|
||||
mint_hive_forge_token(deps.forge, &hive).await
|
||||
}
|
||||
SwarmNodeKind::MintHiveForgeToken { hive } => mint_hive_forge_token(forge, &hive).await,
|
||||
SwarmNodeKind::RenewAgentQueueCredential { agent } => renew_queue_credential(&agent).await,
|
||||
SwarmNodeKind::SetAgentWanted { hive, agent } => match deps.wanted {
|
||||
None => Outcome::Failed(
|
||||
|
|
@ -774,10 +779,10 @@ struct AppState {
|
|||
/// takes without a second hop through `String`.
|
||||
webhook_secret: Option<Arc<str>>,
|
||||
/// Last successful `agent-configs/*` scan, kept current by
|
||||
/// [`config_pr::spawn`]. `None` when this host has no forge configured
|
||||
/// — same "absent means don't ask" shape as `status` and `forge` above,
|
||||
/// not a startup failure.
|
||||
config_prs: Option<Arc<config_pr::ConfigPrCache>>,
|
||||
/// [`config_pr::spawn`]. Empty while this daemon has no forge access —
|
||||
/// same "absent means don't ask" shape as `status` and `forge`, not a
|
||||
/// startup failure — and filled when the forge client is.
|
||||
config_prs: Late<Arc<config_pr::ConfigPrCache>>,
|
||||
/// The swarm's human display name (`services.hyperhive.swarm.name`),
|
||||
/// loaded once at startup (`load_swarm_name`). `None` when the
|
||||
/// operator never set it — a swarm without a display name is a
|
||||
|
|
@ -788,10 +793,9 @@ struct AppState {
|
|||
/// The forge client itself, for the read-only repo/issue-report
|
||||
/// routes (`GET /api/repos`, `GET /api/repos/{org}/{repo}/issue-report`)
|
||||
/// and `POST /api/forge/users/{name}/admin` — distinct from `config_prs`, which holds a *cache* built off this
|
||||
/// same client rather than the client. `None` under the same
|
||||
/// "no forge configured on this host" shape every other
|
||||
/// forge-backed field here uses.
|
||||
forge: Option<Arc<forge::Client>>,
|
||||
/// same client rather than the client. Empty while this daemon has no
|
||||
/// forge access, and filled the moment `forge::enable` builds the client.
|
||||
forge: Late<Arc<forge::Client>>,
|
||||
/// Held by `create_agent` from reading where a name is placed until its
|
||||
/// graph is queued, and by `set_agent_state` from that read until its
|
||||
/// declaration is written, so two placements of one name cannot both
|
||||
|
|
@ -1433,7 +1437,7 @@ async fn get_agents_status(
|
|||
})?;
|
||||
match reader.view(&roster, std::time::SystemTime::now()).await {
|
||||
Ok(mut rows) => {
|
||||
if let Some(cache) = state.config_prs.as_ref() {
|
||||
if let Some(cache) = state.config_prs.get() {
|
||||
let mut config_prs = cache.snapshot();
|
||||
for row in &mut rows {
|
||||
row.config_pr = config_prs.remove(&row.name);
|
||||
|
|
@ -2178,7 +2182,7 @@ async fn get_agent_config_pr(
|
|||
State(state): State<AppState>,
|
||||
Path(name): Path<String>,
|
||||
) -> Result<Json<Option<forge::ConfigPrStatus>>, StatusUnavailable> {
|
||||
let Some(cache) = state.config_prs.as_ref() else {
|
||||
let Some(cache) = state.config_prs.get() else {
|
||||
return Err(StatusUnavailable(
|
||||
"no forge is configured on this host".to_owned(),
|
||||
));
|
||||
|
|
@ -2443,16 +2447,13 @@ fn spawn_matrix_account_backfill(
|
|||
/// registration token, hives' read tokens, agents' `internal/docs` grants, and
|
||||
/// agents' tokens (which also
|
||||
/// sweeps legacy tokens; see `forge::legacy_tokens`) —
|
||||
/// when a forge is configured. Lifted out of `main` for
|
||||
/// once there is a forge client. Lifted out of `main` for
|
||||
/// `clippy::too_many_lines`.
|
||||
fn spawn_forge_workers(
|
||||
jobq: &Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
|
||||
forge_client: Option<Arc<forge::Client>>,
|
||||
client: Arc<forge::Client>,
|
||||
hives: &[HiveEntry],
|
||||
) {
|
||||
let Some(client) = forge_client else {
|
||||
return;
|
||||
};
|
||||
forge::objects::spawn(Arc::clone(&client));
|
||||
forge::runner_token::spawn(Arc::clone(&client));
|
||||
let sched = Arc::clone(jobq);
|
||||
|
|
@ -2648,7 +2649,7 @@ async fn make_forge_admin(
|
|||
let name = hive_types::Ident::parse(&name)
|
||||
.map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))?
|
||||
.into_string();
|
||||
let Some(forge) = state.forge.clone() else {
|
||||
let Some(forge) = state.forge.get().cloned() else {
|
||||
return Err(error_problem(
|
||||
axum::http::StatusCode::SERVICE_UNAVAILABLE,
|
||||
"no forge is configured on this host",
|
||||
|
|
@ -2828,7 +2829,7 @@ async fn get_agent_icon(
|
|||
async fn get_config_prs(
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<std::collections::HashMap<String, forge::ConfigPrStatus>>, StatusUnavailable> {
|
||||
let Some(cache) = state.config_prs.as_ref() else {
|
||||
let Some(cache) = state.config_prs.get() else {
|
||||
return Err(StatusUnavailable(
|
||||
"no forge is configured on this host".to_owned(),
|
||||
));
|
||||
|
|
@ -2862,12 +2863,11 @@ const PUBLIC_URL_ENV: &str = "SWARM_CONTROLLER_PUBLIC_URL";
|
|||
/// daemon keeps running — and the failure is silent at both ends, because the
|
||||
/// forge has nothing to report about a call that never arrived.
|
||||
///
|
||||
/// Silently does nothing when any of the three preconditions is missing —
|
||||
/// each is a legitimate deployment shape (no forge here, no state directory
|
||||
/// to hold a secret, no public vhost), and each is already logged where it
|
||||
/// is discovered.
|
||||
fn register_swarm_webhooks(forge: Option<Arc<forge::Client>>, secret: Option<Arc<str>>) {
|
||||
let Some(forge) = forge else { return };
|
||||
/// Runs once there is a forge client, and silently does nothing when either
|
||||
/// other precondition is missing — each is a legitimate deployment shape (no
|
||||
/// state directory to hold a secret, no public vhost), and each is already
|
||||
/// logged where it is discovered.
|
||||
fn register_swarm_webhooks(forge: Arc<forge::Client>, secret: Option<Arc<str>>) {
|
||||
let Some(secret) = secret else { return };
|
||||
let Ok(public_url) = std::env::var(PUBLIC_URL_ENV) else {
|
||||
tracing::info!(
|
||||
|
|
@ -2953,18 +2953,32 @@ async fn connect_status_reader() -> Result<Option<Arc<status::StatusReader>>> {
|
|||
}
|
||||
}
|
||||
|
||||
/// Clones `forge_client` for [`AppState::forge`] before handing the
|
||||
/// original off to [`register_swarm_webhooks`], which takes it by value —
|
||||
/// split out of `main` purely to keep that function under clippy's
|
||||
/// line-count lint (same rationale as [`connect_status_reader`]'s own doc
|
||||
/// comment), no behavior change.
|
||||
fn keep_forge_for_state(
|
||||
forge_client: Option<Arc<forge::Client>>,
|
||||
/// Fill the forge slots and start everything that runs off the forge client —
|
||||
/// its periodic passes, the config-PR poll and the webhook registration — now
|
||||
/// if the token is in the store, otherwise when `forge::enable`'s background
|
||||
/// re-read finds it. Nothing to start when no forge URL is configured. Lifted
|
||||
/// out of `main` for `clippy::too_many_lines`.
|
||||
async fn start_forge(
|
||||
forge_client: Late<Arc<forge::Client>>,
|
||||
config_prs: Late<Arc<config_pr::ConfigPrCache>>,
|
||||
jobq: Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
|
||||
hives: Vec<HiveEntry>,
|
||||
webhook_secret: Option<Arc<str>>,
|
||||
) -> Option<Arc<forge::Client>> {
|
||||
let state_forge = forge_client.clone();
|
||||
register_swarm_webhooks(forge_client, webhook_secret);
|
||||
state_forge
|
||||
) {
|
||||
let url = match forge::url_from_env() {
|
||||
Ok(Some(url)) => url,
|
||||
Ok(None) => return,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %format!("{e:#}"), "forge misconfigured; repo provisioning is off");
|
||||
return;
|
||||
}
|
||||
};
|
||||
forge::enable(url, forge_client, move |client| {
|
||||
spawn_forge_workers(&jobq, Arc::clone(&client), &hives);
|
||||
config_prs.get_or_init(|| config_pr::spawn(Arc::clone(&client)));
|
||||
register_swarm_webhooks(client, webhook_secret);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
|
|
@ -3019,18 +3033,13 @@ async fn main() -> Result<()> {
|
|||
};
|
||||
// Same shape again: a controller with no forge access still serves
|
||||
// everything else, and the forge-shaped node kinds give an honest
|
||||
// per-job failure rather than this fn refusing to start. A token missing
|
||||
// from the store lands here too, after `from_env`'s retries.
|
||||
let forge_client = match forge::Client::from_env().await {
|
||||
Ok(client) => client.map(Arc::new),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %format!("{e:#}"), "no forge access; repo provisioning is off");
|
||||
None
|
||||
}
|
||||
};
|
||||
// per-job failure rather than this fn refusing to start. Both slots stay
|
||||
// empty until `start_forge` below fills them.
|
||||
let forge_client: Late<Arc<forge::Client>> = Arc::default();
|
||||
let config_prs: Late<Arc<config_pr::ConfigPrCache>> = Arc::default();
|
||||
let deps = WorkerDeps {
|
||||
auth: auth.clone(),
|
||||
forge: forge_client.clone(),
|
||||
forge: Arc::clone(&forge_client),
|
||||
queue: status.as_ref().map(|s| s.queue_client()),
|
||||
wanted: wanted_writer(status.as_ref()),
|
||||
matrix_homeserver: configured_matrix_homeserver(),
|
||||
|
|
@ -3094,9 +3103,14 @@ async fn main() -> Result<()> {
|
|||
);
|
||||
}
|
||||
|
||||
spawn_forge_workers(&jobq, forge_client.clone(), &hives);
|
||||
let config_prs = forge_client.clone().map(config_pr::spawn);
|
||||
let state_forge = keep_forge_for_state(forge_client, webhook_secret.clone());
|
||||
start_forge(
|
||||
Arc::clone(&forge_client),
|
||||
Arc::clone(&config_prs),
|
||||
Arc::clone(&jobq),
|
||||
hives.clone(),
|
||||
webhook_secret.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
spawn_agent_renewal(&jobq, wanted_writer(status.as_ref()), &hives);
|
||||
spawn_per_agent_upkeep(status.as_ref(), &hives);
|
||||
|
|
@ -3123,7 +3137,7 @@ async fn main() -> Result<()> {
|
|||
config_prs,
|
||||
swarm_name: load_swarm_name().map(Arc::from),
|
||||
auth,
|
||||
forge: state_forge,
|
||||
forge: forge_client,
|
||||
create_gate: Arc::default(),
|
||||
accounts: linked_accounts::Accounts::Live,
|
||||
};
|
||||
|
|
@ -3295,13 +3309,13 @@ mod tests {
|
|||
agent_icons: None,
|
||||
jobq: std::sync::Arc::clone(&sched),
|
||||
webhook_secret: None,
|
||||
config_prs: None,
|
||||
config_prs: std::sync::Arc::default(),
|
||||
swarm_name: None,
|
||||
// No bridge: these tests drive agent *creation*, which queues a
|
||||
// job and consults one only for a rule-breaking name. The roster
|
||||
// read is the verb that needs it, and it has its own test below.
|
||||
auth: None,
|
||||
forge: None,
|
||||
forge: std::sync::Arc::default(),
|
||||
create_gate: std::sync::Arc::default(),
|
||||
accounts: super::linked_accounts::Accounts::Fake(std::sync::Arc::default()),
|
||||
};
|
||||
|
|
@ -4197,7 +4211,7 @@ mod tests {
|
|||
|
||||
let deps = WorkerDeps {
|
||||
auth: None,
|
||||
forge: None,
|
||||
forge: std::sync::Arc::default(),
|
||||
queue: None,
|
||||
wanted: None,
|
||||
matrix_homeserver: None,
|
||||
|
|
@ -4250,7 +4264,7 @@ mod tests {
|
|||
|
||||
let deps = WorkerDeps {
|
||||
auth: None,
|
||||
forge: None,
|
||||
forge: std::sync::Arc::default(),
|
||||
queue: None,
|
||||
wanted: None,
|
||||
matrix_homeserver: None,
|
||||
|
|
@ -4313,7 +4327,7 @@ mod tests {
|
|||
|
||||
let deps = WorkerDeps {
|
||||
auth: None,
|
||||
forge: None,
|
||||
forge: std::sync::Arc::default(),
|
||||
queue: None,
|
||||
wanted: None,
|
||||
matrix_homeserver: None,
|
||||
|
|
@ -4364,7 +4378,7 @@ mod tests {
|
|||
|
||||
let deps = WorkerDeps {
|
||||
auth: None,
|
||||
forge: None,
|
||||
forge: std::sync::Arc::default(),
|
||||
queue: None,
|
||||
wanted: None,
|
||||
matrix_homeserver: None,
|
||||
|
|
|
|||
Loading…
Reference in a new issue