vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-controller: bound each periodic forge pass with a deadline

Browse source 
forgejo-api 0.11.1 builds its reqwest client with no timeout and offers
no way to hand it a caller-built one (the `client` field is private,
`new`/`with_user_agent` are the only constructors, `Endpoint` is
sealed). A forge that accepts a connection and then stalls therefore
held each of the six periodic reconcile loops on one request forever,
with no log line.

Wrap each loop's per-tick work in `forge::within_pass_deadline`
(`PASS_DEADLINE`, 2 min, under the 5-minute interval): on elapsed the
pass is dropped, a warn is logged, and the loop waits for its next tick.
Every pass is an idempotent reconcile, so abandoning one mid-way leaves
nothing the next pass does not redo.

Loops: config-pr poll, runner registration token, hive forge tokens,
agent forge tokens (observe + legacy sweep), docs grants, swarm forge
objects.

Refs #5060
This commit is contained in:
atlas 2026-10-09 14:49:50 +02:00 • committed by mara
commit 45f0582163
7 changed files with 85 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

15
swarm-controller/src/forge/agent_token.rs
View file

@ -370,11 +370,18 @@ pub fn spawn(client: Arc<Client>, enqueue: impl Fn(Vec<String>) + Send + 'static
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
loop {
ticker.tick().await;
match client.observe_all().await {
let Some(outcome) = super::within_pass_deadline("agent forge tokens", async {
let observed = client.observe_all().await?;
let legacy = super::legacy_tokens::sweepable(&observed);
client.sweep_legacy_tokens(&legacy).await;
anyhow::Ok(observed)
})
.await
else {
continue;
};
match outcome {
Ok(observed) => {
let legacy = super::legacy_tokens::sweepable(&observed);
client.sweep_legacy_tokens(&legacy).await;
let agents = plan(&observed);
if agents.is_empty() {
tracing::debug!(

8
swarm-controller/src/forge/docs_grant.rs
View file

@ -185,7 +185,13 @@ pub fn spawn(client: Arc<Client>, enqueue: impl Fn(Vec<String>) + Send + 'static
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
loop {
ticker.tick().await;
let agents = run_pass(&client, read_roster().await).await;
let Some(agents) = super::within_pass_deadline("docs grants", async {
run_pass(&client, read_roster().await).await
})
.await
else {
continue;
};
if !agents.is_empty() {
enqueue(agents);
}

8
swarm-controller/src/forge/hive_token.rs
View file

@ -386,7 +386,13 @@ pub fn spawn(
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
loop {
ticker.tick().await;
match client.observe_hives(&hives).await {
let Some(outcome) =
super::within_pass_deadline("hive forge tokens", client.observe_hives(&hives))
.await
else {
continue;
};
match outcome {
Ok((team, observed)) => {
let queued = plan(team, &observed);
if queued.is_empty() {

7
swarm-controller/src/forge/objects.rs
View file

@ -1082,7 +1082,12 @@ pub fn spawn(client: Arc<Client>) {
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
loop {
ticker.tick().await;
let outcome = client.reconcile(&desired).await;
let Some(outcome) =
super::within_pass_deadline("swarm forge objects", client.reconcile(&desired))
.await
else {
continue;
};
if outcome.failed > 0 {
tracing::warn!(
applied = outcome.applied,

10
swarm-controller/src/forge/runner_token.rs
View file

@ -77,7 +77,15 @@ pub fn spawn(client: Arc<Client>) {
let mut last_written: Option<String> = None;
loop {
ticker.tick().await;
match pass(&client, last_written.as_deref()).await {
let Some(outcome) = super::within_pass_deadline(
"runner registration token",
pass(&client, last_written.as_deref()),
)
.await
else {
continue;
};
match outcome {
Ok(Some(written)) => {
tracing::info!("runner registration token: written to the store");
last_written = Some(written);