Watch
0
0
Fork
You've already forked hyperhive
0

swarm-logs-auth, swarm-metrics-auth, swarm-authelia-bridge, swarm-controller: drain http on sigterm

This commit is contained in:
atlas 2026-11-18 03:28:12 +01:00 • committed by mara
commit a089da25a5
7 changed files with 198 additions and 19 deletions

View file

@ -19,6 +19,8 @@ serde.workspace = true
serde_json.workspace = true serde_json.workspace = true
serde_norway = "0.9.42" serde_norway = "0.9.42"
swarm-authelia-bridge-sock.workspace = true swarm-authelia-bridge-sock.workspace = true
# `shutdown_signal`: the bridge drains its in-flight requests on SIGTERM.
swarm-secret-client.workspace = true
tokio.workspace = true tokio.workspace = true
tracing.workspace = true tracing.workspace = true
tracing-subscriber.workspace = true tracing-subscriber.workspace = true

View file

@ -106,6 +106,7 @@ struct AppState {
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
let stop = swarm_secret_client::shutdown_signal();
tracing_subscriber::fmt() tracing_subscriber::fmt()
.with_env_filter( .with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()), tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
@ -130,9 +131,22 @@ async fn main() -> Result<()> {
let listener = bind_socket(&socket)?; let listener = bind_socket(&socket)?;
tracing::info!(socket = %socket.display(), "swarm-authelia-bridge listening"); tracing::info!(socket = %socket.display(), "swarm-authelia-bridge listening");
axum::serve(listener, app) serve(listener, app, stop).await
}
/// Answers on `listener` until `stop` resolves, then lets the requests in hand
/// finish, for at most the drain, and returns.
async fn serve(
listener: tokio::net::UnixListener,
app: Router,
stop: impl Future<Output = ()> + Clone + Send + 'static,
) -> Result<()> {
let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone());
swarm_secret_client::serve_bounded(stop, serving.into_future())
.await .await
.context("serving swarm-authelia-bridge") .transpose()
.context("serving swarm-authelia-bridge")?;
Ok(())
} }
/// Bind the listening socket at `path`, replacing a stale one a crash left /// Bind the listening socket at `path`, replacing a stale one a crash left

View file

@ -269,7 +269,7 @@ async fn main() -> Result<()> {
reviews: Arc::default(), reviews: Arc::default(),
alerts: alerts::from_env(), alerts: alerts::from_env(),
}; };
serve(listener, state).await serve(listener, state, swarm_secret_client::shutdown_signal()).await
} }
/// Registers the `bao.client_cert.*` gauges for the controller's own store /// Registers the `bao.client_cert.*` gauges for the controller's own store
@ -286,11 +286,20 @@ fn register_cert_gauges(provider: Option<&opentelemetry_sdk::metrics::SdkMeterPr
} }
/// Start taking the queued review requests and answering the notification /// Start taking the queued review requests and answering the notification
/// router's, then serve `state` on `listener`. /// router's, then serve `state` on `listener` until `stop` resolves. The
async fn serve(listener: tokio::net::UnixListener, state: AppState) -> Result<()> { /// requests in hand finish for at most the drain; the review queue takes no
review_classifier::review::spawn_queue(&state); /// request after `stop`.
async fn serve(
listener: tokio::net::UnixListener,
state: AppState,
stop: impl Future<Output = ()> + Clone + Send + 'static,
) -> Result<()> {
review_classifier::review::spawn_queue(&state, stop.clone());
request_reviewer::spawn(&state); request_reviewer::spawn(&state);
axum::serve(listener, build_app(state)) let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone());
swarm_secret_client::serve_bounded(stop, serving.into_future())
.await .await
.context("serving swarm-controller") .transpose()
.context("serving swarm-controller")?;
Ok(())
} }

View file

@ -443,7 +443,10 @@ async fn open_consumer(
/// Take the queued review requests and review each, for as long as the /// Take the queued review requests and review each, for as long as the
/// controller runs, once this host has forge access. /// controller runs, once this host has forge access.
pub(crate) fn spawn_queue(state: &AppState) { pub(crate) fn spawn_queue(
state: &AppState,
stop: impl Future<Output = ()> + Clone + Send + 'static,
) {
let Some(client) = state.status.as_ref().map(|s| s.queue_client()) else { let Some(client) = state.status.as_ref().map(|s| s.queue_client()) else {
tracing::warn!("no swarm queue is configured on this host; requested reviews don't run"); tracing::warn!("no swarm queue is configured on this host; requested reviews don't run");
return; return;
@ -466,7 +469,7 @@ pub(crate) fn spawn_queue(state: &AppState) {
runs, runs,
metrics: Metrics::global(), metrics: Metrics::global(),
}; };
consume(&client, TIMING, &reviews, handles).await; consume(&client, TIMING, &reviews, handles, stop).await;
}); });
} }
@ -494,8 +497,9 @@ impl<S, Q: Clone, P> Clone for Handles<S, Q, P> {
} }
/// Take requests from [`REQUEST_STREAM`] one at a time, each once a run /// Take requests from [`REQUEST_STREAM`] one at a time, each once a run
/// place is free, and review each in a task of its own. Never returns; /// place is free, and review each in a task of its own. Returns once `stop`
/// dropping it stops every review it started, and the queue delivers their /// resolves, taking no request after it; dropping it, as the return does, stops
/// every review it started without acking, and the queue delivers their
/// requests again. /// requests again.
/// ///
/// The limit is read from the settings before each wait for a place. A /// The limit is read from the settings before each wait for a place. A
@ -506,6 +510,7 @@ async fn consume<S, Q, P>(
timing: Timing, timing: Timing,
reviews: &Arc<Reviews>, reviews: &Arc<Reviews>,
handles: Handles<S, Q, P>, handles: Handles<S, Q, P>,
stop: impl Future<Output = ()> + Clone,
) where ) where
S: FileSource + PullSource + Send + Sync + 'static, S: FileSource + PullSource + Send + Sync + 'static,
Q: RunSink + SettingsSource, Q: RunSink + SettingsSource,
@ -519,7 +524,11 @@ async fn consume<S, Q, P>(
"opening the review request queue failed; trying again" "opening the review request queue failed; trying again"
), ),
} }
tokio::time::sleep(timing.pull_wait).await; tokio::select! {
biased;
() = stop.clone() => return,
() = tokio::time::sleep(timing.pull_wait) => {}
}
}; };
let mut tasks = JoinSet::new(); let mut tasks = JoinSet::new();
loop { loop {
@ -536,8 +545,17 @@ async fn consume<S, Q, P>(
the last limit read" the last limit read"
), ),
} }
let slot = reviews.slot().await; let slot = tokio::select! {
let message = match next(&consumer, timing.pull_wait).await { biased;
() = stop.clone() => return,
slot = reviews.slot() => slot,
};
let taken = tokio::select! {
biased;
() = stop.clone() => return,
taken = next(&consumer, timing.pull_wait) => taken,
};
let message = match taken {
Ok(Some(message)) => message, Ok(Some(message)) => message,
Ok(None) => continue, Ok(None) => continue,
Err(e) => { Err(e) => {
@ -546,7 +564,11 @@ async fn consume<S, Q, P>(
"taking a review request from the queue failed; trying again" "taking a review request from the queue failed; trying again"
); );
drop(slot); drop(slot);
tokio::time::sleep(timing.pull_wait).await; tokio::select! {
biased;
() = stop.clone() => return,
() = tokio::time::sleep(timing.pull_wait) => {}
}
continue; continue;
} }
}; };
@ -2032,6 +2054,15 @@ mod tests {
/// A controller taking requests, its run places in `reviews`. /// A controller taking requests, its run places in `reviews`.
fn consume(&self, reviews: &Arc<Reviews>) -> JoinHandle<()> { fn consume(&self, reviews: &Arc<Reviews>) -> JoinHandle<()> {
self.consume_until(reviews, std::future::pending())
}
/// A controller that takes no request once `stop` resolves.
fn consume_until(
&self,
reviews: &Arc<Reviews>,
stop: impl Future<Output = ()> + Clone + Send + 'static,
) -> JoinHandle<()> {
let client = self.client.clone(); let client = self.client.clone();
let reviews = Arc::clone(reviews); let reviews = Arc::clone(reviews);
let forge = Arc::clone(&self.rig.forge); let forge = Arc::clone(&self.rig.forge);
@ -2047,7 +2078,7 @@ mod tests {
runs: store, runs: store,
metrics, metrics,
}; };
consume(&client, TEST_TIMING, &reviews, handles).await; consume(&client, TEST_TIMING, &reviews, handles, stop).await;
}) })
} }
@ -2113,6 +2144,36 @@ mod tests {
assert_eq!(q.rig.gate.reached(), 2, "no run after the ack"); assert_eq!(q.rig.gate.reached(), 2, "no run after the ack");
} }
/// After the stop the controller returns and takes no request, and the
/// review it was running stays un-acked, so the queue still holds it.
#[tokio::test]
async fn after_the_stop_no_request_is_taken_and_the_running_one_stays_unacked() {
let q = Queued::new().await;
q.request(7, "abc123").await;
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
let stop = futures_util::FutureExt::shared(async move {
let _ = stop_rx.await;
});
let consumer = q.consume_until(&q.rig.reviews, stop);
until("the first run started", || q.rig.gate.reached() == 1).await;
stop_tx.send(()).expect("stop");
tokio::time::timeout(Duration::from_secs(5), consumer)
.await
.expect("consume returns after the stop")
.expect("consume did not panic");
// A request that arrives now, and the one running, which the queue
// would redeliver after its ack wait: neither runs.
q.request(8, "abc123").await;
tokio::time::sleep(TEST_TIMING.ack_wait * 3).await;
assert_eq!(q.rig.gate.reached(), 1, "no request taken after the stop");
assert!(
q.rig.posted().is_empty(),
"the unfinished review is not posted"
);
assert_eq!(q.waiting().await, 2, "neither request is acked");
}
#[tokio::test] #[tokio::test]
async fn a_newer_request_for_a_pr_replaces_the_waiting_one() { async fn a_newer_request_for_a_pr_replaces_the_waiting_one() {
let q = Queued::new().await; let q = Queued::new().await;

View file

@ -21,3 +21,7 @@ tracing-subscriber.workspace = true
[lints] [lints]
workspace = true workspace = true
[dev-dependencies]
# `Shared`, so the injected stop future can be cloned like `shutdown_signal`'s.
futures-util.workspace = true

View file

@ -195,6 +195,7 @@ async fn handle(
#[tokio::main] #[tokio::main]
async fn main() -> anyhow::Result<()> { async fn main() -> anyhow::Result<()> {
let stop = swarm_secret_client::shutdown_signal();
swarm_secret_client::install_crypto_provider(); swarm_secret_client::install_crypto_provider();
tracing_subscriber::fmt() tracing_subscriber::fmt()
.with_env_filter( .with_env_filter(
@ -235,14 +236,31 @@ async fn main() -> anyhow::Result<()> {
tracing::info!(socket = %args.socket.display(), "swarm-logs-auth listening"); tracing::info!(socket = %args.socket.display(), "swarm-logs-auth listening");
let app = axum::Router::new().fallback(handle).with_state(responder); let app = axum::Router::new().fallback(handle).with_state(responder);
axum::serve(listener, app).await.context("serving") serve(listener, app, stop).await
}
/// Answers on `listener` until `stop` resolves, then lets the requests in hand
/// finish, for at most the drain, and returns.
async fn serve(
listener: tokio::net::UnixListener,
app: axum::Router,
stop: impl Future<Output = ()> + Clone + Send + 'static,
) -> anyhow::Result<()> {
let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone());
swarm_secret_client::serve_bounded(stop, serving.into_future())
.await
.transpose()
.context("serving")?;
Ok(())
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use futures_util::FutureExt as _;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use swarm_secret_client::queue::AgentCredential; use swarm_secret_client::queue::AgentCredential;
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
/// What the mirrored grant leaf answers. /// What the mirrored grant leaf answers.
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
@ -474,4 +492,59 @@ mod tests {
} }
assert_eq!(store.lookups(), 3); assert_eq!(store.lookups(), 3);
} }
/// A request that is being answered when the stop arrives is still
/// answered, and `serve` returns only afterwards.
#[tokio::test]
async fn a_request_in_flight_at_the_stop_is_answered_before_serve_returns() {
let started = Arc::new(tokio::sync::Notify::new());
let release = Arc::new(tokio::sync::Notify::new());
let app = {
let (started, release) = (Arc::clone(&started), Arc::clone(&release));
axum::Router::new().fallback(move || {
let (started, release) = (Arc::clone(&started), Arc::clone(&release));
async move {
started.notify_one();
release.notified().await;
"answered"
}
})
};
let socket =
std::env::temp_dir().join(format!("swarm-logs-auth-stop-{}.sock", std::process::id()));
let _ = std::fs::remove_file(&socket);
let listener = tokio::net::UnixListener::bind(&socket).expect("bind");
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
let stop = async move {
let _ = stop_rx.await;
}
.shared();
let mut serving = tokio::spawn(serve(listener, app, stop));
let mut conn = tokio::net::UnixStream::connect(&socket)
.await
.expect("connect");
conn.write_all(b"GET / HTTP/1.1\r\nhost: x\r\nconnection: close\r\n\r\n")
.await
.expect("send");
started.notified().await;
stop_tx.send(()).expect("stop");
tokio::time::sleep(Duration::from_millis(200)).await;
assert!(
!serving.is_finished(),
"serve returned with a request in flight"
);
release.notify_one();
let mut reply = String::new();
conn.read_to_string(&mut reply).await.expect("reply");
assert!(reply.starts_with("HTTP/1.1 200"), "{reply}");
assert!(reply.ends_with("answered"), "{reply}");
let finished = tokio::time::timeout(Duration::from_secs(5), &mut serving)
.await
.expect("serve returns once the request is answered");
finished.expect("task").expect("serve is Ok");
let _ = std::fs::remove_file(&socket);
}
} }

View file

@ -282,6 +282,7 @@ async fn handle(State(responder): State<Arc<Responder<Store>>>, headers: HeaderM
#[tokio::main] #[tokio::main]
async fn main() -> anyhow::Result<()> { async fn main() -> anyhow::Result<()> {
let stop = swarm_secret_client::shutdown_signal();
swarm_secret_client::install_crypto_provider(); swarm_secret_client::install_crypto_provider();
tracing_subscriber::fmt() tracing_subscriber::fmt()
.with_env_filter( .with_env_filter(
@ -322,7 +323,22 @@ async fn main() -> anyhow::Result<()> {
tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening"); tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening");
let app = axum::Router::new().fallback(handle).with_state(responder); let app = axum::Router::new().fallback(handle).with_state(responder);
axum::serve(listener, app).await.context("serving") serve(listener, app, stop).await
}
/// Answers on `listener` until `stop` resolves, then lets the requests in hand
/// finish, for at most the drain, and returns.
async fn serve(
listener: tokio::net::UnixListener,
app: axum::Router,
stop: impl Future<Output = ()> + Clone + Send + 'static,
) -> anyhow::Result<()> {
let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone());
swarm_secret_client::serve_bounded(stop, serving.into_future())
.await
.transpose()
.context("serving")?;
Ok(())
} }
#[cfg(test)] #[cfg(test)]