swarm-logs-auth, swarm-metrics-auth, swarm-authelia-bridge, swarm-controller: drain http on sigterm
This commit is contained in:
parent
39c29b64e3
commit
a089da25a5
7 changed files with 198 additions and 19 deletions
|
|
@ -19,6 +19,8 @@ serde.workspace = true
|
|||
serde_json.workspace = true
|
||||
serde_norway = "0.9.42"
|
||||
swarm-authelia-bridge-sock.workspace = true
|
||||
# `shutdown_signal`: the bridge drains its in-flight requests on SIGTERM.
|
||||
swarm-secret-client.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ struct AppState {
|
|||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let stop = swarm_secret_client::shutdown_signal();
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
|
||||
|
|
@ -130,9 +131,22 @@ async fn main() -> Result<()> {
|
|||
|
||||
let listener = bind_socket(&socket)?;
|
||||
tracing::info!(socket = %socket.display(), "swarm-authelia-bridge listening");
|
||||
axum::serve(listener, app)
|
||||
serve(listener, app, stop).await
|
||||
}
|
||||
|
||||
/// Answers on `listener` until `stop` resolves, then lets the requests in hand
|
||||
/// finish, for at most the drain, and returns.
|
||||
async fn serve(
|
||||
listener: tokio::net::UnixListener,
|
||||
app: Router,
|
||||
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||
) -> Result<()> {
|
||||
let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone());
|
||||
swarm_secret_client::serve_bounded(stop, serving.into_future())
|
||||
.await
|
||||
.context("serving swarm-authelia-bridge")
|
||||
.transpose()
|
||||
.context("serving swarm-authelia-bridge")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Bind the listening socket at `path`, replacing a stale one a crash left
|
||||
|
|
|
|||
|
|
@ -269,7 +269,7 @@ async fn main() -> Result<()> {
|
|||
reviews: Arc::default(),
|
||||
alerts: alerts::from_env(),
|
||||
};
|
||||
serve(listener, state).await
|
||||
serve(listener, state, swarm_secret_client::shutdown_signal()).await
|
||||
}
|
||||
|
||||
/// Registers the `bao.client_cert.*` gauges for the controller's own store
|
||||
|
|
@ -286,11 +286,20 @@ fn register_cert_gauges(provider: Option<&opentelemetry_sdk::metrics::SdkMeterPr
|
|||
}
|
||||
|
||||
/// Start taking the queued review requests and answering the notification
|
||||
/// router's, then serve `state` on `listener`.
|
||||
async fn serve(listener: tokio::net::UnixListener, state: AppState) -> Result<()> {
|
||||
review_classifier::review::spawn_queue(&state);
|
||||
/// router's, then serve `state` on `listener` until `stop` resolves. The
|
||||
/// requests in hand finish for at most the drain; the review queue takes no
|
||||
/// request after `stop`.
|
||||
async fn serve(
|
||||
listener: tokio::net::UnixListener,
|
||||
state: AppState,
|
||||
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||
) -> Result<()> {
|
||||
review_classifier::review::spawn_queue(&state, stop.clone());
|
||||
request_reviewer::spawn(&state);
|
||||
axum::serve(listener, build_app(state))
|
||||
let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone());
|
||||
swarm_secret_client::serve_bounded(stop, serving.into_future())
|
||||
.await
|
||||
.context("serving swarm-controller")
|
||||
.transpose()
|
||||
.context("serving swarm-controller")?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -443,7 +443,10 @@ async fn open_consumer(
|
|||
|
||||
/// Take the queued review requests and review each, for as long as the
|
||||
/// controller runs, once this host has forge access.
|
||||
pub(crate) fn spawn_queue(state: &AppState) {
|
||||
pub(crate) fn spawn_queue(
|
||||
state: &AppState,
|
||||
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||
) {
|
||||
let Some(client) = state.status.as_ref().map(|s| s.queue_client()) else {
|
||||
tracing::warn!("no swarm queue is configured on this host; requested reviews don't run");
|
||||
return;
|
||||
|
|
@ -466,7 +469,7 @@ pub(crate) fn spawn_queue(state: &AppState) {
|
|||
runs,
|
||||
metrics: Metrics::global(),
|
||||
};
|
||||
consume(&client, TIMING, &reviews, handles).await;
|
||||
consume(&client, TIMING, &reviews, handles, stop).await;
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -494,8 +497,9 @@ impl<S, Q: Clone, P> Clone for Handles<S, Q, P> {
|
|||
}
|
||||
|
||||
/// Take requests from [`REQUEST_STREAM`] one at a time, each once a run
|
||||
/// place is free, and review each in a task of its own. Never returns;
|
||||
/// dropping it stops every review it started, and the queue delivers their
|
||||
/// place is free, and review each in a task of its own. Returns once `stop`
|
||||
/// resolves, taking no request after it; dropping it, as the return does, stops
|
||||
/// every review it started without acking, and the queue delivers their
|
||||
/// requests again.
|
||||
///
|
||||
/// The limit is read from the settings before each wait for a place. A
|
||||
|
|
@ -506,6 +510,7 @@ async fn consume<S, Q, P>(
|
|||
timing: Timing,
|
||||
reviews: &Arc<Reviews>,
|
||||
handles: Handles<S, Q, P>,
|
||||
stop: impl Future<Output = ()> + Clone,
|
||||
) where
|
||||
S: FileSource + PullSource + Send + Sync + 'static,
|
||||
Q: RunSink + SettingsSource,
|
||||
|
|
@ -519,7 +524,11 @@ async fn consume<S, Q, P>(
|
|||
"opening the review request queue failed; trying again"
|
||||
),
|
||||
}
|
||||
tokio::time::sleep(timing.pull_wait).await;
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = stop.clone() => return,
|
||||
() = tokio::time::sleep(timing.pull_wait) => {}
|
||||
}
|
||||
};
|
||||
let mut tasks = JoinSet::new();
|
||||
loop {
|
||||
|
|
@ -536,8 +545,17 @@ async fn consume<S, Q, P>(
|
|||
the last limit read"
|
||||
),
|
||||
}
|
||||
let slot = reviews.slot().await;
|
||||
let message = match next(&consumer, timing.pull_wait).await {
|
||||
let slot = tokio::select! {
|
||||
biased;
|
||||
() = stop.clone() => return,
|
||||
slot = reviews.slot() => slot,
|
||||
};
|
||||
let taken = tokio::select! {
|
||||
biased;
|
||||
() = stop.clone() => return,
|
||||
taken = next(&consumer, timing.pull_wait) => taken,
|
||||
};
|
||||
let message = match taken {
|
||||
Ok(Some(message)) => message,
|
||||
Ok(None) => continue,
|
||||
Err(e) => {
|
||||
|
|
@ -546,7 +564,11 @@ async fn consume<S, Q, P>(
|
|||
"taking a review request from the queue failed; trying again"
|
||||
);
|
||||
drop(slot);
|
||||
tokio::time::sleep(timing.pull_wait).await;
|
||||
tokio::select! {
|
||||
biased;
|
||||
() = stop.clone() => return,
|
||||
() = tokio::time::sleep(timing.pull_wait) => {}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
|
@ -2032,6 +2054,15 @@ mod tests {
|
|||
|
||||
/// A controller taking requests, its run places in `reviews`.
|
||||
fn consume(&self, reviews: &Arc<Reviews>) -> JoinHandle<()> {
|
||||
self.consume_until(reviews, std::future::pending())
|
||||
}
|
||||
|
||||
/// A controller that takes no request once `stop` resolves.
|
||||
fn consume_until(
|
||||
&self,
|
||||
reviews: &Arc<Reviews>,
|
||||
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||
) -> JoinHandle<()> {
|
||||
let client = self.client.clone();
|
||||
let reviews = Arc::clone(reviews);
|
||||
let forge = Arc::clone(&self.rig.forge);
|
||||
|
|
@ -2047,7 +2078,7 @@ mod tests {
|
|||
runs: store,
|
||||
metrics,
|
||||
};
|
||||
consume(&client, TEST_TIMING, &reviews, handles).await;
|
||||
consume(&client, TEST_TIMING, &reviews, handles, stop).await;
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -2113,6 +2144,36 @@ mod tests {
|
|||
assert_eq!(q.rig.gate.reached(), 2, "no run after the ack");
|
||||
}
|
||||
|
||||
/// After the stop the controller returns and takes no request, and the
|
||||
/// review it was running stays un-acked, so the queue still holds it.
|
||||
#[tokio::test]
|
||||
async fn after_the_stop_no_request_is_taken_and_the_running_one_stays_unacked() {
|
||||
let q = Queued::new().await;
|
||||
q.request(7, "abc123").await;
|
||||
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let stop = futures_util::FutureExt::shared(async move {
|
||||
let _ = stop_rx.await;
|
||||
});
|
||||
let consumer = q.consume_until(&q.rig.reviews, stop);
|
||||
until("the first run started", || q.rig.gate.reached() == 1).await;
|
||||
|
||||
stop_tx.send(()).expect("stop");
|
||||
tokio::time::timeout(Duration::from_secs(5), consumer)
|
||||
.await
|
||||
.expect("consume returns after the stop")
|
||||
.expect("consume did not panic");
|
||||
// A request that arrives now, and the one running, which the queue
|
||||
// would redeliver after its ack wait: neither runs.
|
||||
q.request(8, "abc123").await;
|
||||
tokio::time::sleep(TEST_TIMING.ack_wait * 3).await;
|
||||
assert_eq!(q.rig.gate.reached(), 1, "no request taken after the stop");
|
||||
assert!(
|
||||
q.rig.posted().is_empty(),
|
||||
"the unfinished review is not posted"
|
||||
);
|
||||
assert_eq!(q.waiting().await, 2, "neither request is acked");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_newer_request_for_a_pr_replaces_the_waiting_one() {
|
||||
let q = Queued::new().await;
|
||||
|
|
|
|||
|
|
@ -21,3 +21,7 @@ tracing-subscriber.workspace = true
|
|||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
# `Shared`, so the injected stop future can be cloned like `shutdown_signal`'s.
|
||||
futures-util.workspace = true
|
||||
|
|
|
|||
|
|
@ -195,6 +195,7 @@ async fn handle(
|
|||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let stop = swarm_secret_client::shutdown_signal();
|
||||
swarm_secret_client::install_crypto_provider();
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
|
|
@ -235,14 +236,31 @@ async fn main() -> anyhow::Result<()> {
|
|||
tracing::info!(socket = %args.socket.display(), "swarm-logs-auth listening");
|
||||
|
||||
let app = axum::Router::new().fallback(handle).with_state(responder);
|
||||
axum::serve(listener, app).await.context("serving")
|
||||
serve(listener, app, stop).await
|
||||
}
|
||||
|
||||
/// Answers on `listener` until `stop` resolves, then lets the requests in hand
|
||||
/// finish, for at most the drain, and returns.
|
||||
async fn serve(
|
||||
listener: tokio::net::UnixListener,
|
||||
app: axum::Router,
|
||||
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||
) -> anyhow::Result<()> {
|
||||
let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone());
|
||||
swarm_secret_client::serve_bounded(stop, serving.into_future())
|
||||
.await
|
||||
.transpose()
|
||||
.context("serving")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use futures_util::FutureExt as _;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use swarm_secret_client::queue::AgentCredential;
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
/// What the mirrored grant leaf answers.
|
||||
#[derive(Clone, Copy)]
|
||||
|
|
@ -474,4 +492,59 @@ mod tests {
|
|||
}
|
||||
assert_eq!(store.lookups(), 3);
|
||||
}
|
||||
|
||||
/// A request that is being answered when the stop arrives is still
|
||||
/// answered, and `serve` returns only afterwards.
|
||||
#[tokio::test]
|
||||
async fn a_request_in_flight_at_the_stop_is_answered_before_serve_returns() {
|
||||
let started = Arc::new(tokio::sync::Notify::new());
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let app = {
|
||||
let (started, release) = (Arc::clone(&started), Arc::clone(&release));
|
||||
axum::Router::new().fallback(move || {
|
||||
let (started, release) = (Arc::clone(&started), Arc::clone(&release));
|
||||
async move {
|
||||
started.notify_one();
|
||||
release.notified().await;
|
||||
"answered"
|
||||
}
|
||||
})
|
||||
};
|
||||
let socket =
|
||||
std::env::temp_dir().join(format!("swarm-logs-auth-stop-{}.sock", std::process::id()));
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
let listener = tokio::net::UnixListener::bind(&socket).expect("bind");
|
||||
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let stop = async move {
|
||||
let _ = stop_rx.await;
|
||||
}
|
||||
.shared();
|
||||
let mut serving = tokio::spawn(serve(listener, app, stop));
|
||||
|
||||
let mut conn = tokio::net::UnixStream::connect(&socket)
|
||||
.await
|
||||
.expect("connect");
|
||||
conn.write_all(b"GET / HTTP/1.1\r\nhost: x\r\nconnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("send");
|
||||
started.notified().await;
|
||||
|
||||
stop_tx.send(()).expect("stop");
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
assert!(
|
||||
!serving.is_finished(),
|
||||
"serve returned with a request in flight"
|
||||
);
|
||||
|
||||
release.notify_one();
|
||||
let mut reply = String::new();
|
||||
conn.read_to_string(&mut reply).await.expect("reply");
|
||||
assert!(reply.starts_with("HTTP/1.1 200"), "{reply}");
|
||||
assert!(reply.ends_with("answered"), "{reply}");
|
||||
let finished = tokio::time::timeout(Duration::from_secs(5), &mut serving)
|
||||
.await
|
||||
.expect("serve returns once the request is answered");
|
||||
finished.expect("task").expect("serve is Ok");
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -282,6 +282,7 @@ async fn handle(State(responder): State<Arc<Responder<Store>>>, headers: HeaderM
|
|||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let stop = swarm_secret_client::shutdown_signal();
|
||||
swarm_secret_client::install_crypto_provider();
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
|
|
@ -322,7 +323,22 @@ async fn main() -> anyhow::Result<()> {
|
|||
tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening");
|
||||
|
||||
let app = axum::Router::new().fallback(handle).with_state(responder);
|
||||
axum::serve(listener, app).await.context("serving")
|
||||
serve(listener, app, stop).await
|
||||
}
|
||||
|
||||
/// Answers on `listener` until `stop` resolves, then lets the requests in hand
|
||||
/// finish, for at most the drain, and returns.
|
||||
async fn serve(
|
||||
listener: tokio::net::UnixListener,
|
||||
app: axum::Router,
|
||||
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||
) -> anyhow::Result<()> {
|
||||
let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone());
|
||||
swarm_secret_client::serve_bounded(stop, serving.into_future())
|
||||
.await
|
||||
.transpose()
|
||||
.context("serving")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
Loading…
Reference in a new issue