swarm-logs-auth, swarm-metrics-auth, swarm-authelia-bridge, swarm-controller: drain http on sigterm
This commit is contained in:
parent
39c29b64e3
commit
a089da25a5
7 changed files with 198 additions and 19 deletions
|
|
@ -19,6 +19,8 @@ serde.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
serde_norway = "0.9.42"
|
serde_norway = "0.9.42"
|
||||||
swarm-authelia-bridge-sock.workspace = true
|
swarm-authelia-bridge-sock.workspace = true
|
||||||
|
# `shutdown_signal`: the bridge drains its in-flight requests on SIGTERM.
|
||||||
|
swarm-secret-client.workspace = true
|
||||||
tokio.workspace = true
|
tokio.workspace = true
|
||||||
tracing.workspace = true
|
tracing.workspace = true
|
||||||
tracing-subscriber.workspace = true
|
tracing-subscriber.workspace = true
|
||||||
|
|
|
||||||
|
|
@ -106,6 +106,7 @@ struct AppState {
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> Result<()> {
|
||||||
|
let stop = swarm_secret_client::shutdown_signal();
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
.with_env_filter(
|
.with_env_filter(
|
||||||
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
|
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
|
||||||
|
|
@ -130,9 +131,22 @@ async fn main() -> Result<()> {
|
||||||
|
|
||||||
let listener = bind_socket(&socket)?;
|
let listener = bind_socket(&socket)?;
|
||||||
tracing::info!(socket = %socket.display(), "swarm-authelia-bridge listening");
|
tracing::info!(socket = %socket.display(), "swarm-authelia-bridge listening");
|
||||||
axum::serve(listener, app)
|
serve(listener, app, stop).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Answers on `listener` until `stop` resolves, then lets the requests in hand
|
||||||
|
/// finish, for at most the drain, and returns.
|
||||||
|
async fn serve(
|
||||||
|
listener: tokio::net::UnixListener,
|
||||||
|
app: Router,
|
||||||
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||||
|
) -> Result<()> {
|
||||||
|
let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone());
|
||||||
|
swarm_secret_client::serve_bounded(stop, serving.into_future())
|
||||||
.await
|
.await
|
||||||
.context("serving swarm-authelia-bridge")
|
.transpose()
|
||||||
|
.context("serving swarm-authelia-bridge")?;
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Bind the listening socket at `path`, replacing a stale one a crash left
|
/// Bind the listening socket at `path`, replacing a stale one a crash left
|
||||||
|
|
|
||||||
|
|
@ -269,7 +269,7 @@ async fn main() -> Result<()> {
|
||||||
reviews: Arc::default(),
|
reviews: Arc::default(),
|
||||||
alerts: alerts::from_env(),
|
alerts: alerts::from_env(),
|
||||||
};
|
};
|
||||||
serve(listener, state).await
|
serve(listener, state, swarm_secret_client::shutdown_signal()).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Registers the `bao.client_cert.*` gauges for the controller's own store
|
/// Registers the `bao.client_cert.*` gauges for the controller's own store
|
||||||
|
|
@ -286,11 +286,20 @@ fn register_cert_gauges(provider: Option<&opentelemetry_sdk::metrics::SdkMeterPr
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Start taking the queued review requests and answering the notification
|
/// Start taking the queued review requests and answering the notification
|
||||||
/// router's, then serve `state` on `listener`.
|
/// router's, then serve `state` on `listener` until `stop` resolves. The
|
||||||
async fn serve(listener: tokio::net::UnixListener, state: AppState) -> Result<()> {
|
/// requests in hand finish for at most the drain; the review queue takes no
|
||||||
review_classifier::review::spawn_queue(&state);
|
/// request after `stop`.
|
||||||
|
async fn serve(
|
||||||
|
listener: tokio::net::UnixListener,
|
||||||
|
state: AppState,
|
||||||
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||||
|
) -> Result<()> {
|
||||||
|
review_classifier::review::spawn_queue(&state, stop.clone());
|
||||||
request_reviewer::spawn(&state);
|
request_reviewer::spawn(&state);
|
||||||
axum::serve(listener, build_app(state))
|
let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone());
|
||||||
|
swarm_secret_client::serve_bounded(stop, serving.into_future())
|
||||||
.await
|
.await
|
||||||
.context("serving swarm-controller")
|
.transpose()
|
||||||
|
.context("serving swarm-controller")?;
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -443,7 +443,10 @@ async fn open_consumer(
|
||||||
|
|
||||||
/// Take the queued review requests and review each, for as long as the
|
/// Take the queued review requests and review each, for as long as the
|
||||||
/// controller runs, once this host has forge access.
|
/// controller runs, once this host has forge access.
|
||||||
pub(crate) fn spawn_queue(state: &AppState) {
|
pub(crate) fn spawn_queue(
|
||||||
|
state: &AppState,
|
||||||
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||||
|
) {
|
||||||
let Some(client) = state.status.as_ref().map(|s| s.queue_client()) else {
|
let Some(client) = state.status.as_ref().map(|s| s.queue_client()) else {
|
||||||
tracing::warn!("no swarm queue is configured on this host; requested reviews don't run");
|
tracing::warn!("no swarm queue is configured on this host; requested reviews don't run");
|
||||||
return;
|
return;
|
||||||
|
|
@ -466,7 +469,7 @@ pub(crate) fn spawn_queue(state: &AppState) {
|
||||||
runs,
|
runs,
|
||||||
metrics: Metrics::global(),
|
metrics: Metrics::global(),
|
||||||
};
|
};
|
||||||
consume(&client, TIMING, &reviews, handles).await;
|
consume(&client, TIMING, &reviews, handles, stop).await;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -494,8 +497,9 @@ impl<S, Q: Clone, P> Clone for Handles<S, Q, P> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Take requests from [`REQUEST_STREAM`] one at a time, each once a run
|
/// Take requests from [`REQUEST_STREAM`] one at a time, each once a run
|
||||||
/// place is free, and review each in a task of its own. Never returns;
|
/// place is free, and review each in a task of its own. Returns once `stop`
|
||||||
/// dropping it stops every review it started, and the queue delivers their
|
/// resolves, taking no request after it; dropping it, as the return does, stops
|
||||||
|
/// every review it started without acking, and the queue delivers their
|
||||||
/// requests again.
|
/// requests again.
|
||||||
///
|
///
|
||||||
/// The limit is read from the settings before each wait for a place. A
|
/// The limit is read from the settings before each wait for a place. A
|
||||||
|
|
@ -506,6 +510,7 @@ async fn consume<S, Q, P>(
|
||||||
timing: Timing,
|
timing: Timing,
|
||||||
reviews: &Arc<Reviews>,
|
reviews: &Arc<Reviews>,
|
||||||
handles: Handles<S, Q, P>,
|
handles: Handles<S, Q, P>,
|
||||||
|
stop: impl Future<Output = ()> + Clone,
|
||||||
) where
|
) where
|
||||||
S: FileSource + PullSource + Send + Sync + 'static,
|
S: FileSource + PullSource + Send + Sync + 'static,
|
||||||
Q: RunSink + SettingsSource,
|
Q: RunSink + SettingsSource,
|
||||||
|
|
@ -519,7 +524,11 @@ async fn consume<S, Q, P>(
|
||||||
"opening the review request queue failed; trying again"
|
"opening the review request queue failed; trying again"
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
tokio::time::sleep(timing.pull_wait).await;
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
() = stop.clone() => return,
|
||||||
|
() = tokio::time::sleep(timing.pull_wait) => {}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
let mut tasks = JoinSet::new();
|
let mut tasks = JoinSet::new();
|
||||||
loop {
|
loop {
|
||||||
|
|
@ -536,8 +545,17 @@ async fn consume<S, Q, P>(
|
||||||
the last limit read"
|
the last limit read"
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
let slot = reviews.slot().await;
|
let slot = tokio::select! {
|
||||||
let message = match next(&consumer, timing.pull_wait).await {
|
biased;
|
||||||
|
() = stop.clone() => return,
|
||||||
|
slot = reviews.slot() => slot,
|
||||||
|
};
|
||||||
|
let taken = tokio::select! {
|
||||||
|
biased;
|
||||||
|
() = stop.clone() => return,
|
||||||
|
taken = next(&consumer, timing.pull_wait) => taken,
|
||||||
|
};
|
||||||
|
let message = match taken {
|
||||||
Ok(Some(message)) => message,
|
Ok(Some(message)) => message,
|
||||||
Ok(None) => continue,
|
Ok(None) => continue,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
@ -546,7 +564,11 @@ async fn consume<S, Q, P>(
|
||||||
"taking a review request from the queue failed; trying again"
|
"taking a review request from the queue failed; trying again"
|
||||||
);
|
);
|
||||||
drop(slot);
|
drop(slot);
|
||||||
tokio::time::sleep(timing.pull_wait).await;
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
() = stop.clone() => return,
|
||||||
|
() = tokio::time::sleep(timing.pull_wait) => {}
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
@ -2032,6 +2054,15 @@ mod tests {
|
||||||
|
|
||||||
/// A controller taking requests, its run places in `reviews`.
|
/// A controller taking requests, its run places in `reviews`.
|
||||||
fn consume(&self, reviews: &Arc<Reviews>) -> JoinHandle<()> {
|
fn consume(&self, reviews: &Arc<Reviews>) -> JoinHandle<()> {
|
||||||
|
self.consume_until(reviews, std::future::pending())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A controller that takes no request once `stop` resolves.
|
||||||
|
fn consume_until(
|
||||||
|
&self,
|
||||||
|
reviews: &Arc<Reviews>,
|
||||||
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||||
|
) -> JoinHandle<()> {
|
||||||
let client = self.client.clone();
|
let client = self.client.clone();
|
||||||
let reviews = Arc::clone(reviews);
|
let reviews = Arc::clone(reviews);
|
||||||
let forge = Arc::clone(&self.rig.forge);
|
let forge = Arc::clone(&self.rig.forge);
|
||||||
|
|
@ -2047,7 +2078,7 @@ mod tests {
|
||||||
runs: store,
|
runs: store,
|
||||||
metrics,
|
metrics,
|
||||||
};
|
};
|
||||||
consume(&client, TEST_TIMING, &reviews, handles).await;
|
consume(&client, TEST_TIMING, &reviews, handles, stop).await;
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2113,6 +2144,36 @@ mod tests {
|
||||||
assert_eq!(q.rig.gate.reached(), 2, "no run after the ack");
|
assert_eq!(q.rig.gate.reached(), 2, "no run after the ack");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// After the stop the controller returns and takes no request, and the
|
||||||
|
/// review it was running stays un-acked, so the queue still holds it.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn after_the_stop_no_request_is_taken_and_the_running_one_stays_unacked() {
|
||||||
|
let q = Queued::new().await;
|
||||||
|
q.request(7, "abc123").await;
|
||||||
|
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
|
||||||
|
let stop = futures_util::FutureExt::shared(async move {
|
||||||
|
let _ = stop_rx.await;
|
||||||
|
});
|
||||||
|
let consumer = q.consume_until(&q.rig.reviews, stop);
|
||||||
|
until("the first run started", || q.rig.gate.reached() == 1).await;
|
||||||
|
|
||||||
|
stop_tx.send(()).expect("stop");
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), consumer)
|
||||||
|
.await
|
||||||
|
.expect("consume returns after the stop")
|
||||||
|
.expect("consume did not panic");
|
||||||
|
// A request that arrives now, and the one running, which the queue
|
||||||
|
// would redeliver after its ack wait: neither runs.
|
||||||
|
q.request(8, "abc123").await;
|
||||||
|
tokio::time::sleep(TEST_TIMING.ack_wait * 3).await;
|
||||||
|
assert_eq!(q.rig.gate.reached(), 1, "no request taken after the stop");
|
||||||
|
assert!(
|
||||||
|
q.rig.posted().is_empty(),
|
||||||
|
"the unfinished review is not posted"
|
||||||
|
);
|
||||||
|
assert_eq!(q.waiting().await, 2, "neither request is acked");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn a_newer_request_for_a_pr_replaces_the_waiting_one() {
|
async fn a_newer_request_for_a_pr_replaces_the_waiting_one() {
|
||||||
let q = Queued::new().await;
|
let q = Queued::new().await;
|
||||||
|
|
|
||||||
|
|
@ -21,3 +21,7 @@ tracing-subscriber.workspace = true
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
workspace = true
|
workspace = true
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
# `Shared`, so the injected stop future can be cloned like `shutdown_signal`'s.
|
||||||
|
futures-util.workspace = true
|
||||||
|
|
|
||||||
|
|
@ -195,6 +195,7 @@ async fn handle(
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> anyhow::Result<()> {
|
async fn main() -> anyhow::Result<()> {
|
||||||
|
let stop = swarm_secret_client::shutdown_signal();
|
||||||
swarm_secret_client::install_crypto_provider();
|
swarm_secret_client::install_crypto_provider();
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
.with_env_filter(
|
.with_env_filter(
|
||||||
|
|
@ -235,14 +236,31 @@ async fn main() -> anyhow::Result<()> {
|
||||||
tracing::info!(socket = %args.socket.display(), "swarm-logs-auth listening");
|
tracing::info!(socket = %args.socket.display(), "swarm-logs-auth listening");
|
||||||
|
|
||||||
let app = axum::Router::new().fallback(handle).with_state(responder);
|
let app = axum::Router::new().fallback(handle).with_state(responder);
|
||||||
axum::serve(listener, app).await.context("serving")
|
serve(listener, app, stop).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Answers on `listener` until `stop` resolves, then lets the requests in hand
|
||||||
|
/// finish, for at most the drain, and returns.
|
||||||
|
async fn serve(
|
||||||
|
listener: tokio::net::UnixListener,
|
||||||
|
app: axum::Router,
|
||||||
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone());
|
||||||
|
swarm_secret_client::serve_bounded(stop, serving.into_future())
|
||||||
|
.await
|
||||||
|
.transpose()
|
||||||
|
.context("serving")?;
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use futures_util::FutureExt as _;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
use swarm_secret_client::queue::AgentCredential;
|
use swarm_secret_client::queue::AgentCredential;
|
||||||
|
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||||
|
|
||||||
/// What the mirrored grant leaf answers.
|
/// What the mirrored grant leaf answers.
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
|
|
@ -474,4 +492,59 @@ mod tests {
|
||||||
}
|
}
|
||||||
assert_eq!(store.lookups(), 3);
|
assert_eq!(store.lookups(), 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A request that is being answered when the stop arrives is still
|
||||||
|
/// answered, and `serve` returns only afterwards.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_request_in_flight_at_the_stop_is_answered_before_serve_returns() {
|
||||||
|
let started = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let release = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let app = {
|
||||||
|
let (started, release) = (Arc::clone(&started), Arc::clone(&release));
|
||||||
|
axum::Router::new().fallback(move || {
|
||||||
|
let (started, release) = (Arc::clone(&started), Arc::clone(&release));
|
||||||
|
async move {
|
||||||
|
started.notify_one();
|
||||||
|
release.notified().await;
|
||||||
|
"answered"
|
||||||
|
}
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let socket =
|
||||||
|
std::env::temp_dir().join(format!("swarm-logs-auth-stop-{}.sock", std::process::id()));
|
||||||
|
let _ = std::fs::remove_file(&socket);
|
||||||
|
let listener = tokio::net::UnixListener::bind(&socket).expect("bind");
|
||||||
|
let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
|
||||||
|
let stop = async move {
|
||||||
|
let _ = stop_rx.await;
|
||||||
|
}
|
||||||
|
.shared();
|
||||||
|
let mut serving = tokio::spawn(serve(listener, app, stop));
|
||||||
|
|
||||||
|
let mut conn = tokio::net::UnixStream::connect(&socket)
|
||||||
|
.await
|
||||||
|
.expect("connect");
|
||||||
|
conn.write_all(b"GET / HTTP/1.1\r\nhost: x\r\nconnection: close\r\n\r\n")
|
||||||
|
.await
|
||||||
|
.expect("send");
|
||||||
|
started.notified().await;
|
||||||
|
|
||||||
|
stop_tx.send(()).expect("stop");
|
||||||
|
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||||
|
assert!(
|
||||||
|
!serving.is_finished(),
|
||||||
|
"serve returned with a request in flight"
|
||||||
|
);
|
||||||
|
|
||||||
|
release.notify_one();
|
||||||
|
let mut reply = String::new();
|
||||||
|
conn.read_to_string(&mut reply).await.expect("reply");
|
||||||
|
assert!(reply.starts_with("HTTP/1.1 200"), "{reply}");
|
||||||
|
assert!(reply.ends_with("answered"), "{reply}");
|
||||||
|
let finished = tokio::time::timeout(Duration::from_secs(5), &mut serving)
|
||||||
|
.await
|
||||||
|
.expect("serve returns once the request is answered");
|
||||||
|
finished.expect("task").expect("serve is Ok");
|
||||||
|
let _ = std::fs::remove_file(&socket);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -282,6 +282,7 @@ async fn handle(State(responder): State<Arc<Responder<Store>>>, headers: HeaderM
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> anyhow::Result<()> {
|
async fn main() -> anyhow::Result<()> {
|
||||||
|
let stop = swarm_secret_client::shutdown_signal();
|
||||||
swarm_secret_client::install_crypto_provider();
|
swarm_secret_client::install_crypto_provider();
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
.with_env_filter(
|
.with_env_filter(
|
||||||
|
|
@ -322,7 +323,22 @@ async fn main() -> anyhow::Result<()> {
|
||||||
tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening");
|
tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening");
|
||||||
|
|
||||||
let app = axum::Router::new().fallback(handle).with_state(responder);
|
let app = axum::Router::new().fallback(handle).with_state(responder);
|
||||||
axum::serve(listener, app).await.context("serving")
|
serve(listener, app, stop).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Answers on `listener` until `stop` resolves, then lets the requests in hand
|
||||||
|
/// finish, for at most the drain, and returns.
|
||||||
|
async fn serve(
|
||||||
|
listener: tokio::net::UnixListener,
|
||||||
|
app: axum::Router,
|
||||||
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone());
|
||||||
|
swarm_secret_client::serve_bounded(stop, serving.into_future())
|
||||||
|
.await
|
||||||
|
.transpose()
|
||||||
|
.context("serving")?;
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue