diff --git a/Cargo.lock b/Cargo.lock index 0ae9dd629..dc9f423bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5599,6 +5599,7 @@ dependencies = [ name = "swarm-secret-client" version = "0.1.0" dependencies = [ + "axum", "backon", "futures-util", "libc", diff --git a/docs/2-handbook/swarm/services.md b/docs/2-handbook/swarm/services.md index e9a4679c2..d93c0a36e 100644 --- a/docs/2-handbook/swarm/services.md +++ b/docs/2-handbook/swarm/services.md @@ -265,3 +265,38 @@ under a tight sandbox, but its network isn't restricted, which the design page spells out. The settings, limits and reasons: [`stt.md`](../../4-implementation/swarm/stt.md) and [Host setup](../../4-implementation/swarm/stt.md#host-setup). + +### Health endpoints + +The queue daemons (`swarm-stt`, `swarm-review-classifier`, +`swarm-notify-router`, `swarm-builder`, `swarm-claude-login`), +`swarm-nats-auth` and the three auth responders (`swarm-logs-auth`, +`swarm-metrics-auth`, `swarm-authelia-bridge`) can serve two probe endpoints. +They answer 200 or 503 and send no body. + +- `GET /healthz` is 200 while the daemon's main loop is running. The loop + beats as it takes messages, so a loop that stops polling turns the answer to 503 + after 30 seconds (5 minutes for `swarm-notify-router`, whose loop handles a + batch of events between beats). Before the loop's first beat the answer is + 200: waiting on the secret store at startup doesn't count as stuck. A + daemon that only serves requests (the three auth responders) beats from a task + on its runtime, so `/healthz` shows the runtime still runs tasks, not that + its accept loop is healthy. +- `GET /readyz` is 200 while the daemon has what it needs to work: its queue + connection works (`swarm-stt`, `swarm-review-classifier`, + `swarm-notify-router`, `swarm-builder`, `swarm-claude-login`, + `swarm-nats-auth`), or it has bound its socket (the three auth responders). + +Nothing listens by default. Set `SWARM_STT_HEALTH_ADDR`, +`SWARM_REVIEW_CLASSIFIER_HEALTH_ADDR`, `SWARM_NOTIFY_ROUTER_HEALTH_ADDR`, +`SWARM_BUILDER_HEALTH_ADDR`, `SWARM_CLAUDE_LOGIN_HEALTH_ADDR` or +`SWARM_AUTHELIA_BRIDGE_HEALTH_ADDR` to a `host:port` to bind the listener. For +`swarm-nats-auth`, `swarm-logs-auth` and `swarm-metrics-auth`, pass +`--health-addr`, or set `SWARM_NATS_AUTH_HEALTH_ADDR`, +`SWARM_LOGS_AUTH_HEALTH_ADDR` or `SWARM_METRICS_AUTH_HEALTH_ADDR`; the flag +wins. No NixOS module sets one. The listener stops on the same SIGTERM as the +daemon. + +`swarm-controller` serves `GET /readyz` on its existing socket, next to +`/health`. It answers 200 while the queue connection works and the secret store +accepts the controller's login, and 503 otherwise. diff --git a/swarm-authelia-bridge/src/main.rs b/swarm-authelia-bridge/src/main.rs index adbf7b21c..f748d7781 100644 --- a/swarm-authelia-bridge/src/main.rs +++ b/swarm-authelia-bridge/src/main.rs @@ -117,6 +117,12 @@ async fn main() -> Result<()> { .with_ansi(false) .init(); + let health = swarm_secret_client::health::Health::from_env( + "SWARM_AUTHELIA_BRIDGE_HEALTH_ADDR", + stop.clone(), + ) + .await?; + health.beat_on_runtime(); let config = Config::from_env()?; let socket = config.socket.clone(); let state = Arc::new(AppState { @@ -131,6 +137,7 @@ async fn main() -> Result<()> { let listener = bind_socket(&socket)?; tracing::info!(socket = %socket.display(), "swarm-authelia-bridge listening"); + health.ready_when(|| true); serve(listener, app, stop).await } diff --git a/swarm-builder/src/main.rs b/swarm-builder/src/main.rs index ab42f5694..d7a21f935 100644 --- a/swarm-builder/src/main.rs +++ b/swarm-builder/src/main.rs @@ -95,6 +95,11 @@ async fn main() -> Result<()> { let full = format!("{PREFIX}_{name}"); std::env::var(&full).with_context(|| format!("{full} is unset")) }; + let health = swarm_secret_client::health::Health::from_env( + &format!("{PREFIX}_HEALTH_ADDR"), + stop.clone(), + ) + .await?; let client_id = var("OIDC_CLIENT_ID")?; let builder = Arc::new(nix_build::Builder { nix: PathBuf::from(var("NIX")?), @@ -134,6 +139,7 @@ async fn main() -> Result<()> { .map(PathBuf::from), }) .await?; + swarm_queue_client::ready_when_connected(&health, &queue); let requests = queue .queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned()) .await @@ -176,7 +182,7 @@ async fn main() -> Result<()> { serve_meta_builds(&queue, &publisher, Store, &builder, &slot, &metrics).await?; answer_builds( - requests.take_until(stop.clone()), + health.beating(requests.take_until(stop.clone())), publisher, Store, builder, diff --git a/swarm-claude-login/src/main.rs b/swarm-claude-login/src/main.rs index 0c58db0cd..930f9ffa5 100644 --- a/swarm-claude-login/src/main.rs +++ b/swarm-claude-login/src/main.rs @@ -107,6 +107,11 @@ async fn main() -> Result<()> { let full = format!("{PREFIX}_{name}"); std::env::var(&full).with_context(|| format!("{full} is unset")) }; + let health = swarm_secret_client::health::Health::from_env( + &format!("{PREFIX}_HEALTH_ADDR"), + stop.clone(), + ) + .await?; let client_id = var("OIDC_CLIENT_ID")?; let metrics = Arc::new(Metrics::from_env()); let logins = Arc::new(login::Logins::new( @@ -134,13 +139,16 @@ async fn main() -> Result<()> { .map(PathBuf::from), }) .await?; + swarm_queue_client::ready_when_connected(&health, &queue); // No queue group: a running login lives in this process's memory, so // every step for it has to reach this process. - let mut requests = queue - .subscribe(SUBJECT) - .await - .with_context(|| format!("subscribing to {SUBJECT}"))? - .take_until(stop.clone()); + let mut requests = health.beating( + queue + .subscribe(SUBJECT) + .await + .with_context(|| format!("subscribing to {SUBJECT}"))? + .take_until(stop.clone()), + ); tracing::info!(subject = SUBJECT, "claude login minter: awaiting logins"); let mut in_flight = swarm_secret_client::InFlight::new(); diff --git a/swarm-controller/src/routes/mod.rs b/swarm-controller/src/routes/mod.rs index 7ad214479..3821d2ca6 100644 --- a/swarm-controller/src/routes/mod.rs +++ b/swarm-controller/src/routes/mod.rs @@ -7,8 +7,12 @@ pub(crate) mod forge_admin; pub(crate) mod hives; pub(crate) mod jobq_graph; +use std::future::Future; use std::sync::{Arc, Mutex}; +use axum::extract::State; +use axum::http::StatusCode; + use crate::jobq::{Late, SwarmNodeKind, SwarmResourceKind}; use crate::routes::hives::{HiveEntry, ServiceLink}; use crate::{ @@ -28,6 +32,36 @@ pub(crate) async fn health() -> &'static str { concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n") } +/// How long the readiness probe waits for the secret store to answer a login. +const STORE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// Readiness probe: 200 when the swarm queue is connected and the secret store +/// accepts the controller's login, 503 otherwise. No body. +/// +/// Not in the `OpenAPI` document: it is for the supervisor, not for API clients. +pub(crate) async fn readyz(State(state): State) -> StatusCode { + let queue_connected = state + .status + .as_ref() + .is_some_and(|status| swarm_queue_client::ensure_connected(&status.queue_client()).is_ok()); + ready(queue_connected, async { + tokio::time::timeout(STORE_PROBE_TIMEOUT, crate::store::connect()) + .await + .is_ok_and(|login| login.is_ok()) + }) + .await +} + +/// 200 when the queue is connected and `store_reachable` resolves true. The +/// store is not asked while the queue is down. +async fn ready(queue_connected: bool, store_reachable: impl Future) -> StatusCode { + if queue_connected && store_reachable.await { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } +} + #[derive(Clone)] pub(crate) struct AppState { /// Loaded once at startup (`load_hives`); never mutated, so an @@ -204,6 +238,51 @@ pub(crate) fn wanted_error_status(e: &anyhow::Error) -> axum::http::StatusCode { pub(crate) mod tests { use super::*; + /// `/readyz` needs both the queue and the store, and does not ask the store + /// while the queue is down. + #[tokio::test] + async fn ready_needs_the_queue_and_the_store() { + use std::sync::atomic::{AtomicBool, Ordering}; + + assert_eq!(ready(true, async { true }).await, StatusCode::OK); + assert_eq!( + ready(true, async { false }).await, + StatusCode::SERVICE_UNAVAILABLE, + "store unreachable" + ); + let asked = AtomicBool::new(false); + let store = async { + asked.store(true, Ordering::SeqCst); + true + }; + assert_eq!(ready(false, store).await, StatusCode::SERVICE_UNAVAILABLE); + assert!(!asked.load(Ordering::SeqCst), "queue down: store not asked"); + } + + /// Served by the real router: with no queue wired up `/readyz` is 503, + /// while `/health` on the same router stays 200. + #[tokio::test] + async fn the_router_answers_readyz_503_without_a_queue_and_health_200() { + use axum::body::Body; + use axum::http::Request; + use tower::ServiceExt as _; + + let (state, _sched) = state_with_roster(); + let app = crate::startup::build_app(state); + for (path, expected) in [ + ("/readyz", StatusCode::SERVICE_UNAVAILABLE), + ("/health", StatusCode::OK), + ] { + let status = app + .clone() + .oneshot(Request::get(path).body(Body::empty()).expect("a request")) + .await + .expect("answers") + .status(); + assert_eq!(status, expected, "{path}"); + } + } + /// The 503 this route returns is what the operator UIs' shared error /// component renders, so assert the RENDERED response rather than the /// `problem_details` crate: the contract a UI depends on is the content diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs index 00a89226b..86171d7db 100644 --- a/swarm-controller/src/startup.rs +++ b/swarm-controller/src/startup.rs @@ -771,6 +771,7 @@ pub(crate) fn build_app(state: AppState) -> axum::Router { // `Clone`; each request gets its own owned copy for `Json` to // serialize, same as `hive-c0re::dashboard::serve`. router + .route("/readyz", get(routes::readyz)) .route( "/api/openapi.json", get(move || async move { Json(api.clone()) }), diff --git a/swarm-logs-auth/src/main.rs b/swarm-logs-auth/src/main.rs index 9bc40783e..6cebbb6de 100644 --- a/swarm-logs-auth/src/main.rs +++ b/swarm-logs-auth/src/main.rs @@ -55,6 +55,10 @@ struct Args { /// Unix socket to serve on. #[arg(long)] socket: PathBuf, + /// Address to serve `/healthz` and `/readyz` on. Nothing listens unless set; + /// the environment variable `SWARM_LOGS_AUTH_HEALTH_ADDR` is read when the flag is absent. + #[arg(long)] + health_addr: Option, /// Cert-auth role this responder logs in to the store as. #[arg(long, default_value = "swarm-logs-auth")] store_cert_role: String, @@ -206,6 +210,15 @@ async fn main() -> anyhow::Result<()> { .with_ansi(false) .init(); let args = Args::parse(); + let health = swarm_secret_client::health::Health::start( + swarm_secret_client::health::STALE_AFTER, + args.health_addr, + "SWARM_LOGS_AUTH_HEALTH_ADDR", + stop.clone(), + ) + .await + .context("starting the health listener")?; + health.beat_on_runtime(); let settings = swarm_secret_client::client::Settings::from_env() .context("reading the secret store's address and this responder's identity")?; @@ -234,6 +247,7 @@ async fn main() -> anyhow::Result<()> { std::fs::set_permissions(&args.socket, std::fs::Permissions::from_mode(0o666)) .with_context(|| format!("chmod {}", args.socket.display()))?; tracing::info!(socket = %args.socket.display(), "swarm-logs-auth listening"); + health.ready_when(|| true); let app = axum::Router::new().fallback(handle).with_state(responder); serve(listener, app, stop).await diff --git a/swarm-metrics-auth/src/main.rs b/swarm-metrics-auth/src/main.rs index d4845a3ba..9a52f9f32 100644 --- a/swarm-metrics-auth/src/main.rs +++ b/swarm-metrics-auth/src/main.rs @@ -65,6 +65,10 @@ struct Args { /// Unix socket to serve on. #[arg(long)] socket: PathBuf, + /// Address to serve `/healthz` and `/readyz` on. Nothing listens unless set; + /// the environment variable `SWARM_METRICS_AUTH_HEALTH_ADDR` is read when the flag is absent. + #[arg(long)] + health_addr: Option, /// Cert-auth role this responder logs in to the store as. #[arg(long, default_value = "swarm-metrics-auth")] store_cert_role: String, @@ -293,6 +297,15 @@ async fn main() -> anyhow::Result<()> { .with_ansi(false) .init(); let args = Args::parse(); + let health = swarm_secret_client::health::Health::start( + swarm_secret_client::health::STALE_AFTER, + args.health_addr, + "SWARM_METRICS_AUTH_HEALTH_ADDR", + stop.clone(), + ) + .await + .context("starting the health listener")?; + health.beat_on_runtime(); let settings = Settings::from_env() .context("reading the secret store's address and this responder's identity")?; @@ -321,6 +334,7 @@ async fn main() -> anyhow::Result<()> { std::fs::set_permissions(&args.socket, std::fs::Permissions::from_mode(0o666)) .with_context(|| format!("chmod {}", args.socket.display()))?; tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening"); + health.ready_when(|| true); let app = axum::Router::new().fallback(handle).with_state(responder); serve(listener, app, stop).await diff --git a/swarm-nats-auth/src/main.rs b/swarm-nats-auth/src/main.rs index d08c17365..68030379c 100644 --- a/swarm-nats-auth/src/main.rs +++ b/swarm-nats-auth/src/main.rs @@ -178,6 +178,11 @@ struct Args { /// without. #[arg(long, default_value = "swarm-nats-auth")] store_cert_role: String, + /// Address to serve `/healthz` and `/readyz` on. Nothing listens unless set; + /// the environment variable `SWARM_NATS_AUTH_HEALTH_ADDR` is read when the + /// flag is absent. + #[arg(long)] + health_addr: Option, } #[tokio::main] @@ -240,6 +245,14 @@ async fn serve(args: Args, stop: swarm_secret_client::Shutdown) -> anyhow::Resul .with_claude_logins(args.claude_login_clients.clone()) .with_notify_routers(args.notify_router_clients.clone()) .with_stts(args.stt_clients.clone()); + let health = swarm_secret_client::health::Health::start( + swarm_secret_client::health::STALE_AFTER, + args.health_addr, + "SWARM_NATS_AUTH_HEALTH_ADDR", + stop.clone(), + ) + .await + .context("starting the health listener")?; let http = reqwest::Client::new(); let settings = swarm_secret_client::client::Settings::from_env() .context("reading the secret store's address and this responder's identity")?; @@ -261,6 +274,7 @@ async fn serve(args: Args, stop: swarm_secret_client::Shutdown) -> anyhow::Resul .connect(&args.nats_url) .await .with_context(|| format!("connect to {}", args.nats_url))?; + swarm_queue_client::ready_when_connected(&health, &client); let requests = client .subscribe(AUTH_SUBJECT) .await @@ -281,7 +295,7 @@ async fn serve(args: Args, stop: swarm_secret_client::Shutdown) -> anyhow::Resul issuer, }); answer_each( - requests.take_until(stop.clone()), + health.beating(requests.take_until(stop.clone())), CONCURRENT_REQUESTS, move |msg| { let responder = Arc::clone(&responder); diff --git a/swarm-notify-router/src/main.rs b/swarm-notify-router/src/main.rs index 2d484d22b..2e6180de1 100644 --- a/swarm-notify-router/src/main.rs +++ b/swarm-notify-router/src/main.rs @@ -74,6 +74,11 @@ const EVENT_ACK_WAIT: Duration = Duration::from_mins(1); /// again, when the failed event comes back first. const RETRY_DELAY: Duration = Duration::from_secs(30); +/// How long the loop may go without a beat before `/healthz` says it is stuck: a pull +/// waits up to [`PULL_WAIT`], and the events it returns are handled one by one +/// before the next beat. +const LOOP_STALE_AFTER: Duration = Duration::from_mins(5); + /// How long the list of agents is reused before it is read again. const ROSTER_TTL: Duration = Duration::from_mins(1); @@ -86,6 +91,14 @@ async fn main() -> Result<()> { let full = format!("{PREFIX}_{name}"); std::env::var(&full).with_context(|| format!("{full} is unset")) }; + let health = swarm_secret_client::health::Health::start( + LOOP_STALE_AFTER, + None, + &format!("{PREFIX}_HEALTH_ADDR"), + stop.clone(), + ) + .await + .context("starting the health listener")?; let client_id = var("OIDC_CLIENT_ID")?; let secret = swarm_secret_client::read_with_startup_retry("reading the queue client secret", || { @@ -107,6 +120,7 @@ async fn main() -> Result<()> { .map(PathBuf::from), }) .await?; + swarm_queue_client::ready_when_connected(&health, &queue); let mut router = Router::open(&queue).await?; tokio::spawn(api::serve(queue.clone())); @@ -115,6 +129,7 @@ async fn main() -> Result<()> { "notify router: routing forge events" ); while stop.clone().now_or_never().is_none() { + health.beat(); let pulled = swarm_secret_client::serve_bounded(stop.clone(), router.pull_once(stop.clone())).await; match pulled { diff --git a/swarm-queue-client/src/lib.rs b/swarm-queue-client/src/lib.rs index dee6a3d5c..6a2c3a6fb 100644 --- a/swarm-queue-client/src/lib.rs +++ b/swarm-queue-client/src/lib.rs @@ -1093,6 +1093,19 @@ pub fn ensure_connected(client: &async_nats::Client) -> Result<(), Error> { Ok(()) } +/// Makes `health`'s `/readyz` say whether `client` is connected: the same +/// test as [`ensure_connected`], so a client that has never connected, or is +/// reconnecting, is not ready. A daemon whose readiness also needs something +/// else sets its own condition with `Health::ready_when` and calls +/// [`ensure_connected`] in it. +pub fn ready_when_connected( + health: &swarm_secret_client::health::Health, + client: &async_nats::Client, +) { + let client = client.clone(); + health.ready_when(move || ensure_connected(&client).is_ok()); +} + /// The largest payload this connection's server will accept, as the server /// itself announced it at CONNECT. /// @@ -1932,4 +1945,76 @@ mod tests { assert_eq!(mints.load(Ordering::SeqCst), 1, "{event}"); } } + + /// The `/readyz` a queue daemon serves, end to end in this process: 503 + /// while its client has no server to connect to, 200 once one answers. + /// The server is a stand-in that speaks just enough of the protocol for + /// `async-nats` to count itself connected. + #[tokio::test] + async fn readyz_is_503_before_the_client_connects_and_200_after() { + use swarm_secret_client::health::Health; + use tokio::io::{AsyncBufReadExt as _, AsyncReadExt as _, AsyncWriteExt as _, BufReader}; + + async fn status(addr: std::net::SocketAddr, path: &str) -> u16 { + let mut conn = tokio::net::TcpStream::connect(addr).await.unwrap(); + let request = format!("GET {path} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"); + conn.write_all(request.as_bytes()).await.unwrap(); + let mut reply = String::new(); + conn.read_to_string(&mut reply).await.unwrap(); + reply.split_whitespace().nth(1).unwrap().parse().unwrap() + } + + // A port nothing listens on yet. + let queue_addr = { + let held = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + held.local_addr().unwrap() + }; + let client = async_nats::ConnectOptions::new() + .retry_on_initial_connect() + .connect(queue_addr.to_string()) + .await + .unwrap(); + + let health = Health::new(std::time::Duration::from_mins(1)); + ready_when_connected(&health, &client); + let probes = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let probe_addr = probes.local_addr().unwrap(); + tokio::spawn(health.clone().serve(probes, std::future::pending())); + + assert_eq!(status(probe_addr, "/readyz").await, 503, "no server yet"); + assert_eq!( + status(probe_addr, "/healthz").await, + 200, + "alive regardless" + ); + + let server = tokio::net::TcpListener::bind(queue_addr).await.unwrap(); + tokio::spawn(async move { + loop { + let (mut conn, _) = server.accept().await.unwrap(); + let info = r#"INFO {"server_id":"s","server_name":"s","version":"2.10.0","go":"go1","host":"127.0.0.1","port":4222,"headers":true,"max_payload":1048576,"proto":1}"#; + conn.write_all(format!("{info}\r\n").as_bytes()) + .await + .unwrap(); + let mut lines = BufReader::new(conn); + let mut line = String::new(); + while lines.read_line(&mut line).await.unwrap_or(0) > 0 { + if line.starts_with("PING") { + lines.get_mut().write_all(b"PONG\r\n").await.unwrap(); + } + line.clear(); + } + } + }); + + let mut ready = 503; + for _ in 0..100 { + ready = status(probe_addr, "/readyz").await; + if ready == 200 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert_eq!(ready, 200, "connected"); + } } diff --git a/swarm-review-classifier/src/main.rs b/swarm-review-classifier/src/main.rs index dfef53a15..93ec2241c 100644 --- a/swarm-review-classifier/src/main.rs +++ b/swarm-review-classifier/src/main.rs @@ -54,6 +54,11 @@ async fn serve() -> Result<()> { let full = format!("{PREFIX}_{name}"); std::env::var(&full).with_context(|| format!("{full} is unset")) }; + let health = swarm_secret_client::health::Health::from_env( + &format!("{PREFIX}_HEALTH_ADDR"), + stop.clone(), + ) + .await?; let client_id = var("OIDC_CLIENT_ID")?; let trampoline = std::env::current_exe().context("finding this binary for the child trampoline")?; @@ -90,11 +95,14 @@ async fn serve() -> Result<()> { .map(PathBuf::from), }) .await?; - let mut requests = queue - .queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned()) - .await - .with_context(|| format!("subscribing to {SUBJECT}"))? - .take_until(stop.clone()); + swarm_queue_client::ready_when_connected(&health, &queue); + let mut requests = health.beating( + queue + .queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned()) + .await + .with_context(|| format!("subscribing to {SUBJECT}"))? + .take_until(stop.clone()), + ); tracing::info!(subject = SUBJECT, "review classifier: awaiting try-outs"); let models = std::sync::Arc::new(models); diff --git a/swarm-secret-client/Cargo.toml b/swarm-secret-client/Cargo.toml index fd3484b67..a64ec0d9c 100644 --- a/swarm-secret-client/Cargo.toml +++ b/swarm-secret-client/Cargo.toml @@ -4,6 +4,8 @@ version.workspace = true edition.workspace = true [dependencies] +# `health`: the opt-in `/healthz` and `/readyz` listener. +axum.workspace = true # `read_with_startup_retry`'s backoff schedule. backon.workspace = true # `Shared`, so every caller refused while a secret re-read runs waits on that diff --git a/swarm-secret-client/src/health.rs b/swarm-secret-client/src/health.rs new file mode 100644 index 000000000..c9e7d5ce9 --- /dev/null +++ b/swarm-secret-client/src/health.rs @@ -0,0 +1,457 @@ +//! Opt-in liveness and readiness endpoints for a swarm daemon. +//! +//! A daemon that lost its queue connection, or whose main loop is stuck, still +//! looks "running" to systemd and to a container runtime. [`Health`] is the one +//! state a daemon shares with a tiny HTTP listener that says so: +//! +//! - `GET /healthz` is 200 while the daemon's main loop has beaten within +//! `stale_after`, 503 after, and 200 before its first beat, so a long startup is +//! not judged. The loop beats by being polled: wrap its message +//! stream in [`Health::beating`], or call [`Health::beat`] each iteration. A +//! loop stuck in its body stops polling and the beat goes stale, which a +//! constant 200 would not show. A daemon that is a server (no loop of its +//! own) calls [`Health::beat_on_runtime`] instead, which shows the daemon's +//! runtime still schedules tasks, not that the accept loop is healthy. +//! - `GET /readyz` is 200 while the condition set with [`Health::ready_when`] +//! holds, 503 while it does not or none is set yet. +//! +//! The answers carry a status code and no body. Nothing listens unless the +//! operator names an address (see [`Health::start`]), so a daemon that is not +//! given one binds nothing. + +use std::future::Future; +use std::net::SocketAddr; +use std::pin::Pin; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, RwLock}; +use std::task::{Context, Poll}; +use std::time::Duration; + +use axum::extract::State; +use axum::http::StatusCode; +use axum::routing::get; +use futures_util::Stream; +use tokio::time::{Instant, Interval, MissedTickBehavior}; + +/// How often an idle [`Beating`] stream and the runtime beat wake themselves. +pub const BEAT_EVERY: Duration = Duration::from_secs(5); + +/// How long without a beat before `/healthz` answers 503, for a loop that +/// beats at least every [`BEAT_EVERY`]. +pub const STALE_AFTER: Duration = Duration::from_secs(30); + +/// `last_beat` before the main loop has first beaten. Startup, which may wait +/// on the secret store for a long time, is not judged: the saturating +/// subtraction in [`Health::is_alive`] makes the time since this beat zero, so +/// the daemon is alive and not ready. +const NO_BEAT: u64 = u64::MAX; + +type Condition = Box bool + Send + Sync>; + +struct Inner { + born: Instant, + /// Milliseconds after `born` of the last beat, or [`NO_BEAT`]. + last_beat: AtomicU64, + stale_after: Duration, + ready: RwLock>, +} + +/// What a daemon reports and the listener serves. Clones share one state. +#[derive(Clone)] +pub struct Health(Arc); + +impl Health { + /// Alive until the first beat, then judged by the beats; not ready. `stale_after` is how long the main loop may go + /// without a beat before the daemon counts as wedged. + #[must_use] + pub fn new(stale_after: Duration) -> Self { + Self(Arc::new(Inner { + born: Instant::now(), + last_beat: AtomicU64::new(NO_BEAT), + stale_after, + ready: RwLock::new(None), + })) + } + + fn elapsed_ms(&self) -> u64 { + u64::try_from(self.0.born.elapsed().as_millis()).unwrap_or(u64::MAX) + } + + /// Records that the main loop is running. + pub fn beat(&self) { + self.0.last_beat.store(self.elapsed_ms(), Ordering::Relaxed); + } + + /// Sets what `/readyz` checks, replacing an earlier condition. It runs on + /// every probe, so it must not block or do I/O. + pub fn ready_when(&self, condition: impl Fn() -> bool + Send + Sync + 'static) { + *self + .0 + .ready + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Box::new(condition)); + } + + /// Whether the main loop has beaten within `stale_after`. + #[must_use] + pub fn is_alive(&self) -> bool { + let since = self + .elapsed_ms() + .saturating_sub(self.0.last_beat.load(Ordering::Relaxed)); + Duration::from_millis(since) <= self.0.stale_after + } + + /// Whether the readiness condition is set and holds. + #[must_use] + pub fn is_ready(&self) -> bool { + self.0 + .ready + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .is_some_and(|holds| holds()) + } + + /// Wraps the stream a main loop reads from so that each poll beats, and an + /// idle loop wakes itself every [`BEAT_EVERY`] to beat. A loop stuck in + /// the body of its `while let` stops polling, and the beat goes stale. + #[must_use] + pub fn beating(&self, stream: S) -> Beating { + let mut tick = tokio::time::interval(BEAT_EVERY); + tick.set_missed_tick_behavior(MissedTickBehavior::Delay); + Beating { + health: self.clone(), + tick, + inner: stream, + } + } + + /// Beats from a task on the daemon's runtime, for a daemon that is a + /// server and has no loop of its own to beat. Stops beating only when + /// the runtime stops scheduling. + pub fn beat_on_runtime(&self) { + let health = self.clone(); + tokio::spawn(async move { + loop { + health.beat(); + tokio::time::sleep(BEAT_EVERY).await; + } + }); + } + + /// A [`Health`] that serves on the address in `flag`, else in the + /// environment variable `var`. With neither set nothing binds and the + /// daemon behaves as one without this module. The listener stops when + /// `stop` resolves. + /// + /// # Errors + /// + /// [`StartError`] when the address does not parse or cannot be bound: an + /// address the operator asked for and did not get is a startup failure, + /// not a daemon without probes. + pub async fn start( + stale_after: Duration, + flag: Option, + var: &str, + stop: impl Future + Clone + Send + 'static, + ) -> Result { + let (health, _) = Self::start_on(stale_after, addr(flag, var)?, stop).await?; + Ok(health) + } + + /// [`Health::start`] for a daemon with no flag, and the default + /// [`STALE_AFTER`]. + /// + /// # Errors + /// + /// As [`Health::start`]. + pub async fn from_env( + var: &str, + stop: impl Future + Clone + Send + 'static, + ) -> Result { + Self::start(STALE_AFTER, None, var, stop).await + } + + /// [`Health::start`] for an address already resolved; also returns the + /// address bound, `None` when `addr` is. + async fn start_on( + stale_after: Duration, + addr: Option, + stop: impl Future + Clone + Send + 'static, + ) -> Result<(Self, Option), StartError> { + let health = Self::new(stale_after); + let bound = match addr { + Some(addr) => Some( + health + .listen(addr, stop) + .await + .map_err(|source| StartError::Bind { addr, source })?, + ), + None => None, + }; + Ok((health, bound)) + } + + /// Binds `addr` and serves the two endpoints from a task until `stop` + /// resolves, then for at most the drain. Returns the address bound. + async fn listen( + &self, + addr: SocketAddr, + stop: impl Future + Clone + Send + 'static, + ) -> std::io::Result { + let listener = tokio::net::TcpListener::bind(addr).await?; + let bound = listener.local_addr()?; + tracing::info!(addr = %bound, "serving /healthz and /readyz"); + tokio::spawn(self.clone().serve(listener, stop)); + Ok(bound) + } + + /// Serves on `listener` until `stop` resolves, then for at most the drain. + pub async fn serve( + self, + listener: tokio::net::TcpListener, + stop: impl Future + Clone + Send + 'static, + ) { + let app = axum::Router::new() + .route("/healthz", get(healthz)) + .route("/readyz", get(readyz)) + .with_state(self); + let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone()); + let served = + crate::shutdown::serve_bounded(stop, std::future::IntoFuture::into_future(serving)) + .await; + if let Some(Err(e)) = served { + tracing::warn!(error = %e, "the health listener stopped"); + } + } +} + +fn status(up: bool) -> StatusCode { + if up { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } +} + +async fn healthz(State(health): State) -> StatusCode { + status(health.is_alive()) +} + +async fn readyz(State(health): State) -> StatusCode { + status(health.is_ready()) +} + +/// A stream that beats [`Health`] on every poll. See [`Health::beating`]. +pub struct Beating { + health: Health, + tick: Interval, + inner: S, +} + +impl Stream for Beating { + type Item = S::Item; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = &mut *self; + this.health.beat(); + // Registers the wake-up for the next beat; `Delay` makes the second + // poll after a tick Pending, so the loop ends. + while this.tick.poll_tick(cx).is_ready() {} + Pin::new(&mut this.inner).poll_next(cx) + } +} + +/// Why [`Health::start`] could not start the listener. +#[derive(Debug, thiserror::Error)] +pub enum StartError { + #[error("{origin} is not a socket address: {value:?}")] + Addr { origin: String, value: String }, + #[error("binding the health listener on {addr}")] + Bind { + addr: SocketAddr, + #[source] + source: std::io::Error, + }, +} + +/// The address the health listener binds: `flag` when the daemon has a flag +/// and it was given, else the environment variable `var`, else `None`. +/// +/// # Errors +/// +/// [`StartError::Addr`] when `var` is set to something that is not `host:port`. +fn addr(flag: Option, var: &str) -> Result, StartError> { + if flag.is_some() { + return Ok(flag); + } + match std::env::var(var) { + Ok(value) if !value.is_empty() => parse(var, value).map(Some), + _ => Ok(None), + } +} + +fn parse(origin: &str, value: String) -> Result { + value.parse().map_err(|_| StartError::Addr { + origin: origin.to_owned(), + value, + }) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::AtomicBool; + + use futures_util::StreamExt as _; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::sync::oneshot; + + use super::*; + + /// The status code of `GET path` on `addr`, over a bare TCP connection. + pub(crate) async fn get_status(addr: SocketAddr, path: &str) -> u16 { + let mut conn = tokio::net::TcpStream::connect(addr) + .await + .expect("the health listener accepts"); + conn.write_all( + format!("GET {path} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n").as_bytes(), + ) + .await + .expect("sending the request"); + let mut reply = String::new(); + conn.read_to_string(&mut reply).await.expect("the reply"); + reply + .split_whitespace() + .nth(1) + .and_then(|code| code.parse().ok()) + .unwrap_or_else(|| panic!("no status line in {reply:?}")) + } + + async fn listening( + health: &Health, + ) -> (SocketAddr, oneshot::Sender<()>, tokio::task::JoinHandle<()>) { + let (stop_tx, stop_rx) = oneshot::channel::<()>(); + let stop = futures_util::FutureExt::shared(async move { + let _ = stop_rx.await; + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let task = tokio::spawn(health.clone().serve(listener, stop)); + (addr, stop_tx, task) + } + + /// `/readyz` follows the condition, and 503 is what it says before one is + /// set; `/healthz` is 200 while beats are fresh. + #[tokio::test] + async fn readyz_follows_the_condition_and_healthz_the_beat() { + let health = Health::new(Duration::from_mins(1)); + let (addr, _stop, _task) = listening(&health).await; + assert_eq!(get_status(addr, "/healthz").await, 200); + assert_eq!(get_status(addr, "/readyz").await, 503, "no condition set"); + + let up = Arc::new(AtomicBool::new(false)); + let seen = Arc::clone(&up); + health.ready_when(move || seen.load(Ordering::SeqCst)); + assert_eq!(get_status(addr, "/readyz").await, 503, "condition false"); + up.store(true, Ordering::SeqCst); + assert_eq!(get_status(addr, "/readyz").await, 200, "condition true"); + up.store(false, Ordering::SeqCst); + assert_eq!(get_status(addr, "/readyz").await, 503, "back to false"); + assert_eq!(get_status(addr, "/nope").await, 404); + } + + /// `/healthz` goes 503 once the loop stops beating, and 200 again after a + /// beat. + #[tokio::test] + async fn healthz_goes_503_when_the_beat_goes_stale() { + let health = Health::new(Duration::from_millis(300)); + let (addr, _stop, _task) = listening(&health).await; + health.beat(); + assert_eq!(get_status(addr, "/healthz").await, 200); + tokio::time::sleep(Duration::from_millis(700)).await; + assert_eq!(get_status(addr, "/healthz").await, 503, "no beat for 700ms"); + health.beat(); + assert_eq!(get_status(addr, "/healthz").await, 200, "beat again"); + } + + /// The listener stops on the signal: the serving task ends and the port + /// refuses a new connection. + #[tokio::test] + async fn the_listener_stops_on_the_signal() { + let health = Health::new(STALE_AFTER); + let (addr, stop, task) = listening(&health).await; + assert_eq!( + get_status(addr, "/healthz").await, + 200, + "up before the signal" + ); + stop.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(5), task) + .await + .expect("the listener ends on the signal") + .unwrap(); + assert!(tokio::net::TcpStream::connect(addr).await.is_err()); + } + + /// With no address, nothing binds; with one, it does, on the address given. + #[tokio::test] + async fn listen_binds_only_when_given_an_address() { + let (_, none) = Health::start_on(STALE_AFTER, None, std::future::pending()) + .await + .unwrap(); + assert_eq!(none, None); + let (_, bound) = Health::start_on( + STALE_AFTER, + Some("127.0.0.1:0".parse().unwrap()), + std::future::pending(), + ) + .await + .unwrap(); + let bound = bound.expect("a requested address is bound"); + assert_eq!( + get_status(bound, "/healthz").await, + 200, + "control: it serves" + ); + } + + /// A loop that stops polling its beating stream goes stale; one that is + /// idle in `next()` keeps beating on its own. + #[tokio::test(start_paused = true)] + async fn a_beating_stream_beats_while_idle_and_not_while_the_loop_is_stuck() { + let health = Health::new(STALE_AFTER); + tokio::time::sleep(STALE_AFTER * 2).await; + assert!( + health.is_alive(), + "startup is not judged before the first beat" + ); + let (_tx, rx) = tokio::sync::mpsc::channel::(1); + let mut stream = health.beating(tokio_stream_of(rx)); + // Idle in `next()` for far longer than `STALE_AFTER`. + let idle = tokio::time::timeout(STALE_AFTER * 4, stream.next()).await; + assert!(idle.is_err(), "nothing was sent"); + assert!(health.is_alive(), "an idle loop still beats"); + // Stuck: the loop stops polling the stream. + tokio::time::sleep(STALE_AFTER * 2).await; + assert!(!health.is_alive(), "no poll, no beat"); + // Polled again. + let _ = tokio::time::timeout(Duration::from_millis(1), stream.next()).await; + assert!(health.is_alive()); + } + + fn tokio_stream_of( + mut rx: tokio::sync::mpsc::Receiver, + ) -> futures_util::stream::BoxStream<'static, u8> { + futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)).boxed() + } + + #[test] + fn addr_takes_the_flag_else_the_variable_and_names_a_bad_one() { + let flag: SocketAddr = "127.0.0.1:9".parse().unwrap(); + assert_eq!(addr(Some(flag), "HEALTH_TEST_UNSET").unwrap(), Some(flag)); + assert_eq!(addr(None, "HEALTH_TEST_UNSET").unwrap(), None); + assert_eq!(parse("V", "127.0.0.1:9".into()).unwrap(), flag); + let bad = parse("SWARM_X_HEALTH_ADDR", "nonsense".into()).unwrap_err(); + assert!(bad.to_string().contains("SWARM_X_HEALTH_ADDR"), "{bad}"); + } +} diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index d0ea6c07d..5100af2f9 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -45,6 +45,7 @@ pub mod forge; pub mod github; pub mod gitlab; pub mod grafana; +pub mod health; pub mod matrix; #[cfg(feature = "otel")] pub mod meter_provider; diff --git a/swarm-stt/src/main.rs b/swarm-stt/src/main.rs index 90e2af7e6..b9cdbc893 100644 --- a/swarm-stt/src/main.rs +++ b/swarm-stt/src/main.rs @@ -113,6 +113,20 @@ async fn connect_queue(var: Var<'_>) -> Result { .map_err(Into::into) } +/// Removes expired transcripts from `cache` every [`PURGE_EVERY`], for as long as +/// the process runs. +fn spawn_purge(cache: Cache) { + tokio::spawn(async move { + loop { + let removed = cache.purge().await; + if removed > 0 { + tracing::info!(removed, "removed expired transcripts"); + } + tokio::time::sleep(PURGE_EVERY).await; + } + }); +} + #[tokio::main] async fn main() -> Result<()> { let stop = swarm_secret_client::shutdown_signal(); @@ -129,6 +143,11 @@ async fn main() -> Result<()> { Err(_) => Ok(default), } }; + let health = swarm_secret_client::health::Health::from_env( + &format!("{PREFIX}_HEALTH_ADDR"), + stop.clone(), + ) + .await?; let Settings { service, @@ -137,23 +156,18 @@ async fn main() -> Result<()> { } = settings(&var, &number)?; let per_agent: Arc>> = Arc::default(); let queue = connect_queue(&var).await?; + swarm_queue_client::ready_when_connected(&health, &queue); let metrics = Arc::new(Metrics::from_env()); - tokio::spawn(async move { - loop { - let removed = cache.purge().await; - if removed > 0 { - tracing::info!(removed, "removed expired transcripts"); - } - tokio::time::sleep(PURGE_EVERY).await; - } - }); + spawn_purge(cache); - let mut requests = queue - .queue_subscribe(subject_wildcard(), QUEUE_GROUP.to_owned()) - .await - .context("subscribing to the request subject")? - .take_until(stop.clone()); + let mut requests = health.beating( + queue + .queue_subscribe(subject_wildcard(), QUEUE_GROUP.to_owned()) + .await + .context("subscribing to the request subject")? + .take_until(stop.clone()), + ); tracing::info!( subject = subject_wildcard(), "swarm-stt: answering requests"