swarm daemons: opt-in /healthz and /readyz listener, and /readyz on the controller
This commit is contained in:
parent
377bc1d5df
commit
c21b158884
17 changed files with 787 additions and 26 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -5599,6 +5599,7 @@ dependencies = [
|
||||||
name = "swarm-secret-client"
|
name = "swarm-secret-client"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"axum",
|
||||||
"backon",
|
"backon",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"libc",
|
"libc",
|
||||||
|
|
|
||||||
|
|
@ -265,3 +265,38 @@ under a tight sandbox, but its network isn't restricted, which the design page
|
||||||
spells out. The settings, limits and reasons:
|
spells out. The settings, limits and reasons:
|
||||||
[`stt.md`](../../4-implementation/swarm/stt.md) and
|
[`stt.md`](../../4-implementation/swarm/stt.md) and
|
||||||
[Host setup](../../4-implementation/swarm/stt.md#host-setup).
|
[Host setup](../../4-implementation/swarm/stt.md#host-setup).
|
||||||
|
|
||||||
|
### Health endpoints
|
||||||
|
|
||||||
|
The queue daemons (`swarm-stt`, `swarm-review-classifier`,
|
||||||
|
`swarm-notify-router`, `swarm-builder`, `swarm-claude-login`),
|
||||||
|
`swarm-nats-auth` and the three auth responders (`swarm-logs-auth`,
|
||||||
|
`swarm-metrics-auth`, `swarm-authelia-bridge`) can serve two probe endpoints.
|
||||||
|
They answer 200 or 503 and send no body.
|
||||||
|
|
||||||
|
- `GET /healthz` is 200 while the daemon's main loop is running. The loop
|
||||||
|
beats as it takes messages, so a loop that stops polling turns the answer to 503
|
||||||
|
after 30 seconds (5 minutes for `swarm-notify-router`, whose loop handles a
|
||||||
|
batch of events between beats). Before the loop's first beat the answer is
|
||||||
|
200: waiting on the secret store at startup doesn't count as stuck. A
|
||||||
|
daemon that only serves requests (the three auth responders) beats from a task
|
||||||
|
on its runtime, so `/healthz` shows the runtime still runs tasks, not that
|
||||||
|
its accept loop is healthy.
|
||||||
|
- `GET /readyz` is 200 while the daemon has what it needs to work: its queue
|
||||||
|
connection works (`swarm-stt`, `swarm-review-classifier`,
|
||||||
|
`swarm-notify-router`, `swarm-builder`, `swarm-claude-login`,
|
||||||
|
`swarm-nats-auth`), or it has bound its socket (the three auth responders).
|
||||||
|
|
||||||
|
Nothing listens by default. Set `SWARM_STT_HEALTH_ADDR`,
|
||||||
|
`SWARM_REVIEW_CLASSIFIER_HEALTH_ADDR`, `SWARM_NOTIFY_ROUTER_HEALTH_ADDR`,
|
||||||
|
`SWARM_BUILDER_HEALTH_ADDR`, `SWARM_CLAUDE_LOGIN_HEALTH_ADDR` or
|
||||||
|
`SWARM_AUTHELIA_BRIDGE_HEALTH_ADDR` to a `host:port` to bind the listener. For
|
||||||
|
`swarm-nats-auth`, `swarm-logs-auth` and `swarm-metrics-auth`, pass
|
||||||
|
`--health-addr`, or set `SWARM_NATS_AUTH_HEALTH_ADDR`,
|
||||||
|
`SWARM_LOGS_AUTH_HEALTH_ADDR` or `SWARM_METRICS_AUTH_HEALTH_ADDR`; the flag
|
||||||
|
wins. No NixOS module sets one. The listener stops on the same SIGTERM as the
|
||||||
|
daemon.
|
||||||
|
|
||||||
|
`swarm-controller` serves `GET /readyz` on its existing socket, next to
|
||||||
|
`/health`. It answers 200 while the queue connection works and the secret store
|
||||||
|
accepts the controller's login, and 503 otherwise.
|
||||||
|
|
|
||||||
|
|
@ -117,6 +117,12 @@ async fn main() -> Result<()> {
|
||||||
.with_ansi(false)
|
.with_ansi(false)
|
||||||
.init();
|
.init();
|
||||||
|
|
||||||
|
let health = swarm_secret_client::health::Health::from_env(
|
||||||
|
"SWARM_AUTHELIA_BRIDGE_HEALTH_ADDR",
|
||||||
|
stop.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
health.beat_on_runtime();
|
||||||
let config = Config::from_env()?;
|
let config = Config::from_env()?;
|
||||||
let socket = config.socket.clone();
|
let socket = config.socket.clone();
|
||||||
let state = Arc::new(AppState {
|
let state = Arc::new(AppState {
|
||||||
|
|
@ -131,6 +137,7 @@ async fn main() -> Result<()> {
|
||||||
|
|
||||||
let listener = bind_socket(&socket)?;
|
let listener = bind_socket(&socket)?;
|
||||||
tracing::info!(socket = %socket.display(), "swarm-authelia-bridge listening");
|
tracing::info!(socket = %socket.display(), "swarm-authelia-bridge listening");
|
||||||
|
health.ready_when(|| true);
|
||||||
serve(listener, app, stop).await
|
serve(listener, app, stop).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -95,6 +95,11 @@ async fn main() -> Result<()> {
|
||||||
let full = format!("{PREFIX}_{name}");
|
let full = format!("{PREFIX}_{name}");
|
||||||
std::env::var(&full).with_context(|| format!("{full} is unset"))
|
std::env::var(&full).with_context(|| format!("{full} is unset"))
|
||||||
};
|
};
|
||||||
|
let health = swarm_secret_client::health::Health::from_env(
|
||||||
|
&format!("{PREFIX}_HEALTH_ADDR"),
|
||||||
|
stop.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
let client_id = var("OIDC_CLIENT_ID")?;
|
let client_id = var("OIDC_CLIENT_ID")?;
|
||||||
let builder = Arc::new(nix_build::Builder {
|
let builder = Arc::new(nix_build::Builder {
|
||||||
nix: PathBuf::from(var("NIX")?),
|
nix: PathBuf::from(var("NIX")?),
|
||||||
|
|
@ -134,6 +139,7 @@ async fn main() -> Result<()> {
|
||||||
.map(PathBuf::from),
|
.map(PathBuf::from),
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
|
swarm_queue_client::ready_when_connected(&health, &queue);
|
||||||
let requests = queue
|
let requests = queue
|
||||||
.queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned())
|
.queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned())
|
||||||
.await
|
.await
|
||||||
|
|
@ -176,7 +182,7 @@ async fn main() -> Result<()> {
|
||||||
serve_meta_builds(&queue, &publisher, Store, &builder, &slot, &metrics).await?;
|
serve_meta_builds(&queue, &publisher, Store, &builder, &slot, &metrics).await?;
|
||||||
|
|
||||||
answer_builds(
|
answer_builds(
|
||||||
requests.take_until(stop.clone()),
|
health.beating(requests.take_until(stop.clone())),
|
||||||
publisher,
|
publisher,
|
||||||
Store,
|
Store,
|
||||||
builder,
|
builder,
|
||||||
|
|
|
||||||
|
|
@ -107,6 +107,11 @@ async fn main() -> Result<()> {
|
||||||
let full = format!("{PREFIX}_{name}");
|
let full = format!("{PREFIX}_{name}");
|
||||||
std::env::var(&full).with_context(|| format!("{full} is unset"))
|
std::env::var(&full).with_context(|| format!("{full} is unset"))
|
||||||
};
|
};
|
||||||
|
let health = swarm_secret_client::health::Health::from_env(
|
||||||
|
&format!("{PREFIX}_HEALTH_ADDR"),
|
||||||
|
stop.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
let client_id = var("OIDC_CLIENT_ID")?;
|
let client_id = var("OIDC_CLIENT_ID")?;
|
||||||
let metrics = Arc::new(Metrics::from_env());
|
let metrics = Arc::new(Metrics::from_env());
|
||||||
let logins = Arc::new(login::Logins::new(
|
let logins = Arc::new(login::Logins::new(
|
||||||
|
|
@ -134,13 +139,16 @@ async fn main() -> Result<()> {
|
||||||
.map(PathBuf::from),
|
.map(PathBuf::from),
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
|
swarm_queue_client::ready_when_connected(&health, &queue);
|
||||||
// No queue group: a running login lives in this process's memory, so
|
// No queue group: a running login lives in this process's memory, so
|
||||||
// every step for it has to reach this process.
|
// every step for it has to reach this process.
|
||||||
let mut requests = queue
|
let mut requests = health.beating(
|
||||||
.subscribe(SUBJECT)
|
queue
|
||||||
.await
|
.subscribe(SUBJECT)
|
||||||
.with_context(|| format!("subscribing to {SUBJECT}"))?
|
.await
|
||||||
.take_until(stop.clone());
|
.with_context(|| format!("subscribing to {SUBJECT}"))?
|
||||||
|
.take_until(stop.clone()),
|
||||||
|
);
|
||||||
tracing::info!(subject = SUBJECT, "claude login minter: awaiting logins");
|
tracing::info!(subject = SUBJECT, "claude login minter: awaiting logins");
|
||||||
|
|
||||||
let mut in_flight = swarm_secret_client::InFlight::new();
|
let mut in_flight = swarm_secret_client::InFlight::new();
|
||||||
|
|
|
||||||
|
|
@ -7,8 +7,12 @@ pub(crate) mod forge_admin;
|
||||||
pub(crate) mod hives;
|
pub(crate) mod hives;
|
||||||
pub(crate) mod jobq_graph;
|
pub(crate) mod jobq_graph;
|
||||||
|
|
||||||
|
use std::future::Future;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use axum::extract::State;
|
||||||
|
use axum::http::StatusCode;
|
||||||
|
|
||||||
use crate::jobq::{Late, SwarmNodeKind, SwarmResourceKind};
|
use crate::jobq::{Late, SwarmNodeKind, SwarmResourceKind};
|
||||||
use crate::routes::hives::{HiveEntry, ServiceLink};
|
use crate::routes::hives::{HiveEntry, ServiceLink};
|
||||||
use crate::{
|
use crate::{
|
||||||
|
|
@ -28,6 +32,36 @@ pub(crate) async fn health() -> &'static str {
|
||||||
concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n")
|
concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// How long the readiness probe waits for the secret store to answer a login.
|
||||||
|
const STORE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
|
||||||
|
|
||||||
|
/// Readiness probe: 200 when the swarm queue is connected and the secret store
|
||||||
|
/// accepts the controller's login, 503 otherwise. No body.
|
||||||
|
///
|
||||||
|
/// Not in the `OpenAPI` document: it is for the supervisor, not for API clients.
|
||||||
|
pub(crate) async fn readyz(State(state): State<AppState>) -> StatusCode {
|
||||||
|
let queue_connected = state
|
||||||
|
.status
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|status| swarm_queue_client::ensure_connected(&status.queue_client()).is_ok());
|
||||||
|
ready(queue_connected, async {
|
||||||
|
tokio::time::timeout(STORE_PROBE_TIMEOUT, crate::store::connect())
|
||||||
|
.await
|
||||||
|
.is_ok_and(|login| login.is_ok())
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 200 when the queue is connected and `store_reachable` resolves true. The
|
||||||
|
/// store is not asked while the queue is down.
|
||||||
|
async fn ready(queue_connected: bool, store_reachable: impl Future<Output = bool>) -> StatusCode {
|
||||||
|
if queue_connected && store_reachable.await {
|
||||||
|
StatusCode::OK
|
||||||
|
} else {
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub(crate) struct AppState {
|
pub(crate) struct AppState {
|
||||||
/// Loaded once at startup (`load_hives`); never mutated, so an
|
/// Loaded once at startup (`load_hives`); never mutated, so an
|
||||||
|
|
@ -204,6 +238,51 @@ pub(crate) fn wanted_error_status(e: &anyhow::Error) -> axum::http::StatusCode {
|
||||||
pub(crate) mod tests {
|
pub(crate) mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// `/readyz` needs both the queue and the store, and does not ask the store
|
||||||
|
/// while the queue is down.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ready_needs_the_queue_and_the_store() {
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
|
assert_eq!(ready(true, async { true }).await, StatusCode::OK);
|
||||||
|
assert_eq!(
|
||||||
|
ready(true, async { false }).await,
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"store unreachable"
|
||||||
|
);
|
||||||
|
let asked = AtomicBool::new(false);
|
||||||
|
let store = async {
|
||||||
|
asked.store(true, Ordering::SeqCst);
|
||||||
|
true
|
||||||
|
};
|
||||||
|
assert_eq!(ready(false, store).await, StatusCode::SERVICE_UNAVAILABLE);
|
||||||
|
assert!(!asked.load(Ordering::SeqCst), "queue down: store not asked");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Served by the real router: with no queue wired up `/readyz` is 503,
|
||||||
|
/// while `/health` on the same router stays 200.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn the_router_answers_readyz_503_without_a_queue_and_health_200() {
|
||||||
|
use axum::body::Body;
|
||||||
|
use axum::http::Request;
|
||||||
|
use tower::ServiceExt as _;
|
||||||
|
|
||||||
|
let (state, _sched) = state_with_roster();
|
||||||
|
let app = crate::startup::build_app(state);
|
||||||
|
for (path, expected) in [
|
||||||
|
("/readyz", StatusCode::SERVICE_UNAVAILABLE),
|
||||||
|
("/health", StatusCode::OK),
|
||||||
|
] {
|
||||||
|
let status = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(Request::get(path).body(Body::empty()).expect("a request"))
|
||||||
|
.await
|
||||||
|
.expect("answers")
|
||||||
|
.status();
|
||||||
|
assert_eq!(status, expected, "{path}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The 503 this route returns is what the operator UIs' shared error
|
/// The 503 this route returns is what the operator UIs' shared error
|
||||||
/// component renders, so assert the RENDERED response rather than the
|
/// component renders, so assert the RENDERED response rather than the
|
||||||
/// `problem_details` crate: the contract a UI depends on is the content
|
/// `problem_details` crate: the contract a UI depends on is the content
|
||||||
|
|
|
||||||
|
|
@ -771,6 +771,7 @@ pub(crate) fn build_app(state: AppState) -> axum::Router {
|
||||||
// `Clone`; each request gets its own owned copy for `Json` to
|
// `Clone`; each request gets its own owned copy for `Json` to
|
||||||
// serialize, same as `hive-c0re::dashboard::serve`.
|
// serialize, same as `hive-c0re::dashboard::serve`.
|
||||||
router
|
router
|
||||||
|
.route("/readyz", get(routes::readyz))
|
||||||
.route(
|
.route(
|
||||||
"/api/openapi.json",
|
"/api/openapi.json",
|
||||||
get(move || async move { Json(api.clone()) }),
|
get(move || async move { Json(api.clone()) }),
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,10 @@ struct Args {
|
||||||
/// Unix socket to serve on.
|
/// Unix socket to serve on.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
socket: PathBuf,
|
socket: PathBuf,
|
||||||
|
/// Address to serve `/healthz` and `/readyz` on. Nothing listens unless set;
|
||||||
|
/// the environment variable `SWARM_LOGS_AUTH_HEALTH_ADDR` is read when the flag is absent.
|
||||||
|
#[arg(long)]
|
||||||
|
health_addr: Option<std::net::SocketAddr>,
|
||||||
/// Cert-auth role this responder logs in to the store as.
|
/// Cert-auth role this responder logs in to the store as.
|
||||||
#[arg(long, default_value = "swarm-logs-auth")]
|
#[arg(long, default_value = "swarm-logs-auth")]
|
||||||
store_cert_role: String,
|
store_cert_role: String,
|
||||||
|
|
@ -206,6 +210,15 @@ async fn main() -> anyhow::Result<()> {
|
||||||
.with_ansi(false)
|
.with_ansi(false)
|
||||||
.init();
|
.init();
|
||||||
let args = Args::parse();
|
let args = Args::parse();
|
||||||
|
let health = swarm_secret_client::health::Health::start(
|
||||||
|
swarm_secret_client::health::STALE_AFTER,
|
||||||
|
args.health_addr,
|
||||||
|
"SWARM_LOGS_AUTH_HEALTH_ADDR",
|
||||||
|
stop.clone(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.context("starting the health listener")?;
|
||||||
|
health.beat_on_runtime();
|
||||||
|
|
||||||
let settings = swarm_secret_client::client::Settings::from_env()
|
let settings = swarm_secret_client::client::Settings::from_env()
|
||||||
.context("reading the secret store's address and this responder's identity")?;
|
.context("reading the secret store's address and this responder's identity")?;
|
||||||
|
|
@ -234,6 +247,7 @@ async fn main() -> anyhow::Result<()> {
|
||||||
std::fs::set_permissions(&args.socket, std::fs::Permissions::from_mode(0o666))
|
std::fs::set_permissions(&args.socket, std::fs::Permissions::from_mode(0o666))
|
||||||
.with_context(|| format!("chmod {}", args.socket.display()))?;
|
.with_context(|| format!("chmod {}", args.socket.display()))?;
|
||||||
tracing::info!(socket = %args.socket.display(), "swarm-logs-auth listening");
|
tracing::info!(socket = %args.socket.display(), "swarm-logs-auth listening");
|
||||||
|
health.ready_when(|| true);
|
||||||
|
|
||||||
let app = axum::Router::new().fallback(handle).with_state(responder);
|
let app = axum::Router::new().fallback(handle).with_state(responder);
|
||||||
serve(listener, app, stop).await
|
serve(listener, app, stop).await
|
||||||
|
|
|
||||||
|
|
@ -65,6 +65,10 @@ struct Args {
|
||||||
/// Unix socket to serve on.
|
/// Unix socket to serve on.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
socket: PathBuf,
|
socket: PathBuf,
|
||||||
|
/// Address to serve `/healthz` and `/readyz` on. Nothing listens unless set;
|
||||||
|
/// the environment variable `SWARM_METRICS_AUTH_HEALTH_ADDR` is read when the flag is absent.
|
||||||
|
#[arg(long)]
|
||||||
|
health_addr: Option<std::net::SocketAddr>,
|
||||||
/// Cert-auth role this responder logs in to the store as.
|
/// Cert-auth role this responder logs in to the store as.
|
||||||
#[arg(long, default_value = "swarm-metrics-auth")]
|
#[arg(long, default_value = "swarm-metrics-auth")]
|
||||||
store_cert_role: String,
|
store_cert_role: String,
|
||||||
|
|
@ -293,6 +297,15 @@ async fn main() -> anyhow::Result<()> {
|
||||||
.with_ansi(false)
|
.with_ansi(false)
|
||||||
.init();
|
.init();
|
||||||
let args = Args::parse();
|
let args = Args::parse();
|
||||||
|
let health = swarm_secret_client::health::Health::start(
|
||||||
|
swarm_secret_client::health::STALE_AFTER,
|
||||||
|
args.health_addr,
|
||||||
|
"SWARM_METRICS_AUTH_HEALTH_ADDR",
|
||||||
|
stop.clone(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.context("starting the health listener")?;
|
||||||
|
health.beat_on_runtime();
|
||||||
|
|
||||||
let settings = Settings::from_env()
|
let settings = Settings::from_env()
|
||||||
.context("reading the secret store's address and this responder's identity")?;
|
.context("reading the secret store's address and this responder's identity")?;
|
||||||
|
|
@ -321,6 +334,7 @@ async fn main() -> anyhow::Result<()> {
|
||||||
std::fs::set_permissions(&args.socket, std::fs::Permissions::from_mode(0o666))
|
std::fs::set_permissions(&args.socket, std::fs::Permissions::from_mode(0o666))
|
||||||
.with_context(|| format!("chmod {}", args.socket.display()))?;
|
.with_context(|| format!("chmod {}", args.socket.display()))?;
|
||||||
tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening");
|
tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening");
|
||||||
|
health.ready_when(|| true);
|
||||||
|
|
||||||
let app = axum::Router::new().fallback(handle).with_state(responder);
|
let app = axum::Router::new().fallback(handle).with_state(responder);
|
||||||
serve(listener, app, stop).await
|
serve(listener, app, stop).await
|
||||||
|
|
|
||||||
|
|
@ -178,6 +178,11 @@ struct Args {
|
||||||
/// without.
|
/// without.
|
||||||
#[arg(long, default_value = "swarm-nats-auth")]
|
#[arg(long, default_value = "swarm-nats-auth")]
|
||||||
store_cert_role: String,
|
store_cert_role: String,
|
||||||
|
/// Address to serve `/healthz` and `/readyz` on. Nothing listens unless set;
|
||||||
|
/// the environment variable `SWARM_NATS_AUTH_HEALTH_ADDR` is read when the
|
||||||
|
/// flag is absent.
|
||||||
|
#[arg(long)]
|
||||||
|
health_addr: Option<std::net::SocketAddr>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
|
|
@ -240,6 +245,14 @@ async fn serve(args: Args, stop: swarm_secret_client::Shutdown) -> anyhow::Resul
|
||||||
.with_claude_logins(args.claude_login_clients.clone())
|
.with_claude_logins(args.claude_login_clients.clone())
|
||||||
.with_notify_routers(args.notify_router_clients.clone())
|
.with_notify_routers(args.notify_router_clients.clone())
|
||||||
.with_stts(args.stt_clients.clone());
|
.with_stts(args.stt_clients.clone());
|
||||||
|
let health = swarm_secret_client::health::Health::start(
|
||||||
|
swarm_secret_client::health::STALE_AFTER,
|
||||||
|
args.health_addr,
|
||||||
|
"SWARM_NATS_AUTH_HEALTH_ADDR",
|
||||||
|
stop.clone(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.context("starting the health listener")?;
|
||||||
let http = reqwest::Client::new();
|
let http = reqwest::Client::new();
|
||||||
let settings = swarm_secret_client::client::Settings::from_env()
|
let settings = swarm_secret_client::client::Settings::from_env()
|
||||||
.context("reading the secret store's address and this responder's identity")?;
|
.context("reading the secret store's address and this responder's identity")?;
|
||||||
|
|
@ -261,6 +274,7 @@ async fn serve(args: Args, stop: swarm_secret_client::Shutdown) -> anyhow::Resul
|
||||||
.connect(&args.nats_url)
|
.connect(&args.nats_url)
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("connect to {}", args.nats_url))?;
|
.with_context(|| format!("connect to {}", args.nats_url))?;
|
||||||
|
swarm_queue_client::ready_when_connected(&health, &client);
|
||||||
let requests = client
|
let requests = client
|
||||||
.subscribe(AUTH_SUBJECT)
|
.subscribe(AUTH_SUBJECT)
|
||||||
.await
|
.await
|
||||||
|
|
@ -281,7 +295,7 @@ async fn serve(args: Args, stop: swarm_secret_client::Shutdown) -> anyhow::Resul
|
||||||
issuer,
|
issuer,
|
||||||
});
|
});
|
||||||
answer_each(
|
answer_each(
|
||||||
requests.take_until(stop.clone()),
|
health.beating(requests.take_until(stop.clone())),
|
||||||
CONCURRENT_REQUESTS,
|
CONCURRENT_REQUESTS,
|
||||||
move |msg| {
|
move |msg| {
|
||||||
let responder = Arc::clone(&responder);
|
let responder = Arc::clone(&responder);
|
||||||
|
|
|
||||||
|
|
@ -74,6 +74,11 @@ const EVENT_ACK_WAIT: Duration = Duration::from_mins(1);
|
||||||
/// again, when the failed event comes back first.
|
/// again, when the failed event comes back first.
|
||||||
const RETRY_DELAY: Duration = Duration::from_secs(30);
|
const RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
|
/// How long the loop may go without a beat before `/healthz` says it is stuck: a pull
|
||||||
|
/// waits up to [`PULL_WAIT`], and the events it returns are handled one by one
|
||||||
|
/// before the next beat.
|
||||||
|
const LOOP_STALE_AFTER: Duration = Duration::from_mins(5);
|
||||||
|
|
||||||
/// How long the list of agents is reused before it is read again.
|
/// How long the list of agents is reused before it is read again.
|
||||||
const ROSTER_TTL: Duration = Duration::from_mins(1);
|
const ROSTER_TTL: Duration = Duration::from_mins(1);
|
||||||
|
|
||||||
|
|
@ -86,6 +91,14 @@ async fn main() -> Result<()> {
|
||||||
let full = format!("{PREFIX}_{name}");
|
let full = format!("{PREFIX}_{name}");
|
||||||
std::env::var(&full).with_context(|| format!("{full} is unset"))
|
std::env::var(&full).with_context(|| format!("{full} is unset"))
|
||||||
};
|
};
|
||||||
|
let health = swarm_secret_client::health::Health::start(
|
||||||
|
LOOP_STALE_AFTER,
|
||||||
|
None,
|
||||||
|
&format!("{PREFIX}_HEALTH_ADDR"),
|
||||||
|
stop.clone(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.context("starting the health listener")?;
|
||||||
let client_id = var("OIDC_CLIENT_ID")?;
|
let client_id = var("OIDC_CLIENT_ID")?;
|
||||||
let secret =
|
let secret =
|
||||||
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
|
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
|
||||||
|
|
@ -107,6 +120,7 @@ async fn main() -> Result<()> {
|
||||||
.map(PathBuf::from),
|
.map(PathBuf::from),
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
|
swarm_queue_client::ready_when_connected(&health, &queue);
|
||||||
|
|
||||||
let mut router = Router::open(&queue).await?;
|
let mut router = Router::open(&queue).await?;
|
||||||
tokio::spawn(api::serve(queue.clone()));
|
tokio::spawn(api::serve(queue.clone()));
|
||||||
|
|
@ -115,6 +129,7 @@ async fn main() -> Result<()> {
|
||||||
"notify router: routing forge events"
|
"notify router: routing forge events"
|
||||||
);
|
);
|
||||||
while stop.clone().now_or_never().is_none() {
|
while stop.clone().now_or_never().is_none() {
|
||||||
|
health.beat();
|
||||||
let pulled =
|
let pulled =
|
||||||
swarm_secret_client::serve_bounded(stop.clone(), router.pull_once(stop.clone())).await;
|
swarm_secret_client::serve_bounded(stop.clone(), router.pull_once(stop.clone())).await;
|
||||||
match pulled {
|
match pulled {
|
||||||
|
|
|
||||||
|
|
@ -1093,6 +1093,19 @@ pub fn ensure_connected(client: &async_nats::Client) -> Result<(), Error> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Makes `health`'s `/readyz` say whether `client` is connected: the same
|
||||||
|
/// test as [`ensure_connected`], so a client that has never connected, or is
|
||||||
|
/// reconnecting, is not ready. A daemon whose readiness also needs something
|
||||||
|
/// else sets its own condition with `Health::ready_when` and calls
|
||||||
|
/// [`ensure_connected`] in it.
|
||||||
|
pub fn ready_when_connected(
|
||||||
|
health: &swarm_secret_client::health::Health,
|
||||||
|
client: &async_nats::Client,
|
||||||
|
) {
|
||||||
|
let client = client.clone();
|
||||||
|
health.ready_when(move || ensure_connected(&client).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
/// The largest payload this connection's server will accept, as the server
|
/// The largest payload this connection's server will accept, as the server
|
||||||
/// itself announced it at CONNECT.
|
/// itself announced it at CONNECT.
|
||||||
///
|
///
|
||||||
|
|
@ -1932,4 +1945,76 @@ mod tests {
|
||||||
assert_eq!(mints.load(Ordering::SeqCst), 1, "{event}");
|
assert_eq!(mints.load(Ordering::SeqCst), 1, "{event}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The `/readyz` a queue daemon serves, end to end in this process: 503
|
||||||
|
/// while its client has no server to connect to, 200 once one answers.
|
||||||
|
/// The server is a stand-in that speaks just enough of the protocol for
|
||||||
|
/// `async-nats` to count itself connected.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn readyz_is_503_before_the_client_connects_and_200_after() {
|
||||||
|
use swarm_secret_client::health::Health;
|
||||||
|
use tokio::io::{AsyncBufReadExt as _, AsyncReadExt as _, AsyncWriteExt as _, BufReader};
|
||||||
|
|
||||||
|
async fn status(addr: std::net::SocketAddr, path: &str) -> u16 {
|
||||||
|
let mut conn = tokio::net::TcpStream::connect(addr).await.unwrap();
|
||||||
|
let request = format!("GET {path} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n");
|
||||||
|
conn.write_all(request.as_bytes()).await.unwrap();
|
||||||
|
let mut reply = String::new();
|
||||||
|
conn.read_to_string(&mut reply).await.unwrap();
|
||||||
|
reply.split_whitespace().nth(1).unwrap().parse().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
// A port nothing listens on yet.
|
||||||
|
let queue_addr = {
|
||||||
|
let held = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
held.local_addr().unwrap()
|
||||||
|
};
|
||||||
|
let client = async_nats::ConnectOptions::new()
|
||||||
|
.retry_on_initial_connect()
|
||||||
|
.connect(queue_addr.to_string())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let health = Health::new(std::time::Duration::from_mins(1));
|
||||||
|
ready_when_connected(&health, &client);
|
||||||
|
let probes = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let probe_addr = probes.local_addr().unwrap();
|
||||||
|
tokio::spawn(health.clone().serve(probes, std::future::pending()));
|
||||||
|
|
||||||
|
assert_eq!(status(probe_addr, "/readyz").await, 503, "no server yet");
|
||||||
|
assert_eq!(
|
||||||
|
status(probe_addr, "/healthz").await,
|
||||||
|
200,
|
||||||
|
"alive regardless"
|
||||||
|
);
|
||||||
|
|
||||||
|
let server = tokio::net::TcpListener::bind(queue_addr).await.unwrap();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
let (mut conn, _) = server.accept().await.unwrap();
|
||||||
|
let info = r#"INFO {"server_id":"s","server_name":"s","version":"2.10.0","go":"go1","host":"127.0.0.1","port":4222,"headers":true,"max_payload":1048576,"proto":1}"#;
|
||||||
|
conn.write_all(format!("{info}\r\n").as_bytes())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut lines = BufReader::new(conn);
|
||||||
|
let mut line = String::new();
|
||||||
|
while lines.read_line(&mut line).await.unwrap_or(0) > 0 {
|
||||||
|
if line.starts_with("PING") {
|
||||||
|
lines.get_mut().write_all(b"PONG\r\n").await.unwrap();
|
||||||
|
}
|
||||||
|
line.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut ready = 503;
|
||||||
|
for _ in 0..100 {
|
||||||
|
ready = status(probe_addr, "/readyz").await;
|
||||||
|
if ready == 200 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
|
}
|
||||||
|
assert_eq!(ready, 200, "connected");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -54,6 +54,11 @@ async fn serve() -> Result<()> {
|
||||||
let full = format!("{PREFIX}_{name}");
|
let full = format!("{PREFIX}_{name}");
|
||||||
std::env::var(&full).with_context(|| format!("{full} is unset"))
|
std::env::var(&full).with_context(|| format!("{full} is unset"))
|
||||||
};
|
};
|
||||||
|
let health = swarm_secret_client::health::Health::from_env(
|
||||||
|
&format!("{PREFIX}_HEALTH_ADDR"),
|
||||||
|
stop.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
let client_id = var("OIDC_CLIENT_ID")?;
|
let client_id = var("OIDC_CLIENT_ID")?;
|
||||||
let trampoline =
|
let trampoline =
|
||||||
std::env::current_exe().context("finding this binary for the child trampoline")?;
|
std::env::current_exe().context("finding this binary for the child trampoline")?;
|
||||||
|
|
@ -90,11 +95,14 @@ async fn serve() -> Result<()> {
|
||||||
.map(PathBuf::from),
|
.map(PathBuf::from),
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
let mut requests = queue
|
swarm_queue_client::ready_when_connected(&health, &queue);
|
||||||
.queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned())
|
let mut requests = health.beating(
|
||||||
.await
|
queue
|
||||||
.with_context(|| format!("subscribing to {SUBJECT}"))?
|
.queue_subscribe(SUBJECT, QUEUE_GROUP.to_owned())
|
||||||
.take_until(stop.clone());
|
.await
|
||||||
|
.with_context(|| format!("subscribing to {SUBJECT}"))?
|
||||||
|
.take_until(stop.clone()),
|
||||||
|
);
|
||||||
tracing::info!(subject = SUBJECT, "review classifier: awaiting try-outs");
|
tracing::info!(subject = SUBJECT, "review classifier: awaiting try-outs");
|
||||||
|
|
||||||
let models = std::sync::Arc::new(models);
|
let models = std::sync::Arc::new(models);
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,8 @@ version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
# `health`: the opt-in `/healthz` and `/readyz` listener.
|
||||||
|
axum.workspace = true
|
||||||
# `read_with_startup_retry`'s backoff schedule.
|
# `read_with_startup_retry`'s backoff schedule.
|
||||||
backon.workspace = true
|
backon.workspace = true
|
||||||
# `Shared`, so every caller refused while a secret re-read runs waits on that
|
# `Shared`, so every caller refused while a secret re-read runs waits on that
|
||||||
|
|
|
||||||
457
swarm-secret-client/src/health.rs
Normal file
457
swarm-secret-client/src/health.rs
Normal file
|
|
@ -0,0 +1,457 @@
|
||||||
|
//! Opt-in liveness and readiness endpoints for a swarm daemon.
|
||||||
|
//!
|
||||||
|
//! A daemon that lost its queue connection, or whose main loop is stuck, still
|
||||||
|
//! looks "running" to systemd and to a container runtime. [`Health`] is the one
|
||||||
|
//! state a daemon shares with a tiny HTTP listener that says so:
|
||||||
|
//!
|
||||||
|
//! - `GET /healthz` is 200 while the daemon's main loop has beaten within
|
||||||
|
//! `stale_after`, 503 after, and 200 before its first beat, so a long startup is
|
||||||
|
//! not judged. The loop beats by being polled: wrap its message
|
||||||
|
//! stream in [`Health::beating`], or call [`Health::beat`] each iteration. A
|
||||||
|
//! loop stuck in its body stops polling and the beat goes stale, which a
|
||||||
|
//! constant 200 would not show. A daemon that is a server (no loop of its
|
||||||
|
//! own) calls [`Health::beat_on_runtime`] instead, which shows the daemon's
|
||||||
|
//! runtime still schedules tasks, not that the accept loop is healthy.
|
||||||
|
//! - `GET /readyz` is 200 while the condition set with [`Health::ready_when`]
|
||||||
|
//! holds, 503 while it does not or none is set yet.
|
||||||
|
//!
|
||||||
|
//! The answers carry a status code and no body. Nothing listens unless the
|
||||||
|
//! operator names an address (see [`Health::start`]), so a daemon that is not
|
||||||
|
//! given one binds nothing.
|
||||||
|
|
||||||
|
use std::future::Future;
|
||||||
|
use std::net::SocketAddr;
|
||||||
|
use std::pin::Pin;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
use std::sync::{Arc, RwLock};
|
||||||
|
use std::task::{Context, Poll};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use axum::extract::State;
|
||||||
|
use axum::http::StatusCode;
|
||||||
|
use axum::routing::get;
|
||||||
|
use futures_util::Stream;
|
||||||
|
use tokio::time::{Instant, Interval, MissedTickBehavior};
|
||||||
|
|
||||||
|
/// How often an idle [`Beating`] stream and the runtime beat wake themselves.
|
||||||
|
pub const BEAT_EVERY: Duration = Duration::from_secs(5);
|
||||||
|
|
||||||
|
/// How long without a beat before `/healthz` answers 503, for a loop that
|
||||||
|
/// beats at least every [`BEAT_EVERY`].
|
||||||
|
pub const STALE_AFTER: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
|
/// `last_beat` before the main loop has first beaten. Startup, which may wait
|
||||||
|
/// on the secret store for a long time, is not judged: the saturating
|
||||||
|
/// subtraction in [`Health::is_alive`] makes the time since this beat zero, so
|
||||||
|
/// the daemon is alive and not ready.
|
||||||
|
const NO_BEAT: u64 = u64::MAX;
|
||||||
|
|
||||||
|
type Condition = Box<dyn Fn() -> bool + Send + Sync>;
|
||||||
|
|
||||||
|
struct Inner {
|
||||||
|
born: Instant,
|
||||||
|
/// Milliseconds after `born` of the last beat, or [`NO_BEAT`].
|
||||||
|
last_beat: AtomicU64,
|
||||||
|
stale_after: Duration,
|
||||||
|
ready: RwLock<Option<Condition>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What a daemon reports and the listener serves. Clones share one state.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct Health(Arc<Inner>);
|
||||||
|
|
||||||
|
impl Health {
|
||||||
|
/// Alive until the first beat, then judged by the beats; not ready. `stale_after` is how long the main loop may go
|
||||||
|
/// without a beat before the daemon counts as wedged.
|
||||||
|
#[must_use]
|
||||||
|
pub fn new(stale_after: Duration) -> Self {
|
||||||
|
Self(Arc::new(Inner {
|
||||||
|
born: Instant::now(),
|
||||||
|
last_beat: AtomicU64::new(NO_BEAT),
|
||||||
|
stale_after,
|
||||||
|
ready: RwLock::new(None),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn elapsed_ms(&self) -> u64 {
|
||||||
|
u64::try_from(self.0.born.elapsed().as_millis()).unwrap_or(u64::MAX)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records that the main loop is running.
|
||||||
|
pub fn beat(&self) {
|
||||||
|
self.0.last_beat.store(self.elapsed_ms(), Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sets what `/readyz` checks, replacing an earlier condition. It runs on
|
||||||
|
/// every probe, so it must not block or do I/O.
|
||||||
|
pub fn ready_when(&self, condition: impl Fn() -> bool + Send + Sync + 'static) {
|
||||||
|
*self
|
||||||
|
.0
|
||||||
|
.ready
|
||||||
|
.write()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Box::new(condition));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether the main loop has beaten within `stale_after`.
|
||||||
|
#[must_use]
|
||||||
|
pub fn is_alive(&self) -> bool {
|
||||||
|
let since = self
|
||||||
|
.elapsed_ms()
|
||||||
|
.saturating_sub(self.0.last_beat.load(Ordering::Relaxed));
|
||||||
|
Duration::from_millis(since) <= self.0.stale_after
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether the readiness condition is set and holds.
|
||||||
|
#[must_use]
|
||||||
|
pub fn is_ready(&self) -> bool {
|
||||||
|
self.0
|
||||||
|
.ready
|
||||||
|
.read()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|holds| holds())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wraps the stream a main loop reads from so that each poll beats, and an
|
||||||
|
/// idle loop wakes itself every [`BEAT_EVERY`] to beat. A loop stuck in
|
||||||
|
/// the body of its `while let` stops polling, and the beat goes stale.
|
||||||
|
#[must_use]
|
||||||
|
pub fn beating<S: Stream + Unpin>(&self, stream: S) -> Beating<S> {
|
||||||
|
let mut tick = tokio::time::interval(BEAT_EVERY);
|
||||||
|
tick.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
||||||
|
Beating {
|
||||||
|
health: self.clone(),
|
||||||
|
tick,
|
||||||
|
inner: stream,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Beats from a task on the daemon's runtime, for a daemon that is a
|
||||||
|
/// server and has no loop of its own to beat. Stops beating only when
|
||||||
|
/// the runtime stops scheduling.
|
||||||
|
pub fn beat_on_runtime(&self) {
|
||||||
|
let health = self.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
health.beat();
|
||||||
|
tokio::time::sleep(BEAT_EVERY).await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A [`Health`] that serves on the address in `flag`, else in the
|
||||||
|
/// environment variable `var`. With neither set nothing binds and the
|
||||||
|
/// daemon behaves as one without this module. The listener stops when
|
||||||
|
/// `stop` resolves.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// [`StartError`] when the address does not parse or cannot be bound: an
|
||||||
|
/// address the operator asked for and did not get is a startup failure,
|
||||||
|
/// not a daemon without probes.
|
||||||
|
pub async fn start(
|
||||||
|
stale_after: Duration,
|
||||||
|
flag: Option<SocketAddr>,
|
||||||
|
var: &str,
|
||||||
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||||
|
) -> Result<Self, StartError> {
|
||||||
|
let (health, _) = Self::start_on(stale_after, addr(flag, var)?, stop).await?;
|
||||||
|
Ok(health)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`Health::start`] for a daemon with no flag, and the default
|
||||||
|
/// [`STALE_AFTER`].
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// As [`Health::start`].
|
||||||
|
pub async fn from_env(
|
||||||
|
var: &str,
|
||||||
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||||
|
) -> Result<Self, StartError> {
|
||||||
|
Self::start(STALE_AFTER, None, var, stop).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`Health::start`] for an address already resolved; also returns the
|
||||||
|
/// address bound, `None` when `addr` is.
|
||||||
|
async fn start_on(
|
||||||
|
stale_after: Duration,
|
||||||
|
addr: Option<SocketAddr>,
|
||||||
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||||
|
) -> Result<(Self, Option<SocketAddr>), StartError> {
|
||||||
|
let health = Self::new(stale_after);
|
||||||
|
let bound = match addr {
|
||||||
|
Some(addr) => Some(
|
||||||
|
health
|
||||||
|
.listen(addr, stop)
|
||||||
|
.await
|
||||||
|
.map_err(|source| StartError::Bind { addr, source })?,
|
||||||
|
),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
Ok((health, bound))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Binds `addr` and serves the two endpoints from a task until `stop`
|
||||||
|
/// resolves, then for at most the drain. Returns the address bound.
|
||||||
|
async fn listen(
|
||||||
|
&self,
|
||||||
|
addr: SocketAddr,
|
||||||
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||||
|
) -> std::io::Result<SocketAddr> {
|
||||||
|
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||||
|
let bound = listener.local_addr()?;
|
||||||
|
tracing::info!(addr = %bound, "serving /healthz and /readyz");
|
||||||
|
tokio::spawn(self.clone().serve(listener, stop));
|
||||||
|
Ok(bound)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Serves on `listener` until `stop` resolves, then for at most the drain.
|
||||||
|
pub async fn serve(
|
||||||
|
self,
|
||||||
|
listener: tokio::net::TcpListener,
|
||||||
|
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||||
|
) {
|
||||||
|
let app = axum::Router::new()
|
||||||
|
.route("/healthz", get(healthz))
|
||||||
|
.route("/readyz", get(readyz))
|
||||||
|
.with_state(self);
|
||||||
|
let serving = axum::serve(listener, app).with_graceful_shutdown(stop.clone());
|
||||||
|
let served =
|
||||||
|
crate::shutdown::serve_bounded(stop, std::future::IntoFuture::into_future(serving))
|
||||||
|
.await;
|
||||||
|
if let Some(Err(e)) = served {
|
||||||
|
tracing::warn!(error = %e, "the health listener stopped");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn status(up: bool) -> StatusCode {
|
||||||
|
if up {
|
||||||
|
StatusCode::OK
|
||||||
|
} else {
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn healthz(State(health): State<Health>) -> StatusCode {
|
||||||
|
status(health.is_alive())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn readyz(State(health): State<Health>) -> StatusCode {
|
||||||
|
status(health.is_ready())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A stream that beats [`Health`] on every poll. See [`Health::beating`].
|
||||||
|
pub struct Beating<S> {
|
||||||
|
health: Health,
|
||||||
|
tick: Interval,
|
||||||
|
inner: S,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<S: Stream + Unpin> Stream for Beating<S> {
|
||||||
|
type Item = S::Item;
|
||||||
|
|
||||||
|
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<S::Item>> {
|
||||||
|
let this = &mut *self;
|
||||||
|
this.health.beat();
|
||||||
|
// Registers the wake-up for the next beat; `Delay` makes the second
|
||||||
|
// poll after a tick Pending, so the loop ends.
|
||||||
|
while this.tick.poll_tick(cx).is_ready() {}
|
||||||
|
Pin::new(&mut this.inner).poll_next(cx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why [`Health::start`] could not start the listener.
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum StartError {
|
||||||
|
#[error("{origin} is not a socket address: {value:?}")]
|
||||||
|
Addr { origin: String, value: String },
|
||||||
|
#[error("binding the health listener on {addr}")]
|
||||||
|
Bind {
|
||||||
|
addr: SocketAddr,
|
||||||
|
#[source]
|
||||||
|
source: std::io::Error,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The address the health listener binds: `flag` when the daemon has a flag
|
||||||
|
/// and it was given, else the environment variable `var`, else `None`.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// [`StartError::Addr`] when `var` is set to something that is not `host:port`.
|
||||||
|
fn addr(flag: Option<SocketAddr>, var: &str) -> Result<Option<SocketAddr>, StartError> {
|
||||||
|
if flag.is_some() {
|
||||||
|
return Ok(flag);
|
||||||
|
}
|
||||||
|
match std::env::var(var) {
|
||||||
|
Ok(value) if !value.is_empty() => parse(var, value).map(Some),
|
||||||
|
_ => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse(origin: &str, value: String) -> Result<SocketAddr, StartError> {
|
||||||
|
value.parse().map_err(|_| StartError::Addr {
|
||||||
|
origin: origin.to_owned(),
|
||||||
|
value,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::atomic::AtomicBool;
|
||||||
|
|
||||||
|
use futures_util::StreamExt as _;
|
||||||
|
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||||
|
use tokio::sync::oneshot;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The status code of `GET path` on `addr`, over a bare TCP connection.
|
||||||
|
pub(crate) async fn get_status(addr: SocketAddr, path: &str) -> u16 {
|
||||||
|
let mut conn = tokio::net::TcpStream::connect(addr)
|
||||||
|
.await
|
||||||
|
.expect("the health listener accepts");
|
||||||
|
conn.write_all(
|
||||||
|
format!("GET {path} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n").as_bytes(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("sending the request");
|
||||||
|
let mut reply = String::new();
|
||||||
|
conn.read_to_string(&mut reply).await.expect("the reply");
|
||||||
|
reply
|
||||||
|
.split_whitespace()
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|code| code.parse().ok())
|
||||||
|
.unwrap_or_else(|| panic!("no status line in {reply:?}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn listening(
|
||||||
|
health: &Health,
|
||||||
|
) -> (SocketAddr, oneshot::Sender<()>, tokio::task::JoinHandle<()>) {
|
||||||
|
let (stop_tx, stop_rx) = oneshot::channel::<()>();
|
||||||
|
let stop = futures_util::FutureExt::shared(async move {
|
||||||
|
let _ = stop_rx.await;
|
||||||
|
});
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
let task = tokio::spawn(health.clone().serve(listener, stop));
|
||||||
|
(addr, stop_tx, task)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `/readyz` follows the condition, and 503 is what it says before one is
|
||||||
|
/// set; `/healthz` is 200 while beats are fresh.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn readyz_follows_the_condition_and_healthz_the_beat() {
|
||||||
|
let health = Health::new(Duration::from_mins(1));
|
||||||
|
let (addr, _stop, _task) = listening(&health).await;
|
||||||
|
assert_eq!(get_status(addr, "/healthz").await, 200);
|
||||||
|
assert_eq!(get_status(addr, "/readyz").await, 503, "no condition set");
|
||||||
|
|
||||||
|
let up = Arc::new(AtomicBool::new(false));
|
||||||
|
let seen = Arc::clone(&up);
|
||||||
|
health.ready_when(move || seen.load(Ordering::SeqCst));
|
||||||
|
assert_eq!(get_status(addr, "/readyz").await, 503, "condition false");
|
||||||
|
up.store(true, Ordering::SeqCst);
|
||||||
|
assert_eq!(get_status(addr, "/readyz").await, 200, "condition true");
|
||||||
|
up.store(false, Ordering::SeqCst);
|
||||||
|
assert_eq!(get_status(addr, "/readyz").await, 503, "back to false");
|
||||||
|
assert_eq!(get_status(addr, "/nope").await, 404);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `/healthz` goes 503 once the loop stops beating, and 200 again after a
|
||||||
|
/// beat.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn healthz_goes_503_when_the_beat_goes_stale() {
|
||||||
|
let health = Health::new(Duration::from_millis(300));
|
||||||
|
let (addr, _stop, _task) = listening(&health).await;
|
||||||
|
health.beat();
|
||||||
|
assert_eq!(get_status(addr, "/healthz").await, 200);
|
||||||
|
tokio::time::sleep(Duration::from_millis(700)).await;
|
||||||
|
assert_eq!(get_status(addr, "/healthz").await, 503, "no beat for 700ms");
|
||||||
|
health.beat();
|
||||||
|
assert_eq!(get_status(addr, "/healthz").await, 200, "beat again");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The listener stops on the signal: the serving task ends and the port
|
||||||
|
/// refuses a new connection.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn the_listener_stops_on_the_signal() {
|
||||||
|
let health = Health::new(STALE_AFTER);
|
||||||
|
let (addr, stop, task) = listening(&health).await;
|
||||||
|
assert_eq!(
|
||||||
|
get_status(addr, "/healthz").await,
|
||||||
|
200,
|
||||||
|
"up before the signal"
|
||||||
|
);
|
||||||
|
stop.send(()).unwrap();
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), task)
|
||||||
|
.await
|
||||||
|
.expect("the listener ends on the signal")
|
||||||
|
.unwrap();
|
||||||
|
assert!(tokio::net::TcpStream::connect(addr).await.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// With no address, nothing binds; with one, it does, on the address given.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn listen_binds_only_when_given_an_address() {
|
||||||
|
let (_, none) = Health::start_on(STALE_AFTER, None, std::future::pending())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(none, None);
|
||||||
|
let (_, bound) = Health::start_on(
|
||||||
|
STALE_AFTER,
|
||||||
|
Some("127.0.0.1:0".parse().unwrap()),
|
||||||
|
std::future::pending(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let bound = bound.expect("a requested address is bound");
|
||||||
|
assert_eq!(
|
||||||
|
get_status(bound, "/healthz").await,
|
||||||
|
200,
|
||||||
|
"control: it serves"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A loop that stops polling its beating stream goes stale; one that is
|
||||||
|
/// idle in `next()` keeps beating on its own.
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn a_beating_stream_beats_while_idle_and_not_while_the_loop_is_stuck() {
|
||||||
|
let health = Health::new(STALE_AFTER);
|
||||||
|
tokio::time::sleep(STALE_AFTER * 2).await;
|
||||||
|
assert!(
|
||||||
|
health.is_alive(),
|
||||||
|
"startup is not judged before the first beat"
|
||||||
|
);
|
||||||
|
let (_tx, rx) = tokio::sync::mpsc::channel::<u8>(1);
|
||||||
|
let mut stream = health.beating(tokio_stream_of(rx));
|
||||||
|
// Idle in `next()` for far longer than `STALE_AFTER`.
|
||||||
|
let idle = tokio::time::timeout(STALE_AFTER * 4, stream.next()).await;
|
||||||
|
assert!(idle.is_err(), "nothing was sent");
|
||||||
|
assert!(health.is_alive(), "an idle loop still beats");
|
||||||
|
// Stuck: the loop stops polling the stream.
|
||||||
|
tokio::time::sleep(STALE_AFTER * 2).await;
|
||||||
|
assert!(!health.is_alive(), "no poll, no beat");
|
||||||
|
// Polled again.
|
||||||
|
let _ = tokio::time::timeout(Duration::from_millis(1), stream.next()).await;
|
||||||
|
assert!(health.is_alive());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tokio_stream_of(
|
||||||
|
mut rx: tokio::sync::mpsc::Receiver<u8>,
|
||||||
|
) -> futures_util::stream::BoxStream<'static, u8> {
|
||||||
|
futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)).boxed()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn addr_takes_the_flag_else_the_variable_and_names_a_bad_one() {
|
||||||
|
let flag: SocketAddr = "127.0.0.1:9".parse().unwrap();
|
||||||
|
assert_eq!(addr(Some(flag), "HEALTH_TEST_UNSET").unwrap(), Some(flag));
|
||||||
|
assert_eq!(addr(None, "HEALTH_TEST_UNSET").unwrap(), None);
|
||||||
|
assert_eq!(parse("V", "127.0.0.1:9".into()).unwrap(), flag);
|
||||||
|
let bad = parse("SWARM_X_HEALTH_ADDR", "nonsense".into()).unwrap_err();
|
||||||
|
assert!(bad.to_string().contains("SWARM_X_HEALTH_ADDR"), "{bad}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -45,6 +45,7 @@ pub mod forge;
|
||||||
pub mod github;
|
pub mod github;
|
||||||
pub mod gitlab;
|
pub mod gitlab;
|
||||||
pub mod grafana;
|
pub mod grafana;
|
||||||
|
pub mod health;
|
||||||
pub mod matrix;
|
pub mod matrix;
|
||||||
#[cfg(feature = "otel")]
|
#[cfg(feature = "otel")]
|
||||||
pub mod meter_provider;
|
pub mod meter_provider;
|
||||||
|
|
|
||||||
|
|
@ -113,6 +113,20 @@ async fn connect_queue(var: Var<'_>) -> Result<async_nats::Client> {
|
||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Removes expired transcripts from `cache` every [`PURGE_EVERY`], for as long as
|
||||||
|
/// the process runs.
|
||||||
|
fn spawn_purge(cache: Cache) {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
let removed = cache.purge().await;
|
||||||
|
if removed > 0 {
|
||||||
|
tracing::info!(removed, "removed expired transcripts");
|
||||||
|
}
|
||||||
|
tokio::time::sleep(PURGE_EVERY).await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> Result<()> {
|
||||||
let stop = swarm_secret_client::shutdown_signal();
|
let stop = swarm_secret_client::shutdown_signal();
|
||||||
|
|
@ -129,6 +143,11 @@ async fn main() -> Result<()> {
|
||||||
Err(_) => Ok(default),
|
Err(_) => Ok(default),
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let health = swarm_secret_client::health::Health::from_env(
|
||||||
|
&format!("{PREFIX}_HEALTH_ADDR"),
|
||||||
|
stop.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
let Settings {
|
let Settings {
|
||||||
service,
|
service,
|
||||||
|
|
@ -137,23 +156,18 @@ async fn main() -> Result<()> {
|
||||||
} = settings(&var, &number)?;
|
} = settings(&var, &number)?;
|
||||||
let per_agent: Arc<Mutex<HashSet<String>>> = Arc::default();
|
let per_agent: Arc<Mutex<HashSet<String>>> = Arc::default();
|
||||||
let queue = connect_queue(&var).await?;
|
let queue = connect_queue(&var).await?;
|
||||||
|
swarm_queue_client::ready_when_connected(&health, &queue);
|
||||||
let metrics = Arc::new(Metrics::from_env());
|
let metrics = Arc::new(Metrics::from_env());
|
||||||
|
|
||||||
tokio::spawn(async move {
|
spawn_purge(cache);
|
||||||
loop {
|
|
||||||
let removed = cache.purge().await;
|
|
||||||
if removed > 0 {
|
|
||||||
tracing::info!(removed, "removed expired transcripts");
|
|
||||||
}
|
|
||||||
tokio::time::sleep(PURGE_EVERY).await;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let mut requests = queue
|
let mut requests = health.beating(
|
||||||
.queue_subscribe(subject_wildcard(), QUEUE_GROUP.to_owned())
|
queue
|
||||||
.await
|
.queue_subscribe(subject_wildcard(), QUEUE_GROUP.to_owned())
|
||||||
.context("subscribing to the request subject")?
|
.await
|
||||||
.take_until(stop.clone());
|
.context("subscribing to the request subject")?
|
||||||
|
.take_until(stop.clone()),
|
||||||
|
);
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
subject = subject_wildcard(),
|
subject = subject_wildcard(),
|
||||||
"swarm-stt: answering requests"
|
"swarm-stt: answering requests"
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue