swarm daemons: opt-in /healthz and /readyz listener, and /readyz on the controller
This commit is contained in:
parent
377bc1d5df
commit
c21b158884
1 changed files with 787 additions and 26 deletions
|
|
@ -1093,6 +1093,19 @@ pub fn ensure_connected(client: &async_nats::Client) -> Result<(), Error> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Makes `health`'s `/readyz` say whether `client` is connected: the same
|
||||
/// test as [`ensure_connected`], so a client that has never connected, or is
|
||||
/// reconnecting, is not ready. A daemon whose readiness also needs something
|
||||
/// else sets its own condition with `Health::ready_when` and calls
|
||||
/// [`ensure_connected`] in it.
|
||||
pub fn ready_when_connected(
|
||||
health: &swarm_secret_client::health::Health,
|
||||
client: &async_nats::Client,
|
||||
) {
|
||||
let client = client.clone();
|
||||
health.ready_when(move || ensure_connected(&client).is_ok());
|
||||
}
|
||||
|
||||
/// The largest payload this connection's server will accept, as the server
|
||||
/// itself announced it at CONNECT.
|
||||
///
|
||||
|
|
@ -1932,4 +1945,76 @@ mod tests {
|
|||
assert_eq!(mints.load(Ordering::SeqCst), 1, "{event}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The `/readyz` a queue daemon serves, end to end in this process: 503
|
||||
/// while its client has no server to connect to, 200 once one answers.
|
||||
/// The server is a stand-in that speaks just enough of the protocol for
|
||||
/// `async-nats` to count itself connected.
|
||||
#[tokio::test]
|
||||
async fn readyz_is_503_before_the_client_connects_and_200_after() {
|
||||
use swarm_secret_client::health::Health;
|
||||
use tokio::io::{AsyncBufReadExt as _, AsyncReadExt as _, AsyncWriteExt as _, BufReader};
|
||||
|
||||
async fn status(addr: std::net::SocketAddr, path: &str) -> u16 {
|
||||
let mut conn = tokio::net::TcpStream::connect(addr).await.unwrap();
|
||||
let request = format!("GET {path} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n");
|
||||
conn.write_all(request.as_bytes()).await.unwrap();
|
||||
let mut reply = String::new();
|
||||
conn.read_to_string(&mut reply).await.unwrap();
|
||||
reply.split_whitespace().nth(1).unwrap().parse().unwrap()
|
||||
}
|
||||
|
||||
// A port nothing listens on yet.
|
||||
let queue_addr = {
|
||||
let held = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
held.local_addr().unwrap()
|
||||
};
|
||||
let client = async_nats::ConnectOptions::new()
|
||||
.retry_on_initial_connect()
|
||||
.connect(queue_addr.to_string())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let health = Health::new(std::time::Duration::from_mins(1));
|
||||
ready_when_connected(&health, &client);
|
||||
let probes = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let probe_addr = probes.local_addr().unwrap();
|
||||
tokio::spawn(health.clone().serve(probes, std::future::pending()));
|
||||
|
||||
assert_eq!(status(probe_addr, "/readyz").await, 503, "no server yet");
|
||||
assert_eq!(
|
||||
status(probe_addr, "/healthz").await,
|
||||
200,
|
||||
"alive regardless"
|
||||
);
|
||||
|
||||
let server = tokio::net::TcpListener::bind(queue_addr).await.unwrap();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let (mut conn, _) = server.accept().await.unwrap();
|
||||
let info = r#"INFO {"server_id":"s","server_name":"s","version":"2.10.0","go":"go1","host":"127.0.0.1","port":4222,"headers":true,"max_payload":1048576,"proto":1}"#;
|
||||
conn.write_all(format!("{info}\r\n").as_bytes())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut lines = BufReader::new(conn);
|
||||
let mut line = String::new();
|
||||
while lines.read_line(&mut line).await.unwrap_or(0) > 0 {
|
||||
if line.starts_with("PING") {
|
||||
lines.get_mut().write_all(b"PONG\r\n").await.unwrap();
|
||||
}
|
||||
line.clear();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut ready = 503;
|
||||
for _ in 0..100 {
|
||||
ready = status(probe_addr, "/readyz").await;
|
||||
if ready == 200 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
}
|
||||
assert_eq!(ready, 200, "connected");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue