swarm daemons: opt-in /healthz and /readyz listener, and /readyz on the controller
This commit is contained in:
parent
377bc1d5df
commit
c21b158884
1 changed files with 787 additions and 26 deletions
|
|
@ -74,6 +74,11 @@ const EVENT_ACK_WAIT: Duration = Duration::from_mins(1);
|
|||
/// again, when the failed event comes back first.
|
||||
const RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
|
||||
/// How long the loop may go without a beat before `/healthz` says it is stuck: a pull
|
||||
/// waits up to [`PULL_WAIT`], and the events it returns are handled one by one
|
||||
/// before the next beat.
|
||||
const LOOP_STALE_AFTER: Duration = Duration::from_mins(5);
|
||||
|
||||
/// How long the list of agents is reused before it is read again.
|
||||
const ROSTER_TTL: Duration = Duration::from_mins(1);
|
||||
|
||||
|
|
@ -86,6 +91,14 @@ async fn main() -> Result<()> {
|
|||
let full = format!("{PREFIX}_{name}");
|
||||
std::env::var(&full).with_context(|| format!("{full} is unset"))
|
||||
};
|
||||
let health = swarm_secret_client::health::Health::start(
|
||||
LOOP_STALE_AFTER,
|
||||
None,
|
||||
&format!("{PREFIX}_HEALTH_ADDR"),
|
||||
stop.clone(),
|
||||
)
|
||||
.await
|
||||
.context("starting the health listener")?;
|
||||
let client_id = var("OIDC_CLIENT_ID")?;
|
||||
let secret =
|
||||
swarm_secret_client::read_with_startup_retry("reading the queue client secret", || {
|
||||
|
|
@ -107,6 +120,7 @@ async fn main() -> Result<()> {
|
|||
.map(PathBuf::from),
|
||||
})
|
||||
.await?;
|
||||
swarm_queue_client::ready_when_connected(&health, &queue);
|
||||
|
||||
let mut router = Router::open(&queue).await?;
|
||||
tokio::spawn(api::serve(queue.clone()));
|
||||
|
|
@ -115,6 +129,7 @@ async fn main() -> Result<()> {
|
|||
"notify router: routing forge events"
|
||||
);
|
||||
while stop.clone().now_or_never().is_none() {
|
||||
health.beat();
|
||||
let pulled =
|
||||
swarm_secret_client::serve_bounded(stop.clone(), router.pull_once(stop.clone())).await;
|
||||
match pulled {
|
||||
|
|
|
|||
Loading…
Reference in a new issue