swarm-notify-router: export event, todo, review request and consumer metrics
This commit is contained in:
parent
d86ecd68a8
commit
7efcfd0644
1 changed files with 413 additions and 4 deletions
|
|
@ -35,6 +35,7 @@ use std::time::{Duration, Instant};
|
|||
use anyhow::{Context as _, Result, bail};
|
||||
use async_nats::jetstream::{self, consumer::pull, kv, message::PublishMessage};
|
||||
use futures_util::{StreamExt as _, TryStreamExt as _};
|
||||
use metrics::Handled;
|
||||
use swarm_queue_client::{ClientSecret, QueueConfig};
|
||||
use swarm_secret_client::stored_secret::StoredSecret;
|
||||
use swarm_wire::router_api::Kind;
|
||||
|
|
@ -46,6 +47,7 @@ mod dry_run;
|
|||
mod event;
|
||||
mod flow;
|
||||
mod log;
|
||||
mod metrics;
|
||||
mod route;
|
||||
mod settle;
|
||||
mod stored;
|
||||
|
|
@ -111,6 +113,7 @@ async fn main() -> Result<()> {
|
|||
);
|
||||
loop {
|
||||
if let Err(e) = router.pull_once().await {
|
||||
router.metrics.pull_failed();
|
||||
tracing::warn!(
|
||||
error = format!("{e:#}"),
|
||||
"notify router: pulling events failed; retrying"
|
||||
|
|
@ -127,6 +130,7 @@ struct Router {
|
|||
state: kv::Store,
|
||||
flows: stored::Flows,
|
||||
roster: Option<(Instant, BTreeSet<String>)>,
|
||||
metrics: metrics::Metrics,
|
||||
}
|
||||
|
||||
impl Router {
|
||||
|
|
@ -172,11 +176,16 @@ impl Router {
|
|||
state,
|
||||
flows: stored::Flows::builtins(),
|
||||
roster: None,
|
||||
metrics: metrics::Metrics::from_env(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Pull one batch and handle each event in stream order.
|
||||
async fn pull_once(&mut self) -> Result<()> {
|
||||
if let Ok(info) = self.events.info().await {
|
||||
self.metrics
|
||||
.consumer(info.num_pending, info.num_ack_pending, info.num_redelivered);
|
||||
}
|
||||
let batch = self
|
||||
.events
|
||||
.batch()
|
||||
|
|
@ -189,13 +198,27 @@ impl Router {
|
|||
settle::batch(batch, async |m: &jetstream::Message| self.handle(m).await).await
|
||||
}
|
||||
|
||||
/// Handle one event and count how it went.
|
||||
async fn handle(&mut self, message: &jetstream::Message) -> Result<(), Skip> {
|
||||
let started = Instant::now();
|
||||
let result = self.route_event(message).await;
|
||||
let handled = match &result {
|
||||
Ok(handled) => *handled,
|
||||
Err(Skip::Unusable(_)) => Handled::Unusable,
|
||||
Err(Skip::Retry(_)) => Handled::Retried,
|
||||
};
|
||||
let event_type = forge_events::event_type(&message.subject).unwrap_or("unknown");
|
||||
self.metrics.event(event_type, handled, started.elapsed());
|
||||
result.map(|_| ())
|
||||
}
|
||||
|
||||
async fn route_event(&mut self, message: &jetstream::Message) -> Result<Handled, Skip> {
|
||||
let event_type = forge_events::event_type(&message.subject)
|
||||
.ok_or_else(|| Skip::Unusable("not a forge event subject".to_owned()))?
|
||||
.to_owned();
|
||||
let Some(delivery) = event::Delivery::parse(&message.payload) else {
|
||||
// A delivery that names no issue or PR has nothing to route.
|
||||
return Ok(());
|
||||
return Ok(Handled::Ignored);
|
||||
};
|
||||
let seq = message
|
||||
.info()
|
||||
|
|
@ -244,7 +267,11 @@ impl Router {
|
|||
for request in &routed.requests {
|
||||
self.request_review(&cause, request).await;
|
||||
}
|
||||
Ok(())
|
||||
Ok(if flows.is_empty() {
|
||||
Handled::Unmatched
|
||||
} else {
|
||||
Handled::Routed
|
||||
})
|
||||
}
|
||||
|
||||
/// Ask swarm-controller for one review, log what it answered and record
|
||||
|
|
@ -272,6 +299,11 @@ impl Router {
|
|||
},
|
||||
Err(e) => Err(format!("no answer: {e}")),
|
||||
};
|
||||
self.metrics.review_request(match &result {
|
||||
Ok(done) if done == "requested" => "requested",
|
||||
Ok(_) => "skipped",
|
||||
Err(_) => "failed",
|
||||
});
|
||||
log::sink(cause, flow, Kind::RequestReviewer, reviewer, &result);
|
||||
let row = cause.row(flow, &request.node, Kind::RequestReviewer, reviewer, result);
|
||||
log::append(&self.js, &row).await;
|
||||
|
|
@ -281,6 +313,7 @@ impl Router {
|
|||
/// record the delivery, failed or not.
|
||||
async fn queue(&self, seq: u64, cause: &log::Cause, action: &flow::Action) -> Result<()> {
|
||||
let sent = self.publish(seq, action).await;
|
||||
self.metrics.todo(op_name(action.op.op), sent.is_ok());
|
||||
let result = match &sent {
|
||||
Ok(()) => Ok(format!("{} {}", op_name(action.op.op), action.op.key)),
|
||||
Err(e) => Err(format!("{e:#}")),
|
||||
|
|
|
|||
Loading…
Reference in a new issue