diff --git a/docs/4-implementation/swarm/review-classifier.md b/docs/4-implementation/swarm/review-classifier.md index 394b170b8..ee18bd3d3 100644 --- a/docs/4-implementation/swarm/review-classifier.md +++ b/docs/4-implementation/swarm/review-classifier.md @@ -705,6 +705,12 @@ The classifier logs each run's result to its journal, which ships to VictoriaLogs like every swarm service container's ([`services.md`](../../2-handbook/swarm/services.md#logs-victorialogs)). +A run that ends without a verdict logs `review classification run failed` at +`warn`, with the cause in `F_ERROR`. The Grafana alert +`hh-review-classifier-run-failed` fires on that line and keeps firing for 30 +minutes after the last one. It fires on a failed run, so a dead key with no PRs +arriving stays quiet until the next PR. + ## Still open These get settled, and filled in here, as the work reaches them: diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 491606c3f..b6af90bd4 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -1627,6 +1627,19 @@ in for = "0s"; summary = "{{ $labels.hive }}/{{ $labels.agent }} could not connect to the swarm queue and its harness exited; systemd starts it again 30s later"; }) + # `swarm-review-classifier` logs this at `warn` when a run + # ends without a verdict (a rejected or unreadable API key, + # a failed model call), so that PR has no classifier + # review. The cause is in `F_ERROR`. The window is how long + # the rule stays firing after the last failure. + (logCountRule { + uid = "hh-review-classifier-run-failed"; + title = "review classifier run failed"; + windowSeconds = 1800; + expr = ''_time:30m _msg:"review classification run failed" | stats count() as failures''; + for = "0s"; + summary = "a review-classifier run ended without a verdict, so that PR has no classifier review; the cause is in the F_ERROR field of the log line"; + }) (logCountRule { uid = "hh-swarm-agent-state-publish-skipped"; title = "swarm agent state publish skipped"; diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 59a19c587..a205d0daa 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -859,6 +859,24 @@ let && lib.hasInfix ''_msg:="turn failed"'' expr && lib.hasInfix "stats by (hive, agent)" expr; } + { + # A failed classifier run is visible only as this log line, so a + # misspelled `_msg` provisions cleanly and never fires. The literal is + # checked against the logging call's source. + name = "the review-classifier failed-run alert rule matches the classifier's log message"; + ok = + let + groups = + grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups; + rules = lib.concatMap (g: g.rules) groups; + rule = lib.findFirst (r: r.uid == "hh-review-classifier-run-failed") null rules; + expr = (lib.head rule.data).model.expr; + source = builtins.readFile ../../swarm-review-classifier/src/main.rs; + in + rule != null + && lib.hasInfix ''_msg:"review classification run failed"'' expr + && lib.hasInfix ''"review classification run failed"'' source; + } { # The hive-health alerts query metric names that hive-c0re spells in # Rust. A rename on either side provisions cleanly and the rule then