diff --git a/docs/4-implementation/swarm/review-classifier.md b/docs/4-implementation/swarm/review-classifier.md index 8a73b4b89..95a526f45 100644 --- a/docs/4-implementation/swarm/review-classifier.md +++ b/docs/4-implementation/swarm/review-classifier.md @@ -735,11 +735,13 @@ The classifier logs each run's result to its journal, which ships to VictoriaLogs like every swarm service container's ([`services.md`](../../2-handbook/swarm/services.md#logs-victorialogs)). -A run that ends without a verdict logs `review classification run failed` at -`warn`, with the cause in `F_ERROR`. The Grafana alert -`hh-review-classifier-run-failed` fires on that line and keeps firing for 30 -minutes after the last one. It fires on a failed run, so a dead key with no PRs -arriving stays quiet until the next PR. +A run that ends without a verdict logs a `warn` line: `review classification +run failed` with the cause in `F_ERROR`. A request that doesn't decode, and a +reply the classifier can't encode or publish, each log a line of their own +that names the error. +The Grafana alert `hh-review-classifier-run-failed` fires on any of these and +keeps firing for 30 minutes after the last one. It fires on a failed run, so a +dead key with no PRs arriving stays quiet until the next PR. ## Still open diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index b6af90bd4..d872f05c8 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -1627,18 +1627,20 @@ in for = "0s"; summary = "{{ $labels.hive }}/{{ $labels.agent }} could not connect to the swarm queue and its harness exited; systemd starts it again 30s later"; }) - # `swarm-review-classifier` logs this at `warn` when a run - # ends without a verdict (a rejected or unreadable API key, - # a failed model call), so that PR has no classifier - # review. The cause is in `F_ERROR`. The window is how long - # the rule stays firing after the last failure. + # A run that ends without a verdict leaves that PR with no + # classifier review. `swarm-review-classifier` logs four + # `warn` lines for it: `review classification run failed` + # (a rejected or unreadable API key, a failed model call; + # the cause is in `F_ERROR`), an undecodable request, and a + # reply it could not encode or publish. The window is how + # long the rule stays firing after the last one. (logCountRule { uid = "hh-review-classifier-run-failed"; title = "review classifier run failed"; windowSeconds = 1800; - expr = ''_time:30m _msg:"review classification run failed" | stats count() as failures''; + expr = ''_time:30m (_msg:"review classification run failed" OR _msg:"an undecodable try-out; answering with the error" OR _msg:"encoding a try-out's reply failed" OR _msg:"publishing a try-out's reply failed") | stats count() as failures''; for = "0s"; - summary = "a review-classifier run ended without a verdict, so that PR has no classifier review; the cause is in the F_ERROR field of the log line"; + summary = "a review-classifier run ended without a verdict, so that PR has no classifier review; the log line names the cause (F_ERROR on a failed run)"; }) (logCountRule { uid = "hh-swarm-agent-state-publish-skipped"; diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index a205d0daa..97b7a99eb 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -860,9 +860,9 @@ let && lib.hasInfix "stats by (hive, agent)" expr; } { - # A failed classifier run is visible only as this log line, so a - # misspelled `_msg` provisions cleanly and never fires. The literal is - # checked against the logging call's source. + # A classifier run without a verdict is visible only as these log + # lines, so a misspelled `_msg` provisions cleanly and never fires. + # Each literal is checked against the logging calls' source. name = "the review-classifier failed-run alert rule matches the classifier's log message"; ok = let @@ -872,10 +872,15 @@ let rule = lib.findFirst (r: r.uid == "hh-review-classifier-run-failed") null rules; expr = (lib.head rule.data).model.expr; source = builtins.readFile ../../swarm-review-classifier/src/main.rs; + messages = [ + "review classification run failed" + "an undecodable try-out; answering with the error" + "encoding a try-out's reply failed" + "publishing a try-out's reply failed" + ]; in rule != null - && lib.hasInfix ''_msg:"review classification run failed"'' expr - && lib.hasInfix ''"review classification run failed"'' source; + && lib.all (m: lib.hasInfix "_msg:\"${m}\"" expr && lib.hasInfix "\"${m}\"" source) messages; } { # The hive-health alerts query metric names that hive-c0re spells in