mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

the classifier alert also fires on an undecodable request and an unpublished reply

Browse source
This commit is contained in:
atlas 2026-11-11 21:54:03 +01:00 • committed by mara
commit 8847fa48c9
3 changed files with 26 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

12
docs/4-implementation/swarm/review-classifier.md
View file

@ -735,11 +735,13 @@ The classifier logs each run's result to its journal, which ships to
VictoriaLogs like every swarm service container's VictoriaLogs like every swarm service container's
([`services.md`](../../2-handbook/swarm/services.md#logs-victorialogs)). ([`services.md`](../../2-handbook/swarm/services.md#logs-victorialogs)).
A run that ends without a verdict logs `review classification run failed` at A run that ends without a verdict logs a `warn` line: `review classification
`warn`, with the cause in `F_ERROR`. The Grafana alert run failed` with the cause in `F_ERROR`. A request that doesn't decode, and a
`hh-review-classifier-run-failed` fires on that line and keeps firing for 30 reply the classifier can't encode or publish, each log a line of their own
minutes after the last one. It fires on a failed run, so a dead key with no PRs that names the error.
arriving stays quiet until the next PR. The Grafana alert `hh-review-classifier-run-failed` fires on any of these and
keeps firing for 30 minutes after the last one. It fires on a failed run, so a
dead key with no PRs arriving stays quiet until the next PR.
## Still open ## Still open

16
nix/host-modules/swarm-grafana.nix
View file

@ -1627,18 +1627,20 @@ in
for = "0s"; for = "0s";
summary = "{{ $labels.hive }}/{{ $labels.agent }} could not connect to the swarm queue and its harness exited; systemd starts it again 30s later"; summary = "{{ $labels.hive }}/{{ $labels.agent }} could not connect to the swarm queue and its harness exited; systemd starts it again 30s later";
}) })
# `swarm-review-classifier` logs this at `warn` when a run # A run that ends without a verdict leaves that PR with no
# ends without a verdict (a rejected or unreadable API key, # classifier review. `swarm-review-classifier` logs four
# a failed model call), so that PR has no classifier # `warn` lines for it: `review classification run failed`
# review. The cause is in `F_ERROR`. The window is how long # (a rejected or unreadable API key, a failed model call;
# the rule stays firing after the last failure. # the cause is in `F_ERROR`), an undecodable request, and a
# reply it could not encode or publish. The window is how
# long the rule stays firing after the last one.
(logCountRule { (logCountRule {
uid = "hh-review-classifier-run-failed"; uid = "hh-review-classifier-run-failed";
title = "review classifier run failed"; title = "review classifier run failed";
windowSeconds = 1800; windowSeconds = 1800;
expr = ''_time:30m _msg:"review classification run failed" | stats count() as failures''; expr = ''_time:30m (_msg:"review classification run failed" OR _msg:"an undecodable try-out; answering with the error" OR _msg:"encoding a try-out's reply failed" OR _msg:"publishing a try-out's reply failed") | stats count() as failures'';
for = "0s"; for = "0s";
summary = "a review-classifier run ended without a verdict, so that PR has no classifier review; the cause is in the F_ERROR field of the log line"; summary = "a review-classifier run ended without a verdict, so that PR has no classifier review; the log line names the cause (F_ERROR on a failed run)";
}) })
(logCountRule { (logCountRule {
uid = "hh-swarm-agent-state-publish-skipped"; uid = "hh-swarm-agent-state-publish-skipped";

15
nix/module-eval/grafana.nix
View file

@ -860,9 +860,9 @@ let
&& lib.hasInfix "stats by (hive, agent)" expr; && lib.hasInfix "stats by (hive, agent)" expr;
} }
{ {
# A failed classifier run is visible only as this log line, so a # A classifier run without a verdict is visible only as these log
# misspelled `_msg` provisions cleanly and never fires. The literal is # lines, so a misspelled `_msg` provisions cleanly and never fires.
# checked against the logging call's source. # Each literal is checked against the logging calls' source.
name = "the review-classifier failed-run alert rule matches the classifier's log message"; name = "the review-classifier failed-run alert rule matches the classifier's log message";
ok = ok =
let let
@ -872,10 +872,15 @@ let
rule = lib.findFirst (r: r.uid == "hh-review-classifier-run-failed") null rules; rule = lib.findFirst (r: r.uid == "hh-review-classifier-run-failed") null rules;
expr = (lib.head rule.data).model.expr; expr = (lib.head rule.data).model.expr;
source = builtins.readFile ../../swarm-review-classifier/src/main.rs; source = builtins.readFile ../../swarm-review-classifier/src/main.rs;
messages = [
"review classification run failed"
"an undecodable try-out; answering with the error"
"encoding a try-out's reply failed"
"publishing a try-out's reply failed"
];
in in
rule != null rule != null
&& lib.hasInfix ''_msg:"review classification run failed"'' expr && lib.all (m: lib.hasInfix "_msg:\"${m}\"" expr && lib.hasInfix "\"${m}\"" source) messages;
&& lib.hasInfix ''"review classification run failed"'' source;
} }
{ {
# The hive-health alerts query metric names that hive-c0re spells in # The hive-health alerts query metric names that hive-c0re spells in