mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

alert when a review-classifier run fails

Browse source
This commit is contained in:
atlas 2026-11-11 18:39:44 +01:00
commit c5d601e3a2
3 changed files with 37 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
docs/4-implementation/swarm/review-classifier.md
View file

@ -705,6 +705,12 @@ The classifier logs each run's result to its journal, which ships to
VictoriaLogs like every swarm service container's
([`services.md`](../../2-handbook/swarm/services.md#logs-victorialogs)).
A run that ends without a verdict logs `review classification run failed` at
`warn`, with the cause in `F_ERROR`. The Grafana alert
`hh-review-classifier-run-failed` fires on that line and keeps firing for 30
minutes after the last one. It fires on a failed run, so a dead key with no PRs
arriving stays quiet until the next PR.
## Still open
These get settled, and filled in here, as the work reaches them:

13
nix/host-modules/swarm-grafana.nix
View file

@ -1627,6 +1627,19 @@ in
for = "0s";
summary = "{{ $labels.hive }}/{{ $labels.agent }} could not connect to the swarm queue and its harness exited; systemd starts it again 30s later";
})
# `swarm-review-classifier` logs this at `warn` when a run
# ends without a verdict (a rejected or unreadable API key,
# a failed model call), so that PR has no classifier
# review. The cause is in `F_ERROR`. The window is how long
# the rule stays firing after the last failure.
(logCountRule {
uid = "hh-review-classifier-run-failed";
title = "review classifier run failed";
windowSeconds = 1800;
expr = ''_time:30m _msg:"review classification run failed" | stats count() as failures'';
for = "0s";
summary = "a review-classifier run ended without a verdict, so that PR has no classifier review; the cause is in the F_ERROR field of the log line";
})
(logCountRule {
uid = "hh-swarm-agent-state-publish-skipped";
title = "swarm agent state publish skipped";

18
nix/module-eval/grafana.nix
View file

@ -859,6 +859,24 @@ let
&& lib.hasInfix ''_msg:="turn failed"'' expr
&& lib.hasInfix "stats by (hive, agent)" expr;
}
{
# A failed classifier run is visible only as this log line, so a
# misspelled `_msg` provisions cleanly and never fires. The literal is
# checked against the logging call's source.
name = "the review-classifier failed-run alert rule matches the classifier's log message";
ok =
let
groups =
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
rules = lib.concatMap (g: g.rules) groups;
rule = lib.findFirst (r: r.uid == "hh-review-classifier-run-failed") null rules;
expr = (lib.head rule.data).model.expr;
source = builtins.readFile ../../swarm-review-classifier/src/main.rs;
in
rule != null
&& lib.hasInfix ''_msg:"review classification run failed"'' expr
&& lib.hasInfix ''"review classification run failed"'' source;
}
{
# The hive-health alerts query metric names that hive-c0re spells in
# Rust. A rename on either side provisions cleanly and the rule then