alert when a review-classifier run fails

docs/4-implementation/swarm/review-classifier.md

@@ -705,6 +705,12 @@ The classifier logs each run's result to its journal, which ships to
 VictoriaLogs like every swarm service container's
 ([`services.md`](../../2-handbook/swarm/services.md#logs-victorialogs)).
 
+A run that ends without a verdict logs `review classification run failed` at
+`warn`, with the cause in `F_ERROR`. The Grafana alert
+`hh-review-classifier-run-failed` fires on that line and keeps firing for 30
+minutes after the last one. It fires on a failed run, so a dead key with no PRs
+arriving stays quiet until the next PR.
+
 ## Still open
 
 These get settled, and filled in here, as the work reaches them:

nix/host-modules/swarm-grafana.nix

@@ -1627,6 +1627,19 @@ in
                       for = "0s";
                       summary = "{{ $labels.hive }}/{{ $labels.agent }} could not connect to the swarm queue and its harness exited; systemd starts it again 30s later";
                     })
+                    # `swarm-review-classifier` logs this at `warn` when a run
+                    # ends without a verdict (a rejected or unreadable API key,
+                    # a failed model call), so that PR has no classifier
+                    # review. The cause is in `F_ERROR`. The window is how long
+                    # the rule stays firing after the last failure.
+                    (logCountRule {
+                      uid = "hh-review-classifier-run-failed";
+                      title = "review classifier run failed";
+                      windowSeconds = 1800;
+                      expr = ''_time:30m _msg:"review classification run failed" | stats count() as failures'';
+                      for = "0s";
+                      summary = "a review-classifier run ended without a verdict, so that PR has no classifier review; the cause is in the F_ERROR field of the log line";
+                    })
                     (logCountRule {
                       uid = "hh-swarm-agent-state-publish-skipped";
                       title = "swarm agent state publish skipped";

nix/module-eval/grafana.nix

@@ -859,6 +859,24 @@ let
         && lib.hasInfix ''_msg:="turn failed"'' expr
         && lib.hasInfix "stats by (hive, agent)" expr;
     }
+    {
+      # A failed classifier run is visible only as this log line, so a
+      # misspelled `_msg` provisions cleanly and never fires. The literal is
+      # checked against the logging call's source.
+      name = "the review-classifier failed-run alert rule matches the classifier's log message";
+      ok =
+        let
+          groups =
+            grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
+          rules = lib.concatMap (g: g.rules) groups;
+          rule = lib.findFirst (r: r.uid == "hh-review-classifier-run-failed") null rules;
+          expr = (lib.head rule.data).model.expr;
+          source = builtins.readFile ../../swarm-review-classifier/src/main.rs;
+        in
+        rule != null
+        && lib.hasInfix ''_msg:"review classification run failed"'' expr
+        && lib.hasInfix ''"review classification run failed"'' source;
+    }
     {
       # The hive-health alerts query metric names that hive-c0re spells in
       # Rust. A rename on either side provisions cleanly and the rule then