alert when a review-classifier run fails
This commit is contained in:
parent
b89ca3955a
commit
c5d601e3a2
1 changed files with 37 additions and 0 deletions
|
|
@ -859,6 +859,24 @@ let
|
|||
&& lib.hasInfix ''_msg:="turn failed"'' expr
|
||||
&& lib.hasInfix "stats by (hive, agent)" expr;
|
||||
}
|
||||
{
|
||||
# A failed classifier run is visible only as this log line, so a
|
||||
# misspelled `_msg` provisions cleanly and never fires. The literal is
|
||||
# checked against the logging call's source.
|
||||
name = "the review-classifier failed-run alert rule matches the classifier's log message";
|
||||
ok =
|
||||
let
|
||||
groups =
|
||||
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
|
||||
rules = lib.concatMap (g: g.rules) groups;
|
||||
rule = lib.findFirst (r: r.uid == "hh-review-classifier-run-failed") null rules;
|
||||
expr = (lib.head rule.data).model.expr;
|
||||
source = builtins.readFile ../../swarm-review-classifier/src/main.rs;
|
||||
in
|
||||
rule != null
|
||||
&& lib.hasInfix ''_msg:"review classification run failed"'' expr
|
||||
&& lib.hasInfix ''"review classification run failed"'' source;
|
||||
}
|
||||
{
|
||||
# The hive-health alerts query metric names that hive-c0re spells in
|
||||
# Rust. A rename on either side provisions cleanly and the rule then
|
||||
|
|
|
|||
Loading…
Reference in a new issue