build a failed meta build once more, and treat a timeout or missing grant as no verdict
This commit is contained in:
parent
5e64048e53
commit
99d2ed5b9e
1 changed files with 621 additions and 111 deletions
|
|
@ -606,6 +606,32 @@ let
|
|||
]
|
||||
&& query.instant;
|
||||
}
|
||||
{
|
||||
# The held-deploys alert fires on any agent held for 30 minutes and
|
||||
# selects the gauge name `held_deploys.rs` spells in Rust. The disk rule
|
||||
# is the presence control: it goes through the same lookup.
|
||||
# 🔬 Invert-proof: change `for = "30m"` or the selector on
|
||||
# `hh-deploys-held` in swarm-grafana.nix, or `METRIC` in held_deploys.rs.
|
||||
name = "the held-deploys alert fires after 30 minutes on the gauge swarm-controller exports";
|
||||
ok =
|
||||
let
|
||||
groups =
|
||||
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
|
||||
rules = lib.concatMap (g: g.rules) groups;
|
||||
source = builtins.readFile ../../swarm-controller/src/held_deploys.rs;
|
||||
rule = uid: lib.findFirst (r: r.uid == uid) null rules;
|
||||
held = rule "hh-deploys-held";
|
||||
threshold =
|
||||
r:
|
||||
lib.head
|
||||
(lib.head (lib.findFirst (q: q.refId == "C") null r.data).model.conditions).evaluator.params;
|
||||
in
|
||||
threshold held == 0
|
||||
&& held.for == "30m"
|
||||
&& threshold (rule "hh-host-disk-high") == 0.85
|
||||
&& lib.hasInfix ''{__name__="swarm.deploys.held"}'' (lib.head held.data).model.expr
|
||||
&& lib.hasInfix ''"swarm.deploys.held"'' source;
|
||||
}
|
||||
{
|
||||
# 🩸 The regression this guards: moving bao's metrics scrape into its
|
||||
# own in-container collector made that collector stamp every exported
|
||||
|
|
|
|||
Loading…
Reference in a new issue