vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

build a failed meta build once more, and treat a timeout or missing grant as no verdict

Browse source
This commit is contained in:
atlas 2026-10-26 17:53:39 +01:00
commit 99d2ed5b9e
2 changed files with 621 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

13
nix/host-modules/swarm-grafana.nix
View file

@ -1725,6 +1725,19 @@ in
# renew at half their life, so a fraction over 0.8 means
# the renewal has been stuck for days. `hive` and `agent`
# stay in the grouping so the alert names the holder.
# Agents whose deploy the build gate holds back, from the gauge
# swarm-controller exports (`held_deploys.rs`): a build that
# is red, or that keeps getting no answer from the builder.
# Thirty minutes covers a builder restart and the controller's
# ten-minute retries.
(metricThresholdRule {
uid = "hh-deploys-held";
title = "deploys held by the build gate";
expr = ''sum({__name__="swarm.deploys.held"})'';
threshold = 0;
for = "30m";
summary = "swarm-controller is holding back the deploy of one or more agents because the builder's build of internal/meta is red or gets no answer; the failed `build_meta` node on the Jobs page says which";
})
(metricThresholdRule {
uid = "hh-bao-client-cert-lifetime";
title = "bao client certificate past 80% of its lifetime";

26
nix/module-eval/grafana.nix
View file

@ -606,6 +606,32 @@ let
]
&& query.instant;
}
{
# The held-deploys alert fires on any agent held for 30 minutes and
# selects the gauge name `held_deploys.rs` spells in Rust. The disk rule
# is the presence control: it goes through the same lookup.
# 🔬 Invert-proof: change `for = "30m"` or the selector on
# `hh-deploys-held` in swarm-grafana.nix, or `METRIC` in held_deploys.rs.
name = "the held-deploys alert fires after 30 minutes on the gauge swarm-controller exports";
ok =
let
groups =
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
rules = lib.concatMap (g: g.rules) groups;
source = builtins.readFile ../../swarm-controller/src/held_deploys.rs;
rule = uid: lib.findFirst (r: r.uid == uid) null rules;
held = rule "hh-deploys-held";
threshold =
r:
lib.head
(lib.head (lib.findFirst (q: q.refId == "C") null r.data).model.conditions).evaluator.params;
in
threshold held == 0
&& held.for == "30m"
&& threshold (rule "hh-host-disk-high") == 0.85
&& lib.hasInfix ''{__name__="swarm.deploys.held"}'' (lib.head held.data).model.expr
&& lib.hasInfix ''"swarm.deploys.held"'' source;
}
{
# 🩸 The regression this guards: moving bao's metrics scrape into its
# own in-container collector made that collector stamp every exported