vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

export bao client-cert not_before/not_after gauges and alert past 80% of lifetime

Browse source
This commit is contained in:
atlas 2026-10-25 17:41:47 +01:00 • committed by mara
commit 078696aaee
23 changed files with 604 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

18
nix/host-modules/swarm-grafana.nix
View file

@ -1715,6 +1715,24 @@ in
for = "2m";
summary = "the nix store's filesystem on hive {{ $labels.hive }} is over 95% full; garbage-collect now";
})
# The fraction of a store client certificate's validity
# already used, from the `bao.client_cert.*` gauges the
# services export (`swarm-cert-metrics`). Both gauges of a
# series carry identical labels, so the division matches
# one-to-one on all of them without `on()`/`group_left`.
# `last_over_time` rides out the five-minute cadence of
# the one-shot reporter (`swarm-ci-metrics`). Agent leaves
# renew at half their life, so a fraction over 0.8 means
# the renewal has been stuck for days. `hive` and `agent`
# stay in the grouping so the alert names the holder.
(metricThresholdRule {
uid = "hh-bao-client-cert-lifetime";
title = "bao client certificate past 80% of its lifetime";
expr = ''max by ("service.name", cert, hive, agent) ((time() - last_over_time({__name__="bao.client_cert.not_before"}[15m])) / (last_over_time({__name__="bao.client_cert.not_after"}[15m]) - last_over_time({__name__="bao.client_cert.not_before"}[15m])))'';
threshold = 0.8;
for = "10m";
summary = "the store client certificate {{ $labels.cert }} (hive {{ $labels.hive }}, agent {{ $labels.agent }}) is past 80% of its validity";
})
]
# One rule per declared hive rather than one grouped by
# `hive`, so a hive that sends nothing at all still has

36
nix/module-eval/grafana.nix
View file

@ -570,6 +570,42 @@ let
in
lib.all (uid: check uid expected.${uid}) (builtins.attrNames expected);
}
{
# The cert-lifetime alert fires at a lifetime fraction over 0.8 and
# selects the two gauge names `swarm-secret-client` spells in Rust; a
# rename on either side provisions cleanly and the rule then reads
# nothing. The sibling disk rule is the presence control: it goes through
# the same lookup, so a misspelled attribute path fails both arms.
# 🔬 Invert-proof: change `threshold = 0.8` on `hh-bao-client-cert-lifetime`
# in swarm-grafana.nix, or a name in `swarm-secret-client/src/cert.rs`.
name = "the bao client-cert lifetime alert fires over 0.8 on the gauges the services export";
ok =
let
groups =
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
rules = lib.concatMap (g: g.rules) groups;
source = builtins.readFile ../../swarm-secret-client/src/cert.rs;
threshold =
uid:
let
rule = lib.findFirst (r: r.uid == uid) null rules;
condition = lib.findFirst (q: q.refId == "C") null rule.data;
in
lib.head (lib.head condition.model.conditions).evaluator.params;
query =
(lib.head (lib.findFirst (r: r.uid == "hh-bao-client-cert-lifetime") null rules).data).model;
in
threshold "hh-host-disk-high" == 0.85
&& threshold "hh-bao-client-cert-lifetime" == 0.8
&&
lib.all
(metric: lib.hasInfix ''{__name__="${metric}"'' query.expr && lib.hasInfix ''"${metric}"'' source)
[
"bao.client_cert.not_before"
"bao.client_cert.not_after"
]
&& query.instant;
}
{
# 🩸 The regression this guards: moving bao's metrics scrape into its
# own in-container collector made that collector stamp every exported