hive-c0re: hive warnings reach the operator as metrics and alerts, not a banner
The dashboard's top-of-page warning banner is gone. The warning registry
stays, because /health/ready, the swarm status view and the OTEL gauges
read it. What it feeds is now alerted on in Grafana:
- hyperhive.hive.warning{kind,level}: one series per raised warning, so an
alert names the cause. Raising a warning also logs one `hive warning
raised` line with the message.
- hyperhive.hive.nix_store.utilization: the used fraction of the nix
store's filesystem, probed in the gauge callback.
- Alerts: hh-hive-degraded on the existing degraded gauge, hh-hive-warning
per kind (disk excluded), hh-host-disk-high at 85% and
hh-host-disk-critical at 95%.
- pending_logins is now a registry warning, refreshed from the container
rescan, so it reaches the per-kind gauge.
Frontend: server_warnings leaves the state snapshot, and the dashboard,
flow and stats pages keep only the sticky chrome wrapper. The shared
hive-warn and WarnBanner components stay, since other views use them.
A module-eval case checks each alert's metric name against the exporter's
source, so a rename on either side fails the build.
This commit is contained in:
iris
parent
013bbe5adc
commit
4dbcf6fa80
35 changed files with 468 additions and 322 deletions
|
|
@ -142,6 +142,78 @@ let
|
|||
annotations = { inherit summary; };
|
||||
};
|
||||
|
||||
# An alert rule that fires while a metric query's latest value is above
|
||||
# `threshold`. `expr` is an instant MetricsQL query; names with dots are
|
||||
# selected as `{__name__="a.b"}`, the way the dashboards do.
|
||||
#
|
||||
# No data is `OK`: a series that is absent means the condition is not
|
||||
# raised (hive-c0re stops reporting a warning once it clears), and a hive
|
||||
# that stopped exporting at all is the log-absence rules' job.
|
||||
metricThresholdRule =
|
||||
{
|
||||
uid,
|
||||
title,
|
||||
windowSeconds ? 600,
|
||||
expr,
|
||||
threshold,
|
||||
for,
|
||||
summary,
|
||||
}:
|
||||
{
|
||||
inherit uid title for;
|
||||
condition = "C";
|
||||
data = [
|
||||
{
|
||||
refId = "A";
|
||||
relativeTimeRange = {
|
||||
from = windowSeconds;
|
||||
to = 0;
|
||||
};
|
||||
inherit datasourceUid;
|
||||
model = {
|
||||
refId = "A";
|
||||
datasource = {
|
||||
type = "prometheus";
|
||||
uid = datasourceUid;
|
||||
};
|
||||
inherit expr;
|
||||
instant = true;
|
||||
range = false;
|
||||
};
|
||||
}
|
||||
{
|
||||
refId = "B";
|
||||
datasourceUid = "__expr__";
|
||||
model = {
|
||||
refId = "B";
|
||||
type = "reduce";
|
||||
expression = "A";
|
||||
reducer = "last";
|
||||
};
|
||||
}
|
||||
{
|
||||
refId = "C";
|
||||
datasourceUid = "__expr__";
|
||||
model = {
|
||||
refId = "C";
|
||||
type = "threshold";
|
||||
expression = "B";
|
||||
conditions = [
|
||||
{
|
||||
evaluator = {
|
||||
type = "gt";
|
||||
params = [ threshold ];
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
];
|
||||
noDataState = "OK";
|
||||
execErrState = "Error";
|
||||
annotations = { inherit summary; };
|
||||
};
|
||||
|
||||
# `logCountRule` inverted: fires while the count is below one. `expr` must
|
||||
# be an ungrouped `stats count()`, which answers 0 rather than no row when
|
||||
# nothing matched; a grouped count drops the group instead, and a dropped
|
||||
|
|
@ -1600,6 +1672,53 @@ in
|
|||
for = "0s";
|
||||
summary = "{{ $labels._HOSTNAME }} detected agent {{ $labels.F_AGENT }}'s container crash (not an operator-initiated stop)";
|
||||
})
|
||||
# hive-c0re's own readiness verdict (`hyperhive.hive.degraded`
|
||||
# in otel_metrics.rs), the same one `/health/ready` gives:
|
||||
# `1` while any `crit`-level warning is raised. The
|
||||
# per-kind rule below says which one. `for` rides out a
|
||||
# restart-time blip.
|
||||
(metricThresholdRule {
|
||||
uid = "hh-hive-degraded";
|
||||
title = "hive degraded";
|
||||
expr = ''max by (hive) ({__name__="hyperhive.hive.degraded"})'';
|
||||
threshold = 0;
|
||||
for = "2m";
|
||||
summary = "hive {{ $labels.hive }} reports itself degraded: a critical warning is raised; the `hh-hive-warning` alert names it";
|
||||
})
|
||||
# One series per raised warning (`hyperhive.hive.warning`),
|
||||
# so each `kind` is its own alert instance. The text is the
|
||||
# `hive warning raised` line in the hive-c0re log. Disk
|
||||
# pressure is left out: it has its own thresholds below, on
|
||||
# the filesystem metric rather than the registry entry.
|
||||
# Sweeps raise their warning only after several misses, so
|
||||
# `for` is short.
|
||||
(metricThresholdRule {
|
||||
uid = "hh-hive-warning";
|
||||
title = "hive warning raised";
|
||||
expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!="disk_pressure"})'';
|
||||
threshold = 0;
|
||||
for = "2m";
|
||||
summary = "hive {{ $labels.hive }} has a {{ $labels.level }} warning raised: {{ $labels.kind }}; the message is the `hive warning raised` line in the hive-c0re log";
|
||||
})
|
||||
# `hyperhive.hive.nix_store.utilization` is the used fraction of the
|
||||
# filesystem holding the nix store, the one that failed CI
|
||||
# with ENOSPC before the GC guardrails.
|
||||
(metricThresholdRule {
|
||||
uid = "hh-host-disk-high";
|
||||
title = "host disk filling";
|
||||
expr = ''max by (hive) ({__name__="hyperhive.hive.nix_store.utilization"})'';
|
||||
threshold = 0.85;
|
||||
for = "10m";
|
||||
summary = "the nix store's filesystem on hive {{ $labels.hive }} is over 85% full; garbage-collect before it runs out of space";
|
||||
})
|
||||
(metricThresholdRule {
|
||||
uid = "hh-host-disk-critical";
|
||||
title = "host disk almost full";
|
||||
expr = ''max by (hive) ({__name__="hyperhive.hive.nix_store.utilization"})'';
|
||||
threshold = 0.95;
|
||||
for = "2m";
|
||||
summary = "the nix store's filesystem on hive {{ $labels.hive }} is over 95% full; garbage-collect now";
|
||||
})
|
||||
]
|
||||
# One rule per declared hive rather than one grouped by
|
||||
# `hive`, so a hive that sends nothing at all still has
|
||||
|
|
|
|||
|
|
@ -498,6 +498,40 @@ let
|
|||
&& lib.hasInfix ''_msg:="turn failed"'' expr
|
||||
&& lib.hasInfix "stats by (hive, agent)" expr;
|
||||
}
|
||||
{
|
||||
# The hive-health alerts query metric names that hive-c0re spells in
|
||||
# Rust. A rename on either side provisions cleanly and the rule then
|
||||
# reads nothing, so it never fires. Each name here is checked against
|
||||
# the exporter's source rather than a copy of it, and each rule must
|
||||
# query the metrics datasource with an instant query.
|
||||
# 🔬 Invert-proof: rename a gauge in `otel_metrics.rs` and this fails.
|
||||
name = "the hive-health alert rules query metrics hive-c0re exports";
|
||||
ok =
|
||||
let
|
||||
groups =
|
||||
grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups;
|
||||
rules = lib.concatMap (g: g.rules) groups;
|
||||
source = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs;
|
||||
expected = {
|
||||
"hh-hive-degraded" = "hyperhive.hive.degraded";
|
||||
"hh-hive-warning" = "hyperhive.hive.warning";
|
||||
"hh-host-disk-high" = "hyperhive.hive.nix_store.utilization";
|
||||
"hh-host-disk-critical" = "hyperhive.hive.nix_store.utilization";
|
||||
};
|
||||
check =
|
||||
uid: metric:
|
||||
let
|
||||
rule = lib.findFirst (r: r.uid == uid) null rules;
|
||||
query = (lib.head rule.data).model;
|
||||
in
|
||||
rule != null
|
||||
&& lib.hasInfix ''{__name__="${metric}"'' query.expr
|
||||
&& lib.hasInfix ''"${metric}"'' source
|
||||
&& query.instant
|
||||
&& (lib.head rule.data).datasourceUid == "swarm-victoriametrics";
|
||||
in
|
||||
lib.all (uid: check uid expected.${uid}) (builtins.attrNames expected);
|
||||
}
|
||||
{
|
||||
# 🩸 The regression this guards: moving bao's metrics scrape into its
|
||||
# own in-container collector made that collector stamp every exported
|
||||
|
|
|
|||
Loading…
Reference in a new issue