hive-c0re: hive warnings reach the operator as metrics and alerts, not a banner
The dashboard's top-of-page warning banner is gone. The warning registry
stays, because /health/ready, the swarm status view and the OTEL gauges
read it. What it feeds is now alerted on in Grafana:
- hyperhive.hive.warning{kind,level}: one series per raised warning, so an
alert names the cause. Raising a warning also logs one `hive warning
raised` line with the message.
- hyperhive.hive.nix_store.utilization: the used fraction of the nix
store's filesystem, probed in the gauge callback.
- Alerts: hh-hive-degraded on the existing degraded gauge, hh-hive-warning
per kind (disk excluded), hh-host-disk-high at 85% and
hh-host-disk-critical at 95%.
- pending_logins is now a registry warning, refreshed from the container
rescan, so it reaches the per-kind gauge.
Frontend: server_warnings leaves the state snapshot, and the dashboard,
flow and stats pages keep only the sticky chrome wrapper. The shared
hive-warn and WarnBanner components stay, since other views use them.
A module-eval case checks each alert's metric name against the exporter's
source, so a rename on either side fails the build.
This commit is contained in:
parent
013bbe5adc
commit
4dbcf6fa80
35 changed files with 468 additions and 322 deletions
|
|
@ -139,7 +139,7 @@ pub(super) async fn run_node(
|
|||
|
||||
/// Boot-time forge user/token sweep as a DAG node — see
|
||||
/// [`NodeKind::ForgeSweep`]. `forge::ensure_all` already does its own
|
||||
/// per-step error handling and boot-warning banners internally (it's
|
||||
/// per-step error handling and boot warnings internally (it's
|
||||
/// best-effort by design), so this wrapper has nothing left to report;
|
||||
/// it exists purely to make the sweep a visible unit of work.
|
||||
async fn run_forge_sweep() -> Result<()> {
|
||||
|
|
@ -149,7 +149,7 @@ async fn run_forge_sweep() -> Result<()> {
|
|||
|
||||
/// Matrix user/space sweep as a DAG node — see [`NodeKind::MatrixSweep`].
|
||||
/// Every pass reports here, whoever submitted it: as the node's own error on
|
||||
/// the dashboard, and into the debounced banner.
|
||||
/// the dashboard, and into the debounced warning.
|
||||
async fn run_matrix_sweep() -> Result<()> {
|
||||
let ok = tokio::time::timeout(MATRIX_SWEEP_DEADLINE, crate::matrix::ensure_all())
|
||||
.await
|
||||
|
|
@ -164,7 +164,7 @@ async fn run_matrix_sweep() -> Result<()> {
|
|||
health.record_ok();
|
||||
Ok(())
|
||||
} else {
|
||||
health.record_err(matrix_sweep_banner);
|
||||
health.record_err(matrix_sweep_warning);
|
||||
anyhow::bail!("matrix ensure_all: one or more agents failed sync (see logs)")
|
||||
}
|
||||
}
|
||||
|
|
@ -177,7 +177,7 @@ async fn run_matrix_sweep() -> Result<()> {
|
|||
/// [`Resource::MatrixSweep`]: super::resource::Resource::MatrixSweep
|
||||
const MATRIX_SWEEP_DEADLINE: std::time::Duration = std::time::Duration::from_mins(10);
|
||||
|
||||
/// Banner for a matrix sweep that keeps failing. Two misses in a row, so one
|
||||
/// Warning for a matrix sweep that keeps failing. Two misses in a row, so one
|
||||
/// bad pass (homeserver mid-restart, a transient HTTP blip) doesn't flap the
|
||||
/// dashboard; cleared by the next clean pass.
|
||||
fn matrix_sweep_health() -> &'static Mutex<SweepHealth> {
|
||||
|
|
@ -185,7 +185,7 @@ fn matrix_sweep_health() -> &'static Mutex<SweepHealth> {
|
|||
HEALTH.get_or_init(|| Mutex::new(SweepHealth::new("matrix_ensure_all", "warn", 2)))
|
||||
}
|
||||
|
||||
fn matrix_sweep_banner(ctx: sweep_health::SweepFailure) -> String {
|
||||
fn matrix_sweep_warning(ctx: sweep_health::SweepFailure) -> String {
|
||||
let age = ctx.since_last_ok.map_or_else(
|
||||
|| "no success this session".to_owned(),
|
||||
|d| format!("last ok {} ago", sweep_health::fmt_age(d)),
|
||||
|
|
@ -200,7 +200,7 @@ fn matrix_sweep_banner(ctx: sweep_health::SweepFailure) -> String {
|
|||
|
||||
/// `/knowledge` pull as a DAG node — see [`NodeKind::KnowledgePull`]. Every
|
||||
/// pass reports here, whoever submitted it: as the node's own error on the
|
||||
/// dashboard, and into the debounced banner.
|
||||
/// dashboard, and into the debounced warning.
|
||||
async fn run_knowledge_pull(coord: &Arc<Coordinator>) -> Result<()> {
|
||||
let result = tokio::time::timeout(
|
||||
KNOWLEDGE_PULL_DEADLINE,
|
||||
|
|
@ -242,7 +242,7 @@ async fn run_knowledge_pull(coord: &Arc<Coordinator>) -> Result<()> {
|
|||
/// [`Resource::KnowledgeTree`]: super::resource::Resource::KnowledgeTree
|
||||
const KNOWLEDGE_PULL_DEADLINE: std::time::Duration = std::time::Duration::from_mins(5);
|
||||
|
||||
/// Banner for a `/knowledge` pull that keeps failing: three misses in a row,
|
||||
/// Warning for a `/knowledge` pull that keeps failing: three misses in a row,
|
||||
/// cleared by the next successful pull.
|
||||
fn knowledge_pull_health() -> &'static Mutex<SweepHealth> {
|
||||
static HEALTH: OnceLock<Mutex<SweepHealth>> = OnceLock::new();
|
||||
|
|
|
|||
Loading…
Reference in a new issue