diff --git a/docs/integrations/knowledge.md b/docs/integrations/knowledge.md index 37b898f1..866d6189 100644 --- a/docs/integrations/knowledge.md +++ b/docs/integrations/knowledge.md @@ -66,7 +66,7 @@ pull`, so agents see the new content on their next turn. queues a pull on a fixed cadence as a fallback (webhook missed, c0re restarted between pushes). The pull is best-effort — a failure shows as a failed `KnowledgePull` node on the job queue, counts toward - the `knowledge_pull` hive warning, and doesn't affect the rest of the daemon. + the knowledge-pull banner, and doesn't affect the rest of the daemon. Both paths, and the one pull at boot, queue the same `KnowledgePull` job node. It holds the working tree for its duration, so two pulls never run diff --git a/docs/scheduler/coordinator.md b/docs/scheduler/coordinator.md index 774efd48..6b4cbcdb 100644 --- a/docs/scheduler/coordinator.md +++ b/docs/scheduler/coordinator.md @@ -529,8 +529,8 @@ given. Every 30 seconds hive-c0re (`workers::swarm_limits`) reads the key of each agent it has and stores the result in `meta/swarm-resource-limits.json`. Each value passes the validators first. One bad -value or a failed read leaves the file unchanged, and repeated failures raise a hive -warning (the `hyperhive.hive.warning` metric and the `hh-hive-warning` alert). +value or a failed read leaves the file unchanged, and repeated failures raise a +dashboard banner. A hive's queue grant reads every key of the bucket (`$JS.API.DIRECT.GET.KV_agent-limits.>`) and writes none. Scoping it to the diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 9151eb51..66212053 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -444,7 +444,7 @@ whole-swarm spend figure. hive-c0re measures everything above **per agent**, tagged with the hive it -runs in. It measures these five per **hive** instead, and they carry no +runs in. It measures these three per **hive** instead, and they carry no `agent` label — so a hive that hosts no agents still reports, and "this hive is quiet" is distinguishable from "this hive is gone." Select them with @@ -452,41 +452,20 @@ distinguishable from "this hive is gone." Select them with -| Metric | Unit | Kind | Meaning | -| -------------------------------------- | ---- | ----- | -------------------------------------------------------------------------------------------------------------------- | -| `process.uptime` | `s` | gauge | seconds since this hive's `hive-c0re` started exporting; a restart reads as a drop to ~0 | -| `hyperhive.hive.degraded` | `1` | gauge | `1` while the hive reports itself unhealthy — the same verdict `/health/ready` gives and the swarm status view shows | -| `hyperhive.hive.warnings` | `1` | gauge | how many warnings are currently raised, split by a `level` attribute (`warn`, `crit`) | -| `hyperhive.hive.warning` | `1` | gauge | `1` per raised warning, with `kind` and `level` attributes; a cleared warning stops reporting | -| `hyperhive.hive.nix_store.utilization` | `1` | gauge | used fraction (0–1) of the filesystem that holds the nix store; no sample when the probe fails | +| Metric | Unit | Kind | Meaning | +| ------------------------- | ---- | ----- | -------------------------------------------------------------------------------------------------------------------- | +| `process.uptime` | `s` | gauge | seconds since this hive's `hive-c0re` started exporting; a restart reads as a drop to ~0 | +| `hyperhive.hive.degraded` | `1` | gauge | `1` while the hive reports itself unhealthy — the same verdict `/health/ready` gives and the swarm status view shows | +| `hyperhive.hive.warnings` | `1` | gauge | how many warnings are currently raised, split by a `level` attribute (`warn`, `crit`) | -hive-c0re reports both levels of `hyperhive.hive.warnings` every cycle, `0` -included, so a healthy hive is visible as zeros rather than as missing -series. +hive-c0re reports both levels every cycle, `0` included, so a healthy hive is +visible as zeros rather than as missing series. `hyperhive.hive.degraded` is what a dashboard should alert on: it's `hive-c0re`'s own readiness verdict, so it stays in step with `/health/ready` and with what the swarm controller sees. `hyperhive.hive.warnings` is the detail behind it — `warn`-level entries mean "an operator should look" and do -**not** set `degraded`. `hyperhive.hive.warning` names the cause: the `kind` -label is one of a fixed set in the source (for example `disk_pressure`, a -failing sweep such as `swarm_status_publish`, `pending_logins`, -`forge_ensure_core_user`), and the message lives in the -`hive warning raised` line hive-c0re logs when it raises the warning. - -The swarm Grafana (`nix/host-modules/swarm-grafana.nix`) alerts on these: - -| Alert | Fires when | -| ----------------------- | ----------------------------------------------------------------------------------- | -| `hh-hive-degraded` | `hyperhive.hive.degraded` is `1` for 2 minutes | -| `hh-hive-warning` | any `hyperhive.hive.warning` series other than `disk_pressure` exists for 2 minutes | -| `hh-host-disk-high` | `hyperhive.hive.nix_store.utilization` stays above 0.85 for 10 minutes | -| `hh-host-disk-critical` | `hyperhive.hive.nix_store.utilization` stays above 0.95 for 2 minutes | - -The disk alerts replace the old dashboard banner for the nix store, and -`hh-hive-warning` replaces it for every other warning. Disk pressure has its -own pair so the thresholds sit on the metric rather than on the registry -entry. +**not** set `degraded`. Same cadence, transport and resource labels as the container metrics above. diff --git a/docs/swarm/README.md b/docs/swarm/README.md index 6f09cc4f..6c756914 100644 --- a/docs/swarm/README.md +++ b/docs/swarm/README.md @@ -148,8 +148,8 @@ swarm-services certificate. The identity isn't a choice — a hive authenticates as `hive-` and publishes under `hiveName`, the same name that keys `swarm.hives`. -If a hive stops reporting, its `/health/ready` and the `hyperhive.hive.warning` metric are the place to look: a -failure to publish raises a hive warning after three consecutive +If a hive stops reporting, its own dashboard is the place to look: a +failure to publish raises a warning banner there after three consecutive misses. It stays `warn` rather than `crit` on purpose — a hive that can't reach the queue isn't itself unhealthy, so it doesn't start calling itself degraded for being unable to say it's fine. diff --git a/docs/web-ui/dashboard.md b/docs/web-ui/dashboard.md index 0a308514..9ffba8c5 100644 --- a/docs/web-ui/dashboard.md +++ b/docs/web-ui/dashboard.md @@ -31,17 +31,26 @@ dashboard tab strip. K3PT ST4T3 (kept-state entries). - **Banner-thin** (`░▒▓█▓▒░ HYPERHIVE / HIVE-C0RE / WE ARE THE WIRED ░▒▓█▓▒░`) — sits below the tab strip. -- **No warning banner.** No page shows hive-level warnings (host disk - pressure, failing background sweeps, forge boot failures, agents that - need a claude login). They live in a push-based registry - (`hive-c0re`'s `warnings.rs`): any subsystem raises or clears its own - entry through an RAII guard, and each raise logs one - `hive warning raised` line. The operator sees the registry as metrics and - Grafana alerts: see - [Hive-scoped metrics](../scheduler/observability.md#hive-scoped-metrics-hive-c0re). - A container crash is a swarm-level Grafana alert on the crash watcher's - own log line, so a hive with no otel export has no operator-visible - warning signal beyond `/health/ready`. +- **Server-warnings banner** — a generic, sticky top-of-page strip shown + on **every** page (the dashboard + the stand-alone FL0W + and ST4TS pages), injected at the top + of `` by `renderServerWarnings` in `common.js`. Driven by + `state.server_warnings` — a list of `{ kind, level, message }` — and + coloured by `level` (`warn` amber / `crit` red). hive-c0re owns the + threshold + message, so adding a new system warning needs no frontend + change. Warnings are a push-based registry (`hive-c0re`'s + `warnings.rs`): any subsystem raises/clears its own entry via an RAII + guard, and `host_stats::server_warnings()` is just a cheap snapshot of + that registry. Producers today include the host disk-pressure check + (a `statvfs` probe of `/nix`: ≥85% used → `warn`, ≥95% → `crit`, e.g. + `⚠ host nix store N% full (G GiB free) — garbage-collect …`), forge + provisioning/CI-runner boot failures, and an agent-state warning + (`pending_logins`) computed alongside the container snapshot. A + container crash used to have a sibling `agents_crashing` entry here — + removed in favour of a swarm-level grafana alert on the crash + watcher's own log line (`docs/scheduler/observability.md`), so a + hive with no otel export now has no operator-visible crash signal at + all. Hidden when there are no warnings. - **Browser tab title** — `hyperhive // h1ve-c0re` by default; updated to ` / // h1ve-c0re` once `hive_name` / `swarm_name` arrive in the state snapshot. diff --git a/frontend/packages/dashboard/src/common.css b/frontend/packages/dashboard/src/common.css index be07df35..20c59963 100644 --- a/frontend/packages/dashboard/src/common.css +++ b/frontend/packages/dashboard/src/common.css @@ -564,7 +564,7 @@ hive-side-panel .md td { Inverted-gutter layout: the dashboard keeps `` full-bleed and wraps their padded content in a single `.page-content` div that carries the horizontal gutter. Full-width chrome (the tab strip, the - footer) then spans edge-to-edge for free, + server-warnings banner, the footer) then spans edge-to-edge for free, with no negative-margin breakout hacks. Only matches on pages that opt in by adding the wrapper (index.html); FL0W / ST4TS are full-bleed with their own `.-main` padding. */ @@ -573,14 +573,50 @@ hive-side-panel .md td { } /* ─── sticky top region ─────────────────────────────────────────────── - One sticky container per page holding the page chrome (tab bar / page - header). common.js (initStickyTop) wraps the chrome in this on load. */ + One sticky container per page holding the warning banner above the + page chrome (tab bar / page header). common.js (ensureStickyTop) + wraps the page's chrome in this on first render and injects the banner + as its first child, so the banner + chrome stack in a single sticky + context — pinned together — instead of two separate `top:0` stickies + colliding (the banner used to overlay the tab bar). */ .sticky-top { position: sticky; top: 0; z-index: 50; } +/* ─── server warnings banner ────────────────────────────────────────── + Generic top-of-page strip, injected as the first child of `.sticky-top` + by common.js (renderServerWarnings) on every page. One row per warning; + colour comes from the per-warning `level` (warn = amber, crit = red). + Stickiness lives on the `.sticky-top` wrapper, not here. The bar spans + the full width edge-to-edge; the dashboard is full-bleed at the + body level (their 1.5em gutter lives on an inner `.page-content` + wrapper), so the banner is full-width for free. */ +.server-warnings[hidden] { + display: none; +} +.server-warn { + text-align: center; + padding: 0.4em 1em; + font-size: 0.85em; + font-weight: bold; + letter-spacing: 0.02em; +} +/* Tint composited over the opaque elevated-surface grey (not transparent): + the bar is `position: sticky`, so a see-through background would let + page content scroll through behind it. */ +.server-warn-warn { + color: var(--amber); + background: color-mix(in srgb, var(--amber) 16%, var(--bg-elev)); + border-bottom: 1px solid var(--amber); +} +.server-warn-crit { + color: var(--red); + background: color-mix(in srgb, var(--red) 18%, var(--bg-elev)); + border-bottom: 1px solid var(--red); +} + /* ─── stats table ───────────────────────────────────────────────── Shared right-aligned numeric table. Used by /stats.html (busiest agents) — moved here from dashboard.css when ST4TS became its own diff --git a/frontend/packages/dashboard/src/common.js b/frontend/packages/dashboard/src/common.js index 3d1449fd..9052cf38 100644 --- a/frontend/packages/dashboard/src/common.js +++ b/frontend/packages/dashboard/src/common.js @@ -562,12 +562,22 @@ export function appendLinkified(parent, text, refs) { } } -// ─── sticky top region ─────────────────────────────────────────────── -// One sticky container wrapping the page's chrome (tab bar / page -// header), built once by wrapping the existing chrome element. A page -// without a chrome (none today) gets an empty sticky region at the top of -// . -export function initStickyTop() { +// ─── server warnings banner ────────────────────────────────────────── +// A generic top-of-page banner shown on every page (dashboard + the +// stand-alone FL0W / ST4TS pages). The backend decides what to +// warn about — `/api/state.server_warnings` is a list of +// `{ kind, level, message }` — and this just renders it, coloured by +// `level` (`warn` amber / `crit` red). Adding a new system warning is a +// backend-only change. The bar is injected at the top of so no +// page needs to add markup. + +// The sticky top region — one sticky container holding the warning +// banner above the page's chrome (tab bar / page header), so the banner +// stacks with the chrome instead of being overlaid by it (two separate +// `top:0` stickies would otherwise collide). Built once by wrapping the +// page's existing chrome element; pages without a chrome (none today) +// get a banner-only sticky region at the top of . +function ensureStickyTop() { let top = document.querySelector(".sticky-top"); if (top) return top; top = document.createElement("div"); @@ -581,3 +591,51 @@ export function initStickyTop() { } return top; } + +function ensureServerWarningsBar() { + let bar = document.getElementById("server-warnings"); + if (!bar) { + bar = document.createElement("div"); + bar.id = "server-warnings"; + bar.className = "server-warnings"; + bar.setAttribute("role", "alert"); + bar.hidden = true; + ensureStickyTop().prepend(bar); + } + return bar; +} + +/// Render a `server_warnings` list (from /api/state) into the banner. +/// Empty / missing → the bar hides itself. +export function renderServerWarnings(warnings) { + const bar = ensureServerWarningsBar(); + bar.replaceChildren(); + if (!Array.isArray(warnings) || warnings.length === 0) { + bar.hidden = true; + return; + } + for (const w of warnings) { + const row = el("div", { + class: + "server-warn server-warn-" + + (w && w.level === "crit" ? "crit" : "warn"), + }); + appendText(row, "⚠ " + ((w && w.message) || "")); + bar.append(row); + } + bar.hidden = false; +} + +/// One-shot init for pages that don't otherwise poll /api/state: ensure +/// the bar exists, fetch the snapshot once, render. The dashboard (which +/// already polls /api/state) calls `renderServerWarnings` directly for +/// live updates instead. +export function initServerWarnings() { + ensureServerWarningsBar(); + fetch("/api/state") + .then((r) => (r.ok ? r.json() : null)) + .then((s) => renderServerWarnings(s && s.server_warnings)) + .catch(() => { + /* non-fatal: no banner if the snapshot is unreachable */ + }); +} diff --git a/frontend/packages/dashboard/src/dashboard.css b/frontend/packages/dashboard/src/dashboard.css index 9dae6251..f50067d2 100644 --- a/frontend/packages/dashboard/src/dashboard.css +++ b/frontend/packages/dashboard/src/dashboard.css @@ -17,7 +17,7 @@ body.dashboard-shell { /* Full-bleed body (no horizontal gutter). The 1.5em side gutter lives on the inner `.page-content` wrapper (see common.css), so the sticky - chrome bar and the footer span the full + chrome bar, the server-warnings banner, and the footer span the full width edge-to-edge with no negative-margin breakout. `padding-bottom` keeps a little breathing room above the viewport edge (overridden to clear the selection bar via `.has-selection` below). No max-width cap @@ -34,7 +34,8 @@ body.dashboard-shell { align-items: center; gap: 0.75em; /* Stickiness lives on the `.sticky-top` wrapper (common.css) that - common.js wraps this chrome in. */ + common.js wraps this chrome in, alongside the server-warning banner, + so they stack in one sticky context instead of colliding. */ background: color-mix(in srgb, var(--bg) 86%, transparent); -webkit-backdrop-filter: blur(8px) saturate(120%); backdrop-filter: blur(8px) saturate(120%); diff --git a/frontend/packages/dashboard/src/flow.js b/frontend/packages/dashboard/src/flow.js index c721f695..b89cd78d 100644 --- a/frontend/packages/dashboard/src/flow.js +++ b/frontend/packages/dashboard/src/flow.js @@ -10,12 +10,17 @@ // container list). import { create as termCreate } from "@hive/shared/terminal.js"; -import { $, appendLinkified, openStream, initStickyTop } from "./common.js"; +import { + $, + appendLinkified, + openStream, + initServerWarnings, +} from "./common.js"; import { el } from "@hive/shared/dom.js"; import { epochSec } from "./util.js"; (() => { - initStickyTop(); + initServerWarnings(); // ─── local containers cache (for compose autocomplete) ────────────────── // The compose box's @-mention completion suggests known agent names. diff --git a/frontend/packages/dashboard/src/stats.js b/frontend/packages/dashboard/src/stats.js index 93b9f118..c41faa7b 100644 --- a/frontend/packages/dashboard/src/stats.js +++ b/frontend/packages/dashboard/src/stats.js @@ -5,7 +5,7 @@ // this bundle has no chart lib; per-agent trend charts live on each // agent's own /stats page. The window selector is a hash-routed // createTabStrip (#1h / #24h / …), matching the per-agent stats page. -import { $, initStickyTop } from "./common.js"; +import { $, initServerWarnings } from "./common.js"; import { createTabStrip } from "@hive/shared/tabs.js"; let hiveStatsWindow = "24h"; @@ -173,7 +173,7 @@ async function refreshHiveStats() { } } -initStickyTop(); +initServerWarnings(); // Window selector → hash-routed createTabStrip (#1h / #24h / …). No panes // here (the selector swaps data, not DOM); paneFor() returns null for each diff --git a/frontend/packages/dashboard/src/tabs.js b/frontend/packages/dashboard/src/tabs.js index efae029d..7f267b41 100644 --- a/frontend/packages/dashboard/src/tabs.js +++ b/frontend/packages/dashboard/src/tabs.js @@ -11,7 +11,7 @@ // 4G3NTS (containers) domain lives in `./agents.js`. import { marked } from "marked"; -import { $, openStream, initStickyTop } from "./common.js"; +import { $, openStream, renderServerWarnings } from "./common.js"; import { el } from "@hive/shared/dom.js"; import { bindAsyncForms } from "@hive/shared/forms.js"; import { createTabStrip } from "@hive/shared/tabs.js"; @@ -50,8 +50,6 @@ import { renderLinksMenu } from "./links-menu.js"; window.marked = marked; (() => { - initStickyTop(); - // ─── async forms ──────────────────────────────────────────────────────── // Shared `data-async` submit interceptor (in common.js, shared with the other // pages' bundles). On success it re-runs refreshState unless the form @@ -137,6 +135,7 @@ window.marked = marked; // synchronous read (e.g. the compose autocomplete pulls agent // names from here instead of refetching on every keystroke). window.__hyperhive_state = s; + renderServerWarnings(s.server_warnings); // Hive identity: render the swarm/hive name as a headline at the // top of the 4G3NTS pane + update the page title once the // server-side display names are known. `hive_name` / `swarm_name` diff --git a/frontend/packages/shared/src/chrome.css b/frontend/packages/shared/src/chrome.css index 03e50650..8146b297 100644 --- a/frontend/packages/shared/src/chrome.css +++ b/frontend/packages/shared/src/chrome.css @@ -16,7 +16,8 @@ */ .page-header { /* Stickiness lives on the `.sticky-top` wrapper (common.css) that - common.js wraps this header in. */ + common.js wraps this header in, alongside the server-warning banner, + so the two stack instead of colliding. */ background: color-mix(in srgb, var(--bg) 92%, transparent); -webkit-backdrop-filter: blur(8px) saturate(120%); backdrop-filter: blur(8px) saturate(120%); diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index d752f947..cc9c0bb1 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -599,7 +599,6 @@ impl Coordinator { return; } }; - crate::host_stats::refresh_pending_logins(&fresh); let mut last = self.last_containers.lock().await; let mut changed_or_new = Vec::new(); let mut removed = Vec::new(); diff --git a/hive-c0re/src/dashboard/health.rs b/hive-c0re/src/dashboard/health.rs index def12c67..32a669e5 100644 --- a/hive-c0re/src/dashboard/health.rs +++ b/hive-c0re/src/dashboard/health.rs @@ -1,8 +1,9 @@ //! Hive-wide health endpoints for external monitoring: `/health/live` //! (liveness) and `/health/ready` (readiness). //! -//! Deliberately thin: reuses [`crate::warnings`] as the single source of truth -//! for "is anything degraded", rather than tracking health separately. Two +//! Deliberately thin: reuses [`crate::warnings`] — the same registry the +//! dashboard banner already reads — as the single source of truth for +//! "is anything degraded", rather than tracking health separately. Two //! systems independently deciding what counts as unhealthy is how they //! end up disagreeing. //! @@ -11,7 +12,7 @@ //! the *process* down over a transient dependency failure — that's what //! readiness is for. Readiness derives `"degraded"` from any `crit`-level //! warning currently set; `warn`-level entries ride along in the body for -//! detail but don't flip the status, mirroring the warning registry's own severity +//! detail but don't flip the status, mirroring the banner's own severity //! split. //! //! Routed outside `/api/` (see `dashboard/mod.rs`) so the gateway can diff --git a/hive-c0re/src/dashboard/state_snapshot.rs b/hive-c0re/src/dashboard/state_snapshot.rs index 960020a7..a3a62233 100644 --- a/hive-c0re/src/dashboard/state_snapshot.rs +++ b/hive-c0re/src/dashboard/state_snapshot.rs @@ -102,6 +102,12 @@ pub(super) struct StateSnapshot { /// var, set from `services.hyperhive.swarm.name`. `None` when /// unset — chrome omits the swarm segment of the breadcrumb. swarm_name: Option, + /// Server-level warnings for the dashboard's top-of-page banner + /// (currently host disk-pressure; more producers can be added + /// backend-side). Empty when all clear. Built by + /// `host_stats::server_warnings`; the frontend renders this list + /// generically, so new warning kinds need no frontend change. + server_warnings: Vec, } #[derive(Serialize)] @@ -187,6 +193,15 @@ pub(super) async fn api_state( let tombstones = build_tombstone_views(&containers); let port_conflicts = build_port_conflicts(&containers); + // Banner warnings: host probes (disk) + agent-state (pending logins). + // Built before the response struct because the agent-state producer + // borrows `containers`, which moves in below. + let server_warnings = { + let mut w = crate::host_stats::server_warnings(); + w.extend(crate::host_stats::agent_state_warnings(&containers)); + w + }; + axum::Json(StateSnapshot { seq, hostname, @@ -218,6 +233,7 @@ pub(super) async fn api_state( swarm_name: std::env::var("HYPERHIVE_SWARM_NAME") .ok() .filter(|s| !s.is_empty()), + server_warnings, }) } diff --git a/hive-c0re/src/forge/mod.rs b/hive-c0re/src/forge/mod.rs index 029888e5..dc57b764 100644 --- a/hive-c0re/src/forge/mod.rs +++ b/hive-c0re/src/forge/mod.rs @@ -122,7 +122,7 @@ pub(crate) const CONFIG_ORG: &str = "agent-configs"; /// set of boot warnings in [`ensure_all`] keyed by a runtime name (at /// most a handful per process). [`crate::warnings::set_boot_warning`] /// requires a `'static` kind so distinct orgs/repos don't clobber each -/// other's warning entry; leaking a few short strings once per boot is +/// other's banner entry; leaking a few short strings once per boot is /// cheap and bounded, unlike a per-request or per-loop-iteration leak. fn static_kind(s: String) -> &'static str { Box::leak(s.into_boxed_str()) @@ -340,7 +340,7 @@ fn hook_ids_at(hooks: &[forgejo_api::structs::Hook], url: &str) -> Vec { /// confirms the container exists — not that Forgejo is *listening*. A /// `nixos-rebuild` that restarts hive-forge and hive-c0re together races: /// without this gate every `ensure_*` fires at a refused socket and leaves a -/// stale "provisioning failed" warning that never clears (the pass is one-shot). +/// stale "provisioning failed" banner that never clears (the pass is one-shot). /// Returns whether the endpoint became ready before the deadline; on timeout /// the caller proceeds anyway so a genuinely-down forge still surfaces its real /// errors. Uses the unauthenticated `get_version` endpoint (no token yet). @@ -393,7 +393,7 @@ pub async fn ensure_all() { } // The container exists, but its HTTP may not be up yet (a rebuild restarts // hive-forge + hive-c0re together). Wait for the API to answer before the - // provisioning pass so a boot race doesn't leave stale failure warnings. + // provisioning pass so a boot race doesn't leave stale failure banners. wait_until_ready().await; let core_token = match ensure_core_user_and_token().await { Ok(t) => Some(t), diff --git a/hive-c0re/src/forge/repos.rs b/hive-c0re/src/forge/repos.rs index 3fa8d459..a1ce5b4a 100644 --- a/hive-c0re/src/forge/repos.rs +++ b/hive-c0re/src/forge/repos.rs @@ -125,7 +125,7 @@ pub async fn push_meta(dir: &Path) -> Result<()> { let msg = "forge/core/meta push rejected (non-fast-forward): \ remote is ahead of local; leaving remote history intact"; tracing::warn!("{msg}"); - // Raise a persistent hive warning; cleared on next successful push. + // Raise a persistent dashboard banner; cleared on next successful push. if let Ok(mut g) = META_NON_FF_GUARD.lock() { *g = Some(crate::warnings::set_warning( "forge-meta-non-ff", diff --git a/hive-c0re/src/gateway_nginx.rs b/hive-c0re/src/gateway_nginx.rs index 71ae2012..26824385 100644 --- a/hive-c0re/src/gateway_nginx.rs +++ b/hive-c0re/src/gateway_nginx.rs @@ -32,7 +32,7 @@ static LAST_FAILED_RELOAD: AtomicU64 = AtomicU64::new(0); /// Minimum gap between retry attempts after a reload failure (30 s). const RELOAD_RETRY_SECS: u64 = 30; -/// Hive-warning health tracker for the reload sweep — raises a +/// Dashboard-banner health tracker for the reload sweep — raises a /// `gateway_nginx_reload` warning the first time a reload fails (routing is /// broken for whoever depends on the change *right now*, so no debounce /// window) and clears it the moment a reload succeeds again. diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index b9d1b925..e96e62bd 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -139,7 +139,7 @@ pub(super) async fn run_node( /// Boot-time forge user/token sweep as a DAG node — see /// [`NodeKind::ForgeSweep`]. `forge::ensure_all` already does its own -/// per-step error handling and boot warnings internally (it's +/// per-step error handling and boot-warning banners internally (it's /// best-effort by design), so this wrapper has nothing left to report; /// it exists purely to make the sweep a visible unit of work. async fn run_forge_sweep() -> Result<()> { @@ -149,7 +149,7 @@ async fn run_forge_sweep() -> Result<()> { /// Matrix user/space sweep as a DAG node — see [`NodeKind::MatrixSweep`]. /// Every pass reports here, whoever submitted it: as the node's own error on -/// the dashboard, and into the debounced warning. +/// the dashboard, and into the debounced banner. async fn run_matrix_sweep() -> Result<()> { let ok = tokio::time::timeout(MATRIX_SWEEP_DEADLINE, crate::matrix::ensure_all()) .await @@ -164,7 +164,7 @@ async fn run_matrix_sweep() -> Result<()> { health.record_ok(); Ok(()) } else { - health.record_err(matrix_sweep_warning); + health.record_err(matrix_sweep_banner); anyhow::bail!("matrix ensure_all: one or more agents failed sync (see logs)") } } @@ -177,7 +177,7 @@ async fn run_matrix_sweep() -> Result<()> { /// [`Resource::MatrixSweep`]: super::resource::Resource::MatrixSweep const MATRIX_SWEEP_DEADLINE: std::time::Duration = std::time::Duration::from_mins(10); -/// Warning for a matrix sweep that keeps failing. Two misses in a row, so one +/// Banner for a matrix sweep that keeps failing. Two misses in a row, so one /// bad pass (homeserver mid-restart, a transient HTTP blip) doesn't flap the /// dashboard; cleared by the next clean pass. fn matrix_sweep_health() -> &'static Mutex { @@ -185,7 +185,7 @@ fn matrix_sweep_health() -> &'static Mutex { HEALTH.get_or_init(|| Mutex::new(SweepHealth::new("matrix_ensure_all", "warn", 2))) } -fn matrix_sweep_warning(ctx: sweep_health::SweepFailure) -> String { +fn matrix_sweep_banner(ctx: sweep_health::SweepFailure) -> String { let age = ctx.since_last_ok.map_or_else( || "no success this session".to_owned(), |d| format!("last ok {} ago", sweep_health::fmt_age(d)), @@ -200,7 +200,7 @@ fn matrix_sweep_warning(ctx: sweep_health::SweepFailure) -> String { /// `/knowledge` pull as a DAG node — see [`NodeKind::KnowledgePull`]. Every /// pass reports here, whoever submitted it: as the node's own error on the -/// dashboard, and into the debounced warning. +/// dashboard, and into the debounced banner. async fn run_knowledge_pull(coord: &Arc) -> Result<()> { let result = tokio::time::timeout( KNOWLEDGE_PULL_DEADLINE, @@ -242,7 +242,7 @@ async fn run_knowledge_pull(coord: &Arc) -> Result<()> { /// [`Resource::KnowledgeTree`]: super::resource::Resource::KnowledgeTree const KNOWLEDGE_PULL_DEADLINE: std::time::Duration = std::time::Duration::from_mins(5); -/// Warning for a `/knowledge` pull that keeps failing: three misses in a row, +/// Banner for a `/knowledge` pull that keeps failing: three misses in a row, /// cleared by the next successful pull. fn knowledge_pull_health() -> &'static Mutex { static HEALTH: OnceLock> = OnceLock::new(); diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index 36ccac32..8e9ce904 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -367,11 +367,11 @@ pub async fn agents_for_meta_listing() -> Result> { agents_for_meta(None).await } -/// Per-agent "stop failed" warning guards, keyed by agent name. Cleared the +/// Per-agent "stop failed" banner guards, keyed by agent name. Cleared the /// next time that agent's container stops cleanly — a `Mutex>` /// rather than a bare `set_boot_warning` because this condition (unlike a /// one-shot boot step) genuinely resolves later, once an operator has -/// intervened, and the warning should clear with it instead of surviving +/// intervened, and the banner should clear with it instead of surviving /// until the next hive-c0re restart. fn stop_failed_guards() -> &'static std::sync::Mutex> { @@ -402,7 +402,7 @@ pub async fn kill(name: &str) -> Result<()> { .unwrap_or_else(std::sync::PoisonError::into_inner); match result { Ok(()) => { - // A clean stop clears any earlier "this looked wedged" warning + // A clean stop clears any earlier "this looked wedged" banner // for the same agent. guards.remove(name); Ok(()) diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index 31e3fb6c..420340c4 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -284,12 +284,12 @@ async fn cmd_serve( } } }); - // Disk-pressure watch: raise a `disk_pressure` warning while the + // Disk-pressure watch: raise a `disk_pressure` banner warning while the // host nix store is over threshold and clear it when back under, via the // push-based warnings registry. Replaces the old per-`/api/state` // `statvfs`. ~60s cadence; first tick fires immediately. The held guard // lives for the task's lifetime — dropping it (on shutdown) clears the - // warning. + // banner. let mut disk_shutdown = coord.shutdown_rx(); tokio::spawn(async move { let mut disk_guard: Option = None; diff --git a/hive-c0re/src/matrix.rs b/hive-c0re/src/matrix.rs index d8f2d8db..50928906 100644 --- a/hive-c0re/src/matrix.rs +++ b/hive-c0re/src/matrix.rs @@ -787,8 +787,8 @@ pub async fn invite_user( /// Returns `true` when every step of the sweep succeeded, `false` when at /// least one step failed — the caller feeds this into a /// [`crate::stats::sweep_health::SweepHealth`] to raise a debounced -/// hive warning on persistent failure (this sweep re-runs every 30 -/// minutes, so a one-off blip self-heals without ever warning). +/// dashboard banner on persistent failure (this sweep re-runs every 30 +/// minutes, so a one-off blip self-heals without ever bannering). pub async fn ensure_all() -> bool { if !is_present() { tracing::debug!("matrix: hive-matrix container absent, skipping user sweep"); diff --git a/hive-c0re/src/stats/host_stats.rs b/hive-c0re/src/stats/host_stats.rs index 75472431..1787ddc2 100644 --- a/hive-c0re/src/stats/host_stats.rs +++ b/hive-c0re/src/stats/host_stats.rs @@ -1,33 +1,35 @@ -//! Host-system probes that raise hive-level warnings into the push-based -//! [`crate::warnings`] registry. +//! Host-system probes and the derived **server-warning** list that the +//! dashboard renders as a top-of-page banner. //! -//! The first producer is a host disk-usage check: the nix store filling up -//! is what ENOSPC-failed CI before the GC guardrails were documented. -//! hive-c0re runs on the host (not inside a container), so it can `statvfs` -//! the store path directly and warn the operator *before* an ENOSPC, not -//! after. [`refresh_disk_warning`] drives a held guard from a periodic task -//! (see `spawn` in `main.rs`). The same probe backs the -//! `hyperhive.hive.nix_store.utilization` gauge ([`disk_utilization`]), which is -//! what the early-warning Grafana alerts read. +//! The first (and currently only) producer is a host disk-usage check: +//! the nix store filling up is what ENOSPC-failed CI before the GC +//! guardrails were documented. hive-c0re runs on the host (not inside a +//! container), so it can `statvfs` the store path directly and warn the +//! operator *before* an ENOSPC, not after. //! -//! The registry has no dashboard banner any more. A raised warning reaches -//! the operator as the `hyperhive.hive.warning` gauge and a Grafana alert -//! on it (see `docs/scheduler/observability.md`). +//! [`server_warnings`] is the public surface: it returns a flat list of +//! [`ServerWarning`]s for `/api/state`. It is now just a snapshot of the +//! push-based [`crate::warnings`] registry — producers raise/clear their +//! own warnings via an RAII guard rather than being re-probed on every +//! render. The disk-pressure check is reworked onto that model here: +//! [`refresh_disk_warning`] drives a held guard from a periodic task (see +//! `spawn` in `main.rs`) instead of a `statvfs` per `/api/state`. use serde::Serialize; use utoipa::ToSchema; use crate::container_view::ContainerView; -/// One hive-level warning in the [`crate::warnings`] registry. +/// One server-level warning for the dashboard's top-of-page banner. #[derive(Debug, Clone, Serialize, ToSchema)] pub struct ServerWarning { - /// Stable kind id (e.g. `"disk_pressure"`), the `kind` label of the - /// `hyperhive.hive.warning` gauge. + /// Stable kind id (e.g. `"disk_pressure"`) — lets the frontend dedupe + /// or special-case without parsing the message. pub kind: &'static str, - /// Severity: `"warn"` or `"crit"`. `crit` degrades readiness. + /// Severity: `"warn"` (amber) or `"crit"` (red). The banner picks its + /// colour from this; everything else is just the message text. pub level: &'static str, - /// Human-readable, already-formatted message. + /// Human-readable, already-formatted message shown in the banner. pub message: String, } @@ -36,7 +38,17 @@ pub struct ServerWarning { const DISK_WARN_PCT: f64 = 85.0; const DISK_CRIT_PCT: f64 = 95.0; -/// Pure disk-pressure decision: given usage, return the warning +/// Snapshot the current server-level warnings for the dashboard banner — +/// a cheap read of the push-based [`crate::warnings`] registry. Producers +/// (the disk watch below, background converge sweeps, …) raise and clear +/// their own entries via [`crate::warnings::set_warning`]; nothing is +/// re-probed here. +#[must_use] +pub fn server_warnings() -> Vec { + crate::warnings::snapshot() +} + +/// Pure disk-pressure decision: given usage, return the banner /// `(level, message)` or `None` when below the warn threshold. Split from /// the probe + guard plumbing so the threshold + formatting is unit-tested /// without a real `statvfs`. @@ -63,8 +75,9 @@ fn disk_pressure_warning(used_pct: f64, free_bytes: u64) -> Option<(&'static str /// One tick of the disk-pressure watch: probe the store and reconcile the /// held warning `guard` — raise or refresh it while usage is over -/// threshold, drop it (clearing the warning) once back under. Driven by the -/// periodic disk-watch task spawned in `main.rs`. +/// threshold, drop it (clearing the banner) once back under. Driven by the +/// periodic disk-watch task spawned in `main.rs`, replacing the old +/// per-render `statvfs` in [`server_warnings`]. pub fn refresh_disk_warning(guard: &mut Option) { let decision = nix_disk_usage().and_then(|d| disk_pressure_warning(d.used_pct, d.free_bytes)); match decision { @@ -83,44 +96,19 @@ pub fn refresh_disk_warning(guard: &mut Option) { } } -/// Fraction (0–1) of the filesystem holding the nix store that is in use, -/// the value behind the `hyperhive.hive.nix_store.utilization` gauge. `None` when -/// the probe fails, so a failing probe reports no sample rather than zero. -#[must_use] -pub fn disk_utilization() -> Option { - nix_disk_usage().map(|d| d.used_pct / 100.0) -} - -/// The held `pending_logins` warning; see [`refresh_pending_logins`]. -static PENDING_LOGINS: std::sync::Mutex> = - std::sync::Mutex::new(None); - -/// Reconcile the `pending_logins` warning with the current container -/// snapshot: raise or refresh it while any agent needs a claude login, -/// drop it once none does. Driven by the container rescan in the -/// coordinator, which already builds the snapshot every poll. -pub fn refresh_pending_logins(containers: &[ContainerView]) { - let warning = agent_state_warnings(containers).into_iter().next(); - let mut guard = PENDING_LOGINS - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - match (warning, guard.as_ref()) { - (Some(w), Some(g)) => g.update(w.level, w.message), - (Some(w), None) => { - *guard = Some(crate::warnings::set_warning(w.kind, w.level, w.message)); - } - (None, _) => *guard = None, - } -} - -/// Agent-state warnings derived from the live container snapshot: agents -/// that need a claude login. Takes the container list from its caller -/// because that is where the snapshot lives. +/// Agent-state warnings derived from the live container snapshot the +/// dashboard already holds: agents that need a claude login. Kept +/// separate from [`server_warnings`] (host probes) because the caller +/// owns the container list; the dashboard concatenates both into one +/// banner list. /// -/// Container crashes are not a warning here: the crash-watcher logs -/// `container crash detected` and a swarm-level grafana alert -/// (`hh-container-crashed` in `nix/host-modules/swarm-grafana.nix`) matches -/// that line, with no agent in the loop. +/// Container crashes used to have a sibling `agents_crashing` entry here, +/// fed by `Coordinator::recent_crash_counts` — removed in favour of a +/// swarm-level grafana alert on the crash-watcher's own log line (see +/// `nix/host-modules/swarm-grafana.nix`'s `hh-container-crashed` rule and +/// `docs/scheduler/observability.md`). That ships the same signal to the +/// operator without needing an agent (or this per-hive banner) in the +/// loop at all. #[must_use] pub fn agent_state_warnings(containers: &[ContainerView]) -> Vec { let mut out = Vec::new(); @@ -275,46 +263,4 @@ mod tests { let w = agent_state_warnings(&containers); assert!(w[0].message.starts_with("1 agent needs claude login: solo")); } - - /// The registry entry tracks the snapshot it is refreshed from: raised - /// when an agent needs a login, rewritten when the set changes, and gone - /// when none does. This is the only test that touches `pending_logins`, - /// so reading the process-global registry by that kind stays sound under - /// the parallel test runner. - #[test] - fn pending_logins_warning_follows_the_container_snapshot() { - let current = || { - crate::warnings::snapshot() - .into_iter() - .find(|w| w.kind == "pending_logins") - }; - assert!( - current().is_none(), - "nothing raised before the first refresh" - ); - - refresh_pending_logins(&[cv("amy", true)]); - assert!( - current().is_some_and(|w| w.message.starts_with("1 agent needs claude login: amy")), - "an agent needing a login raises the warning" - ); - - refresh_pending_logins(&[cv("amy", true), cv("bob", true)]); - assert!( - current().is_some_and(|w| w.message.starts_with("2 agents need claude login")), - "a second agent rewrites the same entry" - ); - - refresh_pending_logins(&[cv("amy", false), cv("bob", false)]); - assert!(current().is_none(), "no agent needing a login clears it"); - } - - /// The gauge value is a fraction, so a percent leaking through would put - /// every alert threshold 100x off. - #[test] - fn disk_utilization_is_a_fraction() { - if let Some(used) = disk_utilization() { - assert!((0.0..=1.0).contains(&used), "got {used}"); - } - } } diff --git a/hive-c0re/src/stats/otel_metrics.rs b/hive-c0re/src/stats/otel_metrics.rs index a9d3f2a7..70745284 100644 --- a/hive-c0re/src/stats/otel_metrics.rs +++ b/hive-c0re/src/stats/otel_metrics.rs @@ -305,45 +305,6 @@ fn register_hive_instruments(meter: &opentelemetry::metrics::Meter) { } }) .build(); - - // One series per raised warning, `1` while it is raised, labelled with - // its `kind` and `level`. This is what names the cause in an alert: the - // count above says how many, this says which. A cleared warning stops - // reporting rather than reporting `0`, so an alert on it resolves when - // the series goes stale; the kinds are a fixed set in the source, so the - // label has bounded cardinality. - meter - .u64_observable_gauge("hyperhive.hive.warning") - .with_unit("1") - .with_callback(|obs| { - for w in warnings::snapshot() { - obs.observe( - 1, - &[ - KeyValue::new("kind", w.kind), - KeyValue::new("level", w.level), - ], - ); - } - }) - .build(); - - // Fraction of the nix store's filesystem in use. A `hyperhive.` name on - // purpose, not semconv `system.filesystem.utilization`: the hive - // collector's host-metrics scraper covers the `system.filesystem.*` - // family for every mount, and a second series under one of its names - // with different labels could clash. - // Probed in the callback (a `statvfs` is cheap and this runs once an - // export interval); a failed probe reports no sample. - meter - .f64_observable_gauge("hyperhive.hive.nix_store.utilization") - .with_unit("1") - .with_callback(|obs| { - if let Some(used) = crate::host_stats::disk_utilization() { - obs.observe(used, HIVE_ATTRS); - } - }) - .build(); } /// Attributes for the hive-scoped family: **none**. diff --git a/hive-c0re/src/stats/sweep_health.rs b/hive-c0re/src/stats/sweep_health.rs index c792bb30..42637538 100644 --- a/hive-c0re/src/stats/sweep_health.rs +++ b/hive-c0re/src/stats/sweep_health.rs @@ -5,14 +5,14 @@ //! gateway nginx writes, per-agent branch protection, …) historically //! `warn!`'d to the journal and moved on, so a persistent failure was //! invisible to the operator. [`SweepHealth`] routes that failure to the -//! hive warning instead — but *debounced*, per the routing guidance in +//! dashboard banner instead — but *debounced*, per the routing guidance in //! the tracker issue: //! //! - a transient miss (container down mid-sweep, a network blip) shouldn't -//! flap a warning on the first failure → `threshold > 1` raises only after +//! flap a banner on the first failure → `threshold > 1` raises only after //! that many **consecutive** misses; //! - a won't-fix-itself failure (missing team, bad config, auth rejection) -//! should warning immediately → `threshold == 1`. +//! should banner immediately → `threshold == 1`. //! //! The warning clears the moment the sweep next succeeds. A sweep owns one //! `SweepHealth`, calls [`record_ok`](SweepHealth::record_ok) on success and @@ -24,11 +24,11 @@ use std::time::{Duration, Instant}; use crate::warnings::{WarningGuard, set_warning}; /// Per-sweep failure context handed to the [`record_err`](SweepHealth::record_err) -/// message builder so the warning text can include how long the sweep has been +/// message builder so the banner text can include how long the sweep has been /// failing and since when it last worked. #[derive(Debug, Clone, Copy)] pub struct SweepFailure { - /// Number of consecutive failures so far (≥ `threshold` when the warning + /// Number of consecutive failures so far (≥ `threshold` when the banner /// is raised). pub consecutive: u32, /// Time since the last recorded success, if the sweep has ever succeeded @@ -36,7 +36,7 @@ pub struct SweepFailure { pub since_last_ok: Option, } -/// A debounced link between one background sweep and the warning warning. +/// A debounced link between one background sweep and the warning banner. /// Not `Sync`-shared: each sweep task owns its own instance (the underlying /// registry is the shared, thread-safe part). pub struct SweepHealth { @@ -50,8 +50,8 @@ pub struct SweepHealth { impl SweepHealth { /// Track sweep `kind` at severity `level` (`"warn"` / `"crit"`), raising - /// the warning after `threshold` consecutive failures. `threshold` is - /// floored at 1 (0 would warning before any failure). + /// the banner after `threshold` consecutive failures. `threshold` is + /// floored at 1 (0 would banner before any failure). #[must_use] pub fn new(kind: &'static str, level: &'static str, threshold: u32) -> Self { Self { @@ -65,7 +65,7 @@ impl SweepHealth { } /// Record a successful sweep: reset the consecutive-miss counter and drop - /// the guard (clearing any warning entry). + /// the guard (clearing any banner entry). pub fn record_ok(&mut self) { self.misses = 0; self.last_ok = Some(Instant::now()); @@ -74,9 +74,9 @@ impl SweepHealth { } /// Record a failed sweep. Bumps the consecutive-miss counter; once it - /// reaches `threshold`, raises (or refreshes) the warning warning built by + /// reaches `threshold`, raises (or refreshes) the banner warning built by /// `message`. Below the threshold this is a cheap no-op — `message` is - /// only invoked when the warning is actually shown, so callers can format + /// only invoked when the banner is actually shown, so callers can format /// lazily. pub fn record_err(&mut self, message: impl FnOnce(SweepFailure) -> String) { self.misses = self.misses.saturating_add(1); @@ -96,7 +96,7 @@ impl SweepHealth { } /// Compact human-readable age (e.g. `"45s"`, `"12m"`, `"3h"`, `"2d"`) for a -/// warning "last succeeded N ago" suffix. Coarse on purpose — the warning wants +/// banner "last succeeded N ago" suffix. Coarse on purpose — the banner wants /// a glanceable magnitude, not precision. #[must_use] pub fn fmt_age(d: Duration) -> String { @@ -117,7 +117,7 @@ mod tests { use super::*; use crate::warnings::snapshot; - fn warning(kind: &str) -> Option { + fn banner(kind: &str) -> Option { snapshot() .into_iter() .find(|w| w.kind == kind) @@ -125,34 +125,34 @@ mod tests { } #[test] - fn below_threshold_does_not_warn() { + fn below_threshold_does_not_banner() { let mut h = SweepHealth::new("sh_below", "warn", 3); h.record_err(|_| "boom".to_owned()); h.record_err(|_| "boom".to_owned()); assert!(h.guard.is_none()); - assert!(warning("sh_below").is_none()); + assert!(banner("sh_below").is_none()); } #[test] fn raises_at_threshold_and_clears_on_ok() { let mut h = SweepHealth::new("sh_raise", "warn", 2); h.record_err(|c| format!("fail #{}", c.consecutive)); - assert!(warning("sh_raise").is_none(), "one miss < threshold"); + assert!(banner("sh_raise").is_none(), "one miss < threshold"); h.record_err(|c| format!("fail #{}", c.consecutive)); - assert_eq!(warning("sh_raise").as_deref(), Some("fail #2")); + assert_eq!(banner("sh_raise").as_deref(), Some("fail #2")); assert!(h.guard.is_some()); h.record_ok(); assert!(h.guard.is_none()); - assert!(warning("sh_raise").is_none(), "success clears the warning"); + assert!(banner("sh_raise").is_none(), "success clears the banner"); } #[test] - fn threshold_one_warns_immediately() { + fn threshold_one_banners_immediately() { let mut h = SweepHealth::new("sh_immediate", "crit", 1); h.record_err(|_| "gone".to_owned()); - assert_eq!(warning("sh_immediate").as_deref(), Some("gone")); + assert_eq!(banner("sh_immediate").as_deref(), Some("gone")); h.record_ok(); - assert!(warning("sh_immediate").is_none()); + assert!(banner("sh_immediate").is_none()); } #[test] @@ -161,9 +161,9 @@ mod tests { h.record_err(|_| "x".to_owned()); // miss 1 h.record_ok(); // reset h.record_err(|c| format!("c={}", c.consecutive)); // miss 1 again, < threshold - assert!(warning("sh_reset").is_none(), "counter reset after ok"); + assert!(banner("sh_reset").is_none(), "counter reset after ok"); h.record_err(|c| format!("c={}", c.consecutive)); // miss 2 → raise - assert_eq!(warning("sh_reset").as_deref(), Some("c=2")); + assert_eq!(banner("sh_reset").as_deref(), Some("c=2")); h.record_ok(); } @@ -171,9 +171,9 @@ mod tests { fn refresh_updates_message_while_held() { let mut h = SweepHealth::new("sh_refresh", "warn", 1); h.record_err(|c| format!("miss {}", c.consecutive)); - assert_eq!(warning("sh_refresh").as_deref(), Some("miss 1")); + assert_eq!(banner("sh_refresh").as_deref(), Some("miss 1")); h.record_err(|c| format!("miss {}", c.consecutive)); - assert_eq!(warning("sh_refresh").as_deref(), Some("miss 2")); + assert_eq!(banner("sh_refresh").as_deref(), Some("miss 2")); h.record_ok(); } diff --git a/hive-c0re/src/stats/warnings.rs b/hive-c0re/src/stats/warnings.rs index dca7cf8f..ae6e8937 100644 --- a/hive-c0re/src/stats/warnings.rs +++ b/hive-c0re/src/stats/warnings.rs @@ -1,21 +1,21 @@ -//! Process-global hive-warning registry. +//! Process-global server-warning registry backing the dashboard banner. //! -//! Subsystems record hive-level degradation here (disk pressure, a failed -//! converge sweep, agents that can't log in, …). Three consumers read it: -//! `/health/ready` and the swarm status publisher through [`readiness`], and -//! the `hyperhive.hive.degraded` / `hyperhive.hive.warnings` / -//! `hyperhive.hive.warning` gauges in `otel_metrics`, which the Grafana -//! alerts watch. There is no dashboard banner. +//! The dashboard shows a top-of-page banner for hive-level degradation +//! (disk pressure, a failed converge sweep, agents that can't log in, …). +//! Historically each warning was *pulled*: [`host_stats::server_warnings`] +//! recomputed the whole list on every `/api/state` by re-probing (a +//! `statvfs`, a scan of the container snapshot). Background sweeps that +//! failed had nowhere to record it, so they just `warn!`'d to the journal +//! and vanished from anywhere an operator looks. //! -//! The registry is **push**-based: a single `Mutex>` any -//! subsystem writes into. [`snapshot`] is a cheap read of that map. The -//! default API is RAII: [`set_warning`] returns a [`WarningGuard`] whose -//! `Drop` removes the key, so a producer holds the guard exactly as long as -//! the condition is true and lets it drop to clear — no manual dismiss, no -//! stale warning. A failing periodic sweep does `let _w = set_warning(...)` -//! in its failure branch and drops it on the next success. Raising a warning -//! also logs one `warn` line with the message, so the text behind a `kind` -//! label is in the log store. +//! This module replaces the pull model with a **push** one: a single +//! `Mutex>` any subsystem writes into. [`snapshot`] is a +//! cheap read of that map. The default API is RAII: [`set_warning`] returns +//! a [`WarningGuard`] whose `Drop` removes the key, so a producer holds the +//! guard exactly as long as the condition is true and lets it drop to clear +//! — no manual dismiss, no stale banner. A failing periodic sweep does +//! `let _w = set_warning(...)` in its failure branch and drops it on the +//! next success. //! //! Concurrency: all mutation is behind the one mutex. Each guard carries a //! unique id and only touches its own map entry (`Drop` / [`update`] @@ -23,6 +23,7 @@ //! so a re-`set_warning` for the same kind is never clobbered when the //! stale guard later drops. //! +//! [`host_stats::server_warnings`]: crate::host_stats::server_warnings //! [`update`]: WarningGuard::update use std::collections::BTreeMap; @@ -48,7 +49,7 @@ struct Slot { } /// The one process-global registry. `BTreeMap` (not `HashMap`) so -/// [`snapshot`] yields a deterministic, kind-sorted order across +/// [`snapshot`] yields a deterministic, kind-sorted banner order across /// renders. fn registry() -> &'static Mutex> { static REG: OnceLock>> = OnceLock::new(); @@ -80,13 +81,11 @@ pub fn set_warning( message: impl Into, ) -> WarningGuard { let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - let message = message.into(); - tracing::warn!(kind, level, %message, "hive warning raised"); lock().insert( kind, Slot { level, - message, + message: message.into(), owner: id, }, ); @@ -171,24 +170,24 @@ pub fn snapshot() -> Vec { } /// Raise a warning for a one-shot boot-time step that has no periodic -/// retry to observe a later success and clear the warning via +/// retry to observe a later success and clear the banner via /// [`WarningGuard`] drop — e.g. a step inside `forge::ensure_all()`, /// which `tokio::spawn`s once at hive-c0re startup and never runs again /// this process. Intentionally **leaks** the guard for the life of the -/// process: the warning clears the next time hive-c0re restarts (a fresh +/// process: the banner clears the next time hive-c0re restarts (a fresh /// process starts with an empty registry) and re-runs the step, which is /// exactly when a config/environment fix would take effect anyway. /// /// Do not use this for anything that runs periodically or can be /// retried within the same process — hold the [`WarningGuard`] (or use /// [`crate::stats::sweep_health::SweepHealth`]) so a later success can -/// actually clear the warning instead of waiting for a restart. +/// actually clear the banner instead of waiting for a restart. pub fn set_boot_warning(kind: &'static str, level: &'static str, message: impl Into) { std::mem::forget(set_warning(kind, level, message)); } /// RAII handle for one active warning. Drop clears the `kind` from the -/// registry. Obtained from [`set_warning`]. +/// banner. Obtained from [`set_warning`]. #[must_use = "dropping the guard immediately clears the warning; bind it for as long as the condition holds"] pub struct WarningGuard { kind: &'static str, diff --git a/hive-c0re/src/swarm_agent_status.rs b/hive-c0re/src/swarm_agent_status.rs index 27bdb253..04c5fb28 100644 --- a/hive-c0re/src/swarm_agent_status.rs +++ b/hive-c0re/src/swarm_agent_status.rs @@ -27,9 +27,9 @@ use crate::stats::sweep_health::{self, SweepHealth}; /// cadence than the hive's own. pub const PUBLISH_INTERVAL: Duration = crate::swarm_status::PUBLISH_INTERVAL; -/// Consecutive failed sweeps before the sweep raises a hive warning. Mirrors +/// Consecutive failed sweeps before the dashboard banners. Mirrors /// [`crate::swarm_status`]'s constant of the same name and purpose. -const FAILURES_BEFORE_WARNING: u32 = 3; +const FAILURES_BEFORE_BANNER: u32 = 3; /// Start the publish loop, if this deployment wired up a swarm queue. /// @@ -44,8 +44,8 @@ const FAILURES_BEFORE_WARNING: u32 = 3; /// deploy-event listener does. pub fn spawn(mut shutdown: tokio::sync::watch::Receiver) { let Some(hive) = crate::container_view::hive_swarm_names().0 else { - // Already warned once by `swarm_status::spawn`, which runs the - // identical check — a second warning for the same missing env var + // Already bannered once by `swarm_status::spawn`, which runs the + // identical check — a second banner for the same missing env var // would just be the same fact told twice. return; }; @@ -55,11 +55,8 @@ pub fn spawn(mut shutdown: tokio::sync::watch::Receiver) { return; }; - let mut health = SweepHealth::new( - "swarm_agent_status_publish", - "warn", - FAILURES_BEFORE_WARNING, - ); + let mut health = + SweepHealth::new("swarm_agent_status_publish", "warn", FAILURES_BEFORE_BANNER); loop { match publish_all(&client, &hive).await { Ok(published) => { diff --git a/hive-c0re/src/swarm_notices.rs b/hive-c0re/src/swarm_notices.rs index ff9dd3de..97c848df 100644 --- a/hive-c0re/src/swarm_notices.rs +++ b/hive-c0re/src/swarm_notices.rs @@ -14,7 +14,7 @@ //! coordinates is a silent no-op (the ordinary case). A hive that cannot //! reach the queue loses the swarm's visibility of the notice, not the //! host's — `warn!` fires on every failed attempt regardless, and the -//! raises a hive warning only after [`FAILURES_BEFORE_WARNING`] consecutive +//! dashboard banners only after [`FAILURES_BEFORE_BANNER`] consecutive //! misses, the same debounce shape [`crate::swarm_status`] uses and for //! the same reason: a `warn!` that fires every call for three weeks is //! indistinguishable from silence in practice. @@ -24,10 +24,10 @@ use tokio::sync::{Mutex, OnceCell}; use crate::stats::sweep_health::{self, SweepHealth}; -/// Consecutive failed publishes before the sweep raises a hive warning — same +/// Consecutive failed publishes before the dashboard banners — same /// value [`crate::swarm_status`] uses and for the same reason: a debounce -/// against one blip flapping a warning an operator learns to ignore. -const FAILURES_BEFORE_WARNING: u32 = 3; +/// against one blip flapping a banner an operator learns to ignore. +const FAILURES_BEFORE_BANNER: u32 = 3; static HEALTH: OnceCell> = OnceCell::const_new(); @@ -37,7 +37,7 @@ async fn health() -> tokio::sync::MutexGuard<'static, SweepHealth> { Mutex::new(SweepHealth::new( "swarm_notices_publish", "warn", - FAILURES_BEFORE_WARNING, + FAILURES_BEFORE_BANNER, )) }) .await @@ -57,7 +57,7 @@ pub async fn notify(subsystem: &str, key: Option, summary: String, sourc return; }; // Same absent-name condition `swarm_status` bails on — that module - // already warns it under `swarm_status_config` the first time + // already banners it under `swarm_status_config` the first time // either of us hits it; nothing more to add from here. let Some(hive) = crate::container_view::hive_swarm_names().0 else { return; diff --git a/hive-c0re/src/swarm_queue.rs b/hive-c0re/src/swarm_queue.rs index c7d77cb1..4b0e8779 100644 --- a/hive-c0re/src/swarm_queue.rs +++ b/hive-c0re/src/swarm_queue.rs @@ -44,15 +44,15 @@ const FIRST_RETRY_DELAY: Duration = Duration::from_secs(5); /// The longest wait between two attempts. const MAX_RETRY_DELAY: Duration = Duration::from_mins(5); -/// The warning kind a failed attempt raises, cleared by the next success. -const WARNING_KIND: &str = "swarm_queue_config"; +/// The banner kind a failed attempt raises, cleared by the next success. +const BANNER: &str = "swarm_queue_config"; static CLIENT: Retrying = Retrying::new(FIRST_RETRY_DELAY, MAX_RETRY_DELAY); /// The shared swarm-queue client, or `None` when there is none right now. /// /// `None` covers "no queue configured" (the ordinary case, logged once at -/// `info`), a deployment bug in the coordinates (warned, and permanent for +/// `info`), a deployment bug in the coordinates (bannered, and permanent for /// this process), and an attempt that failed or is still backing off. A /// caller with `None` should just skip whatever it was about to publish. /// Never waits out a backoff: see [`wait_for_client`] for that. @@ -95,12 +95,12 @@ enum Outcome { enum Slot { Off, Ready(T), - /// `failures` consecutive attempts have failed. `warning` holds the + /// `failures` consecutive attempts have failed. `banner` holds the /// dashboard warning until the slot leaves this state. Failing { failures: u32, not_before: Instant, - warning: Option, + banner: Option, }, } @@ -163,22 +163,22 @@ impl Retrying { let not_before = Instant::now() + delay; let message = format!("swarm queue is off, retrying in {delay:?}: {e:#}"); tracing::warn!(failures, "{message}"); - // Updating the held guard keeps the warning up without a + // Updating the held guard keeps the banner up without a // moment where it is cleared and re-raised. - let warning = match slot.take() { + let banner = match slot.take() { Some(Slot::Failing { - warning: Some(warning), + banner: Some(banner), .. }) => { - warning.update("warn", message); - warning + banner.update("warn", message); + banner } - _ => crate::warnings::set_warning(WARNING_KIND, "warn", message), + _ => crate::warnings::set_warning(BANNER, "warn", message), }; *slot = Some(Slot::Failing { failures, not_before, - warning: Some(warning), + banner: Some(banner), }); Outcome::NotBefore(not_before) } @@ -204,11 +204,7 @@ async fn connect_once() -> Attempt { } Err(e) => { // A deployment bug, which no retry clears. - crate::warnings::set_boot_warning( - WARNING_KIND, - "warn", - format!("swarm queue is off: {e:#}"), - ); + crate::warnings::set_boot_warning(BANNER, "warn", format!("swarm queue is off: {e:#}")); return Attempt::Off; } }; diff --git a/hive-c0re/src/swarm_status.rs b/hive-c0re/src/swarm_status.rs index c79822a5..d2585e33 100644 --- a/hive-c0re/src/swarm_status.rs +++ b/hive-c0re/src/swarm_status.rs @@ -47,18 +47,18 @@ use crate::stats::sweep_health::{self, SweepHealth}; /// other silently re-tunes the swarm's definition of "quiet". pub const PUBLISH_INTERVAL: Duration = Duration::from_mins(1); -/// Consecutive failed publishes before the sweep raises a hive warning. +/// Consecutive failed publishes before the dashboard banners. /// /// At [`PUBLISH_INTERVAL`] this is ~3 minutes of genuine failure, so a -/// blip does not flap a warning an operator learns to ignore. -const FAILURES_BEFORE_WARNING: u32 = 3; +/// blip does not flap a banner an operator learns to ignore. +const FAILURES_BEFORE_BANNER: u32 = 3; /// Start the publish loop, if this deployment wired up a swarm queue. /// /// The connect itself is shared with every other swarm-queue consumer in /// this process — see [`crate::swarm_queue`] for why one connection and /// not one per consumer, and for where "no queue configured" vs. "queue -/// configured but unreachable" gets warned. This function only decides +/// configured but unreachable" gets bannered. This function only decides /// whether *status* has anything to offer once a client exists. pub fn spawn( coord: std::sync::Arc, @@ -76,7 +76,7 @@ pub fn spawn( tokio::spawn(async move { let Some(client) = crate::swarm_queue::wait_for_client().await else { - // No queue configured, already logged or warned by the + // No queue configured, already logged or bannered by the // shared connector; nothing left to report here. return; }; @@ -117,7 +117,7 @@ pub fn spawn( shutdown.clone(), )); - let mut health = SweepHealth::new("swarm_status_publish", "warn", FAILURES_BEFORE_WARNING); + let mut health = SweepHealth::new("swarm_status_publish", "warn", FAILURES_BEFORE_BANNER); loop { match publish(&client, &hive).await { Ok(()) => health.record_ok(), @@ -197,7 +197,7 @@ async fn drain_swarm_events( let mut sub = match client.subscribe(subject).await { Ok(sub) => sub, Err(e) => { - // Warn rather than a boot warning: the hive is fully functional + // Warn rather than a boot banner: the hive is fully functional // without this, it just falls back to learning about knowledge // changes at its next boot. tracing::warn!( diff --git a/hive-c0re/src/workers/crash_watch.rs b/hive-c0re/src/workers/crash_watch.rs index 8c0f0393..55802e0d 100644 --- a/hive-c0re/src/workers/crash_watch.rs +++ b/hive-c0re/src/workers/crash_watch.rs @@ -149,7 +149,7 @@ fn emit_crash_transitions(coord: &Coordinator, prev: &HashSet, current: // The one place this fires: a swarm-level grafana alert // (`hh-container-crashed` in `nix/host-modules/swarm-grafana.nix`) // matches this exact message against the log store and pages the - // operator directly — no agent (and no per-hive warning) + // operator directly — no agent (and no per-hive dashboard banner) // in the loop. See `docs/scheduler/observability.md`. tracing::warn!(agent = %stopped, "container crash detected"); } diff --git a/hive-c0re/src/workers/permissions_import.rs b/hive-c0re/src/workers/permissions_import.rs index dc5ae49d..815c1b47 100644 --- a/hive-c0re/src/workers/permissions_import.rs +++ b/hive-c0re/src/workers/permissions_import.rs @@ -181,7 +181,7 @@ impl ImportOutcome { matches!(self, Self::NothingPending | Self::Imported) } - /// Why reading is not allowed, for the caller's log and warning. + /// Why reading is not allowed, for the caller's log and banner. #[must_use] pub fn into_error(self) -> Option { match self { diff --git a/hive-c0re/src/workers/swarm_limits.rs b/hive-c0re/src/workers/swarm_limits.rs index b4fe8c3e..d0a41d7d 100644 --- a/hive-c0re/src/workers/swarm_limits.rs +++ b/hive-c0re/src/workers/swarm_limits.rs @@ -24,11 +24,11 @@ use crate::stats::sweep_health::{self, SweepHealth}; pub const READ_INTERVAL: Duration = Duration::from_secs(30); /// At [`READ_INTERVAL`] this is ~90 seconds of failing reads. -const FAILURES_BEFORE_WARNING: u32 = 3; +const FAILURES_BEFORE_BANNER: u32 = 3; /// Read the caps now and every [`READ_INTERVAL`] until shutdown. pub async fn follow(client: async_nats::Client, mut shutdown: tokio::sync::watch::Receiver) { - let mut health = SweepHealth::new("swarm_limits_read", "warn", FAILURES_BEFORE_WARNING); + let mut health = SweepHealth::new("swarm_limits_read", "warn", FAILURES_BEFORE_BANNER); loop { match sync(&client).await { Ok(()) => health.record_ok(), diff --git a/hive-c0re/src/workers/swarm_permissions.rs b/hive-c0re/src/workers/swarm_permissions.rs index bcf35527..5f0cd477 100644 --- a/hive-c0re/src/workers/swarm_permissions.rs +++ b/hive-c0re/src/workers/swarm_permissions.rs @@ -30,7 +30,7 @@ use crate::stats::sweep_health::{self, SweepHealth}; pub const READ_INTERVAL: Duration = Duration::from_secs(30); /// At [`READ_INTERVAL`] this is ~90 seconds of failing reads. -const FAILURES_BEFORE_WARNING: u32 = 3; +const FAILURES_BEFORE_BANNER: u32 = 3; /// Send the meta repo's grants once, then read the permissions now and every /// [`READ_INTERVAL`] until shutdown. @@ -40,7 +40,7 @@ pub async fn follow( hive: String, mut shutdown: tokio::sync::watch::Receiver, ) { - let mut health = SweepHealth::new("swarm_permissions_read", "warn", FAILURES_BEFORE_WARNING); + let mut health = SweepHealth::new("swarm_permissions_read", "warn", FAILURES_BEFORE_BANNER); loop { let import = super::permissions_import::send_pending(&client, &hive).await; let pass = if import.read_allowed() { diff --git a/hive-c0re/src/workers/wanted.rs b/hive-c0re/src/workers/wanted.rs index 33885908..28b599b5 100644 --- a/hive-c0re/src/workers/wanted.rs +++ b/hive-c0re/src/workers/wanted.rs @@ -76,7 +76,7 @@ pub async fn pull(coord: &Arc) -> Result<()> { let client = match tokio::time::timeout_at(deadline, crate::swarm_queue::wait_for_client()).await { // Absent — already reported by the shared connector (an `info` log or - // a `swarm_queue_config` warning). + // a `swarm_queue_config` banner). Ok(None) => return Ok(()), Ok(Some(client)) => client, Err(_) => anyhow::bail!("no swarm queue client within {}s", CONNECT_WAIT.as_secs()), diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index b9f5dc39..25097220 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -142,78 +142,6 @@ let annotations = { inherit summary; }; }; - # An alert rule that fires while a metric query's latest value is above - # `threshold`. `expr` is an instant MetricsQL query; names with dots are - # selected as `{__name__="a.b"}`, the way the dashboards do. - # - # No data is `OK`: a series that is absent means the condition is not - # raised (hive-c0re stops reporting a warning once it clears), and a hive - # that stopped exporting at all is the log-absence rules' job. - metricThresholdRule = - { - uid, - title, - windowSeconds ? 600, - expr, - threshold, - for, - summary, - }: - { - inherit uid title for; - condition = "C"; - data = [ - { - refId = "A"; - relativeTimeRange = { - from = windowSeconds; - to = 0; - }; - inherit datasourceUid; - model = { - refId = "A"; - datasource = { - type = "prometheus"; - uid = datasourceUid; - }; - inherit expr; - instant = true; - range = false; - }; - } - { - refId = "B"; - datasourceUid = "__expr__"; - model = { - refId = "B"; - type = "reduce"; - expression = "A"; - reducer = "last"; - }; - } - { - refId = "C"; - datasourceUid = "__expr__"; - model = { - refId = "C"; - type = "threshold"; - expression = "B"; - conditions = [ - { - evaluator = { - type = "gt"; - params = [ threshold ]; - }; - } - ]; - }; - } - ]; - noDataState = "OK"; - execErrState = "Error"; - annotations = { inherit summary; }; - }; - # `logCountRule` inverted: fires while the count is below one. `expr` must # be an ungrouped `stats count()`, which answers 0 rather than no row when # nothing matched; a grouped count drops the group instead, and a dropped @@ -1672,56 +1600,6 @@ in for = "0s"; summary = "{{ $labels._HOSTNAME }} detected agent {{ $labels.F_AGENT }}'s container crash (not an operator-initiated stop)"; }) - # hive-c0re's own readiness verdict (`hyperhive.hive.degraded` - # in otel_metrics.rs), the same one `/health/ready` gives: - # `1` while any `crit`-level warning is raised. The - # per-kind rule below says which one. `for` rides out a - # restart-time blip. - (metricThresholdRule { - uid = "hh-hive-degraded"; - title = "hive degraded"; - expr = ''max by (hive) ({__name__="hyperhive.hive.degraded"})''; - threshold = 0; - for = "2m"; - summary = "hive {{ $labels.hive }} reports itself degraded: a critical warning is raised; the `hh-hive-warning` alert names it"; - }) - # One series per raised warning (`hyperhive.hive.warning`), - # so each `kind` is its own alert instance. The text is the - # `hive warning raised` line in the hive-c0re log. Disk - # pressure is left out: it has its own thresholds below, on - # the filesystem metric rather than the registry entry. - # Sweeps raise their warning only after several misses, so - # `for` is short. `pending_logins` pages too, on purpose: - # an agent that waits more than 2m for its claude login - # needs a human to act, which is what the banner it - # replaces was for. - (metricThresholdRule { - uid = "hh-hive-warning"; - title = "hive warning raised"; - expr = ''max by (hive, kind, level) ({__name__="hyperhive.hive.warning", kind!="disk_pressure"})''; - threshold = 0; - for = "2m"; - summary = "hive {{ $labels.hive }} has a {{ $labels.level }} warning raised: {{ $labels.kind }}; the message is the `hive warning raised` line in the hive-c0re log"; - }) - # `hyperhive.hive.nix_store.utilization` is the used fraction of the - # filesystem holding the nix store, the one that failed CI - # with ENOSPC before the GC guardrails. - (metricThresholdRule { - uid = "hh-host-disk-high"; - title = "host disk filling"; - expr = ''max by (hive) ({__name__="hyperhive.hive.nix_store.utilization"})''; - threshold = 0.85; - for = "10m"; - summary = "the nix store's filesystem on hive {{ $labels.hive }} is over 85% full; garbage-collect before it runs out of space"; - }) - (metricThresholdRule { - uid = "hh-host-disk-critical"; - title = "host disk almost full"; - expr = ''max by (hive) ({__name__="hyperhive.hive.nix_store.utilization"})''; - threshold = 0.95; - for = "2m"; - summary = "the nix store's filesystem on hive {{ $labels.hive }} is over 95% full; garbage-collect now"; - }) ] # One rule per declared hive rather than one grouped by # `hive`, so a hive that sends nothing at all still has diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index bb826163..ec1058f9 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -498,40 +498,6 @@ let && lib.hasInfix ''_msg:="turn failed"'' expr && lib.hasInfix "stats by (hive, agent)" expr; } - { - # The hive-health alerts query metric names that hive-c0re spells in - # Rust. A rename on either side provisions cleanly and the rule then - # reads nothing, so it never fires. Each name here is checked against - # the exporter's source rather than a copy of it, and each rule must - # query the metrics datasource with an instant query. - # 🔬 Invert-proof: rename a gauge in `otel_metrics.rs` and this fails. - name = "the hive-health alert rules query metrics hive-c0re exports"; - ok = - let - groups = - grafanaHere.containers.swarm-grafana.config.services.grafana.provision.alerting.rules.settings.groups; - rules = lib.concatMap (g: g.rules) groups; - source = builtins.readFile ../../hive-c0re/src/stats/otel_metrics.rs; - expected = { - "hh-hive-degraded" = "hyperhive.hive.degraded"; - "hh-hive-warning" = "hyperhive.hive.warning"; - "hh-host-disk-high" = "hyperhive.hive.nix_store.utilization"; - "hh-host-disk-critical" = "hyperhive.hive.nix_store.utilization"; - }; - check = - uid: metric: - let - rule = lib.findFirst (r: r.uid == uid) null rules; - query = (lib.head rule.data).model; - in - rule != null - && lib.hasInfix ''{__name__="${metric}"'' query.expr - && lib.hasInfix ''"${metric}"'' source - && query.instant - && (lib.head rule.data).datasourceUid == "swarm-victoriametrics"; - in - lib.all (uid: check uid expected.${uid}) (builtins.attrNames expected); - } { # 🩸 The regression this guards: moving bao's metrics scrape into its # own in-container collector made that collector stamp every exported