From b39bf67cb336833749afdb01c72aab1d5e6bf70c Mon Sep 17 00:00:00 2001 From: damocles Date: Fri, 31 Jul 2026 19:30:29 +0200 Subject: [PATCH] add /health/live and /health/ready hive-wide health endpoints --- hive-c0re/src/dashboard/health.rs | 118 +++++++++++++++++++++++ hive-c0re/src/dashboard/mod.rs | 3 + nix/host-modules/hive-gateway/vhosts.nix | 29 ++++-- 3 files changed, 140 insertions(+), 10 deletions(-) create mode 100644 hive-c0re/src/dashboard/health.rs diff --git a/hive-c0re/src/dashboard/health.rs b/hive-c0re/src/dashboard/health.rs new file mode 100644 index 00000000..7d5f14d6 --- /dev/null +++ b/hive-c0re/src/dashboard/health.rs @@ -0,0 +1,118 @@ +//! Hive-wide health endpoints for external monitoring: `/health/live` +//! (liveness) and `/health/ready` (readiness). +//! +//! Deliberately thin: reuses [`crate::warnings`] — the same registry the +//! dashboard banner already reads — as the single source of truth for +//! "is anything degraded", rather than tracking health separately. Two +//! systems independently deciding what counts as unhealthy is how they +//! end up disagreeing. +//! +//! Liveness is trivial by construction: if this handler runs at all, the +//! process is up. A liveness probe that does real work risks flapping +//! the *process* down over a transient dependency failure — that's what +//! readiness is for. Readiness derives `"degraded"` from any `crit`-level +//! warning currently set; `warn`-level entries ride along in the body for +//! detail but don't flip the status, mirroring the banner's own severity +//! split. +//! +//! Routed outside `/api/` (see `dashboard/mod.rs`) so the gateway can +//! carve it out from `dashboardAuth` the same way `/webhook/` already is +//! — an external monitor generally can't do interactive HTTP Basic. + +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, +}; +use serde::Serialize; + +use crate::host_stats::ServerWarning; + +/// `GET /health/live` — liveness. Always `200`; no further checks. +pub(super) async fn get_health_live() -> Response { + ( + StatusCode::OK, + axum::Json(serde_json::json!({ "status": "ok" })), + ) + .into_response() +} + +#[derive(Serialize)] +struct ReadyBody { + status: &'static str, + warnings: Vec, +} + +/// `GET /health/ready` — readiness. `200` with `{"status":"ok", "warnings": +/// [...]}` unless a `crit`-level warning is currently set in +/// [`crate::warnings::snapshot`], in which case `503` with +/// `{"status":"degraded", ...}`. `warnings` always carries the full +/// current list (including `warn`-level entries not affecting the +/// status) so a poller gets detail either way. +pub(super) async fn get_health_ready() -> Response { + let warnings = crate::warnings::snapshot(); + let degraded = warnings.iter().any(|w| w.level == "crit"); + let code = if degraded { + StatusCode::SERVICE_UNAVAILABLE + } else { + StatusCode::OK + }; + let body = ReadyBody { + status: if degraded { "degraded" } else { "ok" }, + warnings, + }; + (code, axum::Json(body)).into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::warnings::set_warning; + + async fn ready_status_and_body(_serial: &()) -> (StatusCode, serde_json::Value) { + let resp = get_health_ready().await; + let status = resp.status(); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("read body"); + let body: serde_json::Value = serde_json::from_slice(&bytes).expect("json body"); + (status, body) + } + + // The warnings registry is process-global (see `stats::warnings`), + // so these two tests share a mutex to avoid racing each other under + // cargo's parallel runner — same pattern `warnings::tests` uses via + // per-test unique kinds, but here the *emptiness* of the whole + // registry is exactly what's under test, so unique kinds aren't + // enough on their own. `tokio::sync::Mutex`, not `std::sync::Mutex` + // — the guard is held across the `.await` in + // `ready_status_and_body`, and a std mutex guard can't cross an + // await point (clippy's `await_holding_lock` catches exactly this). + static SERIAL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + + #[tokio::test] + async fn live_is_always_ok() { + let resp = get_health_live().await; + assert_eq!(resp.status(), StatusCode::OK); + } + + #[tokio::test] + async fn ready_is_ok_with_no_crit_warning() { + let _lock = SERIAL.lock().await; + // A `warn`-level entry must not flip readiness. + let _g = set_warning("t_health_warn", "warn", "just a warn"); + let (status, body) = ready_status_and_body(&()).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["status"], "ok"); + } + + #[tokio::test] + async fn ready_is_degraded_with_a_crit_warning() { + let _lock = SERIAL.lock().await; + let _g = set_warning("t_health_crit", "crit", "on fire"); + let (status, body) = ready_status_and_body(&()).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(body["status"], "degraded"); + let warnings = body["warnings"].as_array().expect("warnings array"); + assert!(warnings.iter().any(|w| w["kind"] == "t_health_crit")); + } +} diff --git a/hive-c0re/src/dashboard/mod.rs b/hive-c0re/src/dashboard/mod.rs index 1470d7fd..55fe7ec7 100644 --- a/hive-c0re/src/dashboard/mod.rs +++ b/hive-c0re/src/dashboard/mod.rs @@ -24,6 +24,7 @@ mod extra_forges; // same type used to build agent paths. Re-exported so submodules + the socket // server reach it as `crate::dashboard::Ident`. pub(crate) use hive_types::Ident; +mod health; mod infra_containers; mod journal; mod lifecycle_ops; @@ -79,6 +80,8 @@ pub async fn serve( // API-only: the gateway static-serves the dashboard dist and proxies // non-static requests here (see hive-gateway.nix). Unmatched paths 404. let app = Router::new() + .route("/health/live", get(health::get_health_live)) + .route("/health/ready", get(health::get_health_ready)) .route("/api/state", get(state_snapshot::api_state)) .route("/api/journal/{name}", get(journal::get_journal)) .route("/api/journal-host", get(journal::get_journal_host)) diff --git a/nix/host-modules/hive-gateway/vhosts.nix b/nix/host-modules/hive-gateway/vhosts.nix index ee5531aa..e9fc2370 100644 --- a/nix/host-modules/hive-gateway/vhosts.nix +++ b/nix/host-modules/hive-gateway/vhosts.nix @@ -259,16 +259,16 @@ let ''; # Dashboard: nginx static-serves the dist, c0re is API-only. Routing - # is by PATH, never content-type. c0re serves exactly two prefixes — - # `/api/` (all dashboard data + actions + the SSE streams) and - # `/webhook/` (knowledge push + config-PR approval triggers, HMAC- - # guarded) — so those proxy to c0re and everything else serves the - # dist with an SPA fallback to index.html. Path routing is - # deterministic where an Accept-header split would make the SAME url - # behave differently by content-type (e.g. `/api/state` fetched with - # `Accept: text/html` wrongly getting index.html). A new top-level - # c0re route prefix (beyond /api + /webhook) needs a matching - # location added here. + # is by PATH, never content-type. c0re serves exactly three prefixes — + # `/api/` (all dashboard data + actions + the SSE streams), `/webhook/` + # (knowledge push + config-PR approval triggers, HMAC-guarded), and + # `/health/` (liveness + readiness) — so those proxy to c0re and + # everything else serves the dist with an SPA fallback to index.html. + # Path routing is deterministic where an Accept-header split would make + # the SAME url behave differently by content-type (e.g. `/api/state` + # fetched with `Accept: text/html` wrongly getting index.html). A new + # top-level c0re route prefix (beyond /api + /webhook + /health) needs + # a matching location added here. dashboardProxyLocation = { "/" = { root = dashboardDist; @@ -294,6 +294,15 @@ let # for these endpoints; hive-c0re verifies it in the handler. proxyPass = "http://${cfg.upstreamHost}:${toString cfg.upstreamPort}"; }; + "/health/" = { + # No dashboardAuth here either, for a different reason than + # /webhook/: an external uptime monitor generally can't do + # interactive HTTP Basic. The endpoints themselves are scoped to + # status + warning kind/message (see hive-c0re/src/dashboard/ + # health.rs) — no tokens, no agent detail — so exposing them + # unauthenticated isn't a new secret surface. + proxyPass = "http://${cfg.upstreamHost}:${toString cfg.upstreamPort}"; + }; }; in {