refactor(hive-c0re): one producer for the readiness verdict
`get_health_ready` computed "degraded iff any warning is crit" inline and wrapped it in a private `ReadyBody`. The swarm status publisher needs the same verdict, and the warnings module's own doc already states why it must not compute its own: two systems independently deciding what counts as unhealthy is how they end up disagreeing. The disagreement would also be silent. Each side would look internally consistent, and the day a second degraded condition is added to one of them, the dashboard and the swarm view would report different things about the same host with nothing to flag it. `warnings::readiness()` is now the single producer and `Readiness` the single type. `ReadyBody` is deleted rather than made public: the endpoint keeps the part that genuinely is its own, the mapping onto an HTTP status code, and serves the shared document as its body.
This commit is contained in:
parent
22659234c4
commit
9c1cfafeb5
2 changed files with 68 additions and 21 deletions
|
|
@ -26,8 +26,6 @@ use axum::{
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use utoipa::ToSchema;
|
use utoipa::ToSchema;
|
||||||
|
|
||||||
use crate::host_stats::ServerWarning;
|
|
||||||
|
|
||||||
#[derive(Serialize, ToSchema)]
|
#[derive(Serialize, ToSchema)]
|
||||||
struct LiveBody {
|
struct LiveBody {
|
||||||
status: &'static str,
|
status: &'static str,
|
||||||
|
|
@ -44,40 +42,37 @@ pub(super) async fn get_health_live() -> Response {
|
||||||
(StatusCode::OK, axum::Json(LiveBody { status: "ok" })).into_response()
|
(StatusCode::OK, axum::Json(LiveBody { status: "ok" })).into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, ToSchema)]
|
|
||||||
struct ReadyBody {
|
|
||||||
status: &'static str,
|
|
||||||
warnings: Vec<ServerWarning>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Readiness.
|
/// Readiness.
|
||||||
///
|
///
|
||||||
/// `200` with `{"status":"ok", "warnings": [...]}` unless a `crit`-level
|
/// `200` with `{"status":"ok", "warnings": [...]}` unless a `crit`-level
|
||||||
/// warning is currently set in [`crate::warnings::snapshot`], in which
|
/// warning is currently set, in which case `503` with
|
||||||
/// case `503` with `{"status":"degraded", ...}`. `warnings` always
|
/// `{"status":"degraded", ...}`. `warnings` always carries the full
|
||||||
/// carries the full current list (including `warn`-level entries not
|
/// current list (including `warn`-level entries not affecting the status)
|
||||||
/// affecting the status) so a poller gets detail either way.
|
/// so a poller gets detail either way.
|
||||||
|
///
|
||||||
|
/// The body is [`crate::warnings::Readiness`] rather than a type of this
|
||||||
|
/// module's own, and the verdict comes from
|
||||||
|
/// [`crate::warnings::readiness`] rather than being computed here. The
|
||||||
|
/// swarm status publisher offers that same document upward, and this
|
||||||
|
/// endpoint deciding "unhealthy" for itself is precisely how the two
|
||||||
|
/// would drift apart. What stays here is the only part that *is* this
|
||||||
|
/// endpoint's: the mapping onto an HTTP status code.
|
||||||
#[utoipa::path(
|
#[utoipa::path(
|
||||||
get,
|
get,
|
||||||
path = "/health/ready",
|
path = "/health/ready",
|
||||||
responses(
|
responses(
|
||||||
(status = 200, description = "no crit-level warning set", body = ReadyBody),
|
(status = 200, description = "no crit-level warning set", body = crate::warnings::Readiness),
|
||||||
(status = 503, description = "at least one crit-level warning set", body = ReadyBody),
|
(status = 503, description = "at least one crit-level warning set", body = crate::warnings::Readiness),
|
||||||
),
|
),
|
||||||
tag = "health"
|
tag = "health"
|
||||||
)]
|
)]
|
||||||
pub(super) async fn get_health_ready() -> Response {
|
pub(super) async fn get_health_ready() -> Response {
|
||||||
let warnings = crate::warnings::snapshot();
|
let body = crate::warnings::readiness();
|
||||||
let degraded = warnings.iter().any(|w| w.level == "crit");
|
let code = if body.is_degraded() {
|
||||||
let code = if degraded {
|
|
||||||
StatusCode::SERVICE_UNAVAILABLE
|
StatusCode::SERVICE_UNAVAILABLE
|
||||||
} else {
|
} else {
|
||||||
StatusCode::OK
|
StatusCode::OK
|
||||||
};
|
};
|
||||||
let body = ReadyBody {
|
|
||||||
status: if degraded { "degraded" } else { "ok" },
|
|
||||||
warnings,
|
|
||||||
};
|
|
||||||
(code, axum::Json(body)).into_response()
|
(code, axum::Json(body)).into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,9 @@ use std::collections::BTreeMap;
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::{Mutex, OnceLock};
|
use std::sync::{Mutex, OnceLock};
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
use utoipa::ToSchema;
|
||||||
|
|
||||||
use crate::host_stats::ServerWarning;
|
use crate::host_stats::ServerWarning;
|
||||||
|
|
||||||
/// Monotonic guard-id source. Each [`set_warning`] mints a fresh id so a
|
/// Monotonic guard-id source. Each [`set_warning`] mints a fresh id so a
|
||||||
|
|
@ -89,6 +92,55 @@ pub fn set_warning(
|
||||||
WarningGuard { kind, id }
|
WarningGuard { kind, id }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `status` value for a hive with no `crit`-level warning set.
|
||||||
|
pub const STATUS_OK: &str = "ok";
|
||||||
|
/// `status` value for a hive with at least one `crit`-level warning set.
|
||||||
|
pub const STATUS_DEGRADED: &str = "degraded";
|
||||||
|
|
||||||
|
/// What this hive currently says about its own health.
|
||||||
|
///
|
||||||
|
/// One type with one producer ([`readiness`]) because there is more than
|
||||||
|
/// one consumer: `/health/ready` answers a poller with it, and the swarm
|
||||||
|
/// status publisher offers the same document upward. **Two consumers each
|
||||||
|
/// deciding for themselves what counts as unhealthy is how they end up
|
||||||
|
/// disagreeing** — and the disagreement would be invisible, since each
|
||||||
|
/// would look internally consistent. The day a second degraded condition
|
||||||
|
/// is added, it is added here and both consumers get it.
|
||||||
|
#[derive(Debug, Clone, Serialize, ToSchema)]
|
||||||
|
pub struct Readiness {
|
||||||
|
/// [`STATUS_OK`] or [`STATUS_DEGRADED`].
|
||||||
|
pub status: &'static str,
|
||||||
|
/// The full current warning list, `warn`-level entries included even
|
||||||
|
/// though they do not affect `status` — a consumer gets the detail
|
||||||
|
/// either way rather than having to ask twice.
|
||||||
|
pub warnings: Vec<ServerWarning>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Readiness {
|
||||||
|
/// Whether anything `crit`-level is set. The predicate lives next to
|
||||||
|
/// the constants that encode it so a caller never spells the string.
|
||||||
|
#[must_use]
|
||||||
|
pub fn is_degraded(&self) -> bool {
|
||||||
|
self.status == STATUS_DEGRADED
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Derive the readiness verdict from the current registry contents.
|
||||||
|
///
|
||||||
|
/// The rule — degraded iff any warning is `crit` — is stated exactly
|
||||||
|
/// once, here. `warn` is deliberately not degrading: it is the level for
|
||||||
|
/// "an operator should look", not "stop sending me work".
|
||||||
|
#[must_use]
|
||||||
|
pub fn readiness() -> Readiness {
|
||||||
|
let warnings = snapshot();
|
||||||
|
let status = if warnings.iter().any(|w| w.level == "crit") {
|
||||||
|
STATUS_DEGRADED
|
||||||
|
} else {
|
||||||
|
STATUS_OK
|
||||||
|
};
|
||||||
|
Readiness { status, warnings }
|
||||||
|
}
|
||||||
|
|
||||||
/// Snapshot of the currently-active warnings, kind-sorted. Cheap read
|
/// Snapshot of the currently-active warnings, kind-sorted. Cheap read
|
||||||
/// behind the registry mutex — safe to call on every `/api/state`.
|
/// behind the registry mutex — safe to call on every `/api/state`.
|
||||||
#[must_use]
|
#[must_use]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue