//! Offering this hive's readiness upward to the swarm. //! //! A hive **offers**; the swarm controller never reaches down to collect. //! That direction is the design and not an implementation detail: the //! gateway has gone down in a way where every recovery channel ran //! through the one broken thing, so a status path that depended on the //! controller would go dark exactly when it is needed. A hive computes //! its own status locally whether or not the swarm can be reached, and //! this task is only the part that carries it. //! //! **It publishes what the hive already says about itself.** //! [`crate::warnings::readiness`] is the same value `/health/ready` //! serves — not a swarm-specific recomputation. Two producers of "is this //! hive OK" would be free to disagree, and the disagreement would surface //! as the dashboard and the swarm view contradicting each other about the //! same host, each internally consistent. //! //! **The key is this hive's `hiveName`.** Nothing here has to arrange for //! that to match the controller's roster: `swarm.nix` asserts that //! `services.hyperhive.hiveName` is a key of `services.hyperhive.swarm.hives`, //! so a hive that evaluates at all publishes under a name the roster //! knows. The alternative — a key the controller has never heard of — //! renders as `unknown` rather than being silently dropped, but the //! assertion means it should not arise. //! //! **Nothing here stamps a time.** Freshness is derived by the reader //! from when the value landed in the bucket, so a hive cannot make itself //! look fresher than it is, and a hive with a wrong clock skews only its //! own payload. use std::time::Duration; use anyhow::{Context, Result}; use crate::stats::sweep_health::{self, SweepHealth}; /// How often this hive offers a snapshot. /// /// **One decision with the controller's `staleAfterSeconds` (default /// 120s), not two.** The ratio is what either option means: at 2, a /// single lost publish still reads `fresh` and two consecutive misses /// read `stale`. Set them equal and any one hiccup alarms; open the gap /// wide and `stale` stops meaning anything. Changing one without the /// other silently re-tunes the swarm's definition of "quiet". pub const PUBLISH_INTERVAL: Duration = Duration::from_mins(1); /// Env var prefix for this daemon's swarm-queue credentials — see /// [`swarm_queue_client::QueueConfig::from_env`]. All four or none. const ENV_PREFIX: &str = "HIVE_C0RE"; /// Consecutive failed publishes before the dashboard banners. /// /// At [`PUBLISH_INTERVAL`] this is ~3 minutes of genuine failure, so a /// blip does not flap a banner an operator learns to ignore. const FAILURES_BEFORE_BANNER: u32 = 3; /// Start the publish loop, if this deployment wired up a swarm queue. /// /// Absent queue config is the ordinary case — most hives are not in a /// swarm — so it is an `info` and not a warning. A *half*-set environment /// is a different thing entirely and [`swarm_queue_client::QueueConfig::from_env`] /// makes it a hard error; it is bannered here rather than swallowed, /// because the failure it otherwise produces is a hive that looks fine /// and silently never reports. pub fn spawn(mut shutdown: tokio::sync::watch::Receiver) { let cfg = match swarm_queue_client::QueueConfig::from_env(ENV_PREFIX) { Ok(Some(cfg)) => cfg, Ok(None) => { tracing::info!("no swarm queue configured; this hive offers no status upward"); return; } Err(e) => { // A one-shot startup step with no later retry to clear it — // exactly what `set_boot_warning` is for. The fix is a // redeploy, which restarts this process anyway. // // `chain`, not `{:#}`: this is the queue client's own error // type, whose Display ignores the alternate flag, so `{:#}` // would show only "swarm queue is half-configured" and drop // which variables are missing. crate::warnings::set_boot_warning( "swarm_status_config", "warn", format!( "swarm status publishing is off: {}", swarm_queue_client::chain(&e) ), ); return; } }; let Some(hive) = crate::container_view::hive_swarm_names().0 else { crate::warnings::set_boot_warning( "swarm_status_config", "warn", "swarm status publishing is off: HYPERHIVE_HIVE_NAME is unset, so this \ hive has no key to publish under", ); return; }; tokio::spawn(async move { // `retry_on_initial_connect` inside, so this returns a client // that may not be connected yet rather than failing on a queue // that comes up second. The publish below is what discovers that, // and it is already the thing that reports it. let client = match swarm_queue_client::connect(cfg).await { Ok(client) => client, Err(e) => { // `chain` for the same reason as above: without it this // banner reads "connecting to the swarm queue at " // and drops the nats error that says why. crate::warnings::set_boot_warning( "swarm_status_config", "warn", format!( "swarm status publishing is off: {}", swarm_queue_client::chain(&e) ), ); return; } }; let mut health = SweepHealth::new("swarm_status_publish", "warn", FAILURES_BEFORE_BANNER); loop { match publish(&client, &hive).await { Ok(()) => health.record_ok(), Err(e) => { tracing::warn!(error = ?e, "swarm status: publish failed"); let err = format!("{e:#}"); health.record_err(|ctx| { let age = ctx.since_last_ok.map_or_else( || "no success this session".to_owned(), |d| format!("last ok {} ago", sweep_health::fmt_age(d)), ); format!( "swarm status publishing is failing ({} consecutive, {age}) \ — the swarm sees this hive as stale, the hive itself is \ unaffected: {err}", ctx.consecutive ) }); } } // Publish first, then wait: a hive that has just come up is // exactly the one whose status someone is looking at, and // sleeping first would make every restart read `stale` for a // full interval. The shutdown arm means a stop is not spent // waiting out that interval either. tokio::select! { () = tokio::time::sleep(PUBLISH_INTERVAL) => {} _ = shutdown.changed() => { tracing::info!("swarm status: shutdown signal received"); break; } } } }); } /// Offer one snapshot: this hive's current readiness, under its own key. async fn publish(client: &async_nats::Client, hive: &str) -> Result<()> { // An unconnected client does not fail a JetStream request, it hangs // on it — which here would hang the loop, shutdown arm included. swarm_queue_client::ensure_connected(client)?; let store = swarm_queue_client::status::open_or_create(client).await?; let payload = serde_json::to_vec(&crate::warnings::readiness()).context("serialising the readiness")?; store .put(hive, payload.into()) .await .with_context(|| format!("publishing the status snapshot for {hive}"))?; Ok(()) }