hyperhive/hive-c0re/src/swarm_status.rs

//! Offering this hive's readiness upward to the swarm.
//!
//! A hive **offers**; the swarm controller never reaches down to collect.
//! That direction is the design and not an implementation detail: the
//! gateway has gone down in a way where every recovery channel ran
//! through the one broken thing, so a status path that depended on the
//! controller would go dark exactly when it is needed. A hive computes
//! its own status locally whether or not the swarm can be reached, and
//! this task is only the part that carries it.
//!
//! **It publishes what the hive already says about itself.**
//! [`crate::warnings::readiness`] is the same value `/health/ready`
//! serves — not a swarm-specific recomputation. Two producers of "is this
//! hive OK" would be free to disagree, and the disagreement would surface
//! as the dashboard and the swarm view contradicting each other about the
//! same host, each internally consistent.
//!
//! **The key is this hive's `hiveName`.** Nothing here has to arrange for
//! that to match the controller's roster: `swarm.nix` asserts that
//! `services.hyperhive.hiveName` is a key of `services.hyperhive.swarm.hives`,
//! so a hive that evaluates at all publishes under a name the roster
//! knows. The alternative — a key the controller has never heard of —
//! renders as `unknown` rather than being silently dropped, but the
//! assertion means it should not arise.
//!
//! **Nothing here stamps a time.** Freshness is derived by the reader
//! from when the value landed in the bucket, so a hive cannot make itself
//! look fresher than it is, and a hive with a wrong clock skews only its
//! own payload.

use std::time::Duration;

use anyhow::{Context, Result};
// `Subscriber` is a `Stream`, so reading the next event needs the extension
// trait — there is no inherent `next()` on it.
use futures_util::StreamExt as _;

use crate::stats::sweep_health::{self, SweepHealth};

/// How often this hive offers a snapshot.
///
/// **One decision with the controller's `staleAfterSeconds` (default
/// 120s), not two.** The ratio is what either option means: at 2, a
/// single lost publish still reads `fresh` and two consecutive misses
/// read `stale`. Set them equal and any one hiccup alarms; open the gap
/// wide and `stale` stops meaning anything. Changing one without the
/// other silently re-tunes the swarm's definition of "quiet".
pub const PUBLISH_INTERVAL: Duration = Duration::from_mins(1);

/// Env var prefix for this daemon's swarm-queue credentials — see
/// [`swarm_queue_client::QueueConfig::from_env`]. All four or none.
const ENV_PREFIX: &str = "HIVE_C0RE";

/// Consecutive failed publishes before the dashboard banners.
///
/// At [`PUBLISH_INTERVAL`] this is ~3 minutes of genuine failure, so a
/// blip does not flap a banner an operator learns to ignore.
const FAILURES_BEFORE_BANNER: u32 = 3;

/// Start the publish loop, if this deployment wired up a swarm queue.
///
/// Absent queue config is the ordinary case — most hives are not in a
/// swarm — so it is an `info` and not a warning. A *half*-set environment
/// is a different thing entirely and [`swarm_queue_client::QueueConfig::from_env`]
/// makes it a hard error; it is bannered here rather than swallowed,
/// because the failure it otherwise produces is a hive that looks fine
/// and silently never reports.
pub fn spawn(
    coord: std::sync::Arc<crate::coordinator::Coordinator>,
    mut shutdown: tokio::sync::watch::Receiver<bool>,
) {
    let cfg = match swarm_queue_client::QueueConfig::from_env(ENV_PREFIX) {
        Ok(Some(cfg)) => cfg,
        Ok(None) => {
            tracing::info!("no swarm queue configured; this hive offers no status upward");
            return;
        }
        Err(e) => {
            // A one-shot startup step with no later retry to clear it —
            // exactly what `set_boot_warning` is for. The fix is a
            // redeploy, which restarts this process anyway.
            //
            // `chain`, not `{:#}`: this is the queue client's own error
            // type, whose Display ignores the alternate flag, so `{:#}`
            // would show only "swarm queue is half-configured" and drop
            // which variables are missing.
            crate::warnings::set_boot_warning(
                "swarm_status_config",
                "warn",
                format!(
                    "swarm status publishing is off: {}",
                    swarm_queue_client::chain(&e)
                ),
            );
            return;
        }
    };

    let Some(hive) = crate::container_view::hive_swarm_names().0 else {
        crate::warnings::set_boot_warning(
            "swarm_status_config",
            "warn",
            "swarm status publishing is off: HYPERHIVE_HIVE_NAME is unset, so this \
             hive has no key to publish under",
        );
        return;
    };

    tokio::spawn(async move {
        // `retry_on_initial_connect` inside, so this returns a client
        // that may not be connected yet rather than failing on a queue
        // that comes up second. The publish below is what discovers that,
        // and it is already the thing that reports it.
        let client = match swarm_queue_client::connect(cfg).await {
            Ok(client) => client,
            Err(e) => {
                // `chain` for the same reason as above: without it this
                // banner reads "connecting to the swarm queue at <url>"
                // and drops the nats error that says why.
                crate::warnings::set_boot_warning(
                    "swarm_status_config",
                    "warn",
                    format!(
                        "swarm status publishing is off: {}",
                        swarm_queue_client::chain(&e)
                    ),
                );
                return;
            }
        };

        // The hive's ONE queue connection, now serving both directions:
        // status goes up, swarm events come down. A second `connect` would
        // double the auth-callout traffic against authelia and give the two
        // paths independent reconnect state, so one could be serving while
        // the other was still down. `async_nats::Client` is a handle, so the
        // clone is cheap.
        tokio::spawn(drain_swarm_events(client.clone(), coord, shutdown.clone()));

        let mut health = SweepHealth::new("swarm_status_publish", "warn", FAILURES_BEFORE_BANNER);
        loop {
            match publish(&client, &hive).await {
                Ok(()) => health.record_ok(),
                Err(e) => {
                    tracing::warn!(error = ?e, "swarm status: publish failed");
                    let err = format!("{e:#}");
                    health.record_err(|ctx| {
                        let age = ctx.since_last_ok.map_or_else(
                            || "no success this session".to_owned(),
                            |d| format!("last ok {} ago", sweep_health::fmt_age(d)),
                        );
                        format!(
                            "swarm status publishing is failing ({} consecutive, {age}) \
                             — the swarm sees this hive as stale, the hive itself is \
                             unaffected: {err}",
                            ctx.consecutive
                        )
                    });
                }
            }
            // Publish first, then wait: a hive that has just come up is
            // exactly the one whose status someone is looking at, and
            // sleeping first would make every restart read `stale` for a
            // full interval. The shutdown arm means a stop is not spent
            // waiting out that interval either.
            tokio::select! {
                () = tokio::time::sleep(PUBLISH_INTERVAL) => {}
                _ = shutdown.changed() => {
                    tracing::info!("swarm status: shutdown signal received");
                    break;
                }
            }
        }
    });
}

/// Listen on the swarm's knowledge-event subject and act on what arrives.
///
/// The controller decides *what a forge delivery means* and addresses the
/// result here; this end does not know a forge exists. Today the one event is
/// **the knowledge repository changed**, and the response is the pull this
/// daemon already runs at boot.
///
/// # There is no payload, and that is deliberate
///
/// The event carries nothing. The webhook handler this replaces read two
/// fields from Forgejo and used neither — both were filters — then ran
/// `git pull`, which re-derives everything from the repository. So it is an
/// edge trigger, and reading a body here would invent a contract nobody owes.
///
/// # What a missed message costs
///
/// Core NATS, so delivery is at-most-once: a hive that is down when the
/// controller publishes never hears it, and its knowledge stays as of its last
/// pull until it next boots. **That is not a regression** — a webhook delivery
/// to a hive that is down is lost identically, and this daemon pulls at startup
/// regardless. `JetStream` would require this end to *publish* to
/// `$JS.API.CONSUMER.CREATE.<stream>`, which the callout policy does not grant,
/// so durability would cost a grant on both sides to remove a failure the boot
/// pull already covers.
///
/// ⚠️ **A refused subscription is indistinguishable from a quiet one.** NATS
/// reports an authorization violation asynchronously on the connection, not as
/// an error from `subscribe`, so this task cannot tell "no events published"
/// from "not allowed to hear them". If a hive stops picking up knowledge
/// changes, the server log is the thing that knows why — nothing here will say.
async fn drain_swarm_events(
    client: async_nats::Client,
    coord: std::sync::Arc<crate::coordinator::Coordinator>,
    mut shutdown: tokio::sync::watch::Receiver<bool>,
) {
    // One subject for the whole swarm, so this hive's own name never enters
    // it: the controller publishes once and core NATS fans out to whoever is
    // subscribed.
    let subject = swarm_queue_client::KNOWLEDGE_SUBJECT;
    let mut sub = match client.subscribe(subject).await {
        Ok(sub) => sub,
        Err(e) => {
            // Warn rather than a boot banner: the hive is fully functional
            // without this, it just falls back to learning about knowledge
            // changes at its next boot.
            tracing::warn!(
                %subject, error = %e,
                "swarm events: subscribe failed; this hive will not hear knowledge changes"
            );
            return;
        }
    };
    tracing::info!(%subject, "swarm events: listening");

    loop {
        tokio::select! {
            msg = sub.next() => {
                if msg.is_none() {
                    // The subscription ended — the connection went away for
                    // good. Returning is right: `async-nats` reconnects
                    // underneath a live subscription, so a closed stream is
                    // not a blip this should spin on.
                    tracing::warn!(%subject, "swarm events: subscription closed");
                    return;
                }
                tracing::info!(%subject, "swarm events: knowledge change announced, pulling");
                if let Err(e) = crate::workers::knowledge::pull(&coord).await {
                    tracing::warn!(error = ?e, "swarm events: knowledge pull failed");
                }
            }
            _ = shutdown.changed() => {
                tracing::info!("swarm events: shutdown signal received");
                return;
            }
        }
    }
}

/// Offer one snapshot: this hive's current readiness, under its own key.
async fn publish(client: &async_nats::Client, hive: &str) -> Result<()> {
    // An unconnected client does not fail a JetStream request, it hangs
    // on it — which here would hang the loop, shutdown arm included.
    swarm_queue_client::ensure_connected(client)?;

    let store = swarm_queue_client::status::open_or_create(client).await?;
    let payload =
        serde_json::to_vec(&crate::warnings::readiness()).context("serialising the readiness")?;
    store
        .put(hive, payload.into())
        .await
        .with_context(|| format!("publishing the status snapshot for {hive}"))?;
    Ok(())
}