hyperhive/swarm-queue-client/src/lib.rs

//! Connecting to the swarm message queue as an authenticated client.
//!
//! Shared by every process that needs the queue — the swarm controller reads
//! hive status out of it, a hive publishes its own status into it — because
//! the *connect* is identical for all of them and only the use differs.
//! Duplicating it per binary would put credential handling in two places, and
//! a token-refresh fix would then have to be found twice.
//!
//! The queue admits every non-responder client through `auth_callout`: a
//! client presents a token at CONNECT, the callout responder introspects it
//! against authelia and mints a user JWT if it is good. So each participant is
//! an ordinary client that needs an identity of its own — the controller is
//! not a hive, and the per-hive clients issued from the roster are not its to
//! use.
//!
//! Two things about that shape drive everything here:
//!
//! - **A token expires.** Authelia issues `client_credentials` access tokens
//!   with `expires_in: 3599`. Authentication happens at CONNECT, so a
//!   long-lived connection is fine — but a *reconnect* an hour later needs a
//!   token that was minted an hour later.
//! - **`async-nats` re-runs an auth callback per connection attempt** (it is
//!   handed that attempt's nonce). So the refresh belongs in the callback and
//!   not in a timer: there is no window in which the client presents a token
//!   that has expired since it was minted.
//!
//! ⚠️ That second point is also a trap that cost a production incident —
//! **anything in a retry path runs at the FAILURE rate.** See `CachedToken`
//! and `MAX_RECONNECT_DELAY`.

use std::path::PathBuf;

/// Everything that can go wrong reaching the swarm queue.
///
/// `thiserror` and not `anyhow` because this is a library: a caller gets a
/// type it can match on, and the binaries that consume it keep using
/// `anyhow` — `?` converts for free, so nothing downstream is more verbose
/// for it. The same split `hive-claude` uses.
///
/// The variants are the failures an operator acts on differently: a
/// half-configured environment is a deployment bug, a refused token is an
/// identity-provider config problem, an unreachable queue is a network
/// one. Collapsing them into one string would make that distinction a
/// matter of reading prose.
#[derive(Debug, thiserror::Error)]
pub enum Error {
    #[error(
        "swarm queue is half-configured: {prefix}_NATS_URL, \
         {prefix}_OIDC_TOKEN_ENDPOINT, {prefix}_OIDC_CLIENT_ID and \
         {prefix}_OIDC_CLIENT_SECRET_FILE must be set together or not at all"
    )]
    PartialConfig { prefix: String },

    #[error("reading the queue client secret from {path}")]
    ClientSecret {
        path: String,
        #[source]
        source: std::io::Error,
    },

    #[error("building the token-endpoint HTTP client")]
    HttpClient(#[source] reqwest::Error),

    /// Distinct from `HttpClient` because the operator's next move differs:
    /// this one names a path they configured, and it fires before any
    /// network call.
    #[error("reading the token-endpoint CA certificate from {path}")]
    CaFile {
        path: String,
        #[source]
        source: std::io::Error,
    },

    #[error("parsing the token-endpoint CA certificate from {path} as PEM")]
    CaParse {
        path: String,
        #[source]
        source: reqwest::Error,
    },

    #[error("requesting an access token from authelia")]
    TokenRequest(#[source] reqwest::Error),

    /// The body carries authelia's own `error_description`, and it is far
    /// more useful than the status alone: a wrong grant says
    /// `unauthorized_client`, a wrong secret says `invalid_client`, and
    /// those point at different config.
    #[error("authelia refused the token request ({status}): {body}")]
    TokenRefused {
        status: reqwest::StatusCode,
        body: String,
    },

    #[error("parsing authelia's token response")]
    TokenResponse(#[source] serde_json::Error),

    #[error("connecting to the swarm queue at {url}")]
    Connect {
        url: String,
        #[source]
        source: async_nats::ConnectError,
    },

    /// Distinct from [`Error::Connect`] on purpose: that one is a connect
    /// that was attempted and refused, this one is a request made before
    /// any connection exists. The second is a caller-ordering problem and
    /// the first is a deployment one.
    #[error("not connected to the swarm queue (client state: {0:?})")]
    NotConnected(async_nats::connection::State),

    #[cfg(feature = "kv")]
    #[error("creating the {bucket} bucket")]
    CreateBucket {
        // `String` rather than `&'static str`: a per-hive bucket name is built
        // at runtime (`wanted::bucket`), so the error has to own it.
        bucket: String,
        #[source]
        source: async_nats::jetstream::context::CreateKeyValueError,
    },

    #[cfg(feature = "notices")]
    #[error("creating the {stream} stream")]
    CreateStream {
        stream: &'static str,
        #[source]
        source: async_nats::jetstream::context::CreateStreamError,
    },
}

/// Render an error and its source chain on one line.
///
/// **Use this instead of `{:#}` anywhere an [`Error`] is rendered without
/// first being `?`-converted into an `anyhow::Error`.** `anyhow`'s
/// `Display` special-cases `f.alternate()` to walk the source chain;
/// `thiserror`'s derive does not, so `{e:#}` and `{e}` render
/// identically for this type. A call site that formatted an
/// `anyhow::Error` with `{:#}` and now holds an [`Error`] therefore keeps
/// compiling, keeps looking right, and silently drops the cause — which
/// is the half that says *why*, and is exactly what a one-shot boot
/// warning with no retry needs most.
///
/// Public for that reason: the fix cannot live only inside this crate's
/// own auth callback while the callers it was written for reach for
/// `{:#}` and get nothing.
pub fn chain(error: &dyn std::error::Error) -> String {
    let mut rendered = error.to_string();
    let mut source = error.source();
    while let Some(cause) = source {
        rendered.push_str(": ");
        rendered.push_str(&cause.to_string());
        source = cause.source();
    }
    rendered
}

/// The hive-status KV bucket, shared by the hive that writes it and the
/// controller that reads it. See the module doc for why a bucket name and
/// its config belong to neither end alone.
///
/// The module itself is unconditional; only the parts that *open* the bucket
/// need the `kv` feature. The name is a `&str` with no dependencies, and a
/// third end names it too — the auth-callout responder, which derives the
/// subjects a hive may publish to from it without ever speaking `jetstream`.
/// Gating the name behind `kv` would have forced that consumer to choose
/// between pulling a JetStream stack it does not use and copying the literal,
/// which is the disagreement this module exists to prevent.
pub mod status;

/// The bucket the controller declares each hive's wanted agent set in — the
/// opposite direction from [`status`], and deliberately not a mirror of it.
/// Status is *observed* and republishes itself; this is *declared* and nothing
/// recreates it, which is why the module documents its own durability and its
/// "absence is not a deletion order" rule rather than inheriting either.
pub mod wanted;

/// The bucket per-agent status snapshots are published into — one key per
/// `(hive, agent)`, deliberately a sibling of [`status`] rather than a field
/// inside it. See the module doc for why the two must not merge.
pub mod agent_status;

/// The subject the swarm controller publishes on when the hive-wide knowledge
/// repository has changed. One writer, many readers — every hive subscribes.
///
/// Here rather than in a module of its own for the same reason as the bucket
/// name above: three crates must agree on the string, and the one that agrees
/// hardest — the auth-callout responder, which decides whether a publish is
/// permitted at all — speaks neither `jetstream` nor `kv`.
pub const KNOWLEDGE_SUBJECT: &str = "$SWARM.knowledge";

/// The subject the swarm controller publishes on to ask `hive` to rebuild one
/// of its agents. Lives here for the same three-crate reason as
/// [`KNOWLEDGE_SUBJECT`], but unlike it this is a **family, one subject per
/// hive** — a hive subscribes to its own and is never woken by a deploy meant
/// for someone else.
///
/// ⚠️ **That split is about noise, not confidentiality.** The auth-callout
/// responder scopes *publish* only and leaves `sub` unrestricted, so a hive
/// that wanted another's messages could still subscribe to them. What the
/// family buys is that it does not receive them by default.
#[must_use]
pub fn deploy_subject(hive: &str) -> String {
    format!("{DEPLOY_SUBJECT_PREFIX}.{hive}")
}

/// The publish grant covering every [`deploy_subject`], for the one client
/// that may send them. A wildcard rather than a subject per hive because the
/// responder has no roster — it cannot enumerate hives, and a grant it had to
/// keep in step with one would be a second place to get the list wrong.
pub const DEPLOY_SUBJECT_WILDCARD: &str = "$SWARM.deploy.*";

/// Shared by [`deploy_subject`] and [`DEPLOY_SUBJECT_WILDCARD`] so the two
/// cannot drift into naming different families.
const DEPLOY_SUBJECT_PREFIX: &str = "$SWARM.deploy";

/// The subject the controller publishes on to tell `hive` that a credential
/// for one of its agents is waiting in the secret store. Same per-hive family
/// as [`deploy_subject`], here for the same three-crate reason.
///
/// 🔑 **The message NAMES a credential and never carries one**, and the note
/// on [`deploy_subject`] is why: the family split buys quiet, not
/// confidentiality — the auth-callout responder scopes `pub` and leaves `sub`
/// unrestricted, so any hive that wanted another's messages could subscribe to
/// them. A secret in this payload would be readable swarm-wide. The hive reads
/// the value from the store under its own identity instead, where the store's
/// policy is the thing that actually scopes it.
#[must_use]
pub fn credential_subject(hive: &str) -> String {
    format!("{CREDENTIAL_SUBJECT_PREFIX}.{hive}")
}

/// The publish grant covering every [`credential_subject`] — a wildcard for
/// the same no-roster reason as [`DEPLOY_SUBJECT_WILDCARD`].
pub const CREDENTIAL_SUBJECT_WILDCARD: &str = "$SWARM.credential.*";

/// Shared by [`credential_subject`] and [`CREDENTIAL_SUBJECT_WILDCARD`] so the
/// two cannot drift into naming different families.
const CREDENTIAL_SUBJECT_PREFIX: &str = "$SWARM.credential";

/// What a [`credential_subject`] message says: which agent's credential
/// changed, and which account it belongs to. Deliberately the whole payload —
/// anything more would be either derivable by the reader or a secret that
/// must not be on the wire.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct CredentialNotice {
    /// The agent whose state dir receives the credential.
    pub agent: String,
    /// The external account the credential authenticates as.
    pub account: String,
}

/// What a [`deploy_subject`] message carries.
///
/// Only the agent: the subject already names the hive, and repeating it here
/// would be two places stating one fact, free to disagree.
///
/// ⚠️ **A trigger, not the config.** The hive already tracks the agent's
/// config repo; putting desired state on the wire would make this message a
/// second source of truth for something git already owns, and a hive that
/// missed a message would then be wrong rather than merely late.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct DeployRequest {
    /// The agent to rebuild, as the swarm knows it.
    pub agent: String,
}

/// The hive-notices stream, shared by the hive that publishes and
/// whatever eventually consumes it. Behind the `notices` feature, same
/// reason `status` is behind `kv` — see the module doc.
#[cfg(feature = "notices")]
pub mod notices;

/// Only the fields this needs; authelia returns several.
#[derive(serde::Deserialize)]
struct TokenResponse {
    access_token: String,
    /// Seconds the token stays valid — authelia says 3599 for
    /// `client_credentials`. Read rather than assumed, because it is what
    /// decides when the cache below stops handing the same token back, and a
    /// hardcoded hour would silently become wrong the day the identity
    /// provider is retuned.
    expires_in: u64,
}

/// A minted token and the moment it stops being usable.
///
/// Exists because the auth callback runs **per connection attempt**, not per
/// hour: without a cache, every reconnect mints, and a queue that cannot
/// connect turns its retry loop into a token-request loop against the
/// identity provider. That is not hypothetical — the first version of this
/// crate did exactly that, and authelia answered `429 Too Many Requests`
/// continuously until the client was changed.
struct CachedToken {
    token: String,
    expires_at: std::time::Instant,
}

/// Re-mint this long before expiry rather than at it. A token that expires
/// mid-CONNECT is refused by the server, which costs a whole reconnect cycle
/// to discover — far more than minting slightly early costs.
const TOKEN_REFRESH_SKEW: std::time::Duration = std::time::Duration::from_mins(2);

/// Cap on how long `async-nats` waits between reconnect attempts.
///
/// ⚠️ Deliberately far above the library's default, and the default is the
/// bug: `reconnect_delay_callback_default` backs off exponentially but clamps
/// at **4 seconds** (`connector.rs`), forever. Every attempt runs the auth
/// callback, so an unreachable queue means one token request every 4s
/// indefinitely — measured in production as a continuous
/// `429 Too Many Requests / temporarily_unavailable` from authelia, which then
/// keeps *itself* alive: the rate limit outlives whatever first broke the
/// connection, and its log volume buries the original cause.
///
/// The cost of raising it is slower recovery from a brief outage. That is the
/// right trade for an infrastructure service: a minute of staleness is a
/// dashboard reading `stale`, whereas a hot loop against the identity provider
/// degrades every other client of it.
const MAX_RECONNECT_DELAY: std::time::Duration = std::time::Duration::from_mins(1);

/// Where the controller finds the queue and what it authenticates with.
///
/// Every field comes from an environment variable the NixOS module sets, the
/// same way `load_hives` takes the roster — a config change is a redeploy, and
/// this process reads no file it was not pointed at.
#[derive(Debug, Clone)]
pub struct QueueConfig {
    /// `nats://host:port` for the swarm queue.
    pub url: String,
    /// Authelia's token endpoint, e.g. `https://auth.<swarm>/api/oidc/token`.
    pub token_endpoint: String,
    /// The controller's own `OAuth2` client id.
    pub client_id: String,
    /// File holding the client secret's PLAINTEXT.
    ///
    /// A path and not a value: the secret is minted on the authelia host and
    /// read here, and putting it in the environment would publish it to
    /// anything that can read `/proc/<pid>/environ`.
    pub client_secret_file: PathBuf,
    /// Extra trust anchor for the token endpoint, when it is not signed by
    /// a publicly-trusted CA.
    ///
    /// Optional, and deliberately NOT part of the all-or-none group below: a
    /// swarm fronted by a public certificate needs no extra anchor, and
    /// making this required would break that deployment to fix ours. Absent
    /// means "the platform's roots are enough", which is the correct default
    /// for a client that might talk to anything.
    ///
    /// ⚠️ Without it, a swarm using its own CA fails at TLS with
    /// `invalid peer certificate: UnknownIssuer` — the anchor exists on the
    /// host and this client simply never looked at it.
    pub ca_file: Option<PathBuf>,
}

impl QueueConfig {
    /// Read the config from `<prefix>_NATS_URL`, `<prefix>_OIDC_TOKEN_ENDPOINT`,
    /// `<prefix>_OIDC_CLIENT_ID` and `<prefix>_OIDC_CLIENT_SECRET_FILE`, or
    /// `None` when the queue was not wired up for this deployment.
    ///
    /// The prefix is a parameter rather than a constant because the variables
    /// belong to the *consuming unit* — a NixOS module sets them alongside its
    /// other options, and two daemons sharing one name would be a worse
    /// coupling than passing four characters. What is shared is the RULE
    /// below, not the spelling.
    ///
    /// `None` rather than an error on purpose: a daemon serves its other
    /// surfaces on hosts where the queue is not enabled, and refusing to start
    /// there would trade a missing feature for a dead process. What must NOT
    /// happen is a *half* configuration silently behaving like an absent one —
    /// hence the explicit partial check below.
    pub fn from_env(prefix: &str) -> Result<Option<Self>, Error> {
        let url = std::env::var(format!("{prefix}_NATS_URL")).ok();
        let token_endpoint = std::env::var(format!("{prefix}_OIDC_TOKEN_ENDPOINT")).ok();
        let client_id = std::env::var(format!("{prefix}_OIDC_CLIENT_ID")).ok();
        let secret = std::env::var(format!("{prefix}_OIDC_CLIENT_SECRET_FILE")).ok();

        // Read outside the match on purpose: this one is INDEPENDENT of the
        // all-or-none rule, so it must not participate in the tuple that
        // decides whether the queue is configured at all. A CA path with no
        // queue is meaningless rather than half-configured.
        let ca_file = std::env::var(format!("{prefix}_OIDC_CA_FILE"))
            .ok()
            .map(PathBuf::from);

        match (url, token_endpoint, client_id, secret) {
            (None, None, None, None) => Ok(None),
            (Some(url), Some(token_endpoint), Some(client_id), Some(secret)) => Ok(Some(Self {
                url,
                token_endpoint,
                client_id,
                client_secret_file: PathBuf::from(secret),
                ca_file,
            })),
            // A partially-set environment is a deployment bug, and the failure
            // it would otherwise produce is the expensive kind: the process
            // comes up "fine", never connects, and the data it was supposed to
            // move silently stops moving. Naming the variables costs one line.
            _ => Err(Error::PartialConfig {
                prefix: prefix.to_owned(),
            }),
        }
    }
}

/// Mint a fresh access token for the controller's own client.
///
/// `client_credentials`, because there is no user here: the controller
/// authenticates as itself. Authelia refuses the `openid` scope for this grant
/// (a machine client receives an access token and never an id-token), so no
/// scope is requested.
async fn mint_token(
    http: &reqwest::Client,
    cfg: &QueueConfig,
    audience: Option<&str>,
) -> Result<CachedToken, Error> {
    // Read per call rather than caching: the file is small, and a cached
    // secret would survive a rotation that the operator believes took effect.
    let secret = tokio::fs::read_to_string(&cfg.client_secret_file)
        .await
        .map_err(|source| Error::ClientSecret {
            path: cfg.client_secret_file.display().to_string(),
            source,
        })?;

    let response = token_request(http, cfg, secret.trim(), audience)
        .send()
        .await
        .map_err(Error::TokenRequest)?;

    let status = response.status();
    let body = response.text().await.unwrap_or_default();
    parse_token_response(status, &body)
}

/// Turn a raw token-endpoint response into a [`CachedToken`] — the half of
/// [`mint_token`] that is pure and therefore shared with the blocking sibling
/// below: neither the success/failure check nor the JSON shape cares which
/// HTTP client fetched the bytes.
///
/// Takes the body already read out, not the response itself: `reqwest`'s
/// blocking and async response types have no common trait for that, and
/// forcing one here would mean this function is not actually callable from
/// both callers, which defeats the reason it exists.
fn parse_token_response(status: reqwest::StatusCode, body: &str) -> Result<CachedToken, Error> {
    // Why the body and not just the status: see `Error::TokenRefused`.
    if !status.is_success() {
        return Err(Error::TokenRefused {
            status,
            body: body.to_owned(),
        });
    }

    let parsed: TokenResponse = serde_json::from_str(body).map_err(Error::TokenResponse)?;
    Ok(CachedToken {
        token: parsed.access_token,
        expires_at: std::time::Instant::now() + std::time::Duration::from_secs(parsed.expires_in),
    })
}

/// Build the token request: `client_credentials`, authenticated with HTTP
/// Basic.
///
/// 🩸 **The credentials go in the `Authorization` header, not the form body.**
/// Both are legal OAuth 2.0 — `client_secret_basic` and `client_secret_post` —
/// but a client registration names *one*, and authelia's default (and ours) is
/// Basic. Sending them in the body got every token request refused with
/// `Client authentication failed … the registered client is configured to only
/// support 'client_secret_basic'`, which reached the operator as an endless
/// `429` because the retries tripped a rate limiter whose penalty grew faster
/// than the retry interval. The 429 then arrived *before* the credentials were
/// ever evaluated, so the one line naming the real cause appeared once an hour.
///
/// RFC 6749 §2.3.1 says clients SHOULD use Basic, both introspection callers in
/// this workspace already do, and a secret in a header is one fewer place for a
/// proxy to log it.
///
/// Split out of [`mint_token`] so the request's *shape* is testable without a
/// running identity provider — see the tests at the bottom of this file.
///
/// `audience` is RFC 8707 resource-indicator territory, and it is opt-in:
/// omitted, this identity gets whatever audience authelia defaults a
/// scopeless `client_credentials` grant to (the queue connection's own
/// case — it has always worked without asking). A caller proving this
/// identity to a SPECIFIC audience-checked receiver — the swarm-otel
/// `oidc/swarm` authenticator being the first one — has to ask for it by
/// name, the same way `swarm-otel.nix`'s own prometheus scrape config
/// already does per target (`endpoint_params.audience`): a token minted
/// without asking carries `aud: []`, and an audience-checked receiver
/// refuses that just as readily as the wrong one.
fn token_request(
    http: &reqwest::Client,
    cfg: &QueueConfig,
    secret: &str,
    audience: Option<&str>,
) -> reqwest::RequestBuilder {
    let mut form = vec![("grant_type", "client_credentials")];
    if let Some(audience) = audience {
        form.push(("audience", audience));
    }
    http.post(&cfg.token_endpoint)
        .basic_auth(&cfg.client_id, Some(secret))
        .form(&form)
}

/// Read + parse `path` as a PEM root certificate. Shared by the async and
/// blocking HTTP client builders below — `reqwest::Certificate::from_pem`
/// takes the same bytes regardless of which client type ends up trusting
/// it, so the CA loading has no reason to exist twice.
fn load_ca_cert(path: &std::path::Path) -> Result<reqwest::Certificate, Error> {
    let pem = std::fs::read(path).map_err(|source| Error::CaFile {
        path: path.display().to_string(),
        source,
    })?;
    reqwest::Certificate::from_pem(&pem).map_err(|source| Error::CaParse {
        path: path.display().to_string(),
        source,
    })
}

/// Build the HTTP client used to reach `cfg.token_endpoint`, trusting
/// `cfg.ca_file` when set. Shared by [`connect`]'s auth callback and by
/// [`mint_token_for`] — anything presenting this identity's credentials to
/// the token endpoint needs the swarm's own CA trusted the same way, so the
/// trust-anchor logic lives here once rather than once per caller.
fn build_http_client(cfg: &QueueConfig) -> Result<reqwest::Client, Error> {
    let mut builder = reqwest::Client::builder().timeout(std::time::Duration::from_secs(10));
    if let Some(path) = &cfg.ca_file {
        builder = builder.add_root_certificate(load_ca_cert(path)?);
    }
    builder.build().map_err(Error::HttpClient)
}

/// Mint a fresh token for `cfg`'s identity and hand back just the string —
/// no caching, a fresh HTTP client per call.
///
/// Public because the queue connection is not the only thing this identity
/// authenticates: "one identity per principal" means a caller that already
/// holds a [`QueueConfig`] for its queue connection authenticates anywhere
/// else it needs to prove who it is from the exact same client, rather than
/// a second identity being provisioned per destination. No caching here
/// unlike [`connect`]'s callback: that one exists because `async-nats` reruns
/// its callback per reconnect *attempt*, a hot path this isn't — a caller
/// outside that loop (e.g. `swarm-controller::auth`'s bridge client) mints
/// per call, same as this crate did before the reconnect-storm fix added the
/// cache.
///
/// `audience` is passed straight to the private `token_request` helper —
/// see its doc for why it is optional and when a caller needs it. `None`
/// reproduces this
/// function's behaviour before the parameter existed, so every caller from
/// before that added it (the queue connect path, `auth.rs`'s bridge client)
/// is unaffected.
pub async fn mint_token_for(cfg: &QueueConfig, audience: Option<&str>) -> Result<String, Error> {
    let http = build_http_client(cfg)?;
    Ok(mint_token(&http, cfg, audience).await?.token)
}

/// Blocking sibling of [`mint_token_for`], for a caller with no tokio
/// reactor to `.await` on.
///
/// That caller is real, not hypothetical: an OTLP exporter's `HttpClient`
/// implementation runs on the `PeriodicReader`'s own thread, which this
/// workspace deliberately keeps reactor-free (`reqwest-blocking-client` was
/// chosen over the async client for exactly that reason — see
/// `swarm-controller::vcs_metrics`'s module doc). Calling the async
/// [`mint_token_for`] from there would need a runtime that thread does not
/// have; this function uses `reqwest::blocking::Client` end to end instead,
/// so it never needs one.
///
/// Same request shape and same no-caching behaviour as [`mint_token_for`] —
/// see that function's doc for why both of those are the right call here.
pub fn mint_token_for_blocking(cfg: &QueueConfig, audience: Option<&str>) -> Result<String, Error> {
    let http = build_blocking_http_client(cfg)?;

    // Read per call rather than caching — see `mint_token`'s identical
    // comment on the async path; the reasoning does not change with the
    // client type.
    let secret =
        std::fs::read_to_string(&cfg.client_secret_file).map_err(|source| Error::ClientSecret {
            path: cfg.client_secret_file.display().to_string(),
            source,
        })?;

    let mut form = vec![("grant_type", "client_credentials")];
    if let Some(audience) = audience {
        form.push(("audience", audience));
    }
    let response = http
        .post(&cfg.token_endpoint)
        .basic_auth(&cfg.client_id, Some(secret.trim()))
        .form(&form)
        .send()
        .map_err(Error::TokenRequest)?;

    let status = response.status();
    let body = response.text().unwrap_or_default();
    Ok(parse_token_response(status, &body)?.token)
}

/// Blocking counterpart to [`build_http_client`] — same CA-trust logic
/// (via the shared [`load_ca_cert`]), a `reqwest::blocking::Client` instead
/// of the async one. See [`mint_token_for_blocking`] for why a caller needs
/// this rather than the async builder.
fn build_blocking_http_client(cfg: &QueueConfig) -> Result<reqwest::blocking::Client, Error> {
    let mut builder =
        reqwest::blocking::Client::builder().timeout(std::time::Duration::from_secs(10));
    if let Some(path) = &cfg.ca_file {
        builder = builder.add_root_certificate(load_ca_cert(path)?);
    }
    builder.build().map_err(Error::HttpClient)
}

/// Fail fast unless the client is actually connected.
///
/// **Call this before every `JetStream` request.** `retry_on_initial_connect`
/// means the client exists before it is usable, and a `JetStream` request made
/// in that window does not fail — it *waits*, for longer than any caller
/// should (measured: still going at 15s against a queue that simply refuses
/// the credential). On a request path that hangs a poll; on a periodic task it
/// hangs the task, including the shutdown branch it never reaches.
///
/// Testing for `!= Connected` rather than `== Disconnected` is the whole
/// point: a client that has never connected once sits in `Pending`, so the
/// `Disconnected` test passes it straight through to the hang it was written
/// to prevent. That is exactly the case that matters — a process whose
/// credential is wrong from boot never reaches `Disconnected`, because it was
/// never connected to begin with.
///
/// Naming the state is also the better error: "not connected" is actionable,
/// a timeout is not.
pub fn ensure_connected(client: &async_nats::Client) -> Result<(), Error> {
    let state = client.connection_state();
    if state != async_nats::connection::State::Connected {
        return Err(Error::NotConnected(state));
    }
    Ok(())
}

/// Connect to the swarm queue, presenting a token on each connection attempt
/// and minting a fresh one only when the cached one is near expiry.
///
/// The rejected alternative — mint once, pass a static `auth_token`, own the
/// reconnect loop — fails in the way this subsystem exists to prevent: the
/// token expires, the controller keeps serving, its status data quietly stops
/// updating, and nothing says so until someone reads a dashboard.
pub async fn connect(cfg: QueueConfig) -> Result<async_nats::Client, Error> {
    // Trusts `cfg.ca_file` when set (the swarm's own CA, when the token
    // endpoint is signed by it) — see `build_http_client`. A timeout too,
    // because this client runs INSIDE the auth callback: a token endpoint
    // that accepts the connection and then never answers would hang the
    // callback, and with it the connection attempt that invoked it, with no
    // retry and nothing in the log to say why. Failing fast lets
    // `async-nats` do what it already does well — back off and try again.
    let http = build_http_client(&cfg)?;
    let url = cfg.url.clone();

    // Shared across every invocation of the callback below, which is the
    // whole point: the callback fires per connection ATTEMPT, so without
    // somewhere to remember the last token, "mint on demand" and "mint on
    // every retry" are the same code.
    let cache: std::sync::Arc<tokio::sync::Mutex<Option<CachedToken>>> =
        std::sync::Arc::new(tokio::sync::Mutex::new(None));

    let client = async_nats::ConnectOptions::with_auth_callback(move |_nonce| {
        let http = http.clone();
        let cfg = cfg.clone();
        let cache = cache.clone();
        async move {
            let mut slot = cache.lock().await;

            // Reuse while there is comfortably more than the skew left. The
            // freshness property this callback exists for is preserved — a
            // reconnect an hour later finds an expired entry and re-mints —
            // but a reconnect ten seconds later does NOT ask the IdP again.
            let reuse = slot
                .as_ref()
                .filter(|t| {
                    t.expires_at
                        .saturating_duration_since(std::time::Instant::now())
                        > TOKEN_REFRESH_SKEW
                })
                .map(|t| t.token.clone());

            let token = if let Some(token) = reuse {
                token
            } else {
                let minted = mint_token(&http, &cfg, None)
                    .await
                    // The callback's error type carries a string, so the
                    // source chain would be lost; flatten it rather than
                    // dropping it.
                    .map_err(|e| async_nats::AuthError::new(chain(&e)))?;
                let token = minted.token.clone();
                *slot = Some(minted);
                token
            };

            let mut auth = async_nats::Auth::new();
            auth.token = Some(token);
            Ok(auth)
        }
    })
    // See `MAX_RECONNECT_DELAY`. The library's default clamps at 4s forever,
    // which turns an unreachable queue into a permanent 4s poll — and, before
    // the cache above, a permanent 4s token-request loop against authelia.
    // Exponential from 500ms so a momentary blip still reconnects promptly.
    .reconnect_delay_callback(|attempts| {
        let exp = u32::try_from(attempts.saturating_sub(1)).unwrap_or(u32::MAX);
        std::cmp::min(
            std::time::Duration::from_millis(
                500u64.saturating_mul(2u64.saturating_pow(exp.min(8))),
            ),
            MAX_RECONNECT_DELAY,
        )
    })
    // The controller and the queue are separate units on (possibly)
    // separate hosts, and nothing orders them. Without this, a queue that
    // comes up one second later leaves the controller permanently
    // queue-less until someone restarts it — a boot-order race that
    // presents as "status has been unavailable since Tuesday".
    //
    // It also composes with the callback above rather than fighting it:
    // each background attempt runs the callback, which hands over a token
    // that is still valid — minting a new one only when the cached one is
    // near expiry. (Before the cache, "runs the callback" meant "mints",
    // and this retry loop was a token-request loop. See
    // `MAX_RECONNECT_DELAY`.)
    .retry_on_initial_connect()
    .connect(&url)
    .await
    .map_err(|source| Error::Connect {
        url: url.clone(),
        source,
    })?;

    Ok(client)
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The all-unset case is the common one — most hosts do not run the queue.
    ///
    /// Uses a prefix no deployment sets, so it cannot pass vacuously by
    /// running inside a configured environment (the guard below covers the
    /// same ground, and both are cheap).
    #[test]
    fn an_absent_environment_is_not_an_error() {
        for k in [
            "SWARM_QUEUE_TEST_NATS_URL",
            "SWARM_QUEUE_TEST_OIDC_TOKEN_ENDPOINT",
            "SWARM_QUEUE_TEST_OIDC_CLIENT_ID",
            "SWARM_QUEUE_TEST_OIDC_CLIENT_SECRET_FILE",
        ] {
            assert!(std::env::var(k).is_err(), "{k} must be unset for this test");
        }
        assert!(
            QueueConfig::from_env("SWARM_QUEUE_TEST")
                .expect("absent is not an error")
                .is_none()
        );
    }

    /// The half-set case is the one the rule exists for: a deployment bug that
    /// would otherwise look exactly like "no queue configured".
    ///
    /// SAFETY: single-threaded mutation of a process env var under a prefix no
    /// other test or deployment uses; removed before returning.
    #[test]
    fn a_half_set_environment_is_a_hard_error() {
        unsafe {
            std::env::set_var("SWARM_QUEUE_HALF_NATS_URL", "nats://127.0.0.1:4222");
        }
        let err = QueueConfig::from_env("SWARM_QUEUE_HALF")
            .expect_err("a partial set must not read as absent");
        let msg = format!("{err}");
        assert!(
            msg.contains("SWARM_QUEUE_HALF_OIDC_CLIENT_ID"),
            "the error must name the missing variables, got: {msg}"
        );
        unsafe {
            std::env::remove_var("SWARM_QUEUE_HALF_NATS_URL");
        }
    }

    fn token_cfg() -> QueueConfig {
        QueueConfig {
            url: "nats://127.0.0.1:4222".to_owned(),
            token_endpoint: "https://auth.example.com/api/oidc/token".to_owned(),
            client_id: "hive-alpha".to_owned(),
            client_secret_file: PathBuf::from("/nonexistent"),
            ca_file: None,
        }
    }

    /// 🩸 THE REGRESSION TEST FOR AN OUTAGE THAT RAN FOR WEEKS.
    ///
    /// The credentials used to go in the form body (`client_secret_post`).
    /// Authelia's client registration allows only `client_secret_basic`, so
    /// every token request was refused — and the refusals tripped a rate
    /// limiter whose 429 then arrived *before* the credentials were evaluated,
    /// so the error naming the cause appeared roughly once an hour inside a
    /// continuous storm of a different error.
    ///
    /// This asserts the *shape of the request* rather than a server's reply,
    /// which is the whole point: it fails on the old code with no identity
    /// provider, no deployment and no network.
    /// A client for inspecting a request, never for sending one.
    ///
    /// 🩸 `reqwest::Client::new()` **panics in the nix build sandbox**, which
    /// has no system CA store: `ClientBuilder::build()` reaches
    /// `rustls_platform_verifier::Verifier::new()` and fails with "No CA
    /// certificates were loaded from the system", and `new()` is
    /// `build().expect(..)`. The test passed locally — a devshell has
    /// `/etc/ssl/certs` — and failed in CI.
    ///
    /// Turning verification off takes the `!certs_verification` branch, which
    /// installs a no-op verifier and never consults the platform store, so
    /// this builds anywhere. It is sound *here specifically* because nothing
    /// is ever sent: the request is built and its bytes inspected.
    fn offline_client() -> reqwest::Client {
        reqwest::Client::builder()
            .danger_accept_invalid_certs(true)
            .build()
            .expect("a client that verifies nothing needs no system trust store")
    }

    #[test]
    fn the_token_request_authenticates_with_http_basic() {
        let req = token_request(&offline_client(), &token_cfg(), "s3cret", None)
            .build()
            .expect("the token request must build");

        let auth = req
            .headers()
            .get(reqwest::header::AUTHORIZATION)
            .expect("credentials must travel in the Authorization header")
            .to_str()
            .expect("the header is ascii");
        assert!(
            auth.starts_with("Basic "),
            "must be client_secret_basic, got: {auth}"
        );

        let body = std::str::from_utf8(
            req.body()
                .and_then(reqwest::Body::as_bytes)
                .expect("the request has an in-memory body"),
        )
        .expect("the body is utf-8");
        assert!(
            body.contains("grant_type=client_credentials"),
            "the grant type still belongs in the body, got: {body}"
        );
        // The half that was actually broken: a secret in the body is both the
        // wrong auth method for our registration and a value proxies log.
        assert!(
            !body.contains("client_secret"),
            "the secret must not be in the request body, got: {body}"
        );
        assert!(
            !body.contains("client_id"),
            "the client id belongs in the Basic credentials, got: {body}"
        );
    }

    /// The queue/bridge shape (`audience: None`) must stay unchanged by the
    /// parameter's addition — no `audience` field appears in the body at
    /// all, not even empty.
    #[test]
    fn no_audience_means_no_audience_field() {
        let req = token_request(&offline_client(), &token_cfg(), "s3cret", None)
            .build()
            .expect("the token request must build");
        let body = std::str::from_utf8(
            req.body()
                .and_then(reqwest::Body::as_bytes)
                .expect("the request has an in-memory body"),
        )
        .expect("the body is utf-8");
        assert!(
            !body.contains("audience"),
            "omitting the audience must not even send an empty field, got: {body}"
        );
    }

    /// A caller that DOES ask for an audience gets it in the form body,
    /// verbatim — this is the half `swarm-otel`'s `oidc/<owner>`
    /// authenticators actually check.
    #[test]
    fn an_audience_is_sent_verbatim() {
        let req = token_request(
            &offline_client(),
            &token_cfg(),
            "s3cret",
            Some("https://otel.example/swarm"),
        )
        .build()
        .expect("the token request must build");
        let body = std::str::from_utf8(
            req.body()
                .and_then(reqwest::Body::as_bytes)
                .expect("the request has an in-memory body"),
        )
        .expect("the body is utf-8");
        assert!(
            body.contains("audience=https%3A%2F%2Fotel.example%2Fswarm"),
            "the requested audience must reach the form body, got: {body}"
        );
    }
}