swarm-nats-auth: re-read the OIDC client secret only on a 400 or 401

The responder re-read its secret on any non-2xx introspection answer.
With the re-read backoff added in the previous commit, an IdP outage
(503s, 502s from nginx, 429s) made each re-read find the unchanged
secret, so the delay grew to minutes. A real 401 after the rotation
then landed inside that window, and the responder denied every new
NATS connect until it ran out.

`Answer::Refused` now carries the status. `OidcSecret::introspect`
re-reads only when `stored_secret::is_credential_refusal` says the
status is 400 or 401, the same predicate `ClientSecret::refused` uses on
the client path. Every other non-2xx is a denial with no re-read.

The client-path test for a 429 now runs the mint against a loopback
endpoint answering 429 and 503, instead of calling `refused()` directly.
swarm-queue-client's README no longer names `ClientSecret::Value`.

Part of #5017 (slice Q).

swarm-queue-client/README.md

@@ -44,9 +44,10 @@ The client secret is never in the environment, which would publish it to
 anything that can read `/proc/<pid>/environ`. A caller passes it either as a
 **path** (`ClientSecret::File`), read per token request rather than cached, so
 a rotation the operator believes took effect actually did, or as a value it
-fetched itself (`ClientSecret::Value`), held in memory. `swarm-controller` and
-`hive-c0re` both do the latter with the secret they read from the swarm's
-secret store.
+fetched itself (`ClientSecret::Stored`), held in memory and fetched again
+through the caller's callback when the token endpoint refuses it as a
+credential. `swarm-controller`, `hive-c0re` and `swarm-review-classifier` do the
+latter with the secret they read from the swarm's secret store.
 
 ## Token request shape: HTTP Basic, `audience`, `scope`
 

swarm-queue-client/src/lib.rs

@@ -505,15 +505,11 @@ pub enum ClientSecret {
 
 impl ClientSecret {
     /// The secret to read again after `error`: a held one the token endpoint
-    /// refused as a credential (400 or 401). A 429 or a 5xx says nothing
-    /// about the secret.
+    /// refused as a credential ([`stored_secret::is_credential_refusal`]).
     fn refused(&self, error: &Error) -> Option<&stored_secret::StoredSecret> {
         match (self, error) {
             (Self::Stored(stored), Error::TokenRefused { status, .. })
-                if matches!(
-                    *status,
-                    reqwest::StatusCode::BAD_REQUEST | reqwest::StatusCode::UNAUTHORIZED
-                ) =>
+                if stored_secret::is_credential_refusal(*status) =>
             {
                 Some(stored)
             }
@@ -1159,22 +1155,37 @@ mod tests {
 
     /// A refusal that is not about the credential reads nothing.
     #[tokio::test]
-    async fn a_rate_limited_mint_does_not_re_read() {
-        let store = stored_secret::tests::Fake::holding("secret-b");
-        let cfg = QueueConfig {
-            client_secret: ClientSecret::Stored(store.secret("hive-alpha", "secret-a")),
-            ..token_cfg()
-        };
-        let limited = Error::TokenRefused {
-            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
-            body: String::new(),
-        };
-        assert!(cfg.client_secret.refused(&limited).is_none());
-        let refused = Error::TokenRefused {
-            status: reqwest::StatusCode::UNAUTHORIZED,
-            body: String::new(),
-        };
-        assert!(cfg.client_secret.refused(&refused).is_some());
+    async fn a_rate_limited_or_failing_mint_does_not_re_read() {
+        use axum::http::StatusCode;
+
+        let http = loopback_client();
+        for status in [
+            StatusCode::TOO_MANY_REQUESTS,
+            StatusCode::SERVICE_UNAVAILABLE,
+        ] {
+            let app = axum::Router::new().fallback(move || async move { (status, "") });
+            let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+                .await
+                .expect("bind");
+            let url = format!("http://{}/token", listener.local_addr().expect("addr"));
+            tokio::spawn(async move { axum::serve(listener, app).await });
+
+            let store = stored_secret::tests::Fake::holding("secret-b");
+            let cfg = QueueConfig {
+                token_endpoint: url,
+                client_secret: ClientSecret::Stored(store.secret("hive-alpha", "secret-a")),
+                ..token_cfg()
+            };
+            let err = mint_token(&http, &cfg, None, None)
+                .await
+                .err()
+                .expect("refused");
+            assert!(
+                matches!(err, Error::TokenRefused { status: s, .. } if s == status.as_u16()),
+                "{err}"
+            );
+            assert_eq!(store.reads(), 0, "{status}");
+        }
     }
 
     /// The blocking mint re-reads the same way, from a thread with no

swarm-queue-client/src/stored_secret.rs

@@ -205,6 +205,17 @@ impl Inner {
     }
 }
 
+/// Whether an `IdP` answering `status` refused the client's credential (400 or
+/// 401), the one answer a re-read can fix. A 429 or a 5xx says nothing about
+/// the secret, and re-reading on one would spend the backoff on an outage.
+#[must_use]
+pub fn is_credential_refusal(status: reqwest::StatusCode) -> bool {
+    matches!(
+        status,
+        reqwest::StatusCode::BAD_REQUEST | reqwest::StatusCode::UNAUTHORIZED
+    )
+}
+
 /// [`FIRST_REREAD_DELAY`] doubled per fruitless re-read after the first,
 /// capped at [`MAX_REREAD_DELAY`].
 fn delay_after(fruitless: u32) -> Duration {