swarm-queue-client: trim token_request comment block under the 30-line lint

The scope-parameter addition pushed the doc comment on token_request to 39
lines. Moved the HTTP-Basic incident story and the audience/scope rationale
to the crate README's new "Token request shape" section (docs/ is markdown,
exempt from the lint); the code comment keeps the pointer plus the one-line
summary of the invariant. Refs #4464

swarm-controller/src/auth.rs

@@ -98,7 +98,7 @@ impl AuthBridge {
         // configured CA, if any) — deliberately not `self.http`, which is
         // the bridge's own client and has nothing to do with authelia's
         // token endpoint's trust anchors.
-        let token = swarm_queue_client::mint_token_for(&self.queue_cfg, None)
+        let token = swarm_queue_client::mint_token_for(&self.queue_cfg, None, None)
             .await
             .context("minting a bearer token for swarm-authelia-bridge")?;
 

swarm-controller/src/otel_http_client.rs

@@ -78,7 +78,7 @@ impl HttpClient for AuthenticatedHttpClient {
     /// why this specific caller deliberately runs on a thread with nothing
     /// to yield to.
     async fn send_bytes(&self, mut request: Request<Bytes>) -> Result<Response<Bytes>, HttpError> {
-        let token = mint_token_for_blocking(&self.cfg, Some(&self.audience))?;
+        let token = mint_token_for_blocking(&self.cfg, Some(&self.audience), None)?;
         request.headers_mut().insert(
             http::header::AUTHORIZATION,
             http::HeaderValue::from_str(&format!("Bearer {token}"))?,

swarm-logs/README.md

@@ -50,6 +50,11 @@ hold `authelia.bearer.authz`. Both are set in `swarm-authelia.nix`'s
 `agentClients`; without them authelia answers `invalid_target` at the token
 endpoint, or the gateway answers 401 with no explanation.
 
+⚠️ Registration is not issuance, so the token request _asks_ for both: the
+audience and the scope are named in the `client_credentials` form, because a
+client that is registered for a scope it does not request is handed a token
+carrying none, and the gateway refuses that with the same bare 401.
+
 ## Configuration
 
 Supplied by `nix/agent-modules/logs.nix`, which wraps the binary — the same

swarm-logs/src/query.rs

@@ -46,13 +46,25 @@ pub fn run(
     format: Format,
     out: &mut impl Write,
 ) -> Result<()> {
-    // Minted per invocation, with the query URL as the audience. Both halves
+    // Minted per invocation, with the query URL as the audience and the
-    // are load-bearing: authelia refuses a token carrying no audience at the
+    // `authelia.bearer.authz` scope. All three are load-bearing: authelia
-    // authz endpoint the gateway's `auth_request` calls, and the audience it
+    // refuses a token carrying no audience at the authz endpoint the
-    // checks is the URL being requested — so the string sent here and the
+    // gateway's `auth_request` calls, and the audience it checks is the URL
-    // string requested below must be one binding, which is why `Config` holds
+    // being requested — so the string sent here and the string requested
-    // exactly one.
+    // below must be one binding, which is why `Config` holds exactly one.
-    let token = swarm_queue_client::mint_token_for_blocking(&cfg.queue, Some(&cfg.query_url))
+    //
+    // The scope has to be ASKED for, not merely registered: the agent client
+    // is granted `authelia.bearer.authz` by the `agentClients` entry in
+    // `swarm-authelia.nix`, but an OAuth2 server issues no scope the client
+    // never requested, and a scopeless token is refused at the authz endpoint
+    // exactly as an unauthenticated one is — a bare nginx 401 with nothing in
+    // it that names the scope. `swarm-otel.nix` records the same failure
+    // against the collector's client, on the same string.
+    let token = swarm_queue_client::mint_token_for_blocking(
+        &cfg.queue,
+        Some(&cfg.query_url),
+        Some("authelia.bearer.authz"),
+    )
     .map_err(|e| anyhow::anyhow!("{}", swarm_queue_client::chain(&e)))
     .context("minting an access token for the swarm log store")?;
 

swarm-queue-client/README.md

@@ -48,6 +48,41 @@ would publish it to anything that can read `/proc/<pid>/environ`. It is read
 per token request rather than cached, so a rotation the operator believes took
 effect actually did.
 
+## Token request shape: HTTP Basic, `audience`, `scope`
+
+`token_request` builds a `client_credentials` grant, authenticated with HTTP
+Basic — **never** the form body. Both are legal OAuth 2.0
+(`client_secret_basic` vs `client_secret_post`), but a client registration
+names one, and authelia's default (and ours) is Basic. Sending the
+credentials in the body once got every token request refused with `Client
+authentication failed … the registered client is configured to only support
+'client_secret_basic'`, which reached the operator as an endless `429`
+because the retries tripped a rate limiter whose penalty grew faster than the
+retry interval — the 429 arrived _before_ the credentials were ever
+evaluated, so the line naming the real cause appeared once an hour. RFC 6749
+§2.3.1 says clients SHOULD use Basic, both introspection callers in this
+workspace already do, and a secret in a header is one fewer place for a proxy
+to log it.
+
+`audience` (RFC 8707 resource indicators) and `scope` are both opt-in;
+`None` for either reproduces the request this crate sent before the
+parameter existed. Omitted, `audience` gets whatever authelia defaults a
+scopeless `client_credentials` grant to — the queue connection's own case,
+which has always worked without asking. A caller proving this identity to a
+specifically audience-checked receiver (the swarm-otel `oidc/swarm`
+authenticator being the first one) has to ask by name, the same way
+`swarm-otel.nix`'s own prometheus scrape config already does per target
+(`endpoint_params.audience`): a token minted without asking carries `aud:
+[]`, and an audience-checked receiver refuses that just as readily as the
+wrong one. `scope` follows the same rule and travels with `audience`:
+registration is not issuance, and the authorisation server grants no scope
+the client never requested. A caller reaching a destination behind
+authelia's `/api/authz/auth-request` needs `authelia.bearer.authz` here
+however completely the client is registered for it — the failure
+`swarm-otel.nix` records against its own client (a scopeless token refused
+at introspection with "the requested scope is invalid, unknown, or
+malformed") is the same one, one layer down.
+
 ## What this crate does not do
 
 It ends at a connected client. `jetstream`/`kv` are **off by default** — what a

swarm-queue-client/src/lib.rs

@@ -404,12 +404,14 @@ impl QueueConfig {
 ///
 /// `client_credentials`, because there is no user here: the controller
 /// authenticates as itself. Authelia refuses the `openid` scope for this grant
-/// (a machine client receives an access token and never an id-token), so no
+/// (a machine client receives an access token and never an id-token), so a
-/// scope is requested.
+/// caller passing `None` for `scope` requests none at all — see
+/// [`token_request`] for the one scope a caller does have to ask for.
 async fn mint_token(
     http: &reqwest::Client,
     cfg: &QueueConfig,
     audience: Option<&str>,
+    scope: Option<&str>,
 ) -> Result<CachedToken, Error> {
     // Read per call rather than caching: the file is small, and a cached
     // secret would survive a rotation that the operator believes took effect.
@@ -420,7 +422,7 @@ async fn mint_token(
             source,
         })?;
 
-    let response = token_request(http, cfg, secret.trim(), audience)
+    let response = token_request(http, cfg, secret.trim(), audience, scope)
         .send()
         .await
         .map_err(Error::TokenRequest)?;
@@ -458,43 +460,32 @@ fn parse_token_response(status: reqwest::StatusCode, body: &str) -> Result<Cache
 /// Build the token request: `client_credentials`, authenticated with HTTP
 /// Basic.
 ///
-/// 🩸 **The credentials go in the `Authorization` header, not the form body.**
+/// 🩸 **The credentials go in the `Authorization` header, not the form
-/// Both are legal OAuth 2.0 — `client_secret_basic` and `client_secret_post` —
+/// body.** A client registration names one flavor, and authelia's default
-/// but a client registration names *one*, and authelia's default (and ours) is
+/// (and ours) is Basic — the other one once produced an hour of misleading
-/// Basic. Sending them in the body got every token request refused with
+/// `429`s before the real cause (every request refused at the auth step)
-/// `Client authentication failed … the registered client is configured to only
+/// surfaced. `audience` and `scope` are both opt-in, RFC 8707 /
-/// support 'client_secret_basic'`, which reached the operator as an endless
+/// registered-scope territory; `None` for either reproduces the request
-/// `429` because the retries tripped a rate limiter whose penalty grew faster
+/// this crate sent before the parameter existed. See the crate README's
-/// than the retry interval. The 429 then arrived *before* the credentials were
+/// "Token request shape" section for the incident, the RFC citations, and
-/// ever evaluated, so the one line naming the real cause appeared once an hour.
+/// why each parameter exists.
-///
-/// RFC 6749 §2.3.1 says clients SHOULD use Basic, both introspection callers in
-/// this workspace already do, and a secret in a header is one fewer place for a
-/// proxy to log it.
 ///
 /// Split out of [`mint_token`] so the request's *shape* is testable without a
 /// running identity provider — see the tests at the bottom of this file.
-///
-/// `audience` is RFC 8707 resource-indicator territory, and it is opt-in:
-/// omitted, this identity gets whatever audience authelia defaults a
-/// scopeless `client_credentials` grant to (the queue connection's own
-/// case — it has always worked without asking). A caller proving this
-/// identity to a SPECIFIC audience-checked receiver — the swarm-otel
-/// `oidc/swarm` authenticator being the first one — has to ask for it by
-/// name, the same way `swarm-otel.nix`'s own prometheus scrape config
-/// already does per target (`endpoint_params.audience`): a token minted
-/// without asking carries `aud: []`, and an audience-checked receiver
-/// refuses that just as readily as the wrong one.
 fn token_request(
     http: &reqwest::Client,
     cfg: &QueueConfig,
     secret: &str,
     audience: Option<&str>,
+    scope: Option<&str>,
 ) -> reqwest::RequestBuilder {
     let mut form = vec![("grant_type", "client_credentials")];
     if let Some(audience) = audience {
         form.push(("audience", audience));
     }
+    if let Some(scope) = scope {
+        form.push(("scope", scope));
+    }
     http.post(&cfg.token_endpoint)
         .basic_auth(&cfg.client_id, Some(secret))
         .form(&form)
@@ -542,15 +533,19 @@ fn build_http_client(cfg: &QueueConfig) -> Result<reqwest::Client, Error> {
 /// per call, same as this crate did before the reconnect-storm fix added the
 /// cache.
 ///
-/// `audience` is passed straight to the private `token_request` helper —
+/// `audience` and `scope` are passed straight to the private `token_request`
-/// see its doc for why it is optional and when a caller needs it. `None`
+/// helper — see its doc for why each is optional and when a caller needs it.
-/// reproduces this
+/// `None` for both reproduces this
-/// function's behaviour before the parameter existed, so every caller from
+/// function's behaviour before the parameters existed, so every caller from
-/// before that added it (the queue connect path, `auth.rs`'s bridge client)
+/// before they were added (the queue connect path, `auth.rs`'s bridge client)
 /// is unaffected.
-pub async fn mint_token_for(cfg: &QueueConfig, audience: Option<&str>) -> Result<String, Error> {
+pub async fn mint_token_for(
+    cfg: &QueueConfig,
+    audience: Option<&str>,
+    scope: Option<&str>,
+) -> Result<String, Error> {
     let http = build_http_client(cfg)?;
-    Ok(mint_token(&http, cfg, audience).await?.token)
+    Ok(mint_token(&http, cfg, audience, scope).await?.token)
 }
 
 /// Blocking sibling of [`mint_token_for`], for a caller with no tokio
@@ -567,7 +562,11 @@ pub async fn mint_token_for(cfg: &QueueConfig, audience: Option<&str>) -> Result
 ///
 /// Same request shape and same no-caching behaviour as [`mint_token_for`] —
 /// see that function's doc for why both of those are the right call here.
-pub fn mint_token_for_blocking(cfg: &QueueConfig, audience: Option<&str>) -> Result<String, Error> {
+pub fn mint_token_for_blocking(
+    cfg: &QueueConfig,
+    audience: Option<&str>,
+    scope: Option<&str>,
+) -> Result<String, Error> {
     let http = build_blocking_http_client(cfg)?;
 
     // Read per call rather than caching — see `mint_token`'s identical
@@ -583,6 +582,9 @@ pub fn mint_token_for_blocking(cfg: &QueueConfig, audience: Option<&str>) -> Res
     if let Some(audience) = audience {
         form.push(("audience", audience));
     }
+    if let Some(scope) = scope {
+        form.push(("scope", scope));
+    }
     let response = http
         .post(&cfg.token_endpoint)
         .basic_auth(&cfg.client_id, Some(secret.trim()))
@@ -702,7 +704,7 @@ pub async fn connect(cfg: QueueConfig) -> Result<async_nats::Client, Error> {
             let token = if let Some(token) = reuse {
                 token
             } else {
-                let minted = mint_token(&http, &cfg, None)
+                let minted = mint_token(&http, &cfg, None, None)
                     .await
                     // The callback's error type carries a string, so the
                     // source chain would be lost; flatten it rather than
@@ -846,7 +848,7 @@ mod tests {
 
     #[test]
     fn the_token_request_authenticates_with_http_basic() {
-        let req = token_request(&offline_client(), &token_cfg(), "s3cret", None)
+        let req = token_request(&offline_client(), &token_cfg(), "s3cret", None, None)
             .build()
             .expect("the token request must build");
 
@@ -883,12 +885,12 @@ mod tests {
         );
     }
 
-    /// The queue/bridge shape (`audience: None`) must stay unchanged by the
+    /// The queue/bridge shape (`audience: None`, `scope: None`) must stay
-    /// parameter's addition — no `audience` field appears in the body at
+    /// unchanged by the parameters' addition — no `audience` field appears
-    /// all, not even empty.
+    /// in the body at all, not even empty.
     #[test]
     fn no_audience_means_no_audience_field() {
-        let req = token_request(&offline_client(), &token_cfg(), "s3cret", None)
+        let req = token_request(&offline_client(), &token_cfg(), "s3cret", None, None)
             .build()
             .expect("the token request must build");
         let body = std::str::from_utf8(
@@ -913,6 +915,7 @@ mod tests {
             &token_cfg(),
             "s3cret",
             Some("https://otel.example/swarm"),
+            None,
         )
         .build()
         .expect("the token request must build");
@@ -927,4 +930,50 @@ mod tests {
             "the requested audience must reach the form body, got: {body}"
         );
     }
+
+    /// No scope asked for, no `scope` field — the queue connection and the
+    /// OTLP push both mint this way, and a `scope=` authelia has no mapping
+    /// for is refused rather than ignored.
+    #[test]
+    fn no_scope_means_no_scope_field() {
+        let req = token_request(&offline_client(), &token_cfg(), "s3cret", None, None)
+            .build()
+            .expect("the token request must build");
+        let body = std::str::from_utf8(
+            req.body()
+                .and_then(reqwest::Body::as_bytes)
+                .expect("the request has an in-memory body"),
+        )
+        .expect("the body is utf-8");
+        assert!(
+            !body.contains("scope"),
+            "omitting the scope must not even send an empty field, got: {body}"
+        );
+    }
+
+    /// The bug behind the log store's 401: a client REGISTERED for
+    /// `authelia.bearer.authz` still receives a token carrying no scope
+    /// unless the request asks, and authelia's authz endpoint refuses that.
+    #[test]
+    fn a_scope_is_sent_verbatim() {
+        let req = token_request(
+            &offline_client(),
+            &token_cfg(),
+            "s3cret",
+            Some("https://logs.example/select/logsql/query"),
+            Some("authelia.bearer.authz"),
+        )
+        .build()
+        .expect("the token request must build");
+        let body = std::str::from_utf8(
+            req.body()
+                .and_then(reqwest::Body::as_bytes)
+                .expect("the request has an in-memory body"),
+        )
+        .expect("the body is utf-8");
+        assert!(
+            body.contains("scope=authelia.bearer.authz"),
+            "the requested scope must reach the form body, got: {body}"
+        );
+    }
 }