From ef9339da16d3136de9beab08ab712f7b6e8963f9 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 16 Aug 2026 20:45:51 +0200 Subject: [PATCH] fix(#3349): point the swarm-queue client at the hive's trust bundle The queue client built a bare reqwest::Client, so it trusted only the platform roots. Against a swarm whose authelia is signed by the swarm CA that is fatal: minting a token dies with 'invalid peer certificate: UnknownIssuer', inside the auth callback, on a four-second retry loop, with the queue never connecting. The anchor was never missing. hive-tls.nix assembles trust-bundle.pem and already hands it to hive-c0re as HIVE_TLS_CA_PATH; nothing pointed the queue client at it. QueueConfig gains an optional ca_file from _OIDC_CA_FILE, read outside the all-or-none tuple on purpose: a CA path with no queue is meaningless rather than half-configured, and requiring it would break a swarm fronted by a public certificate in order to fix one that is not. add_root_certificate extends the default roots rather than replacing them, so both deployments work. A bad path fails loudly instead of falling back to the platform roots. An operator who names a CA file wants that anchor; a silent fallback turns their typo into UnknownIssuer five layers away. hive-tls.nix names the bundle for both clients, beside the line that already does it for hive-c0re, rather than having each consumer re-derive the path. --- nix/host-modules/hive-tls.nix | 22 ++++++++++++ swarm-queue-client/src/lib.rs | 67 ++++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 2199b169..fec2c70d 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -563,5 +563,27 @@ in # spelled out where the bundle is written above; no key path is ever # exposed (an agent that could read one could mint trusted certs). systemd.services.hive-c0re.environment.HIVE_TLS_CA_PATH = "${cfg.stateDir}/trust-bundle.pem"; + + # The same anchor, named for the swarm-queue clients that need it when + # they mint a token from authelia over TLS. Declared HERE, beside the + # bundle, rather than in each consumer's module: the path is this + # module's fact, and two consumers re-deriving `${stateDir}/…` would be + # two places to fix the day it moves. + # + # ⚠️ This is what was missing. `swarm-queue-client` built a bare + # `reqwest::Client`, so it trusted only the platform roots and died at + # `invalid peer certificate: UnknownIssuer` against a swarm whose + # authelia is signed by the swarm CA — while this very bundle sat on + # disk, already assembled, already handed to hive-c0re under a different + # variable name. The anchor was never missing; nothing pointed the queue + # client at it. + # + # Set unconditionally within this module's `active` guard, exactly like + # the line above: where there is no hive CA this module contributes + # nothing at all, and the clients then fall back to the platform roots — + # which is correct for a swarm fronted by a public certificate. + systemd.services.hive-c0re.environment.HIVE_C0RE_OIDC_CA_FILE = "${cfg.stateDir}/trust-bundle.pem"; + systemd.services.swarm-controller.environment.SWARM_CONTROLLER_OIDC_CA_FILE = + "${cfg.stateDir}/trust-bundle.pem"; }; } diff --git a/swarm-queue-client/src/lib.rs b/swarm-queue-client/src/lib.rs index d6d0ce62..81f3098d 100644 --- a/swarm-queue-client/src/lib.rs +++ b/swarm-queue-client/src/lib.rs @@ -62,6 +62,23 @@ pub enum Error { #[error("building the token-endpoint HTTP client")] HttpClient(#[source] reqwest::Error), + /// Distinct from `HttpClient` because the operator's next move differs: + /// this one names a path they configured, and it fires before any + /// network call. + #[error("reading the token-endpoint CA certificate from {path}")] + CaFile { + path: String, + #[source] + source: std::io::Error, + }, + + #[error("parsing the token-endpoint CA certificate from {path} as PEM")] + CaParse { + path: String, + #[source] + source: reqwest::Error, + }, + #[error("requesting an access token from authelia")] TokenRequest(#[source] reqwest::Error), @@ -158,6 +175,19 @@ pub struct QueueConfig { /// read here, and putting it in the environment would publish it to /// anything that can read `/proc//environ`. pub client_secret_file: PathBuf, + /// Extra trust anchor for the token endpoint, when it is not signed by + /// a publicly-trusted CA. + /// + /// Optional, and deliberately NOT part of the all-or-none group below: a + /// swarm fronted by a public certificate needs no extra anchor, and + /// making this required would break that deployment to fix ours. Absent + /// means "the platform's roots are enough", which is the correct default + /// for a client that might talk to anything. + /// + /// ⚠️ Without it, a swarm using its own CA fails at TLS with + /// `invalid peer certificate: UnknownIssuer` — the anchor exists on the + /// host and this client simply never looked at it. + pub ca_file: Option, } impl QueueConfig { @@ -182,6 +212,14 @@ impl QueueConfig { let client_id = std::env::var(format!("{prefix}_OIDC_CLIENT_ID")).ok(); let secret = std::env::var(format!("{prefix}_OIDC_CLIENT_SECRET_FILE")).ok(); + // Read outside the match on purpose: this one is INDEPENDENT of the + // all-or-none rule, so it must not participate in the tuple that + // decides whether the queue is configured at all. A CA path with no + // queue is meaningless rather than half-configured. + let ca_file = std::env::var(format!("{prefix}_OIDC_CA_FILE")) + .ok() + .map(PathBuf::from); + match (url, token_endpoint, client_id, secret) { (None, None, None, None) => Ok(None), (Some(url), Some(token_endpoint), Some(client_id), Some(secret)) => Ok(Some(Self { @@ -189,6 +227,7 @@ impl QueueConfig { token_endpoint, client_id, client_secret_file: PathBuf::from(secret), + ca_file, })), // A partially-set environment is a deployment bug, and the failure // it would otherwise produce is the expensive kind: the process @@ -273,10 +312,30 @@ pub async fn connect(cfg: QueueConfig) -> Result { // no retry and nothing in the log to say why. Failing fast lets // `async-nats` do what it already does well — back off and try again. // 10s is generous for a form POST to a local IdP. - let http = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(10)) - .build() - .map_err(Error::HttpClient)?; + let mut builder = reqwest::Client::builder().timeout(std::time::Duration::from_secs(10)); + + // The swarm's own CA, when the token endpoint is signed by it. ADDED, not + // substituted: `add_root_certificate` extends the default set rather than + // replacing it, so a swarm can front authelia publicly and still have + // this work. + // + // Failing here rather than falling back to the platform roots is the + // point — an operator who named a CA file wants that anchor, and a + // silent fallback would turn their typo into `UnknownIssuer` five layers + // away, inside an auth callback, on a retry loop. + if let Some(path) = &cfg.ca_file { + let pem = std::fs::read(path).map_err(|source| Error::CaFile { + path: path.display().to_string(), + source, + })?; + let cert = reqwest::Certificate::from_pem(&pem).map_err(|source| Error::CaParse { + path: path.display().to_string(), + source, + })?; + builder = builder.add_root_certificate(cert); + } + + let http = builder.build().map_err(Error::HttpClient)?; let url = cfg.url.clone(); let client = async_nats::ConnectOptions::with_auth_callback(move |_nonce| {