//! This responder's own OIDC client secret, the credential it introspects //! with. Read from the secret store as the responder starts and held in //! memory, never in a file, `argv` or the environment. //! //! The secret can change under a running responder: authelia's host //! re-publishes it when it is re-minted. So when the `IdP` refuses an //! introspection (a non-2xx), the secret is read from the store again and, //! when it changed, the introspection is tried once more. Both fit in the one //! request's [`INTROSPECTION_TIMEOUT`](crate::introspect::INTROSPECTION_TIMEOUT). //! A re-read that does not is left running: that request is denied, and the //! next one introspects with what the re-read landed. One re-read runs at a //! time; a refusal while one is running waits for it rather than starting //! another. use std::future::Future; use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; use anyhow::{Context, Result}; use futures_util::FutureExt; use futures_util::future::{BoxFuture, Shared}; use swarm_secret_client::client::{DEFAULT_CERT_MOUNT, Settings}; use swarm_secret_client::path::{Kind, principal_prefix}; use tokio::task::AbortHandle; use tokio::time::Instant; use crate::introspect::Answer; use crate::keystore::KeyStore; /// The path under the store's KV mount holding `client_id`'s secret. /// `swarm-secret-publisher.nix` writes it, and `natsAuthReaders` in /// `swarm-bao.nix` grants this responder's role the read. /// /// # Errors /// When `client_id` is not a single path segment. pub fn path(client_id: &str) -> Result { let prefix = principal_prefix(Kind::Service, client_id)?; Ok(format!("{prefix}/oidc/client")) } /// The secret at `path`, with surrounding whitespace trimmed: the publisher /// stores the minted file as is, and a trailing newline is not part of the /// secret. Errors name the path, never the value. /// /// # Errors /// When the store cannot be read, or holds nothing or only whitespace there. pub async fn read(store: &impl KeyStore, path: &str) -> Result { let stored = store.read(path).await?.with_context(|| { format!("nothing is stored at {path}; swarm-secret-publish on authelia's host writes it") })?; let trimmed = stored.trim(); if trimmed.is_empty() { anyhow::bail!("{path} holds an empty secret"); } Ok(trimmed.to_owned()) } /// Where a re-read of the secret comes from. pub trait Source: Send + Sync + 'static { /// The secret as the store holds it now. fn read(&self) -> impl Future> + Send; } /// The secret store, logged in to per read: a re-read can come long after /// the start's token has expired. pub struct Store { settings: Settings, cert_role: String, path: String, } impl Store { /// The secret at `path`, read as `cert_role` from the store at `settings`. pub fn new(settings: Settings, cert_role: String, path: String) -> Self { Self { settings, cert_role, path, } } } impl Source for Store { async fn read(&self) -> Result { let store = swarm_secret_client::SecretStore::connect( &self.settings, &self.cert_role, DEFAULT_CERT_MOUNT, ) .await .context("logging in to the secret store")?; read(&store, &self.path).await } } /// The longest one re-read runs. The store client sets no timeout of its /// own, and while a re-read runs every refusal waits on it instead of /// starting another. const REREAD_TIMEOUT: Duration = Duration::from_secs(10); /// A re-read in progress, shared by every request waiting on it. It yields /// whether the store was read; a failed or timed-out read logs for itself. type Reread = Shared>; /// The secret in use, and the re-read that replaces it. No `Debug`: it holds /// the secret. pub struct OidcSecret { source: Arc, current: Arc>, reread: Mutex>, } impl OidcSecret { /// `initial`, re-read from `source` when the `IdP` refuses it. pub fn new(source: S, initial: String) -> Self { Self { source: Arc::new(source), current: Arc::new(RwLock::new(initial)), reread: Mutex::new(None), } } fn current(&self) -> String { self.current.read().expect("unpoisoned").clone() } /// The caller `attempt` names, introspecting with the current secret. /// When the `IdP` refuses that, the secret is re-read and, if it changed /// and `deadline` has not passed, `attempt` runs once more with the new /// one. `attempt` bounds itself by the same `deadline`. /// /// # Errors /// When an attempt could not be made at all; the caller denies. pub async fn introspect(&self, deadline: Instant, attempt: F) -> Result> where F: Fn(String) -> Fut, Fut: Future>, { let used = self.current(); if let Answer::Caller(caller) = attempt(used.clone()).await? { return Ok(caller); } match tokio::time::timeout_at(deadline, self.reread()).await { Err(_) => { tracing::warn!( "re-reading the OIDC client secret did not finish inside the introspection budget; \ denying, and the next request introspects with what it reads" ); return Ok(None); } Ok(false) => return Ok(None), Ok(true) => {} } let fresh = self.current(); if fresh == used { tracing::warn!("the stored OIDC client secret is the one the IdP refused; denying"); return Ok(None); } tracing::info!("re-read a changed OIDC client secret; introspecting again"); match attempt(fresh).await? { Answer::Caller(caller) => Ok(caller), Answer::Refused => Ok(None), } } /// The running re-read, or a new one when none is running. Spawned, so it /// finishes even when every request waiting on it has given up. fn reread(&self) -> Reread { let mut slot = self.reread.lock().expect("unpoisoned"); if let Some((handle, running)) = slot.as_ref() && !handle.is_finished() { return running.clone(); } let source = Arc::clone(&self.source); let current = Arc::clone(&self.current); let task = tokio::spawn(async move { match tokio::time::timeout(REREAD_TIMEOUT, source.read()).await { Ok(Ok(secret)) => { *current.write().expect("unpoisoned") = secret; true } Ok(Err(e)) => { tracing::warn!( error = format!("{e:#}"), "could not re-read the OIDC client secret; keeping the one in use" ); false } Err(_) => { tracing::warn!( timeout = ?REREAD_TIMEOUT, "re-reading the OIDC client secret timed out; keeping the one in use" ); false } } }); let handle = task.abort_handle(); let shared = task .map(|read| { read.unwrap_or_else(|e| { tracing::warn!( error = %e, "the OIDC client secret re-read task died; keeping the one in use" ); false }) }) .boxed() .shared(); *slot = Some((handle, shared.clone())); shared } } #[cfg(test)] mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; use super::*; use crate::introspect::INTROSPECTION_TIMEOUT; use crate::keystore::tests::Fake as FakeKeys; /// A store holding `secret`, or failing, that takes `delay` to answer. struct Fake { secret: Option<&'static str>, delay: Duration, reads: Arc, } impl Fake { fn holding(secret: &'static str) -> Self { Self { secret: Some(secret), delay: Duration::ZERO, reads: Arc::default(), } } } impl Source for Fake { async fn read(&self) -> Result { self.reads.fetch_add(1, Ordering::SeqCst); tokio::time::sleep(self.delay).await; self.secret.map(str::to_owned).context("store unreachable") } } /// An `IdP` that admits as `hive-alpha` only when introspected with /// `registered`, and refuses every other secret, counting its calls. fn idp<'a>( registered: &'static str, calls: &'a AtomicUsize, ) -> impl Fn(String) -> std::future::Ready> + 'a { move |secret| { calls.fetch_add(1, Ordering::SeqCst); std::future::ready(Ok(if secret == registered { Answer::Caller(Some("hive-alpha".to_owned())) } else { Answer::Refused })) } } fn budget() -> Instant { Instant::now() + INTROSPECTION_TIMEOUT } #[test] fn the_path_is_the_one_the_publisher_writes() { assert_eq!( path("swarm-nats").expect("path"), "swarm/services/swarm-nats/oidc/client" ); assert!(path("../swarm-grafana").is_err()); } #[tokio::test] async fn the_stored_secret_is_trimmed_and_an_empty_or_absent_one_refused() { let at = "swarm/services/swarm-nats/oidc/client"; let keys = FakeKeys::holding(&[(at, "s3cret\n")]); assert_eq!(read(&keys, at).await.expect("stored"), "s3cret"); keys.put(at, " \n"); assert!(read(&keys, at).await.is_err()); assert!(read(&FakeKeys::default(), at).await.is_err()); } #[tokio::test] async fn an_answered_introspection_reads_nothing() { let store = Fake::holding("new"); let reads = Arc::clone(&store.reads); let secret = OidcSecret::new(store, "old".to_owned()); let calls = AtomicUsize::new(0); let caller = secret .introspect(budget(), idp("old", &calls)) .await .expect("answered"); assert_eq!(caller.as_deref(), Some("hive-alpha")); assert_eq!(calls.load(Ordering::SeqCst), 1); assert_eq!(reads.load(Ordering::SeqCst), 0); } /// The rotation the re-read exists for: the `IdP` refuses the secret held /// since start, the store holds the new one, and the same request is /// admitted with it. #[tokio::test] async fn a_refused_secret_is_re_read_and_the_request_retried_once() { let store = Fake::holding("new"); let reads = Arc::clone(&store.reads); let secret = OidcSecret::new(store, "old".to_owned()); let calls = AtomicUsize::new(0); let caller = secret .introspect(budget(), idp("new", &calls)) .await .expect("answered"); assert_eq!(caller.as_deref(), Some("hive-alpha")); assert_eq!(calls.load(Ordering::SeqCst), 2); assert_eq!(reads.load(Ordering::SeqCst), 1); } #[tokio::test] async fn a_refusal_after_the_re_read_is_a_denial_with_no_third_attempt() { let store = Fake::holding("newer"); let reads = Arc::clone(&store.reads); let secret = OidcSecret::new(store, "old".to_owned()); let calls = AtomicUsize::new(0); let caller = secret .introspect(budget(), idp("unknown", &calls)) .await .expect("answered"); assert_eq!(caller, None); assert_eq!(calls.load(Ordering::SeqCst), 2); assert_eq!(reads.load(Ordering::SeqCst), 1); } #[tokio::test] async fn an_unchanged_secret_is_not_retried() { let secret = OidcSecret::new(Fake::holding("old"), "old".to_owned()); let calls = AtomicUsize::new(0); let caller = secret .introspect(budget(), idp("unknown", &calls)) .await .expect("answered"); assert_eq!(caller, None); assert_eq!(calls.load(Ordering::SeqCst), 1); } thread_local! { static LOG: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; } /// What `tracing` records on the calling thread. The subscriber is /// process-wide, because a scoped one loses to the callsite cache when /// another test reaches the same event first; the buffer is per thread, /// which is where a current-thread test runtime runs the spawned re-read. struct Logs; impl std::io::Write for Logs { fn write(&mut self, buf: &[u8]) -> std::io::Result { LOG.with(|log| log.borrow_mut().extend_from_slice(buf)); Ok(buf.len()) } fn flush(&mut self) -> std::io::Result<()> { Ok(()) } } impl Logs { /// Starts this thread's capture empty. fn capture() { static INSTALLED: std::sync::Once = std::sync::Once::new(); INSTALLED.call_once(|| { tracing_subscriber::fmt() .with_writer(|| Logs) .with_ansi(false) .init(); }); LOG.with(|log| log.borrow_mut().clear()); } fn text() -> String { LOG.with(|log| String::from_utf8(log.borrow().clone()).expect("utf-8")) } } #[tokio::test] async fn a_failed_re_read_denies_and_keeps_the_secret_in_use() { Logs::capture(); let store = Fake { secret: None, delay: Duration::ZERO, reads: Arc::default(), }; let secret = OidcSecret::new(store, "old".to_owned()); let calls = AtomicUsize::new(0); let caller = secret .introspect(budget(), idp("new", &calls)) .await .expect("answered"); assert_eq!(caller, None); assert_eq!(calls.load(Ordering::SeqCst), 1); assert_eq!(secret.current(), "old"); let text = Logs::text(); assert!(text.contains("could not re-read the OIDC client secret")); assert!(!text.contains("is the one the IdP refused"), "{text}"); assert!(!text.contains("old"), "no secret in the log: {text}"); } /// The store was read and holds what the `IdP` already refused: the one /// case that says so. #[tokio::test] async fn the_unchanged_secret_is_logged_as_the_one_the_idp_refused() { Logs::capture(); let secret = OidcSecret::new(Fake::holding("old"), "old".to_owned()); let calls = AtomicUsize::new(0); let caller = secret .introspect(budget(), idp("new", &calls)) .await .expect("answered"); assert_eq!(caller, None); let text = Logs::text(); assert!(text.contains("is the one the IdP refused"), "{text}"); assert!(!text.contains("could not re-read"), "{text}"); } /// A store slower than the budget: the request is denied at its /// deadline, the re-read finishes behind it, and the next request is /// admitted on its first attempt without reading again. #[tokio::test(start_paused = true)] async fn a_re_read_past_the_budget_serves_the_next_request() { let store = Fake { secret: Some("new"), delay: Duration::from_secs(3), reads: Arc::default(), }; let reads = Arc::clone(&store.reads); let secret = OidcSecret::new(store, "old".to_owned()); let calls = AtomicUsize::new(0); let started = Instant::now(); let deadline = started + INTROSPECTION_TIMEOUT; let first = secret .introspect(deadline, idp("new", &calls)) .await .expect("answered"); assert_eq!(first, None); assert_eq!(Instant::now(), deadline); assert_eq!(calls.load(Ordering::SeqCst), 1); tokio::time::sleep(Duration::from_secs(2)).await; let next = secret .introspect(budget(), idp("new", &calls)) .await .expect("answered"); assert_eq!(next.as_deref(), Some("hive-alpha")); assert_eq!(calls.load(Ordering::SeqCst), 2); assert_eq!(reads.load(Ordering::SeqCst), 1); } /// A refusal while a re-read is still running waits on that one, and is /// admitted when it lands inside this request's budget. #[tokio::test(start_paused = true)] async fn a_refusal_during_a_running_re_read_joins_it() { let store = Fake { secret: Some("new"), delay: Duration::from_secs(2), reads: Arc::default(), }; let reads = Arc::clone(&store.reads); let secret = OidcSecret::new(store, "old".to_owned()); let calls = AtomicUsize::new(0); let first = secret .introspect(budget(), idp("new", &calls)) .await .expect("answered"); assert_eq!(first, None); let second = secret .introspect(budget(), idp("new", &calls)) .await .expect("answered"); assert_eq!(second.as_deref(), Some("hive-alpha")); assert_eq!(calls.load(Ordering::SeqCst), 3); assert_eq!(reads.load(Ordering::SeqCst), 1); } /// A re-read that stalls is given up at [`REREAD_TIMEOUT`]: the secret in /// use stays, and the next refusal starts a fresh re-read rather than /// waiting on the stalled one. #[tokio::test(start_paused = true)] async fn a_stalled_re_read_is_dropped_at_its_timeout() { let store = Fake { secret: Some("new"), delay: REREAD_TIMEOUT * 6, reads: Arc::default(), }; let reads = Arc::clone(&store.reads); let secret = OidcSecret::new(store, "old".to_owned()); let calls = AtomicUsize::new(0); let first = secret .introspect(budget(), idp("new", &calls)) .await .expect("answered"); assert_eq!(first, None); tokio::time::sleep(REREAD_TIMEOUT).await; assert_eq!(secret.current(), "old"); let second = secret .introspect(budget(), idp("new", &calls)) .await .expect("answered"); assert_eq!(second, None); assert_eq!(reads.load(Ordering::SeqCst), 2); } /// An `IdP` on a loopback port that admits as `hive-alpha` a request /// whose basic auth is `swarm-nats:{registered}`, and answers any other /// with a 401. async fn stub_idp(registered: &str) -> String { use axum::http::{HeaderMap, StatusCode, header::AUTHORIZATION}; let expected = format!( "Basic {}", data_encoding::BASE64.encode(format!("swarm-nats:{registered}").as_bytes()) ); let app = axum::Router::new().fallback(move |headers: HeaderMap| async move { if headers.get(AUTHORIZATION).and_then(|v| v.to_str().ok()) == Some(&*expected) { ( StatusCode::OK, r#"{"active":true,"client_id":"hive-alpha"}"#, ) } else { (StatusCode::UNAUTHORIZED, "") } }); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("bind"); let url = format!("http://{}/introspect", listener.local_addr().expect("addr")); tokio::spawn(async move { axum::serve(listener, app).await }); url } /// The `IdP`'s 401 to the secret held since start is what starts the /// re-read: driven through the real HTTP introspection, not an [`Answer`] /// handed in. #[tokio::test] async fn an_idp_401_re_reads_the_secret_and_admits_with_the_new_one() { let url = stub_idp("new").await; let http = reqwest::Client::builder() .no_proxy() .build() .expect("client"); let store = Fake::holding("new"); let reads = Arc::clone(&store.reads); let secret = OidcSecret::new(store, "old".to_owned()); let deadline = budget(); let caller = secret .introspect(deadline, |own| { let (http, url) = (&http, &url); async move { crate::introspect::identify_caller( http, url, "swarm-nats", &own, "a-token", deadline, ) .await } }) .await .expect("answered"); assert_eq!(caller.as_deref(), Some("hive-alpha")); assert_eq!(reads.load(Ordering::SeqCst), 1); } }