swarm-nats-auth: read its own OIDC client secret from the store
The responder reads its OIDC client secret at `swarm/services/<client-id>/oidc/client` with the store login it already opens for the callout keys, and holds it in memory. `--client-secret-file` is removed. When introspection answers non-2xx, the responder re-reads the secret from the store once and, if it changed, retries the introspection once. Both fit inside the request's 1.5s introspection budget, under the server's 2s callout timeout: the budget is now one deadline for the whole request. A re-read that does not finish by then keeps running in the background, the request is denied, and the next request introspects with what it read. One re-read runs at a time; a refusal while one runs waits on it. In swarm-nats.nix the `swarm-bao-nats-auth-oidc` fetch unit, the flag and the `oidc-client.secret` LoadCredential entry are gone, along with the unit's policy ordering in glue-bao-readers-policy-order.nix. A tmpfiles `r` rule removes the copy a container still holds at /var/lib/nixos-containers/swarm-nats/var/lib/swarm-nats-auth/oidc-client.secret; nothing reads it. No policy change: the `swarm-nats-auth` role already reads this path (`natsAuthReaders` in swarm-bao.nix), which is the role the removed fetch unit logged in as. Deploy order: none, and no operator step. The store's grant already exists, so the queue host can be deployed before, after or without the store's host and authelia's. Rolling back re-creates the fetch unit and the file; the store leaf is untouched.
This commit is contained in:
atlas
parent
a66670be87
commit
6646329da5
12 changed files with 557 additions and 253 deletions
397
swarm-nats-auth/src/oidc_secret.rs
Normal file
397
swarm-nats-auth/src/oidc_secret.rs
Normal file
|
|
@ -0,0 +1,397 @@
|
|||
//! This responder's own OIDC client secret, the credential it introspects
|
||||
//! with. Read from the secret store as the responder starts and held in
|
||||
//! memory, never in a file, `argv` or the environment.
|
||||
//!
|
||||
//! The secret can change under a running responder: authelia's host
|
||||
//! re-publishes it when it is re-minted. So when the `IdP` refuses an
|
||||
//! introspection (a non-2xx), the secret is read from the store again and,
|
||||
//! when it changed, the introspection is tried once more. Both fit in the one
|
||||
//! request's [`INTROSPECTION_TIMEOUT`](crate::introspect::INTROSPECTION_TIMEOUT).
|
||||
//! A re-read that does not is left running: that request is denied, and the
|
||||
//! next one introspects with what the re-read landed. One re-read runs at a
|
||||
//! time; a refusal while one is running waits for it rather than starting
|
||||
//! another.
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use futures_util::FutureExt;
|
||||
use futures_util::future::{BoxFuture, Shared};
|
||||
use swarm_secret_client::client::{DEFAULT_CERT_MOUNT, Settings};
|
||||
use swarm_secret_client::path::{Kind, principal_prefix};
|
||||
use tokio::task::AbortHandle;
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::introspect::Answer;
|
||||
use crate::keystore::KeyStore;
|
||||
|
||||
/// The path under the store's KV mount holding `client_id`'s secret.
|
||||
/// `swarm-secret-publisher.nix` writes it, and `natsAuthReaders` in
|
||||
/// `swarm-bao.nix` grants this responder's role the read.
|
||||
///
|
||||
/// # Errors
|
||||
/// When `client_id` is not a single path segment.
|
||||
pub fn path(client_id: &str) -> Result<String> {
|
||||
let prefix = principal_prefix(Kind::Service, client_id)?;
|
||||
Ok(format!("{prefix}/oidc/client"))
|
||||
}
|
||||
|
||||
/// The secret at `path`, with surrounding whitespace trimmed: the publisher
|
||||
/// stores the minted file as is, and a trailing newline is not part of the
|
||||
/// secret. Errors name the path, never the value.
|
||||
///
|
||||
/// # Errors
|
||||
/// When the store cannot be read, or holds nothing or only whitespace there.
|
||||
pub async fn read(store: &impl KeyStore, path: &str) -> Result<String> {
|
||||
let stored = store.read(path).await?.with_context(|| {
|
||||
format!("nothing is stored at {path}; swarm-secret-publish on authelia's host writes it")
|
||||
})?;
|
||||
let trimmed = stored.trim();
|
||||
if trimmed.is_empty() {
|
||||
anyhow::bail!("{path} holds an empty secret");
|
||||
}
|
||||
Ok(trimmed.to_owned())
|
||||
}
|
||||
|
||||
/// Where a re-read of the secret comes from.
|
||||
pub trait Source: Send + Sync + 'static {
|
||||
/// The secret as the store holds it now.
|
||||
fn read(&self) -> impl Future<Output = Result<String>> + Send;
|
||||
}
|
||||
|
||||
/// The secret store, logged in to per read: a re-read can come long after
|
||||
/// the start's token has expired.
|
||||
pub struct Store {
|
||||
settings: Settings,
|
||||
cert_role: String,
|
||||
path: String,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// The secret at `path`, read as `cert_role` from the store at `settings`.
|
||||
pub fn new(settings: Settings, cert_role: String, path: String) -> Self {
|
||||
Self {
|
||||
settings,
|
||||
cert_role,
|
||||
path,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Source for Store {
|
||||
async fn read(&self) -> Result<String> {
|
||||
let store = swarm_secret_client::SecretStore::connect(
|
||||
&self.settings,
|
||||
&self.cert_role,
|
||||
DEFAULT_CERT_MOUNT,
|
||||
)
|
||||
.await
|
||||
.context("logging in to the secret store")?;
|
||||
read(&store, &self.path).await
|
||||
}
|
||||
}
|
||||
|
||||
/// A re-read in progress, shared by every request waiting on it.
|
||||
type Reread = Shared<BoxFuture<'static, ()>>;
|
||||
|
||||
/// The secret in use, and the re-read that replaces it. No `Debug`: it holds
|
||||
/// the secret.
|
||||
pub struct OidcSecret<S> {
|
||||
source: Arc<S>,
|
||||
current: Arc<RwLock<String>>,
|
||||
reread: Mutex<Option<(AbortHandle, Reread)>>,
|
||||
}
|
||||
|
||||
impl<S: Source> OidcSecret<S> {
|
||||
/// `initial`, re-read from `source` when the `IdP` refuses it.
|
||||
pub fn new(source: S, initial: String) -> Self {
|
||||
Self {
|
||||
source: Arc::new(source),
|
||||
current: Arc::new(RwLock::new(initial)),
|
||||
reread: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn current(&self) -> String {
|
||||
self.current.read().expect("unpoisoned").clone()
|
||||
}
|
||||
|
||||
/// The caller `attempt` names, introspecting with the current secret.
|
||||
/// When the `IdP` refuses that, the secret is re-read and, if it changed
|
||||
/// and `deadline` has not passed, `attempt` runs once more with the new
|
||||
/// one. `attempt` bounds itself by the same `deadline`.
|
||||
///
|
||||
/// # Errors
|
||||
/// When an attempt could not be made at all; the caller denies.
|
||||
pub async fn introspect<F, Fut>(&self, deadline: Instant, attempt: F) -> Result<Option<String>>
|
||||
where
|
||||
F: Fn(String) -> Fut,
|
||||
Fut: Future<Output = Result<Answer>>,
|
||||
{
|
||||
let used = self.current();
|
||||
if let Answer::Caller(caller) = attempt(used.clone()).await? {
|
||||
return Ok(caller);
|
||||
}
|
||||
if tokio::time::timeout_at(deadline, self.reread())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
"re-reading the OIDC client secret did not finish inside the introspection budget; \
|
||||
denying, and the next request introspects with what it reads"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
let fresh = self.current();
|
||||
if fresh == used {
|
||||
tracing::warn!("the stored OIDC client secret is the one the IdP refused; denying");
|
||||
return Ok(None);
|
||||
}
|
||||
tracing::info!("re-read a changed OIDC client secret; introspecting again");
|
||||
match attempt(fresh).await? {
|
||||
Answer::Caller(caller) => Ok(caller),
|
||||
Answer::Refused => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// The running re-read, or a new one when none is running. Spawned, so it
|
||||
/// finishes even when every request waiting on it has given up.
|
||||
fn reread(&self) -> Reread {
|
||||
let mut slot = self.reread.lock().expect("unpoisoned");
|
||||
if let Some((handle, running)) = slot.as_ref()
|
||||
&& !handle.is_finished()
|
||||
{
|
||||
return running.clone();
|
||||
}
|
||||
let source = Arc::clone(&self.source);
|
||||
let current = Arc::clone(&self.current);
|
||||
let task = tokio::spawn(async move {
|
||||
match source.read().await {
|
||||
Ok(secret) => *current.write().expect("unpoisoned") = secret,
|
||||
Err(e) => tracing::warn!(
|
||||
error = format!("{e:#}"),
|
||||
"could not re-read the OIDC client secret; keeping the one in use"
|
||||
),
|
||||
}
|
||||
});
|
||||
let handle = task.abort_handle();
|
||||
let shared = task.map(|_| ()).boxed().shared();
|
||||
*slot = Some((handle, shared.clone()));
|
||||
shared
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use super::*;
|
||||
use crate::introspect::INTROSPECTION_TIMEOUT;
|
||||
use crate::keystore::tests::Fake as FakeKeys;
|
||||
|
||||
/// A store holding `secret`, or failing, that takes `delay` to answer.
|
||||
struct Fake {
|
||||
secret: Option<&'static str>,
|
||||
delay: Duration,
|
||||
reads: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Fake {
|
||||
fn holding(secret: &'static str) -> Self {
|
||||
Self {
|
||||
secret: Some(secret),
|
||||
delay: Duration::ZERO,
|
||||
reads: Arc::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Source for Fake {
|
||||
async fn read(&self) -> Result<String> {
|
||||
self.reads.fetch_add(1, Ordering::SeqCst);
|
||||
tokio::time::sleep(self.delay).await;
|
||||
self.secret.map(str::to_owned).context("store unreachable")
|
||||
}
|
||||
}
|
||||
|
||||
/// An `IdP` that admits as `hive-alpha` only when introspected with
|
||||
/// `registered`, and refuses every other secret, counting its calls.
|
||||
fn idp<'a>(
|
||||
registered: &'static str,
|
||||
calls: &'a AtomicUsize,
|
||||
) -> impl Fn(String) -> std::future::Ready<Result<Answer>> + 'a {
|
||||
move |secret| {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
std::future::ready(Ok(if secret == registered {
|
||||
Answer::Caller(Some("hive-alpha".to_owned()))
|
||||
} else {
|
||||
Answer::Refused
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
fn budget() -> Instant {
|
||||
Instant::now() + INTROSPECTION_TIMEOUT
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_path_is_the_one_the_publisher_writes() {
|
||||
assert_eq!(
|
||||
path("swarm-nats").expect("path"),
|
||||
"swarm/services/swarm-nats/oidc/client"
|
||||
);
|
||||
assert!(path("../swarm-grafana").is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_stored_secret_is_trimmed_and_an_empty_or_absent_one_refused() {
|
||||
let at = "swarm/services/swarm-nats/oidc/client";
|
||||
let keys = FakeKeys::holding(&[(at, "s3cret\n")]);
|
||||
assert_eq!(read(&keys, at).await.expect("stored"), "s3cret");
|
||||
keys.put(at, " \n");
|
||||
assert!(read(&keys, at).await.is_err());
|
||||
assert!(read(&FakeKeys::default(), at).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_answered_introspection_reads_nothing() {
|
||||
let store = Fake::holding("new");
|
||||
let reads = Arc::clone(&store.reads);
|
||||
let secret = OidcSecret::new(store, "old".to_owned());
|
||||
let calls = AtomicUsize::new(0);
|
||||
let caller = secret
|
||||
.introspect(budget(), idp("old", &calls))
|
||||
.await
|
||||
.expect("answered");
|
||||
assert_eq!(caller.as_deref(), Some("hive-alpha"));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(reads.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// The rotation the re-read exists for: the `IdP` refuses the secret held
|
||||
/// since start, the store holds the new one, and the same request is
|
||||
/// admitted with it.
|
||||
#[tokio::test]
|
||||
async fn a_refused_secret_is_re_read_and_the_request_retried_once() {
|
||||
let store = Fake::holding("new");
|
||||
let reads = Arc::clone(&store.reads);
|
||||
let secret = OidcSecret::new(store, "old".to_owned());
|
||||
let calls = AtomicUsize::new(0);
|
||||
let caller = secret
|
||||
.introspect(budget(), idp("new", &calls))
|
||||
.await
|
||||
.expect("answered");
|
||||
assert_eq!(caller.as_deref(), Some("hive-alpha"));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(reads.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_refusal_after_the_re_read_is_a_denial_with_no_third_attempt() {
|
||||
let store = Fake::holding("newer");
|
||||
let reads = Arc::clone(&store.reads);
|
||||
let secret = OidcSecret::new(store, "old".to_owned());
|
||||
let calls = AtomicUsize::new(0);
|
||||
let caller = secret
|
||||
.introspect(budget(), idp("unknown", &calls))
|
||||
.await
|
||||
.expect("answered");
|
||||
assert_eq!(caller, None);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(reads.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unchanged_secret_is_not_retried() {
|
||||
let secret = OidcSecret::new(Fake::holding("old"), "old".to_owned());
|
||||
let calls = AtomicUsize::new(0);
|
||||
let caller = secret
|
||||
.introspect(budget(), idp("unknown", &calls))
|
||||
.await
|
||||
.expect("answered");
|
||||
assert_eq!(caller, None);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_re_read_denies_and_keeps_the_secret_in_use() {
|
||||
let store = Fake {
|
||||
secret: None,
|
||||
delay: Duration::ZERO,
|
||||
reads: Arc::default(),
|
||||
};
|
||||
let secret = OidcSecret::new(store, "old".to_owned());
|
||||
let calls = AtomicUsize::new(0);
|
||||
let caller = secret
|
||||
.introspect(budget(), idp("new", &calls))
|
||||
.await
|
||||
.expect("answered");
|
||||
assert_eq!(caller, None);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(secret.current(), "old");
|
||||
}
|
||||
|
||||
/// A store slower than the budget: the request is denied at its
|
||||
/// deadline, the re-read finishes behind it, and the next request is
|
||||
/// admitted on its first attempt without reading again.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_re_read_past_the_budget_serves_the_next_request() {
|
||||
let store = Fake {
|
||||
secret: Some("new"),
|
||||
delay: Duration::from_secs(3),
|
||||
reads: Arc::default(),
|
||||
};
|
||||
let reads = Arc::clone(&store.reads);
|
||||
let secret = OidcSecret::new(store, "old".to_owned());
|
||||
let calls = AtomicUsize::new(0);
|
||||
|
||||
let started = Instant::now();
|
||||
let deadline = started + INTROSPECTION_TIMEOUT;
|
||||
let first = secret
|
||||
.introspect(deadline, idp("new", &calls))
|
||||
.await
|
||||
.expect("answered");
|
||||
assert_eq!(first, None);
|
||||
assert_eq!(Instant::now(), deadline);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
let next = secret
|
||||
.introspect(budget(), idp("new", &calls))
|
||||
.await
|
||||
.expect("answered");
|
||||
assert_eq!(next.as_deref(), Some("hive-alpha"));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(reads.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
/// A refusal while a re-read is still running waits on that one, and is
|
||||
/// admitted when it lands inside this request's budget.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_refusal_during_a_running_re_read_joins_it() {
|
||||
let store = Fake {
|
||||
secret: Some("new"),
|
||||
delay: Duration::from_secs(2),
|
||||
reads: Arc::default(),
|
||||
};
|
||||
let reads = Arc::clone(&store.reads);
|
||||
let secret = OidcSecret::new(store, "old".to_owned());
|
||||
let calls = AtomicUsize::new(0);
|
||||
|
||||
let first = secret
|
||||
.introspect(budget(), idp("new", &calls))
|
||||
.await
|
||||
.expect("answered");
|
||||
assert_eq!(first, None);
|
||||
|
||||
let second = secret
|
||||
.introspect(budget(), idp("new", &calls))
|
||||
.await
|
||||
.expect("answered");
|
||||
assert_eq!(second.as_deref(), Some("hive-alpha"));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 3);
|
||||
assert_eq!(reads.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue