//! The credentials squid authenticates its clients with: one per agent, at //! `swarm_secret_client::proxy::agent_credential_path`, and one per hive for //! that hive's squid self-check, at `proxy::selfcheck_path`. //! //! Minted here because the agent's store policy is read-only and a hive's //! policy cannot create under another principal's prefix. [`spawn`]'s pass runs //! at start and every [`RECONCILE_INTERVAL`] over every agent that holds a store //! identity, so an agent that existed before this pass was added gets one on the //! next tick, with no restart. //! //! ⚠️ **Never replaced.** A path that holds anything is left alone, and the write //! is a `cas=0` create. Replacing a credential is a rotation: squid's users file //! is rendered from whatever the store holds, and a rotation has to reach every //! client that carries the old one in the same step. use anyhow::{Context, Result}; use serde::de::IgnoredAny; use swarm_secret_client::{Error, SecretStore, client::DEFAULT_CERT_MOUNT, policy, proxy}; /// How often [`spawn`] re-checks every agent and hive. const RECONCILE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(5); /// What [`ensure`] did at one path. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Outcome { /// The path already held a credential. Present, /// This pass wrote one. Minted, /// Another writer's create landed between this pass's read and its own. LostRace, } /// The two store calls [`ensure`] makes, so a test can stand in for the store. pub(crate) trait CredentialStore { /// Whether anything is stored at `path`, as [`SecretStore::read_optional`] /// answers it. The value is decoded as [`IgnoredAny`] and never held. fn exists(&self, path: &str) -> impl Future> + Send; /// As [`SecretStore::create`]. fn create( &self, path: &str, value: &proxy::Credential, ) -> impl Future> + Send; } impl CredentialStore for SecretStore { async fn exists(&self, path: &str) -> Result { Ok(self.read_optional::(path).await?.is_some()) } async fn create(&self, path: &str, value: &proxy::Credential) -> Result<(), Error> { SecretStore::create(self, path, value).await } } /// Make sure `path` holds a credential, writing one only if it holds nothing. /// /// A create that fails is read back: if the path now holds something, a /// concurrent writer won and that is the credential. /// /// # Errors /// When the store refuses the read, or the create fails and the path is still /// empty. pub(crate) async fn ensure(store: &impl CredentialStore, path: &str) -> Result { if store .exists(path) .await .with_context(|| format!("reading {path}"))? { return Ok(Outcome::Present); } let credential = proxy::Credential { value: crate::agent_identity::generate_queue_secret()?, }; let Err(e) = store.create(path, &credential).await else { return Ok(Outcome::Minted); }; if store .exists(path) .await .with_context(|| format!("reading {path} back after a failed create"))? { return Ok(Outcome::LostRace); } Err(anyhow::Error::new(e).context(format!("creating {path}"))) } /// [`ensure`] at each of `paths`, which `what` names in the log. Returns how /// many it minted. A failure at one path is logged and the rest still run. async fn ensure_all( store: &impl CredentialStore, what: &str, paths: Vec<(String, Result)>, ) -> usize { let mut minted = 0; for (name, path) in paths { let path = match path { Ok(path) => path, Err(e) => { tracing::warn!(name, error = %e, "{what}: unusable name"); continue; } }; match ensure(store, &path).await { Ok(Outcome::Present) => {} Ok(Outcome::Minted) => { minted += 1; tracing::info!(name, %path, "{what} minted"); } Ok(Outcome::LostRace) => { tracing::info!(name, %path, "{what}: another writer created it first"); } Err(e) => tracing::warn!(name, error = %format!("{e:#}"), "{what}: mint failed"), } } minted } /// One pass: a credential for every agent in `agents` and a self-check /// credential for every hive in `hives`. Returns how many were minted. async fn mint_missing(store: &impl CredentialStore, agents: &[String], hives: &[String]) -> usize { let for_agents = agents .iter() .map(|a| (a.clone(), proxy::agent_credential_path(a))) .collect(); let for_hives = hives .iter() .map(|h| (h.clone(), proxy::selfcheck_path(h))) .collect(); ensure_all(store, "agent proxy credential", for_agents).await + ensure_all(store, "hive squid self-check credential", for_hives).await } /// The agents that hold a store identity: exactly the ones whose hive reads /// what a pass writes. async fn roster(store: &SecretStore) -> Result> { let roles = store .list_cert_roles(DEFAULT_CERT_MOUNT) .await .context("listing the store's cert-auth roles")?; Ok(policy::agents_from_role_names(&roles)) } /// Run a pass now and every [`RECONCILE_INTERVAL`] after. A pass that fails is /// logged and retried on the next tick; it never stops the daemon. pub fn spawn(hives: Vec) { tokio::spawn(async move { let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); loop { ticker.tick().await; let pass = async { let store = crate::store::connect() .await .context("logging in to the swarm secret store")?; let agents = roster(&store).await?; anyhow::Ok((agents.len(), mint_missing(&store, &agents, &hives).await)) }; match pass.await { Ok((agents, minted)) if minted > 0 => { tracing::info!( "proxy credentials: minted {minted} (agents and hive self-checks); {agents} agents hold a store identity" ); } Ok((agents, minted)) => { tracing::debug!(agents, minted, "proxy credentials: pass done"); } Err(e) => tracing::warn!( error = %format!("{e:#}"), retry_in_s = RECONCILE_INTERVAL.as_secs(), "proxy credentials: pass failed; retrying next tick" ), } } }); } #[cfg(test)] mod tests { use std::collections::HashMap; use std::sync::Mutex; use super::*; /// An in-memory store that refuses a create over a held path the way a /// `cas=0` write does, and counts the creates it was asked for. #[derive(Default)] struct Fake { held: Mutex>, creates: Mutex, /// A path whose create fails and leaves nothing behind. broken: Option, /// Makes the next create find the path taken by someone else. race: Option, } impl Fake { fn holding(path: &str, value: &str) -> Self { let fake = Self::default(); fake.held .lock() .unwrap() .insert(path.to_owned(), value.to_owned()); fake } fn value(&self, path: &str) -> Option { self.held.lock().unwrap().get(path).cloned() } } fn refused() -> Error { Error::MissingEnv("BAO_ADDR") } impl CredentialStore for Fake { async fn exists(&self, path: &str) -> Result { Ok(self.held.lock().unwrap().contains_key(path)) } async fn create(&self, path: &str, value: &proxy::Credential) -> Result<(), Error> { *self.creates.lock().unwrap() += 1; let mut held = self.held.lock().unwrap(); if self.broken.as_deref() == Some(path) { return Err(refused()); } if let Some(winner) = &self.race { held.insert(path.to_owned(), winner.clone()); return Err(refused()); } if held.contains_key(path) { return Err(refused()); } held.insert(path.to_owned(), value.value.clone()); Ok(()) } } const ATLAS: &str = "swarm/agents/atlas/proxy-credential"; const ARGUS: &str = "swarm/agents/argus/proxy-credential"; const H1: &str = "swarm/hives/h1/squid-selfcheck"; fn names(names: &[&str]) -> Vec { names.iter().map(|n| (*n).to_owned()).collect() } #[tokio::test] async fn an_absent_credential_is_created() { let store = Fake::default(); assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::Minted); let value = store.value(ATLAS).expect("written"); // 32 bytes, base64url without padding. assert_eq!(value.len(), 43, "{value}"); assert!( value .bytes() .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') ); } #[tokio::test] async fn a_present_credential_is_untouched_and_never_updated() { let store = Fake::holding(ATLAS, "existing"); assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::Present); assert_eq!(store.value(ATLAS).as_deref(), Some("existing")); assert_eq!(*store.creates.lock().unwrap(), 0, "no write was attempted"); } #[tokio::test] async fn a_lost_race_keeps_the_winners_value() { let store = Fake { race: Some("winner".to_owned()), ..Fake::default() }; assert_eq!(ensure(&store, ATLAS).await.unwrap(), Outcome::LostRace); assert_eq!(store.value(ATLAS).as_deref(), Some("winner")); } #[tokio::test] async fn a_failed_create_with_nothing_behind_it_is_an_error() { let store = Fake { broken: Some(ATLAS.to_owned()), ..Fake::default() }; assert!(ensure(&store, ATLAS).await.is_err()); assert_eq!(store.value(ATLAS), None); } #[tokio::test] async fn a_pass_mints_for_every_agent_and_hive_that_lacks_one() { let store = Fake::holding(ARGUS, "existing"); let minted = mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await; assert_eq!(minted, 2); assert!(store.value(ATLAS).is_some()); assert!(store.value(H1).is_some()); assert_eq!(store.value(ARGUS).as_deref(), Some("existing")); // Control: the second pass has nothing left to mint, and writes nothing. let creates = *store.creates.lock().unwrap(); assert_eq!( mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await, 0 ); assert_eq!(*store.creates.lock().unwrap(), creates); } #[tokio::test] async fn one_agent_failing_does_not_stop_the_rest() { let store = Fake { broken: Some(ATLAS.to_owned()), ..Fake::default() }; let minted = mint_missing(&store, &names(&["atlas", "argus"]), &names(&["h1"])).await; assert_eq!(minted, 2); assert!(store.value(ARGUS).is_some()); assert!(store.value(H1).is_some()); } #[tokio::test] async fn an_unusable_agent_name_mints_nothing() { let store = Fake::default(); assert_eq!(mint_missing(&store, &names(&["../argus"]), &[]).await, 0); assert!(store.held.lock().unwrap().is_empty()); } }