//! The optional TCP listener: TLS with a required client certificate. //! //! The controller authenticates nobody per request (its routes trust the //! transport, and `Remote-User` is a header the gateway sets), so a TCP port //! is only safe if only the gateway can complete a handshake on it. This //! listener therefore accepts a connection only when the client presents a //! certificate that //! //! 1. chains to the swarm services root (`SWARM_CONTROLLER_TCP_CLIENT_CA`), //! 2. carries the `clientAuth` extended key usage, and //! 3. names the gateway's client identity (`SWARM_CONTROLLER_TCP_CLIENT_NAME`) //! in a DNS subject alternative name. //! //! The swarm-services leaf the gateway serves browsers with chains to the same //! root but is `serverAuth` only and names other hosts, so it fails the second //! and third checks. Everything that fails is refused during the handshake, //! before a byte of HTTP is read. //! //! The server certificate and key are read from files and re-read when either //! changes, so a rotation needs no restart. The client CA is read once at //! start. use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::{Duration, SystemTime}; use anyhow::{Context, Result, bail}; use rustls::crypto::CryptoProvider; use rustls::pki_types::pem::PemObject as _; use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, UnixTime}; use rustls::server::danger::{ClientCertVerified, ClientCertVerifier}; use rustls::server::{ClientHello, ResolvesServerCert, WebPkiClientVerifier}; use rustls::sign::CertifiedKey; use rustls::{ CertificateError, DigitallySignedStruct, DistinguishedName, Error as TlsError, RootCertStore, ServerConfig, SignatureScheme, }; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::mpsc; /// Address to bind, `host:port`. Unset means no TCP listener at all. const LISTEN_ENV: &str = "SWARM_CONTROLLER_TCP_LISTEN"; /// PEM certificate chain this listener serves. const CERT_ENV: &str = "SWARM_CONTROLLER_TCP_CERT"; /// PEM private key for [`CERT_ENV`]. const KEY_ENV: &str = "SWARM_CONTROLLER_TCP_KEY"; /// PEM file holding the root client certificates must chain to. const CLIENT_CA_ENV: &str = "SWARM_CONTROLLER_TCP_CLIENT_CA"; /// The DNS name the one permitted client certificate carries. const CLIENT_NAME_ENV: &str = "SWARM_CONTROLLER_TCP_CLIENT_NAME"; /// How long a peer gets to finish the handshake. A connection that stalls /// holds a task and a file descriptor, nothing else. const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); /// What the five variables name. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TcpTlsConfig { pub listen: SocketAddr, pub cert: PathBuf, pub key: PathBuf, pub client_ca: PathBuf, pub client_name: String, } impl TcpTlsConfig { /// `None` when [`LISTEN_ENV`] is unset. The other four are then not read, /// but one of them set without it is an error: a half-configured listener /// must not read as "off". pub fn from_env() -> Result> { Self::from_lookup(|name| std::env::var(name).ok().filter(|v| !v.is_empty())) } fn from_lookup(get: impl Fn(&str) -> Option) -> Result> { let rest = [CERT_ENV, KEY_ENV, CLIENT_CA_ENV, CLIENT_NAME_ENV]; let Some(listen) = get(LISTEN_ENV) else { if let Some(set) = rest.iter().find(|n| get(n).is_some()) { bail!("{set} is set but {LISTEN_ENV} is not; the TCP listener needs all five"); } return Ok(None); }; let need = |name: &str| { get(name).with_context(|| format!("{name} is unset, but {LISTEN_ENV} is set")) }; Ok(Some(Self { listen: listen .parse() .with_context(|| format!("{LISTEN_ENV} is not a socket address: {listen}"))?, cert: need(CERT_ENV)?.into(), key: need(KEY_ENV)?.into(), client_ca: need(CLIENT_CA_ENV)?.into(), client_name: need(CLIENT_NAME_ENV)?, })) } } /// The server certificate, re-read from disk when its files change. /// /// A failed re-read keeps serving the last good pair and logs: a rotation /// caught half-written must not take the listener down. #[derive(Debug)] struct ReloadingCert { cert: PathBuf, key: PathBuf, provider: Arc, cached: Mutex, } #[derive(Debug)] struct Cached { stamp: Stamp, key: Arc, } /// What identifies one version of the pair on disk. type Stamp = [Option<(SystemTime, u64)>; 2]; impl ReloadingCert { fn load(cert: &Path, key: &Path, provider: Arc) -> Result { let stamp = stamp(cert, key); let loaded = read_pair(cert, key, &provider)?; Ok(Self { cert: cert.to_path_buf(), key: key.to_path_buf(), provider, cached: Mutex::new(Cached { stamp, key: loaded }), }) } fn current(&self) -> Arc { let mut cached = self.cached.lock().unwrap_or_else(PoisonError::into_inner); let now = stamp(&self.cert, &self.key); if now != cached.stamp { match read_pair(&self.cert, &self.key, &self.provider) { Ok(key) => { tracing::info!(cert = %self.cert.display(), "tcp listener: server certificate reloaded"); *cached = Cached { stamp: now, key }; } Err(e) => tracing::warn!( error = %format!("{e:#}"), "tcp listener: server certificate changed on disk but does not load; keeping the previous one" ), } } Arc::clone(&cached.key) } } impl ResolvesServerCert for ReloadingCert { fn resolve(&self, _hello: ClientHello<'_>) -> Option> { Some(self.current()) } } fn stamp(cert: &Path, key: &Path) -> Stamp { let one = |p: &Path| { let m = std::fs::metadata(p).ok()?; Some((m.modified().ok()?, m.len())) }; [one(cert), one(key)] } fn read_pair(cert: &Path, key: &Path, provider: &CryptoProvider) -> Result> { let chain = CertificateDer::pem_file_iter(cert) .with_context(|| format!("reading {}", cert.display()))? .collect::, _>>() .with_context(|| format!("parsing {}", cert.display()))?; if chain.is_empty() { bail!("{} holds no certificate", cert.display()); } let key = PrivateKeyDer::from_pem_file(key) .with_context(|| format!("reading the private key {}", key.display()))?; let signing = provider .key_provider .load_private_key(key) .context("the private key is not one rustls can sign with")?; let pair = CertifiedKey::new(chain, signing); pair.keys_match() .context("the private key does not belong to the certificate")?; Ok(Arc::new(pair)) } /// Accepts exactly the certificates `inner` accepts that also name `name`. /// /// `inner` has already checked the chain, the validity window, the signature /// and the `clientAuth` usage. The name is what makes the gateway's identity /// narrower than "anything the swarm root ever signed for a client". #[derive(Debug)] struct NamedClient { inner: Arc, name: ServerName<'static>, } impl ClientCertVerifier for NamedClient { fn offer_client_auth(&self) -> bool { self.inner.offer_client_auth() } fn client_auth_mandatory(&self) -> bool { self.inner.client_auth_mandatory() } fn root_hint_subjects(&self) -> &[DistinguishedName] { self.inner.root_hint_subjects() } fn verify_client_cert( &self, end_entity: &CertificateDer<'_>, intermediates: &[CertificateDer<'_>], now: UnixTime, ) -> Result { let verified = self .inner .verify_client_cert(end_entity, intermediates, now)?; let cert = webpki::EndEntityCert::try_from(end_entity) .map_err(|_| TlsError::InvalidCertificate(CertificateError::BadEncoding))?; cert.verify_is_valid_for_subject_name(&self.name) .map_err(|_| TlsError::InvalidCertificate(CertificateError::NotValidForName))?; Ok(verified) } fn verify_tls12_signature( &self, message: &[u8], cert: &CertificateDer<'_>, dss: &DigitallySignedStruct, ) -> Result { self.inner.verify_tls12_signature(message, cert, dss) } fn verify_tls13_signature( &self, message: &[u8], cert: &CertificateDer<'_>, dss: &DigitallySignedStruct, ) -> Result { self.inner.verify_tls13_signature(message, cert, dss) } fn supported_verify_schemes(&self) -> Vec { self.inner.supported_verify_schemes() } } fn client_roots(path: &Path) -> Result { let mut roots = RootCertStore::empty(); for cert in CertificateDer::pem_file_iter(path) .with_context(|| format!("reading {}", path.display()))? { let cert = cert.with_context(|| format!("parsing {}", path.display()))?; roots.add(cert).with_context(|| { format!( "{} holds a certificate that is not a usable root", path.display() ) })?; } if roots.is_empty() { bail!( "{} holds no certificate, so no client could ever be accepted", path.display() ); } Ok(roots) } /// The rustls config for `cfg`: TLS only, client certificate mandatory and /// narrowed to the gateway's name, server certificate reloaded from disk. pub fn server_config(cfg: &TcpTlsConfig) -> Result> { let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); let inner = WebPkiClientVerifier::builder_with_provider( Arc::new(client_roots(&cfg.client_ca)?), Arc::clone(&provider), ) .build() .context("building the client certificate verifier")?; let name = ServerName::try_from(cfg.client_name.clone()) .with_context(|| format!("{CLIENT_NAME_ENV} is not a DNS name: {}", cfg.client_name))?; let resolver = ReloadingCert::load(&cfg.cert, &cfg.key, Arc::clone(&provider))?; let mut config = ServerConfig::builder_with_provider(provider) .with_safe_default_protocol_versions() .context("selecting TLS versions")? .with_client_cert_verifier(Arc::new(NamedClient { inner, name })) .with_cert_resolver(Arc::new(resolver)); config.alpn_protocols = vec![b"http/1.1".to_vec()]; Ok(Arc::new(config)) } /// An [`axum::serve::Listener`] yielding only connections that completed the /// handshake, client certificate included. /// /// Handshakes run in their own tasks so a peer that stalls one cannot hold up /// the accept loop. #[derive(Debug)] pub struct TlsListener { rx: mpsc::Receiver<(tokio_rustls::server::TlsStream, SocketAddr)>, local: SocketAddr, } impl TlsListener { pub fn new(tcp: TcpListener, config: Arc) -> Result { let local = tcp.local_addr().context("reading the bound address")?; let (tx, rx) = mpsc::channel(64); let acceptor = tokio_rustls::TlsAcceptor::from(config); tokio::spawn(async move { while !tx.is_closed() { let (stream, peer) = match tcp.accept().await { Ok(accepted) => accepted, Err(e) => { // EMFILE and friends: back off rather than spin. tracing::warn!(error = %e, "tcp listener: accept failed"); tokio::time::sleep(Duration::from_millis(100)).await; continue; } }; let acceptor = acceptor.clone(); let tx = tx.clone(); tokio::spawn(async move { match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await { Ok(Ok(tls)) => { let _ = tx.send((tls, peer)).await; } Ok(Err(e)) => { tracing::warn!(%peer, error = %e, "tcp listener: handshake refused"); } Err(_) => tracing::warn!(%peer, "tcp listener: handshake timed out"), } }); } }); Ok(Self { rx, local }) } } impl axum::serve::Listener for TlsListener { type Io = tokio_rustls::server::TlsStream; type Addr = SocketAddr; async fn accept(&mut self) -> (Self::Io, Self::Addr) { // The sender lives as long as the accept task, which only ends when // this receiver is dropped. self.rx .recv() .await .expect("the accept task outlives the listener") } fn local_addr(&self) -> std::io::Result { Ok(self.local) } } /// Binds `cfg.listen` and returns the listener, ready for `axum::serve`. pub async fn bind(cfg: &TcpTlsConfig) -> Result { let config = server_config(cfg)?; let tcp = TcpListener::bind(cfg.listen) .await .with_context(|| format!("binding {}", cfg.listen))?; tracing::info!(addr = %cfg.listen, client = %cfg.client_name, "swarm-controller listening on tcp"); TlsListener::new(tcp, config) } #[cfg(test)] mod tests { use super::*; use axum::Router; use axum::routing::get; use rcgen::{ BasicConstraints, CertificateParams, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair, KeyUsagePurpose, }; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; const SERVER_NAME: &str = "controller.test"; const GATEWAY_NAME: &str = "gateway-client.test"; struct Ca { issuer: Issuer<'static, KeyPair>, pem: String, } struct Leaf { cert: String, key: String, } fn ca() -> Ca { let mut params = CertificateParams::new(Vec::new()).unwrap(); params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); params.key_usages = vec![KeyUsagePurpose::KeyCertSign]; let key = KeyPair::generate().unwrap(); let pem = params.self_signed(&key).unwrap().pem(); Ca { issuer: Issuer::new(params, key), pem, } } fn leaf(ca: &Ca, name: &str, usage: ExtendedKeyUsagePurpose) -> Leaf { let mut params = CertificateParams::new(vec![name.to_owned()]).unwrap(); params.extended_key_usages = vec![usage]; let key = KeyPair::generate().unwrap(); Leaf { cert: params.signed_by(&key, &ca.issuer).unwrap().pem(), key: key.serialize_pem(), } } /// A listener on `127.0.0.1:0` trusting `ca`, named like the gateway. async fn serve(dir: &Path, ca: &Ca, server: &Leaf) -> SocketAddr { let write = |name: &str, body: &str| { let p = dir.join(name); std::fs::write(&p, body).unwrap(); p }; let cfg = TcpTlsConfig { listen: "127.0.0.1:0".parse().unwrap(), cert: write("cert.pem", &server.cert), key: write("key.pem", &server.key), client_ca: write("ca.pem", &ca.pem), client_name: GATEWAY_NAME.to_owned(), }; let listener = bind(&cfg).await.unwrap(); let addr = axum::serve::Listener::local_addr(&listener).unwrap(); let app = Router::new().route("/", get(|| async { "controller" })); tokio::spawn(async move { axum::serve(listener, app).await }); addr } /// One request with `client` presented, or none. `Err` when the TCP or TLS /// layer refused at any point: with TLS 1.3 the client finishes its half /// of the handshake before the server's verdict arrives, so a refusal is /// seen on the first read, not on connect. async fn get_with(addr: SocketAddr, ca: &Ca, client: Option<&Leaf>) -> std::io::Result { let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); let mut roots = RootCertStore::empty(); roots .add(CertificateDer::from_pem_slice(ca.pem.as_bytes()).unwrap()) .unwrap(); let builder = rustls::ClientConfig::builder_with_provider(provider) .with_safe_default_protocol_versions() .unwrap() .with_root_certificates(roots); let config = match client { None => builder.with_no_client_auth(), Some(l) => builder .with_client_auth_cert( CertificateDer::pem_slice_iter(l.cert.as_bytes()) .collect::>() .unwrap(), PrivateKeyDer::from_pem_slice(l.key.as_bytes()).unwrap(), ) .unwrap(), }; let tcp = TcpStream::connect(addr).await?; let mut tls = tokio_rustls::TlsConnector::from(Arc::new(config)) .connect(ServerName::try_from(SERVER_NAME).unwrap(), tcp) .await?; tls.write_all(b"GET / HTTP/1.1\r\nHost: controller.test\r\nConnection: close\r\n\r\n") .await?; let mut out = String::new(); tls.read_to_string(&mut out).await?; Ok(out) } struct Fixture { _dir: tempfile::TempDir, ca: Ca, addr: SocketAddr, } async fn fixture() -> Fixture { let dir = tempfile::tempdir().unwrap(); let ca = ca(); let server = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth); let addr = serve(dir.path(), &ca, &server).await; Fixture { _dir: dir, ca, addr, } } #[tokio::test] async fn accepts_the_gateway_identity() { let f = fixture().await; let gateway = leaf(&f.ca, GATEWAY_NAME, ExtendedKeyUsagePurpose::ClientAuth); let body = get_with(f.addr, &f.ca, Some(&gateway)).await.unwrap(); assert!(body.starts_with("HTTP/1.1 200"), "{body}"); assert!(body.ends_with("controller"), "{body}"); } #[tokio::test] async fn refuses_a_client_with_no_certificate() { let f = fixture().await; assert!(get_with(f.addr, &f.ca, None).await.is_err()); } #[tokio::test] async fn refuses_a_client_certificate_naming_someone_else() { let f = fixture().await; let other = leaf( &f.ca, "other-client.test", ExtendedKeyUsagePurpose::ClientAuth, ); assert!(get_with(f.addr, &f.ca, Some(&other)).await.is_err()); } #[tokio::test] async fn refuses_a_server_only_leaf_even_under_the_gateway_name() { let f = fixture().await; let services = leaf(&f.ca, GATEWAY_NAME, ExtendedKeyUsagePurpose::ServerAuth); assert!(get_with(f.addr, &f.ca, Some(&services)).await.is_err()); } #[tokio::test] async fn refuses_a_services_style_leaf() { let f = fixture().await; let services = leaf(&f.ca, "swarm.test", ExtendedKeyUsagePurpose::ServerAuth); assert!(get_with(f.addr, &f.ca, Some(&services)).await.is_err()); } #[tokio::test] async fn refuses_a_client_certificate_from_another_root() { let f = fixture().await; let stranger = leaf(&ca(), GATEWAY_NAME, ExtendedKeyUsagePurpose::ClientAuth); assert!(get_with(f.addr, &f.ca, Some(&stranger)).await.is_err()); } #[test] fn reload_picks_up_a_rotated_pair_and_survives_a_broken_one() { let dir = tempfile::tempdir().unwrap(); let ca = ca(); let first = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth); let second = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth); let (cert, key) = (dir.path().join("c.pem"), dir.path().join("k.pem")); let put = |l: &Leaf, secs: u64| { std::fs::write(&cert, &l.cert).unwrap(); std::fs::write(&key, &l.key).unwrap(); let t = SystemTime::UNIX_EPOCH + Duration::from_secs(secs); for p in [&cert, &key] { std::fs::File::options() .write(true) .open(p) .unwrap() .set_modified(t) .unwrap(); } }; put(&first, 1_000_000); let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); let r = ReloadingCert::load(&cert, &key, provider).unwrap(); let before = r.current(); assert!( Arc::ptr_eq(&before, &r.current()), "unchanged files are not re-read" ); put(&second, 2_000_000); let after = r.current(); assert!(!Arc::ptr_eq(&before, &after), "a rotated pair is served"); std::fs::write(&cert, "not a certificate").unwrap(); assert!( Arc::ptr_eq(&after, &r.current()), "a broken pair keeps the last good one" ); } #[test] fn config_is_all_or_nothing() { let env = |pairs: &'static [(&'static str, &'static str)]| { move |n: &str| { pairs .iter() .find(|(k, _)| *k == n) .map(|(_, v)| (*v).to_owned()) } }; assert_eq!(TcpTlsConfig::from_lookup(env(&[])).unwrap(), None); assert!( TcpTlsConfig::from_lookup(env(&[(CERT_ENV, "/c")])).is_err(), "a stray variable without the listen address is not 'off'" ); assert!( TcpTlsConfig::from_lookup(env(&[(LISTEN_ENV, "0.0.0.0:8443")])).is_err(), "a listen address without the TLS material is not a listener" ); let full = TcpTlsConfig::from_lookup(env(&[ (LISTEN_ENV, "0.0.0.0:8443"), (CERT_ENV, "/c"), (KEY_ENV, "/k"), (CLIENT_CA_ENV, "/ca"), (CLIENT_NAME_ENV, GATEWAY_NAME), ])) .unwrap() .unwrap(); assert_eq!(full.listen.port(), 8443); assert_eq!(full.client_name, GATEWAY_NAME); } }