From 17fffaae795d6ed49d279cea64875ceaf837a7a0 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 15 Nov 2026 15:56:59 +0100 Subject: [PATCH] swarm-controller serves tls on tcp to the gateway client identity only, when configured --- Cargo.lock | 193 +++++++++- Cargo.toml | 16 + swarm-controller/Cargo.toml | 7 + swarm-controller/src/main.rs | 55 ++- swarm-controller/src/tcp_tls.rs | 601 ++++++++++++++++++++++++++++++++ 5 files changed, 861 insertions(+), 11 deletions(-) create mode 100644 swarm-controller/src/tcp_tls.rs diff --git a/Cargo.lock b/Cargo.lock index 31b96c45c..0b127158a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,6 +165,45 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dbc3a507a82b17ba0d98f6ce8fd6954ea0c8152e98009d36a40d8dcc8ce078a" +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure 0.13.2", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -310,6 +349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53" dependencies = [ "aws-lc-sys", + "untrusted 0.7.1", "zeroize", ] @@ -410,6 +450,15 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "2.13.2" @@ -1096,6 +1145,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "der_derive" version = "0.7.3" @@ -3385,6 +3448,12 @@ dependencies = [ "serde", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.9.1" @@ -3473,6 +3542,16 @@ dependencies = [ "signatory", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -3482,12 +3561,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3546,6 +3644,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3707,6 +3814,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -4240,6 +4357,20 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "readlock" version = "0.1.11" @@ -4369,7 +4500,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.17", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -4598,6 +4729,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustify" version = "0.7.0" @@ -4719,7 +4859,7 @@ dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -5385,7 +5525,10 @@ dependencies = [ "opentelemetry-otlp", "opentelemetry_sdk", "problem_details", + "rcgen", "reqwest", + "rustls", + "rustls-webpki", "serde", "serde_json", "serde_norway", @@ -5401,6 +5544,7 @@ dependencies = [ "swarm-wire", "tempfile", "tokio", + "tokio-rustls", "tower", "tracing", "tracing-subscriber", @@ -5739,6 +5883,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "synstructure" version = "0.14.0" @@ -6303,6 +6458,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39abd59bf32521c7f2301b52d05a6a2c975b6003521cbd0c6dc1582f0a22104" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -6939,12 +7100,40 @@ dependencies = [ "spki", ] +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "aws-lc-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + [[package]] name = "xxhash-rust" version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec", + "time", +] + [[package]] name = "yoke" version = "0.8.4" diff --git a/Cargo.toml b/Cargo.toml index 3e5059f72..4d3baee5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -164,6 +164,22 @@ tokio = { version = "1", features = [ "time", ] } tokio-stream = { version = "0.1", features = ["sync"] } +# The swarm-controller's TCP listener terminates TLS itself and checks the +# client certificate's name (`swarm-controller/src/tcp_tls.rs`). Both crates +# are already in the tree through reqwest; named here for the handshake and for +# the name check rustls leaves to the application on a client certificate. +tokio-rustls = { version = "0.26", default-features = false, features = [ + "aws_lc_rs", +] } +rustls-webpki = { version = "0.103", default-features = false, features = [ + "std", + "aws-lc-rs", +] } +# Certificates generated in test code, so no key or certificate is checked in. +rcgen = { version = "0.14", default-features = false, features = [ + "aws_lc_rs", + "pem", +] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } # The journald sink `hive-log` installs when the process runs under a diff --git a/swarm-controller/Cargo.toml b/swarm-controller/Cargo.toml index d04c6ba39..59f8501a1 100644 --- a/swarm-controller/Cargo.toml +++ b/swarm-controller/Cargo.toml @@ -149,6 +149,11 @@ async-trait.workspace = true bytes.workspace = true http.workspace = true tokio.workspace = true +# The TCP listener's TLS (`tcp_tls.rs`): the handshake, and the client +# certificate's name check. +tokio-rustls.workspace = true +rustls.workspace = true +rustls-webpki.workspace = true tracing.workspace = true url.workspace = true utoipa.workspace = true @@ -158,6 +163,8 @@ utoipa-axum.workspace = true # tests wait out its five-minute bound on paused time. [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } +# Test CA and leaves, generated per test (`tcp_tls.rs`). +rcgen.workspace = true # A fresh directory per test `nats-server`, for its store and ports file. tempfile = "3" # `InMemoryMetricExporter`, to read back what `review_classifier::metrics` records. diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 6e3a4c87d..11398f4ab 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -1,7 +1,8 @@ //! Swarm-level controller daemon. Runs as the unprivileged //! `swarm-controller` user on whichever host the operator flips //! `services.hyperhive.deploy.swarm-controller.enable` on, and serves HTTP over a -//! unix socket that the hive-gateway's nginx proxies to. +//! unix socket that the hive-gateway's nginx proxies to, and also over TLS on TCP when +//! `SWARM_CONTROLLER_TCP_LISTEN` is set (`tcp_tls`). //! //! Configuration is read-only and loaded once at startup from env vars the //! NixOS module sets (`services.hyperhive.swarm.controller`) — see @@ -93,6 +94,7 @@ mod startup; mod status; mod store; mod subagent_term; +mod tcp_tls; mod term_stream; #[cfg(test)] mod test_nats; @@ -269,7 +271,14 @@ async fn main() -> Result<()> { reviews: Arc::default(), alerts: alerts::from_env(), }; - serve(listener, state, stop, health).await + serve( + listener, + tcp_tls::TcpTlsConfig::from_env()?, + state, + stop, + health, + ) + .await } /// The value a startup step produced, or exit 0 when the stop signal ended the @@ -318,11 +327,14 @@ fn register_cert_gauges(provider: Option<&opentelemetry_sdk::metrics::SdkMeterPr } /// Start taking the queued review requests and answering the notification -/// router's, then serve `state` on `listener` until `stop` resolves. The -/// requests in hand finish for at most the drain; the review queue takes no -/// request after `stop`. `health` answers `/readyz` as the socket's does. +/// router's, then serve `state` on `listener`, and on `tcp` too when it is +/// configured, until `stop` resolves. Both answer from the same router and +/// both drain on `stop`. The requests in hand finish for at most the drain; +/// the review queue takes no request after `stop`. `health` answers `/readyz` +/// as the socket's does. async fn serve( listener: tokio::net::UnixListener, + tcp: Option, state: AppState, stop: impl Future + Clone + Send + 'static, health: swarm_secret_client::health::Health, @@ -331,10 +343,35 @@ async fn serve( health.ready_when(routes::probe_condition(state.status.clone(), store)); review_classifier::review::spawn_queue(&state, stop.clone()); request_reviewer::spawn(&state); - let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone()); - swarm_secret_client::serve_bounded(stop, serving.into_future()) + let app = build_app(state); + let socket = axum::serve(listener, app.clone()).with_graceful_shutdown(stop.clone()); + let bound = stop.clone(); + let serving = async move { + let Some(tcp) = tcp else { + return socket + .into_future() + .await + .context("serving swarm-controller"); + }; + let tcp = tcp_tls::bind(&tcp).await?; + let tcp = axum::serve(tcp, app).with_graceful_shutdown(stop); + tokio::try_join!( + async { + socket + .into_future() + .await + .context("serving swarm-controller on its socket") + }, + async { + tcp.into_future() + .await + .context("serving swarm-controller on tcp") + }, + ) + .map(|_| ()) + }; + swarm_secret_client::serve_bounded(bound, serving) .await - .transpose() - .context("serving swarm-controller")?; + .transpose()?; Ok(()) } diff --git a/swarm-controller/src/tcp_tls.rs b/swarm-controller/src/tcp_tls.rs new file mode 100644 index 000000000..b07827aa0 --- /dev/null +++ b/swarm-controller/src/tcp_tls.rs @@ -0,0 +1,601 @@ +//! The optional TCP listener: TLS with a required client certificate. +//! +//! The controller authenticates nobody per request (its routes trust the +//! transport, and `Remote-User` is a header the gateway sets), so a TCP port +//! is only safe if only the gateway can complete a handshake on it. This +//! listener therefore accepts a connection only when the client presents a +//! certificate that +//! +//! 1. chains to the swarm services root (`SWARM_CONTROLLER_TCP_CLIENT_CA`), +//! 2. carries the `clientAuth` extended key usage, and +//! 3. names the gateway's client identity (`SWARM_CONTROLLER_TCP_CLIENT_NAME`) +//! in a DNS subject alternative name. +//! +//! The swarm-services leaf the gateway serves browsers with chains to the same +//! root but is `serverAuth` only and names other hosts, so it fails the second +//! and third checks. Everything that fails is refused during the handshake, +//! before a byte of HTTP is read. +//! +//! The server certificate and key are read from files and re-read when either +//! changes, so a rotation needs no restart. The client CA is read once at +//! start. + +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, SystemTime}; + +use anyhow::{Context, Result, bail}; +use rustls::crypto::CryptoProvider; +use rustls::pki_types::pem::PemObject as _; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, UnixTime}; +use rustls::server::danger::{ClientCertVerified, ClientCertVerifier}; +use rustls::server::{ClientHello, ResolvesServerCert, WebPkiClientVerifier}; +use rustls::sign::CertifiedKey; +use rustls::{ + CertificateError, DigitallySignedStruct, DistinguishedName, Error as TlsError, RootCertStore, + ServerConfig, SignatureScheme, +}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::mpsc; + +/// Address to bind, `host:port`. Unset means no TCP listener at all. +const LISTEN_ENV: &str = "SWARM_CONTROLLER_TCP_LISTEN"; +/// PEM certificate chain this listener serves. +const CERT_ENV: &str = "SWARM_CONTROLLER_TCP_CERT"; +/// PEM private key for [`CERT_ENV`]. +const KEY_ENV: &str = "SWARM_CONTROLLER_TCP_KEY"; +/// PEM file holding the root client certificates must chain to. +const CLIENT_CA_ENV: &str = "SWARM_CONTROLLER_TCP_CLIENT_CA"; +/// The DNS name the one permitted client certificate carries. +const CLIENT_NAME_ENV: &str = "SWARM_CONTROLLER_TCP_CLIENT_NAME"; + +/// How long a peer gets to finish the handshake. A connection that stalls +/// holds a task and a file descriptor, nothing else. +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); + +/// What the five variables name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TcpTlsConfig { + pub listen: SocketAddr, + pub cert: PathBuf, + pub key: PathBuf, + pub client_ca: PathBuf, + pub client_name: String, +} + +impl TcpTlsConfig { + /// `None` when [`LISTEN_ENV`] is unset. The other four are then not read, + /// but one of them set without it is an error: a half-configured listener + /// must not read as "off". + pub fn from_env() -> Result> { + Self::from_lookup(|name| std::env::var(name).ok().filter(|v| !v.is_empty())) + } + + fn from_lookup(get: impl Fn(&str) -> Option) -> Result> { + let rest = [CERT_ENV, KEY_ENV, CLIENT_CA_ENV, CLIENT_NAME_ENV]; + let Some(listen) = get(LISTEN_ENV) else { + if let Some(set) = rest.iter().find(|n| get(n).is_some()) { + bail!("{set} is set but {LISTEN_ENV} is not; the TCP listener needs all five"); + } + return Ok(None); + }; + let need = |name: &str| { + get(name).with_context(|| format!("{name} is unset, but {LISTEN_ENV} is set")) + }; + Ok(Some(Self { + listen: listen + .parse() + .with_context(|| format!("{LISTEN_ENV} is not a socket address: {listen}"))?, + cert: need(CERT_ENV)?.into(), + key: need(KEY_ENV)?.into(), + client_ca: need(CLIENT_CA_ENV)?.into(), + client_name: need(CLIENT_NAME_ENV)?, + })) + } +} + +/// The server certificate, re-read from disk when its files change. +/// +/// A failed re-read keeps serving the last good pair and logs: a rotation +/// caught half-written must not take the listener down. +#[derive(Debug)] +struct ReloadingCert { + cert: PathBuf, + key: PathBuf, + provider: Arc, + cached: Mutex, +} + +#[derive(Debug)] +struct Cached { + stamp: Stamp, + key: Arc, +} + +/// What identifies one version of the pair on disk. +type Stamp = [Option<(SystemTime, u64)>; 2]; + +impl ReloadingCert { + fn load(cert: &Path, key: &Path, provider: Arc) -> Result { + let stamp = stamp(cert, key); + let loaded = read_pair(cert, key, &provider)?; + Ok(Self { + cert: cert.to_path_buf(), + key: key.to_path_buf(), + provider, + cached: Mutex::new(Cached { stamp, key: loaded }), + }) + } + + fn current(&self) -> Arc { + let mut cached = self.cached.lock().unwrap_or_else(PoisonError::into_inner); + let now = stamp(&self.cert, &self.key); + if now != cached.stamp { + match read_pair(&self.cert, &self.key, &self.provider) { + Ok(key) => { + tracing::info!(cert = %self.cert.display(), "tcp listener: server certificate reloaded"); + *cached = Cached { stamp: now, key }; + } + Err(e) => tracing::warn!( + error = %format!("{e:#}"), + "tcp listener: server certificate changed on disk but does not load; keeping the previous one" + ), + } + } + Arc::clone(&cached.key) + } +} + +impl ResolvesServerCert for ReloadingCert { + fn resolve(&self, _hello: ClientHello<'_>) -> Option> { + Some(self.current()) + } +} + +fn stamp(cert: &Path, key: &Path) -> Stamp { + let one = |p: &Path| { + let m = std::fs::metadata(p).ok()?; + Some((m.modified().ok()?, m.len())) + }; + [one(cert), one(key)] +} + +fn read_pair(cert: &Path, key: &Path, provider: &CryptoProvider) -> Result> { + let chain = CertificateDer::pem_file_iter(cert) + .with_context(|| format!("reading {}", cert.display()))? + .collect::, _>>() + .with_context(|| format!("parsing {}", cert.display()))?; + if chain.is_empty() { + bail!("{} holds no certificate", cert.display()); + } + let key = PrivateKeyDer::from_pem_file(key) + .with_context(|| format!("reading the private key {}", key.display()))?; + let signing = provider + .key_provider + .load_private_key(key) + .context("the private key is not one rustls can sign with")?; + Ok(Arc::new(CertifiedKey::new(chain, signing))) +} + +/// Accepts exactly the certificates `inner` accepts that also name `name`. +/// +/// `inner` has already checked the chain, the validity window, the signature +/// and the `clientAuth` usage. The name is what makes the gateway's identity +/// narrower than "anything the swarm root ever signed for a client". +#[derive(Debug)] +struct NamedClient { + inner: Arc, + name: ServerName<'static>, +} + +impl ClientCertVerifier for NamedClient { + fn offer_client_auth(&self) -> bool { + self.inner.offer_client_auth() + } + + fn client_auth_mandatory(&self) -> bool { + self.inner.client_auth_mandatory() + } + + fn root_hint_subjects(&self) -> &[DistinguishedName] { + self.inner.root_hint_subjects() + } + + fn verify_client_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + now: UnixTime, + ) -> Result { + let verified = self + .inner + .verify_client_cert(end_entity, intermediates, now)?; + let cert = webpki::EndEntityCert::try_from(end_entity) + .map_err(|_| TlsError::InvalidCertificate(CertificateError::BadEncoding))?; + cert.verify_is_valid_for_subject_name(&self.name) + .map_err(|_| TlsError::InvalidCertificate(CertificateError::NotValidForName))?; + Ok(verified) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + self.inner.verify_tls12_signature(message, cert, dss) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + self.inner.verify_tls13_signature(message, cert, dss) + } + + fn supported_verify_schemes(&self) -> Vec { + self.inner.supported_verify_schemes() + } +} + +fn client_roots(path: &Path) -> Result { + let mut roots = RootCertStore::empty(); + for cert in CertificateDer::pem_file_iter(path) + .with_context(|| format!("reading {}", path.display()))? + { + let cert = cert.with_context(|| format!("parsing {}", path.display()))?; + roots.add(cert).with_context(|| { + format!( + "{} holds a certificate that is not a usable root", + path.display() + ) + })?; + } + if roots.is_empty() { + bail!( + "{} holds no certificate, so no client could ever be accepted", + path.display() + ); + } + Ok(roots) +} + +/// The rustls config for `cfg`: TLS only, client certificate mandatory and +/// narrowed to the gateway's name, server certificate reloaded from disk. +pub fn server_config(cfg: &TcpTlsConfig) -> Result> { + let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let inner = WebPkiClientVerifier::builder_with_provider( + Arc::new(client_roots(&cfg.client_ca)?), + Arc::clone(&provider), + ) + .build() + .context("building the client certificate verifier")?; + let name = ServerName::try_from(cfg.client_name.clone()) + .with_context(|| format!("{CLIENT_NAME_ENV} is not a DNS name: {}", cfg.client_name))?; + let resolver = ReloadingCert::load(&cfg.cert, &cfg.key, Arc::clone(&provider))?; + let mut config = ServerConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .context("selecting TLS versions")? + .with_client_cert_verifier(Arc::new(NamedClient { inner, name })) + .with_cert_resolver(Arc::new(resolver)); + config.alpn_protocols = vec![b"http/1.1".to_vec()]; + Ok(Arc::new(config)) +} + +/// An [`axum::serve::Listener`] yielding only connections that completed the +/// handshake, client certificate included. +/// +/// Handshakes run in their own tasks so a peer that stalls one cannot hold up +/// the accept loop. +#[derive(Debug)] +pub struct TlsListener { + rx: mpsc::Receiver<(tokio_rustls::server::TlsStream, SocketAddr)>, + local: SocketAddr, +} + +impl TlsListener { + pub fn new(tcp: TcpListener, config: Arc) -> Result { + let local = tcp.local_addr().context("reading the bound address")?; + let (tx, rx) = mpsc::channel(64); + let acceptor = tokio_rustls::TlsAcceptor::from(config); + tokio::spawn(async move { + while !tx.is_closed() { + let (stream, peer) = match tcp.accept().await { + Ok(accepted) => accepted, + Err(e) => { + // EMFILE and friends: back off rather than spin. + tracing::warn!(error = %e, "tcp listener: accept failed"); + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + }; + let acceptor = acceptor.clone(); + let tx = tx.clone(); + tokio::spawn(async move { + match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await { + Ok(Ok(tls)) => { + let _ = tx.send((tls, peer)).await; + } + Ok(Err(e)) => { + tracing::warn!(%peer, error = %e, "tcp listener: handshake refused"); + } + Err(_) => tracing::warn!(%peer, "tcp listener: handshake timed out"), + } + }); + } + }); + Ok(Self { rx, local }) + } +} + +impl axum::serve::Listener for TlsListener { + type Io = tokio_rustls::server::TlsStream; + type Addr = SocketAddr; + + async fn accept(&mut self) -> (Self::Io, Self::Addr) { + // The sender lives as long as the accept task, which only ends when + // this receiver is dropped. + self.rx + .recv() + .await + .expect("the accept task outlives the listener") + } + + fn local_addr(&self) -> std::io::Result { + Ok(self.local) + } +} + +/// Binds `cfg.listen` and returns the listener, ready for `axum::serve`. +pub async fn bind(cfg: &TcpTlsConfig) -> Result { + let config = server_config(cfg)?; + let tcp = TcpListener::bind(cfg.listen) + .await + .with_context(|| format!("binding {}", cfg.listen))?; + tracing::info!(addr = %cfg.listen, client = %cfg.client_name, "swarm-controller listening on tcp"); + TlsListener::new(tcp, config) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::Router; + use axum::routing::get; + use rcgen::{ + BasicConstraints, CertificateParams, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair, + KeyUsagePurpose, + }; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + const SERVER_NAME: &str = "controller.test"; + const GATEWAY_NAME: &str = "gateway-client.test"; + + struct Ca { + issuer: Issuer<'static, KeyPair>, + pem: String, + } + + struct Leaf { + cert: String, + key: String, + } + + fn ca() -> Ca { + let mut params = CertificateParams::new(Vec::new()).unwrap(); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages = vec![KeyUsagePurpose::KeyCertSign]; + let key = KeyPair::generate().unwrap(); + let pem = params.self_signed(&key).unwrap().pem(); + Ca { + issuer: Issuer::new(params, key), + pem, + } + } + + fn leaf(ca: &Ca, name: &str, usage: ExtendedKeyUsagePurpose) -> Leaf { + let mut params = CertificateParams::new(vec![name.to_owned()]).unwrap(); + params.extended_key_usages = vec![usage]; + let key = KeyPair::generate().unwrap(); + Leaf { + cert: params.signed_by(&key, &ca.issuer).unwrap().pem(), + key: key.serialize_pem(), + } + } + + /// A listener on `127.0.0.1:0` trusting `ca`, named like the gateway. + async fn serve(dir: &Path, ca: &Ca, server: &Leaf) -> SocketAddr { + let write = |name: &str, body: &str| { + let p = dir.join(name); + std::fs::write(&p, body).unwrap(); + p + }; + let cfg = TcpTlsConfig { + listen: "127.0.0.1:0".parse().unwrap(), + cert: write("cert.pem", &server.cert), + key: write("key.pem", &server.key), + client_ca: write("ca.pem", &ca.pem), + client_name: GATEWAY_NAME.to_owned(), + }; + let listener = bind(&cfg).await.unwrap(); + let addr = axum::serve::Listener::local_addr(&listener).unwrap(); + let app = Router::new().route("/", get(|| async { "controller" })); + tokio::spawn(async move { axum::serve(listener, app).await }); + addr + } + + /// One request with `client` presented, or none. `Err` when the TCP or TLS + /// layer refused at any point: with TLS 1.3 the client finishes its half + /// of the handshake before the server's verdict arrives, so a refusal is + /// seen on the first read, not on connect. + async fn get_with(addr: SocketAddr, ca: &Ca, client: Option<&Leaf>) -> std::io::Result { + let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let mut roots = RootCertStore::empty(); + roots + .add(CertificateDer::from_pem_slice(ca.pem.as_bytes()).unwrap()) + .unwrap(); + let builder = rustls::ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(roots); + let config = match client { + None => builder.with_no_client_auth(), + Some(l) => builder + .with_client_auth_cert( + CertificateDer::pem_slice_iter(l.cert.as_bytes()) + .collect::>() + .unwrap(), + PrivateKeyDer::from_pem_slice(l.key.as_bytes()).unwrap(), + ) + .unwrap(), + }; + let tcp = TcpStream::connect(addr).await?; + let mut tls = tokio_rustls::TlsConnector::from(Arc::new(config)) + .connect(ServerName::try_from(SERVER_NAME).unwrap(), tcp) + .await?; + tls.write_all(b"GET / HTTP/1.1\r\nHost: controller.test\r\nConnection: close\r\n\r\n") + .await?; + let mut out = String::new(); + tls.read_to_string(&mut out).await?; + Ok(out) + } + + struct Fixture { + _dir: tempfile::TempDir, + ca: Ca, + addr: SocketAddr, + } + + async fn fixture() -> Fixture { + let dir = tempfile::tempdir().unwrap(); + let ca = ca(); + let server = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth); + let addr = serve(dir.path(), &ca, &server).await; + Fixture { + _dir: dir, + ca, + addr, + } + } + + #[tokio::test] + async fn accepts_the_gateway_identity() { + let f = fixture().await; + let gateway = leaf(&f.ca, GATEWAY_NAME, ExtendedKeyUsagePurpose::ClientAuth); + let body = get_with(f.addr, &f.ca, Some(&gateway)).await.unwrap(); + assert!(body.starts_with("HTTP/1.1 200"), "{body}"); + assert!(body.ends_with("controller"), "{body}"); + } + + #[tokio::test] + async fn refuses_a_client_with_no_certificate() { + let f = fixture().await; + assert!(get_with(f.addr, &f.ca, None).await.is_err()); + } + + #[tokio::test] + async fn refuses_a_client_certificate_naming_someone_else() { + let f = fixture().await; + let other = leaf( + &f.ca, + "other-client.test", + ExtendedKeyUsagePurpose::ClientAuth, + ); + assert!(get_with(f.addr, &f.ca, Some(&other)).await.is_err()); + } + + #[tokio::test] + async fn refuses_a_server_only_leaf_even_under_the_gateway_name() { + let f = fixture().await; + let services = leaf(&f.ca, GATEWAY_NAME, ExtendedKeyUsagePurpose::ServerAuth); + assert!(get_with(f.addr, &f.ca, Some(&services)).await.is_err()); + } + + #[tokio::test] + async fn refuses_a_services_style_leaf() { + let f = fixture().await; + let services = leaf(&f.ca, "swarm.test", ExtendedKeyUsagePurpose::ServerAuth); + assert!(get_with(f.addr, &f.ca, Some(&services)).await.is_err()); + } + + #[tokio::test] + async fn refuses_a_client_certificate_from_another_root() { + let f = fixture().await; + let stranger = leaf(&ca(), GATEWAY_NAME, ExtendedKeyUsagePurpose::ClientAuth); + assert!(get_with(f.addr, &f.ca, Some(&stranger)).await.is_err()); + } + + #[test] + fn reload_picks_up_a_rotated_pair_and_survives_a_broken_one() { + let dir = tempfile::tempdir().unwrap(); + let ca = ca(); + let first = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth); + let second = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth); + let (cert, key) = (dir.path().join("c.pem"), dir.path().join("k.pem")); + let put = |l: &Leaf, secs: u64| { + std::fs::write(&cert, &l.cert).unwrap(); + std::fs::write(&key, &l.key).unwrap(); + let t = SystemTime::UNIX_EPOCH + Duration::from_secs(secs); + for p in [&cert, &key] { + std::fs::File::options() + .write(true) + .open(p) + .unwrap() + .set_modified(t) + .unwrap(); + } + }; + put(&first, 1_000_000); + let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let r = ReloadingCert::load(&cert, &key, provider).unwrap(); + let before = r.current(); + assert!( + Arc::ptr_eq(&before, &r.current()), + "unchanged files are not re-read" + ); + + put(&second, 2_000_000); + let after = r.current(); + assert!(!Arc::ptr_eq(&before, &after), "a rotated pair is served"); + + std::fs::write(&cert, "not a certificate").unwrap(); + assert!( + Arc::ptr_eq(&after, &r.current()), + "a broken pair keeps the last good one" + ); + } + + #[test] + fn config_is_all_or_nothing() { + let env = |pairs: &'static [(&'static str, &'static str)]| { + move |n: &str| { + pairs + .iter() + .find(|(k, _)| *k == n) + .map(|(_, v)| (*v).to_owned()) + } + }; + assert_eq!(TcpTlsConfig::from_lookup(env(&[])).unwrap(), None); + assert!( + TcpTlsConfig::from_lookup(env(&[(CERT_ENV, "/c")])).is_err(), + "a stray variable without the listen address is not 'off'" + ); + assert!( + TcpTlsConfig::from_lookup(env(&[(LISTEN_ENV, "0.0.0.0:8443")])).is_err(), + "a listen address without the TLS material is not a listener" + ); + let full = TcpTlsConfig::from_lookup(env(&[ + (LISTEN_ENV, "0.0.0.0:8443"), + (CERT_ENV, "/c"), + (KEY_ENV, "/k"), + (CLIENT_CA_ENV, "/ca"), + (CLIENT_NAME_ENV, GATEWAY_NAME), + ])) + .unwrap() + .unwrap(); + assert_eq!(full.listen.port(), 8443); + assert_eq!(full.client_name, GATEWAY_NAME); + } +}