Watch
0
0
Fork
You've already forked hyperhive
0

swarm-controller serves tls on tcp to the gateway client identity only, when configured

This commit is contained in:
atlas 2026-11-15 15:56:59 +01:00 • committed by mara
commit 17fffaae79
5 changed files with 861 additions and 11 deletions

193
Cargo.lock generated
View file

@ -165,6 +165,45 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dbc3a507a82b17ba0d98f6ce8fd6954ea0c8152e98009d36a40d8dcc8ce078a"
[[package]]
name = "asn1-rs"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8"
dependencies = [
"asn1-rs-derive",
"asn1-rs-impl",
"displaydoc",
"nom",
"num-traits",
"rusticata-macros",
"thiserror 2.0.21",
"time",
]
[[package]]
name = "asn1-rs-derive"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"synstructure 0.13.2",
]
[[package]]
name = "asn1-rs-impl"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "assert-json-diff"
version = "2.0.2"
@ -310,6 +349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53"
dependencies = [
"aws-lc-sys",
"untrusted 0.7.1",
"zeroize",
]
@ -410,6 +450,15 @@ version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]]
name = "bit-vec"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51"
dependencies = [
"serde",
]
[[package]]
name = "bitflags"
version = "2.13.2"
@ -1096,6 +1145,20 @@ dependencies = [
"zeroize",
]
[[package]]
name = "der-parser"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6"
dependencies = [
"asn1-rs",
"displaydoc",
"nom",
"num-bigint",
"num-traits",
"rusticata-macros",
]
[[package]]
name = "der_derive"
version = "0.7.3"
@ -3385,6 +3448,12 @@ dependencies = [
"serde",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.9.1"
@ -3473,6 +3542,16 @@ dependencies = [
"signatory",
]
[[package]]
name = "nom"
version = "7.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
dependencies = [
"memchr",
"minimal-lexical",
]
[[package]]
name = "nu-ansi-term"
version = "0.50.3"
@ -3482,12 +3561,31 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "num-bigint"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367"
dependencies = [
"num-integer",
"num-traits",
]
[[package]]
name = "num-conv"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
[[package]]
name = "num-integer"
version = "0.1.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
dependencies = [
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@ -3546,6 +3644,15 @@ dependencies = [
"memchr",
]
[[package]]
name = "oid-registry"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7"
dependencies = [
"asn1-rs",
]
[[package]]
name = "once_cell"
version = "1.21.4"
@ -3707,6 +3814,16 @@ dependencies = [
"digest 0.10.7",
]
[[package]]
name = "pem"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120"
dependencies = [
"base64 0.23.1",
"serde_core",
]
[[package]]
name = "pem-rfc7468"
version = "0.7.0"
@ -4240,6 +4357,20 @@ dependencies = [
"rand_core 0.9.5",
]
[[package]]
name = "rcgen"
version = "0.14.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8"
dependencies = [
"aws-lc-rs",
"pem",
"rustls-pki-types",
"time",
"x509-parser",
"yasna",
]
[[package]]
name = "readlock"
version = "0.1.11"
@ -4369,7 +4500,7 @@ dependencies = [
"cfg-if",
"getrandom 0.2.17",
"libc",
"untrusted",
"untrusted 0.9.0",
"windows-sys 0.52.0",
]
@ -4598,6 +4729,15 @@ dependencies = [
"semver",
]
[[package]]
name = "rusticata-macros"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"
dependencies = [
"nom",
]
[[package]]
name = "rustify"
version = "0.7.0"
@ -4719,7 +4859,7 @@ dependencies = [
"aws-lc-rs",
"ring",
"rustls-pki-types",
"untrusted",
"untrusted 0.9.0",
]
[[package]]
@ -5385,7 +5525,10 @@ dependencies = [
"opentelemetry-otlp",
"opentelemetry_sdk",
"problem_details",
"rcgen",
"reqwest",
"rustls",
"rustls-webpki",
"serde",
"serde_json",
"serde_norway",
@ -5401,6 +5544,7 @@ dependencies = [
"swarm-wire",
"tempfile",
"tokio",
"tokio-rustls",
"tower",
"tracing",
"tracing-subscriber",
@ -5739,6 +5883,17 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "synstructure"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "synstructure"
version = "0.14.0"
@ -6303,6 +6458,12 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39abd59bf32521c7f2301b52d05a6a2c975b6003521cbd0c6dc1582f0a22104"
[[package]]
name = "untrusted"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "untrusted"
version = "0.9.0"
@ -6939,12 +7100,40 @@ dependencies = [
"spki",
]
[[package]]
name = "x509-parser"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202"
dependencies = [
"asn1-rs",
"aws-lc-rs",
"data-encoding",
"der-parser",
"lazy_static",
"nom",
"oid-registry",
"rusticata-macros",
"thiserror 2.0.21",
"time",
]
[[package]]
name = "xxhash-rust"
version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7"
[[package]]
name = "yasna"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282"
dependencies = [
"bit-vec",
"time",
]
[[package]]
name = "yoke"
version = "0.8.4"

View file

@ -164,6 +164,22 @@ tokio = { version = "1", features = [
"time",
] }
tokio-stream = { version = "0.1", features = ["sync"] }
# The swarm-controller's TCP listener terminates TLS itself and checks the
# client certificate's name (`swarm-controller/src/tcp_tls.rs`). Both crates
# are already in the tree through reqwest; named here for the handshake and for
# the name check rustls leaves to the application on a client certificate.
tokio-rustls = { version = "0.26", default-features = false, features = [
"aws_lc_rs",
] }
rustls-webpki = { version = "0.103", default-features = false, features = [
"std",
"aws-lc-rs",
] }
# Certificates generated in test code, so no key or certificate is checked in.
rcgen = { version = "0.14", default-features = false, features = [
"aws_lc_rs",
"pem",
] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
# The journald sink `hive-log` installs when the process runs under a

View file

@ -149,6 +149,11 @@ async-trait.workspace = true
bytes.workspace = true
http.workspace = true
tokio.workspace = true
# The TCP listener's TLS (`tcp_tls.rs`): the handshake, and the client
# certificate's name check.
tokio-rustls.workspace = true
rustls.workspace = true
rustls-webpki.workspace = true
tracing.workspace = true
url.workspace = true
utoipa.workspace = true
@ -158,6 +163,8 @@ utoipa-axum.workspace = true
# tests wait out its five-minute bound on paused time.
[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
# Test CA and leaves, generated per test (`tcp_tls.rs`).
rcgen.workspace = true
# A fresh directory per test `nats-server`, for its store and ports file.
tempfile = "3"
# `InMemoryMetricExporter`, to read back what `review_classifier::metrics` records.

View file

@ -1,7 +1,8 @@
//! Swarm-level controller daemon. Runs as the unprivileged
//! `swarm-controller` user on whichever host the operator flips
//! `services.hyperhive.deploy.swarm-controller.enable` on, and serves HTTP over a
//! unix socket that the hive-gateway's nginx proxies to.
//! unix socket that the hive-gateway's nginx proxies to, and also over TLS on TCP when
//! `SWARM_CONTROLLER_TCP_LISTEN` is set (`tcp_tls`).
//!
//! Configuration is read-only and loaded once at startup from env vars the
//! NixOS module sets (`services.hyperhive.swarm.controller`) — see
@ -93,6 +94,7 @@ mod startup;
mod status;
mod store;
mod subagent_term;
mod tcp_tls;
mod term_stream;
#[cfg(test)]
mod test_nats;
@ -269,7 +271,14 @@ async fn main() -> Result<()> {
reviews: Arc::default(),
alerts: alerts::from_env(),
};
serve(listener, state, stop, health).await
serve(
listener,
tcp_tls::TcpTlsConfig::from_env()?,
state,
stop,
health,
)
.await
}
/// The value a startup step produced, or exit 0 when the stop signal ended the
@ -318,11 +327,14 @@ fn register_cert_gauges(provider: Option<&opentelemetry_sdk::metrics::SdkMeterPr
}
/// Start taking the queued review requests and answering the notification
/// router's, then serve `state` on `listener` until `stop` resolves. The
/// requests in hand finish for at most the drain; the review queue takes no
/// request after `stop`. `health` answers `/readyz` as the socket's does.
/// router's, then serve `state` on `listener`, and on `tcp` too when it is
/// configured, until `stop` resolves. Both answer from the same router and
/// both drain on `stop`. The requests in hand finish for at most the drain;
/// the review queue takes no request after `stop`. `health` answers `/readyz`
/// as the socket's does.
async fn serve(
listener: tokio::net::UnixListener,
tcp: Option<tcp_tls::TcpTlsConfig>,
state: AppState,
stop: impl Future<Output = ()> + Clone + Send + 'static,
health: swarm_secret_client::health::Health,
@ -331,10 +343,35 @@ async fn serve(
health.ready_when(routes::probe_condition(state.status.clone(), store));
review_classifier::review::spawn_queue(&state, stop.clone());
request_reviewer::spawn(&state);
let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone());
swarm_secret_client::serve_bounded(stop, serving.into_future())
let app = build_app(state);
let socket = axum::serve(listener, app.clone()).with_graceful_shutdown(stop.clone());
let bound = stop.clone();
let serving = async move {
let Some(tcp) = tcp else {
return socket
.into_future()
.await
.context("serving swarm-controller");
};
let tcp = tcp_tls::bind(&tcp).await?;
let tcp = axum::serve(tcp, app).with_graceful_shutdown(stop);
tokio::try_join!(
async {
socket
.into_future()
.await
.context("serving swarm-controller on its socket")
},
async {
tcp.into_future()
.await
.context("serving swarm-controller on tcp")
},
)
.map(|_| ())
};
swarm_secret_client::serve_bounded(bound, serving)
.await
.transpose()
.context("serving swarm-controller")?;
.transpose()?;
Ok(())
}

View file

@ -0,0 +1,601 @@
//! The optional TCP listener: TLS with a required client certificate.
//!
//! The controller authenticates nobody per request (its routes trust the
//! transport, and `Remote-User` is a header the gateway sets), so a TCP port
//! is only safe if only the gateway can complete a handshake on it. This
//! listener therefore accepts a connection only when the client presents a
//! certificate that
//!
//! 1. chains to the swarm services root (`SWARM_CONTROLLER_TCP_CLIENT_CA`),
//! 2. carries the `clientAuth` extended key usage, and
//! 3. names the gateway's client identity (`SWARM_CONTROLLER_TCP_CLIENT_NAME`)
//! in a DNS subject alternative name.
//!
//! The swarm-services leaf the gateway serves browsers with chains to the same
//! root but is `serverAuth` only and names other hosts, so it fails the second
//! and third checks. Everything that fails is refused during the handshake,
//! before a byte of HTTP is read.
//!
//! The server certificate and key are read from files and re-read when either
//! changes, so a rotation needs no restart. The client CA is read once at
//! start.
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::{Duration, SystemTime};
use anyhow::{Context, Result, bail};
use rustls::crypto::CryptoProvider;
use rustls::pki_types::pem::PemObject as _;
use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, UnixTime};
use rustls::server::danger::{ClientCertVerified, ClientCertVerifier};
use rustls::server::{ClientHello, ResolvesServerCert, WebPkiClientVerifier};
use rustls::sign::CertifiedKey;
use rustls::{
CertificateError, DigitallySignedStruct, DistinguishedName, Error as TlsError, RootCertStore,
ServerConfig, SignatureScheme,
};
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::mpsc;
/// Address to bind, `host:port`. Unset means no TCP listener at all.
const LISTEN_ENV: &str = "SWARM_CONTROLLER_TCP_LISTEN";
/// PEM certificate chain this listener serves.
const CERT_ENV: &str = "SWARM_CONTROLLER_TCP_CERT";
/// PEM private key for [`CERT_ENV`].
const KEY_ENV: &str = "SWARM_CONTROLLER_TCP_KEY";
/// PEM file holding the root client certificates must chain to.
const CLIENT_CA_ENV: &str = "SWARM_CONTROLLER_TCP_CLIENT_CA";
/// The DNS name the one permitted client certificate carries.
const CLIENT_NAME_ENV: &str = "SWARM_CONTROLLER_TCP_CLIENT_NAME";
/// How long a peer gets to finish the handshake. A connection that stalls
/// holds a task and a file descriptor, nothing else.
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
/// What the five variables name.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TcpTlsConfig {
pub listen: SocketAddr,
pub cert: PathBuf,
pub key: PathBuf,
pub client_ca: PathBuf,
pub client_name: String,
}
impl TcpTlsConfig {
/// `None` when [`LISTEN_ENV`] is unset. The other four are then not read,
/// but one of them set without it is an error: a half-configured listener
/// must not read as "off".
pub fn from_env() -> Result<Option<Self>> {
Self::from_lookup(|name| std::env::var(name).ok().filter(|v| !v.is_empty()))
}
fn from_lookup(get: impl Fn(&str) -> Option<String>) -> Result<Option<Self>> {
let rest = [CERT_ENV, KEY_ENV, CLIENT_CA_ENV, CLIENT_NAME_ENV];
let Some(listen) = get(LISTEN_ENV) else {
if let Some(set) = rest.iter().find(|n| get(n).is_some()) {
bail!("{set} is set but {LISTEN_ENV} is not; the TCP listener needs all five");
}
return Ok(None);
};
let need = |name: &str| {
get(name).with_context(|| format!("{name} is unset, but {LISTEN_ENV} is set"))
};
Ok(Some(Self {
listen: listen
.parse()
.with_context(|| format!("{LISTEN_ENV} is not a socket address: {listen}"))?,
cert: need(CERT_ENV)?.into(),
key: need(KEY_ENV)?.into(),
client_ca: need(CLIENT_CA_ENV)?.into(),
client_name: need(CLIENT_NAME_ENV)?,
}))
}
}
/// The server certificate, re-read from disk when its files change.
///
/// A failed re-read keeps serving the last good pair and logs: a rotation
/// caught half-written must not take the listener down.
#[derive(Debug)]
struct ReloadingCert {
cert: PathBuf,
key: PathBuf,
provider: Arc<CryptoProvider>,
cached: Mutex<Cached>,
}
#[derive(Debug)]
struct Cached {
stamp: Stamp,
key: Arc<CertifiedKey>,
}
/// What identifies one version of the pair on disk.
type Stamp = [Option<(SystemTime, u64)>; 2];
impl ReloadingCert {
fn load(cert: &Path, key: &Path, provider: Arc<CryptoProvider>) -> Result<Self> {
let stamp = stamp(cert, key);
let loaded = read_pair(cert, key, &provider)?;
Ok(Self {
cert: cert.to_path_buf(),
key: key.to_path_buf(),
provider,
cached: Mutex::new(Cached { stamp, key: loaded }),
})
}
fn current(&self) -> Arc<CertifiedKey> {
let mut cached = self.cached.lock().unwrap_or_else(PoisonError::into_inner);
let now = stamp(&self.cert, &self.key);
if now != cached.stamp {
match read_pair(&self.cert, &self.key, &self.provider) {
Ok(key) => {
tracing::info!(cert = %self.cert.display(), "tcp listener: server certificate reloaded");
*cached = Cached { stamp: now, key };
}
Err(e) => tracing::warn!(
error = %format!("{e:#}"),
"tcp listener: server certificate changed on disk but does not load; keeping the previous one"
),
}
}
Arc::clone(&cached.key)
}
}
impl ResolvesServerCert for ReloadingCert {
fn resolve(&self, _hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
Some(self.current())
}
}
fn stamp(cert: &Path, key: &Path) -> Stamp {
let one = |p: &Path| {
let m = std::fs::metadata(p).ok()?;
Some((m.modified().ok()?, m.len()))
};
[one(cert), one(key)]
}
fn read_pair(cert: &Path, key: &Path, provider: &CryptoProvider) -> Result<Arc<CertifiedKey>> {
let chain = CertificateDer::pem_file_iter(cert)
.with_context(|| format!("reading {}", cert.display()))?
.collect::<Result<Vec<_>, _>>()
.with_context(|| format!("parsing {}", cert.display()))?;
if chain.is_empty() {
bail!("{} holds no certificate", cert.display());
}
let key = PrivateKeyDer::from_pem_file(key)
.with_context(|| format!("reading the private key {}", key.display()))?;
let signing = provider
.key_provider
.load_private_key(key)
.context("the private key is not one rustls can sign with")?;
Ok(Arc::new(CertifiedKey::new(chain, signing)))
}
/// Accepts exactly the certificates `inner` accepts that also name `name`.
///
/// `inner` has already checked the chain, the validity window, the signature
/// and the `clientAuth` usage. The name is what makes the gateway's identity
/// narrower than "anything the swarm root ever signed for a client".
#[derive(Debug)]
struct NamedClient {
inner: Arc<dyn ClientCertVerifier>,
name: ServerName<'static>,
}
impl ClientCertVerifier for NamedClient {
fn offer_client_auth(&self) -> bool {
self.inner.offer_client_auth()
}
fn client_auth_mandatory(&self) -> bool {
self.inner.client_auth_mandatory()
}
fn root_hint_subjects(&self) -> &[DistinguishedName] {
self.inner.root_hint_subjects()
}
fn verify_client_cert(
&self,
end_entity: &CertificateDer<'_>,
intermediates: &[CertificateDer<'_>],
now: UnixTime,
) -> Result<ClientCertVerified, TlsError> {
let verified = self
.inner
.verify_client_cert(end_entity, intermediates, now)?;
let cert = webpki::EndEntityCert::try_from(end_entity)
.map_err(|_| TlsError::InvalidCertificate(CertificateError::BadEncoding))?;
cert.verify_is_valid_for_subject_name(&self.name)
.map_err(|_| TlsError::InvalidCertificate(CertificateError::NotValidForName))?;
Ok(verified)
}
fn verify_tls12_signature(
&self,
message: &[u8],
cert: &CertificateDer<'_>,
dss: &DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, TlsError> {
self.inner.verify_tls12_signature(message, cert, dss)
}
fn verify_tls13_signature(
&self,
message: &[u8],
cert: &CertificateDer<'_>,
dss: &DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, TlsError> {
self.inner.verify_tls13_signature(message, cert, dss)
}
fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
self.inner.supported_verify_schemes()
}
}
fn client_roots(path: &Path) -> Result<RootCertStore> {
let mut roots = RootCertStore::empty();
for cert in CertificateDer::pem_file_iter(path)
.with_context(|| format!("reading {}", path.display()))?
{
let cert = cert.with_context(|| format!("parsing {}", path.display()))?;
roots.add(cert).with_context(|| {
format!(
"{} holds a certificate that is not a usable root",
path.display()
)
})?;
}
if roots.is_empty() {
bail!(
"{} holds no certificate, so no client could ever be accepted",
path.display()
);
}
Ok(roots)
}
/// The rustls config for `cfg`: TLS only, client certificate mandatory and
/// narrowed to the gateway's name, server certificate reloaded from disk.
pub fn server_config(cfg: &TcpTlsConfig) -> Result<Arc<ServerConfig>> {
let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider());
let inner = WebPkiClientVerifier::builder_with_provider(
Arc::new(client_roots(&cfg.client_ca)?),
Arc::clone(&provider),
)
.build()
.context("building the client certificate verifier")?;
let name = ServerName::try_from(cfg.client_name.clone())
.with_context(|| format!("{CLIENT_NAME_ENV} is not a DNS name: {}", cfg.client_name))?;
let resolver = ReloadingCert::load(&cfg.cert, &cfg.key, Arc::clone(&provider))?;
let mut config = ServerConfig::builder_with_provider(provider)
.with_safe_default_protocol_versions()
.context("selecting TLS versions")?
.with_client_cert_verifier(Arc::new(NamedClient { inner, name }))
.with_cert_resolver(Arc::new(resolver));
config.alpn_protocols = vec![b"http/1.1".to_vec()];
Ok(Arc::new(config))
}
/// An [`axum::serve::Listener`] yielding only connections that completed the
/// handshake, client certificate included.
///
/// Handshakes run in their own tasks so a peer that stalls one cannot hold up
/// the accept loop.
#[derive(Debug)]
pub struct TlsListener {
rx: mpsc::Receiver<(tokio_rustls::server::TlsStream<TcpStream>, SocketAddr)>,
local: SocketAddr,
}
impl TlsListener {
pub fn new(tcp: TcpListener, config: Arc<ServerConfig>) -> Result<Self> {
let local = tcp.local_addr().context("reading the bound address")?;
let (tx, rx) = mpsc::channel(64);
let acceptor = tokio_rustls::TlsAcceptor::from(config);
tokio::spawn(async move {
while !tx.is_closed() {
let (stream, peer) = match tcp.accept().await {
Ok(accepted) => accepted,
Err(e) => {
// EMFILE and friends: back off rather than spin.
tracing::warn!(error = %e, "tcp listener: accept failed");
tokio::time::sleep(Duration::from_millis(100)).await;
continue;
}
};
let acceptor = acceptor.clone();
let tx = tx.clone();
tokio::spawn(async move {
match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await {
Ok(Ok(tls)) => {
let _ = tx.send((tls, peer)).await;
}
Ok(Err(e)) => {
tracing::warn!(%peer, error = %e, "tcp listener: handshake refused");
}
Err(_) => tracing::warn!(%peer, "tcp listener: handshake timed out"),
}
});
}
});
Ok(Self { rx, local })
}
}
impl axum::serve::Listener for TlsListener {
type Io = tokio_rustls::server::TlsStream<TcpStream>;
type Addr = SocketAddr;
async fn accept(&mut self) -> (Self::Io, Self::Addr) {
// The sender lives as long as the accept task, which only ends when
// this receiver is dropped.
self.rx
.recv()
.await
.expect("the accept task outlives the listener")
}
fn local_addr(&self) -> std::io::Result<Self::Addr> {
Ok(self.local)
}
}
/// Binds `cfg.listen` and returns the listener, ready for `axum::serve`.
pub async fn bind(cfg: &TcpTlsConfig) -> Result<TlsListener> {
let config = server_config(cfg)?;
let tcp = TcpListener::bind(cfg.listen)
.await
.with_context(|| format!("binding {}", cfg.listen))?;
tracing::info!(addr = %cfg.listen, client = %cfg.client_name, "swarm-controller listening on tcp");
TlsListener::new(tcp, config)
}
#[cfg(test)]
mod tests {
use super::*;
use axum::Router;
use axum::routing::get;
use rcgen::{
BasicConstraints, CertificateParams, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair,
KeyUsagePurpose,
};
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
const SERVER_NAME: &str = "controller.test";
const GATEWAY_NAME: &str = "gateway-client.test";
struct Ca {
issuer: Issuer<'static, KeyPair>,
pem: String,
}
struct Leaf {
cert: String,
key: String,
}
fn ca() -> Ca {
let mut params = CertificateParams::new(Vec::new()).unwrap();
params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
params.key_usages = vec![KeyUsagePurpose::KeyCertSign];
let key = KeyPair::generate().unwrap();
let pem = params.self_signed(&key).unwrap().pem();
Ca {
issuer: Issuer::new(params, key),
pem,
}
}
fn leaf(ca: &Ca, name: &str, usage: ExtendedKeyUsagePurpose) -> Leaf {
let mut params = CertificateParams::new(vec![name.to_owned()]).unwrap();
params.extended_key_usages = vec![usage];
let key = KeyPair::generate().unwrap();
Leaf {
cert: params.signed_by(&key, &ca.issuer).unwrap().pem(),
key: key.serialize_pem(),
}
}
/// A listener on `127.0.0.1:0` trusting `ca`, named like the gateway.
async fn serve(dir: &Path, ca: &Ca, server: &Leaf) -> SocketAddr {
let write = |name: &str, body: &str| {
let p = dir.join(name);
std::fs::write(&p, body).unwrap();
p
};
let cfg = TcpTlsConfig {
listen: "127.0.0.1:0".parse().unwrap(),
cert: write("cert.pem", &server.cert),
key: write("key.pem", &server.key),
client_ca: write("ca.pem", &ca.pem),
client_name: GATEWAY_NAME.to_owned(),
};
let listener = bind(&cfg).await.unwrap();
let addr = axum::serve::Listener::local_addr(&listener).unwrap();
let app = Router::new().route("/", get(|| async { "controller" }));
tokio::spawn(async move { axum::serve(listener, app).await });
addr
}
/// One request with `client` presented, or none. `Err` when the TCP or TLS
/// layer refused at any point: with TLS 1.3 the client finishes its half
/// of the handshake before the server's verdict arrives, so a refusal is
/// seen on the first read, not on connect.
async fn get_with(addr: SocketAddr, ca: &Ca, client: Option<&Leaf>) -> std::io::Result<String> {
let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider());
let mut roots = RootCertStore::empty();
roots
.add(CertificateDer::from_pem_slice(ca.pem.as_bytes()).unwrap())
.unwrap();
let builder = rustls::ClientConfig::builder_with_provider(provider)
.with_safe_default_protocol_versions()
.unwrap()
.with_root_certificates(roots);
let config = match client {
None => builder.with_no_client_auth(),
Some(l) => builder
.with_client_auth_cert(
CertificateDer::pem_slice_iter(l.cert.as_bytes())
.collect::<Result<_, _>>()
.unwrap(),
PrivateKeyDer::from_pem_slice(l.key.as_bytes()).unwrap(),
)
.unwrap(),
};
let tcp = TcpStream::connect(addr).await?;
let mut tls = tokio_rustls::TlsConnector::from(Arc::new(config))
.connect(ServerName::try_from(SERVER_NAME).unwrap(), tcp)
.await?;
tls.write_all(b"GET / HTTP/1.1\r\nHost: controller.test\r\nConnection: close\r\n\r\n")
.await?;
let mut out = String::new();
tls.read_to_string(&mut out).await?;
Ok(out)
}
struct Fixture {
_dir: tempfile::TempDir,
ca: Ca,
addr: SocketAddr,
}
async fn fixture() -> Fixture {
let dir = tempfile::tempdir().unwrap();
let ca = ca();
let server = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth);
let addr = serve(dir.path(), &ca, &server).await;
Fixture {
_dir: dir,
ca,
addr,
}
}
#[tokio::test]
async fn accepts_the_gateway_identity() {
let f = fixture().await;
let gateway = leaf(&f.ca, GATEWAY_NAME, ExtendedKeyUsagePurpose::ClientAuth);
let body = get_with(f.addr, &f.ca, Some(&gateway)).await.unwrap();
assert!(body.starts_with("HTTP/1.1 200"), "{body}");
assert!(body.ends_with("controller"), "{body}");
}
#[tokio::test]
async fn refuses_a_client_with_no_certificate() {
let f = fixture().await;
assert!(get_with(f.addr, &f.ca, None).await.is_err());
}
#[tokio::test]
async fn refuses_a_client_certificate_naming_someone_else() {
let f = fixture().await;
let other = leaf(
&f.ca,
"other-client.test",
ExtendedKeyUsagePurpose::ClientAuth,
);
assert!(get_with(f.addr, &f.ca, Some(&other)).await.is_err());
}
#[tokio::test]
async fn refuses_a_server_only_leaf_even_under_the_gateway_name() {
let f = fixture().await;
let services = leaf(&f.ca, GATEWAY_NAME, ExtendedKeyUsagePurpose::ServerAuth);
assert!(get_with(f.addr, &f.ca, Some(&services)).await.is_err());
}
#[tokio::test]
async fn refuses_a_services_style_leaf() {
let f = fixture().await;
let services = leaf(&f.ca, "swarm.test", ExtendedKeyUsagePurpose::ServerAuth);
assert!(get_with(f.addr, &f.ca, Some(&services)).await.is_err());
}
#[tokio::test]
async fn refuses_a_client_certificate_from_another_root() {
let f = fixture().await;
let stranger = leaf(&ca(), GATEWAY_NAME, ExtendedKeyUsagePurpose::ClientAuth);
assert!(get_with(f.addr, &f.ca, Some(&stranger)).await.is_err());
}
#[test]
fn reload_picks_up_a_rotated_pair_and_survives_a_broken_one() {
let dir = tempfile::tempdir().unwrap();
let ca = ca();
let first = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth);
let second = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth);
let (cert, key) = (dir.path().join("c.pem"), dir.path().join("k.pem"));
let put = |l: &Leaf, secs: u64| {
std::fs::write(&cert, &l.cert).unwrap();
std::fs::write(&key, &l.key).unwrap();
let t = SystemTime::UNIX_EPOCH + Duration::from_secs(secs);
for p in [&cert, &key] {
std::fs::File::options()
.write(true)
.open(p)
.unwrap()
.set_modified(t)
.unwrap();
}
};
put(&first, 1_000_000);
let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider());
let r = ReloadingCert::load(&cert, &key, provider).unwrap();
let before = r.current();
assert!(
Arc::ptr_eq(&before, &r.current()),
"unchanged files are not re-read"
);
put(&second, 2_000_000);
let after = r.current();
assert!(!Arc::ptr_eq(&before, &after), "a rotated pair is served");
std::fs::write(&cert, "not a certificate").unwrap();
assert!(
Arc::ptr_eq(&after, &r.current()),
"a broken pair keeps the last good one"
);
}
#[test]
fn config_is_all_or_nothing() {
let env = |pairs: &'static [(&'static str, &'static str)]| {
move |n: &str| {
pairs
.iter()
.find(|(k, _)| *k == n)
.map(|(_, v)| (*v).to_owned())
}
};
assert_eq!(TcpTlsConfig::from_lookup(env(&[])).unwrap(), None);
assert!(
TcpTlsConfig::from_lookup(env(&[(CERT_ENV, "/c")])).is_err(),
"a stray variable without the listen address is not 'off'"
);
assert!(
TcpTlsConfig::from_lookup(env(&[(LISTEN_ENV, "0.0.0.0:8443")])).is_err(),
"a listen address without the TLS material is not a listener"
);
let full = TcpTlsConfig::from_lookup(env(&[
(LISTEN_ENV, "0.0.0.0:8443"),
(CERT_ENV, "/c"),
(KEY_ENV, "/k"),
(CLIENT_CA_ENV, "/ca"),
(CLIENT_NAME_ENV, GATEWAY_NAME),
]))
.unwrap()
.unwrap();
assert_eq!(full.listen.port(), 8443);
assert_eq!(full.client_name, GATEWAY_NAME);
}
}