swarm-controller serves tls on tcp to the gateway client identity only, when configured
This commit is contained in:
parent
f4421b216f
commit
17fffaae79
2 changed files with 861 additions and 11 deletions
|
|
@ -1,7 +1,8 @@
|
|||
//! Swarm-level controller daemon. Runs as the unprivileged
|
||||
//! `swarm-controller` user on whichever host the operator flips
|
||||
//! `services.hyperhive.deploy.swarm-controller.enable` on, and serves HTTP over a
|
||||
//! unix socket that the hive-gateway's nginx proxies to.
|
||||
//! unix socket that the hive-gateway's nginx proxies to, and also over TLS on TCP when
|
||||
//! `SWARM_CONTROLLER_TCP_LISTEN` is set (`tcp_tls`).
|
||||
//!
|
||||
//! Configuration is read-only and loaded once at startup from env vars the
|
||||
//! NixOS module sets (`services.hyperhive.swarm.controller`) — see
|
||||
|
|
@ -93,6 +94,7 @@ mod startup;
|
|||
mod status;
|
||||
mod store;
|
||||
mod subagent_term;
|
||||
mod tcp_tls;
|
||||
mod term_stream;
|
||||
#[cfg(test)]
|
||||
mod test_nats;
|
||||
|
|
@ -269,7 +271,14 @@ async fn main() -> Result<()> {
|
|||
reviews: Arc::default(),
|
||||
alerts: alerts::from_env(),
|
||||
};
|
||||
serve(listener, state, stop, health).await
|
||||
serve(
|
||||
listener,
|
||||
tcp_tls::TcpTlsConfig::from_env()?,
|
||||
state,
|
||||
stop,
|
||||
health,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// The value a startup step produced, or exit 0 when the stop signal ended the
|
||||
|
|
@ -318,11 +327,14 @@ fn register_cert_gauges(provider: Option<&opentelemetry_sdk::metrics::SdkMeterPr
|
|||
}
|
||||
|
||||
/// Start taking the queued review requests and answering the notification
|
||||
/// router's, then serve `state` on `listener` until `stop` resolves. The
|
||||
/// requests in hand finish for at most the drain; the review queue takes no
|
||||
/// request after `stop`. `health` answers `/readyz` as the socket's does.
|
||||
/// router's, then serve `state` on `listener`, and on `tcp` too when it is
|
||||
/// configured, until `stop` resolves. Both answer from the same router and
|
||||
/// both drain on `stop`. The requests in hand finish for at most the drain;
|
||||
/// the review queue takes no request after `stop`. `health` answers `/readyz`
|
||||
/// as the socket's does.
|
||||
async fn serve(
|
||||
listener: tokio::net::UnixListener,
|
||||
tcp: Option<tcp_tls::TcpTlsConfig>,
|
||||
state: AppState,
|
||||
stop: impl Future<Output = ()> + Clone + Send + 'static,
|
||||
health: swarm_secret_client::health::Health,
|
||||
|
|
@ -331,10 +343,35 @@ async fn serve(
|
|||
health.ready_when(routes::probe_condition(state.status.clone(), store));
|
||||
review_classifier::review::spawn_queue(&state, stop.clone());
|
||||
request_reviewer::spawn(&state);
|
||||
let serving = axum::serve(listener, build_app(state)).with_graceful_shutdown(stop.clone());
|
||||
swarm_secret_client::serve_bounded(stop, serving.into_future())
|
||||
let app = build_app(state);
|
||||
let socket = axum::serve(listener, app.clone()).with_graceful_shutdown(stop.clone());
|
||||
let bound = stop.clone();
|
||||
let serving = async move {
|
||||
let Some(tcp) = tcp else {
|
||||
return socket
|
||||
.into_future()
|
||||
.await
|
||||
.context("serving swarm-controller");
|
||||
};
|
||||
let tcp = tcp_tls::bind(&tcp).await?;
|
||||
let tcp = axum::serve(tcp, app).with_graceful_shutdown(stop);
|
||||
tokio::try_join!(
|
||||
async {
|
||||
socket
|
||||
.into_future()
|
||||
.await
|
||||
.context("serving swarm-controller on its socket")
|
||||
},
|
||||
async {
|
||||
tcp.into_future()
|
||||
.await
|
||||
.context("serving swarm-controller on tcp")
|
||||
},
|
||||
)
|
||||
.map(|_| ())
|
||||
};
|
||||
swarm_secret_client::serve_bounded(bound, serving)
|
||||
.await
|
||||
.transpose()
|
||||
.context("serving swarm-controller")?;
|
||||
.transpose()?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
601
swarm-controller/src/tcp_tls.rs
Normal file
601
swarm-controller/src/tcp_tls.rs
Normal file
|
|
@ -0,0 +1,601 @@
|
|||
//! The optional TCP listener: TLS with a required client certificate.
|
||||
//!
|
||||
//! The controller authenticates nobody per request (its routes trust the
|
||||
//! transport, and `Remote-User` is a header the gateway sets), so a TCP port
|
||||
//! is only safe if only the gateway can complete a handshake on it. This
|
||||
//! listener therefore accepts a connection only when the client presents a
|
||||
//! certificate that
|
||||
//!
|
||||
//! 1. chains to the swarm services root (`SWARM_CONTROLLER_TCP_CLIENT_CA`),
|
||||
//! 2. carries the `clientAuth` extended key usage, and
|
||||
//! 3. names the gateway's client identity (`SWARM_CONTROLLER_TCP_CLIENT_NAME`)
|
||||
//! in a DNS subject alternative name.
|
||||
//!
|
||||
//! The swarm-services leaf the gateway serves browsers with chains to the same
|
||||
//! root but is `serverAuth` only and names other hosts, so it fails the second
|
||||
//! and third checks. Everything that fails is refused during the handshake,
|
||||
//! before a byte of HTTP is read.
|
||||
//!
|
||||
//! The server certificate and key are read from files and re-read when either
|
||||
//! changes, so a rotation needs no restart. The client CA is read once at
|
||||
//! start.
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use rustls::crypto::CryptoProvider;
|
||||
use rustls::pki_types::pem::PemObject as _;
|
||||
use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, UnixTime};
|
||||
use rustls::server::danger::{ClientCertVerified, ClientCertVerifier};
|
||||
use rustls::server::{ClientHello, ResolvesServerCert, WebPkiClientVerifier};
|
||||
use rustls::sign::CertifiedKey;
|
||||
use rustls::{
|
||||
CertificateError, DigitallySignedStruct, DistinguishedName, Error as TlsError, RootCertStore,
|
||||
ServerConfig, SignatureScheme,
|
||||
};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
/// Address to bind, `host:port`. Unset means no TCP listener at all.
|
||||
const LISTEN_ENV: &str = "SWARM_CONTROLLER_TCP_LISTEN";
|
||||
/// PEM certificate chain this listener serves.
|
||||
const CERT_ENV: &str = "SWARM_CONTROLLER_TCP_CERT";
|
||||
/// PEM private key for [`CERT_ENV`].
|
||||
const KEY_ENV: &str = "SWARM_CONTROLLER_TCP_KEY";
|
||||
/// PEM file holding the root client certificates must chain to.
|
||||
const CLIENT_CA_ENV: &str = "SWARM_CONTROLLER_TCP_CLIENT_CA";
|
||||
/// The DNS name the one permitted client certificate carries.
|
||||
const CLIENT_NAME_ENV: &str = "SWARM_CONTROLLER_TCP_CLIENT_NAME";
|
||||
|
||||
/// How long a peer gets to finish the handshake. A connection that stalls
|
||||
/// holds a task and a file descriptor, nothing else.
|
||||
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// What the five variables name.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TcpTlsConfig {
|
||||
pub listen: SocketAddr,
|
||||
pub cert: PathBuf,
|
||||
pub key: PathBuf,
|
||||
pub client_ca: PathBuf,
|
||||
pub client_name: String,
|
||||
}
|
||||
|
||||
impl TcpTlsConfig {
|
||||
/// `None` when [`LISTEN_ENV`] is unset. The other four are then not read,
|
||||
/// but one of them set without it is an error: a half-configured listener
|
||||
/// must not read as "off".
|
||||
pub fn from_env() -> Result<Option<Self>> {
|
||||
Self::from_lookup(|name| std::env::var(name).ok().filter(|v| !v.is_empty()))
|
||||
}
|
||||
|
||||
fn from_lookup(get: impl Fn(&str) -> Option<String>) -> Result<Option<Self>> {
|
||||
let rest = [CERT_ENV, KEY_ENV, CLIENT_CA_ENV, CLIENT_NAME_ENV];
|
||||
let Some(listen) = get(LISTEN_ENV) else {
|
||||
if let Some(set) = rest.iter().find(|n| get(n).is_some()) {
|
||||
bail!("{set} is set but {LISTEN_ENV} is not; the TCP listener needs all five");
|
||||
}
|
||||
return Ok(None);
|
||||
};
|
||||
let need = |name: &str| {
|
||||
get(name).with_context(|| format!("{name} is unset, but {LISTEN_ENV} is set"))
|
||||
};
|
||||
Ok(Some(Self {
|
||||
listen: listen
|
||||
.parse()
|
||||
.with_context(|| format!("{LISTEN_ENV} is not a socket address: {listen}"))?,
|
||||
cert: need(CERT_ENV)?.into(),
|
||||
key: need(KEY_ENV)?.into(),
|
||||
client_ca: need(CLIENT_CA_ENV)?.into(),
|
||||
client_name: need(CLIENT_NAME_ENV)?,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
/// The server certificate, re-read from disk when its files change.
|
||||
///
|
||||
/// A failed re-read keeps serving the last good pair and logs: a rotation
|
||||
/// caught half-written must not take the listener down.
|
||||
#[derive(Debug)]
|
||||
struct ReloadingCert {
|
||||
cert: PathBuf,
|
||||
key: PathBuf,
|
||||
provider: Arc<CryptoProvider>,
|
||||
cached: Mutex<Cached>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Cached {
|
||||
stamp: Stamp,
|
||||
key: Arc<CertifiedKey>,
|
||||
}
|
||||
|
||||
/// What identifies one version of the pair on disk.
|
||||
type Stamp = [Option<(SystemTime, u64)>; 2];
|
||||
|
||||
impl ReloadingCert {
|
||||
fn load(cert: &Path, key: &Path, provider: Arc<CryptoProvider>) -> Result<Self> {
|
||||
let stamp = stamp(cert, key);
|
||||
let loaded = read_pair(cert, key, &provider)?;
|
||||
Ok(Self {
|
||||
cert: cert.to_path_buf(),
|
||||
key: key.to_path_buf(),
|
||||
provider,
|
||||
cached: Mutex::new(Cached { stamp, key: loaded }),
|
||||
})
|
||||
}
|
||||
|
||||
fn current(&self) -> Arc<CertifiedKey> {
|
||||
let mut cached = self.cached.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let now = stamp(&self.cert, &self.key);
|
||||
if now != cached.stamp {
|
||||
match read_pair(&self.cert, &self.key, &self.provider) {
|
||||
Ok(key) => {
|
||||
tracing::info!(cert = %self.cert.display(), "tcp listener: server certificate reloaded");
|
||||
*cached = Cached { stamp: now, key };
|
||||
}
|
||||
Err(e) => tracing::warn!(
|
||||
error = %format!("{e:#}"),
|
||||
"tcp listener: server certificate changed on disk but does not load; keeping the previous one"
|
||||
),
|
||||
}
|
||||
}
|
||||
Arc::clone(&cached.key)
|
||||
}
|
||||
}
|
||||
|
||||
impl ResolvesServerCert for ReloadingCert {
|
||||
fn resolve(&self, _hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
|
||||
Some(self.current())
|
||||
}
|
||||
}
|
||||
|
||||
fn stamp(cert: &Path, key: &Path) -> Stamp {
|
||||
let one = |p: &Path| {
|
||||
let m = std::fs::metadata(p).ok()?;
|
||||
Some((m.modified().ok()?, m.len()))
|
||||
};
|
||||
[one(cert), one(key)]
|
||||
}
|
||||
|
||||
fn read_pair(cert: &Path, key: &Path, provider: &CryptoProvider) -> Result<Arc<CertifiedKey>> {
|
||||
let chain = CertificateDer::pem_file_iter(cert)
|
||||
.with_context(|| format!("reading {}", cert.display()))?
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.with_context(|| format!("parsing {}", cert.display()))?;
|
||||
if chain.is_empty() {
|
||||
bail!("{} holds no certificate", cert.display());
|
||||
}
|
||||
let key = PrivateKeyDer::from_pem_file(key)
|
||||
.with_context(|| format!("reading the private key {}", key.display()))?;
|
||||
let signing = provider
|
||||
.key_provider
|
||||
.load_private_key(key)
|
||||
.context("the private key is not one rustls can sign with")?;
|
||||
Ok(Arc::new(CertifiedKey::new(chain, signing)))
|
||||
}
|
||||
|
||||
/// Accepts exactly the certificates `inner` accepts that also name `name`.
|
||||
///
|
||||
/// `inner` has already checked the chain, the validity window, the signature
|
||||
/// and the `clientAuth` usage. The name is what makes the gateway's identity
|
||||
/// narrower than "anything the swarm root ever signed for a client".
|
||||
#[derive(Debug)]
|
||||
struct NamedClient {
|
||||
inner: Arc<dyn ClientCertVerifier>,
|
||||
name: ServerName<'static>,
|
||||
}
|
||||
|
||||
impl ClientCertVerifier for NamedClient {
|
||||
fn offer_client_auth(&self) -> bool {
|
||||
self.inner.offer_client_auth()
|
||||
}
|
||||
|
||||
fn client_auth_mandatory(&self) -> bool {
|
||||
self.inner.client_auth_mandatory()
|
||||
}
|
||||
|
||||
fn root_hint_subjects(&self) -> &[DistinguishedName] {
|
||||
self.inner.root_hint_subjects()
|
||||
}
|
||||
|
||||
fn verify_client_cert(
|
||||
&self,
|
||||
end_entity: &CertificateDer<'_>,
|
||||
intermediates: &[CertificateDer<'_>],
|
||||
now: UnixTime,
|
||||
) -> Result<ClientCertVerified, TlsError> {
|
||||
let verified = self
|
||||
.inner
|
||||
.verify_client_cert(end_entity, intermediates, now)?;
|
||||
let cert = webpki::EndEntityCert::try_from(end_entity)
|
||||
.map_err(|_| TlsError::InvalidCertificate(CertificateError::BadEncoding))?;
|
||||
cert.verify_is_valid_for_subject_name(&self.name)
|
||||
.map_err(|_| TlsError::InvalidCertificate(CertificateError::NotValidForName))?;
|
||||
Ok(verified)
|
||||
}
|
||||
|
||||
fn verify_tls12_signature(
|
||||
&self,
|
||||
message: &[u8],
|
||||
cert: &CertificateDer<'_>,
|
||||
dss: &DigitallySignedStruct,
|
||||
) -> Result<rustls::client::danger::HandshakeSignatureValid, TlsError> {
|
||||
self.inner.verify_tls12_signature(message, cert, dss)
|
||||
}
|
||||
|
||||
fn verify_tls13_signature(
|
||||
&self,
|
||||
message: &[u8],
|
||||
cert: &CertificateDer<'_>,
|
||||
dss: &DigitallySignedStruct,
|
||||
) -> Result<rustls::client::danger::HandshakeSignatureValid, TlsError> {
|
||||
self.inner.verify_tls13_signature(message, cert, dss)
|
||||
}
|
||||
|
||||
fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
|
||||
self.inner.supported_verify_schemes()
|
||||
}
|
||||
}
|
||||
|
||||
fn client_roots(path: &Path) -> Result<RootCertStore> {
|
||||
let mut roots = RootCertStore::empty();
|
||||
for cert in CertificateDer::pem_file_iter(path)
|
||||
.with_context(|| format!("reading {}", path.display()))?
|
||||
{
|
||||
let cert = cert.with_context(|| format!("parsing {}", path.display()))?;
|
||||
roots.add(cert).with_context(|| {
|
||||
format!(
|
||||
"{} holds a certificate that is not a usable root",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
if roots.is_empty() {
|
||||
bail!(
|
||||
"{} holds no certificate, so no client could ever be accepted",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
Ok(roots)
|
||||
}
|
||||
|
||||
/// The rustls config for `cfg`: TLS only, client certificate mandatory and
|
||||
/// narrowed to the gateway's name, server certificate reloaded from disk.
|
||||
pub fn server_config(cfg: &TcpTlsConfig) -> Result<Arc<ServerConfig>> {
|
||||
let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider());
|
||||
let inner = WebPkiClientVerifier::builder_with_provider(
|
||||
Arc::new(client_roots(&cfg.client_ca)?),
|
||||
Arc::clone(&provider),
|
||||
)
|
||||
.build()
|
||||
.context("building the client certificate verifier")?;
|
||||
let name = ServerName::try_from(cfg.client_name.clone())
|
||||
.with_context(|| format!("{CLIENT_NAME_ENV} is not a DNS name: {}", cfg.client_name))?;
|
||||
let resolver = ReloadingCert::load(&cfg.cert, &cfg.key, Arc::clone(&provider))?;
|
||||
let mut config = ServerConfig::builder_with_provider(provider)
|
||||
.with_safe_default_protocol_versions()
|
||||
.context("selecting TLS versions")?
|
||||
.with_client_cert_verifier(Arc::new(NamedClient { inner, name }))
|
||||
.with_cert_resolver(Arc::new(resolver));
|
||||
config.alpn_protocols = vec![b"http/1.1".to_vec()];
|
||||
Ok(Arc::new(config))
|
||||
}
|
||||
|
||||
/// An [`axum::serve::Listener`] yielding only connections that completed the
|
||||
/// handshake, client certificate included.
|
||||
///
|
||||
/// Handshakes run in their own tasks so a peer that stalls one cannot hold up
|
||||
/// the accept loop.
|
||||
#[derive(Debug)]
|
||||
pub struct TlsListener {
|
||||
rx: mpsc::Receiver<(tokio_rustls::server::TlsStream<TcpStream>, SocketAddr)>,
|
||||
local: SocketAddr,
|
||||
}
|
||||
|
||||
impl TlsListener {
|
||||
pub fn new(tcp: TcpListener, config: Arc<ServerConfig>) -> Result<Self> {
|
||||
let local = tcp.local_addr().context("reading the bound address")?;
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
let acceptor = tokio_rustls::TlsAcceptor::from(config);
|
||||
tokio::spawn(async move {
|
||||
while !tx.is_closed() {
|
||||
let (stream, peer) = match tcp.accept().await {
|
||||
Ok(accepted) => accepted,
|
||||
Err(e) => {
|
||||
// EMFILE and friends: back off rather than spin.
|
||||
tracing::warn!(error = %e, "tcp listener: accept failed");
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let acceptor = acceptor.clone();
|
||||
let tx = tx.clone();
|
||||
tokio::spawn(async move {
|
||||
match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await {
|
||||
Ok(Ok(tls)) => {
|
||||
let _ = tx.send((tls, peer)).await;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!(%peer, error = %e, "tcp listener: handshake refused");
|
||||
}
|
||||
Err(_) => tracing::warn!(%peer, "tcp listener: handshake timed out"),
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
Ok(Self { rx, local })
|
||||
}
|
||||
}
|
||||
|
||||
impl axum::serve::Listener for TlsListener {
|
||||
type Io = tokio_rustls::server::TlsStream<TcpStream>;
|
||||
type Addr = SocketAddr;
|
||||
|
||||
async fn accept(&mut self) -> (Self::Io, Self::Addr) {
|
||||
// The sender lives as long as the accept task, which only ends when
|
||||
// this receiver is dropped.
|
||||
self.rx
|
||||
.recv()
|
||||
.await
|
||||
.expect("the accept task outlives the listener")
|
||||
}
|
||||
|
||||
fn local_addr(&self) -> std::io::Result<Self::Addr> {
|
||||
Ok(self.local)
|
||||
}
|
||||
}
|
||||
|
||||
/// Binds `cfg.listen` and returns the listener, ready for `axum::serve`.
|
||||
pub async fn bind(cfg: &TcpTlsConfig) -> Result<TlsListener> {
|
||||
let config = server_config(cfg)?;
|
||||
let tcp = TcpListener::bind(cfg.listen)
|
||||
.await
|
||||
.with_context(|| format!("binding {}", cfg.listen))?;
|
||||
tracing::info!(addr = %cfg.listen, client = %cfg.client_name, "swarm-controller listening on tcp");
|
||||
TlsListener::new(tcp, config)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::Router;
|
||||
use axum::routing::get;
|
||||
use rcgen::{
|
||||
BasicConstraints, CertificateParams, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair,
|
||||
KeyUsagePurpose,
|
||||
};
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
const SERVER_NAME: &str = "controller.test";
|
||||
const GATEWAY_NAME: &str = "gateway-client.test";
|
||||
|
||||
struct Ca {
|
||||
issuer: Issuer<'static, KeyPair>,
|
||||
pem: String,
|
||||
}
|
||||
|
||||
struct Leaf {
|
||||
cert: String,
|
||||
key: String,
|
||||
}
|
||||
|
||||
fn ca() -> Ca {
|
||||
let mut params = CertificateParams::new(Vec::new()).unwrap();
|
||||
params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
|
||||
params.key_usages = vec![KeyUsagePurpose::KeyCertSign];
|
||||
let key = KeyPair::generate().unwrap();
|
||||
let pem = params.self_signed(&key).unwrap().pem();
|
||||
Ca {
|
||||
issuer: Issuer::new(params, key),
|
||||
pem,
|
||||
}
|
||||
}
|
||||
|
||||
fn leaf(ca: &Ca, name: &str, usage: ExtendedKeyUsagePurpose) -> Leaf {
|
||||
let mut params = CertificateParams::new(vec![name.to_owned()]).unwrap();
|
||||
params.extended_key_usages = vec![usage];
|
||||
let key = KeyPair::generate().unwrap();
|
||||
Leaf {
|
||||
cert: params.signed_by(&key, &ca.issuer).unwrap().pem(),
|
||||
key: key.serialize_pem(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A listener on `127.0.0.1:0` trusting `ca`, named like the gateway.
|
||||
async fn serve(dir: &Path, ca: &Ca, server: &Leaf) -> SocketAddr {
|
||||
let write = |name: &str, body: &str| {
|
||||
let p = dir.join(name);
|
||||
std::fs::write(&p, body).unwrap();
|
||||
p
|
||||
};
|
||||
let cfg = TcpTlsConfig {
|
||||
listen: "127.0.0.1:0".parse().unwrap(),
|
||||
cert: write("cert.pem", &server.cert),
|
||||
key: write("key.pem", &server.key),
|
||||
client_ca: write("ca.pem", &ca.pem),
|
||||
client_name: GATEWAY_NAME.to_owned(),
|
||||
};
|
||||
let listener = bind(&cfg).await.unwrap();
|
||||
let addr = axum::serve::Listener::local_addr(&listener).unwrap();
|
||||
let app = Router::new().route("/", get(|| async { "controller" }));
|
||||
tokio::spawn(async move { axum::serve(listener, app).await });
|
||||
addr
|
||||
}
|
||||
|
||||
/// One request with `client` presented, or none. `Err` when the TCP or TLS
|
||||
/// layer refused at any point: with TLS 1.3 the client finishes its half
|
||||
/// of the handshake before the server's verdict arrives, so a refusal is
|
||||
/// seen on the first read, not on connect.
|
||||
async fn get_with(addr: SocketAddr, ca: &Ca, client: Option<&Leaf>) -> std::io::Result<String> {
|
||||
let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider());
|
||||
let mut roots = RootCertStore::empty();
|
||||
roots
|
||||
.add(CertificateDer::from_pem_slice(ca.pem.as_bytes()).unwrap())
|
||||
.unwrap();
|
||||
let builder = rustls::ClientConfig::builder_with_provider(provider)
|
||||
.with_safe_default_protocol_versions()
|
||||
.unwrap()
|
||||
.with_root_certificates(roots);
|
||||
let config = match client {
|
||||
None => builder.with_no_client_auth(),
|
||||
Some(l) => builder
|
||||
.with_client_auth_cert(
|
||||
CertificateDer::pem_slice_iter(l.cert.as_bytes())
|
||||
.collect::<Result<_, _>>()
|
||||
.unwrap(),
|
||||
PrivateKeyDer::from_pem_slice(l.key.as_bytes()).unwrap(),
|
||||
)
|
||||
.unwrap(),
|
||||
};
|
||||
let tcp = TcpStream::connect(addr).await?;
|
||||
let mut tls = tokio_rustls::TlsConnector::from(Arc::new(config))
|
||||
.connect(ServerName::try_from(SERVER_NAME).unwrap(), tcp)
|
||||
.await?;
|
||||
tls.write_all(b"GET / HTTP/1.1\r\nHost: controller.test\r\nConnection: close\r\n\r\n")
|
||||
.await?;
|
||||
let mut out = String::new();
|
||||
tls.read_to_string(&mut out).await?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
_dir: tempfile::TempDir,
|
||||
ca: Ca,
|
||||
addr: SocketAddr,
|
||||
}
|
||||
|
||||
async fn fixture() -> Fixture {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let ca = ca();
|
||||
let server = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth);
|
||||
let addr = serve(dir.path(), &ca, &server).await;
|
||||
Fixture {
|
||||
_dir: dir,
|
||||
ca,
|
||||
addr,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepts_the_gateway_identity() {
|
||||
let f = fixture().await;
|
||||
let gateway = leaf(&f.ca, GATEWAY_NAME, ExtendedKeyUsagePurpose::ClientAuth);
|
||||
let body = get_with(f.addr, &f.ca, Some(&gateway)).await.unwrap();
|
||||
assert!(body.starts_with("HTTP/1.1 200"), "{body}");
|
||||
assert!(body.ends_with("controller"), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refuses_a_client_with_no_certificate() {
|
||||
let f = fixture().await;
|
||||
assert!(get_with(f.addr, &f.ca, None).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refuses_a_client_certificate_naming_someone_else() {
|
||||
let f = fixture().await;
|
||||
let other = leaf(
|
||||
&f.ca,
|
||||
"other-client.test",
|
||||
ExtendedKeyUsagePurpose::ClientAuth,
|
||||
);
|
||||
assert!(get_with(f.addr, &f.ca, Some(&other)).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refuses_a_server_only_leaf_even_under_the_gateway_name() {
|
||||
let f = fixture().await;
|
||||
let services = leaf(&f.ca, GATEWAY_NAME, ExtendedKeyUsagePurpose::ServerAuth);
|
||||
assert!(get_with(f.addr, &f.ca, Some(&services)).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refuses_a_services_style_leaf() {
|
||||
let f = fixture().await;
|
||||
let services = leaf(&f.ca, "swarm.test", ExtendedKeyUsagePurpose::ServerAuth);
|
||||
assert!(get_with(f.addr, &f.ca, Some(&services)).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refuses_a_client_certificate_from_another_root() {
|
||||
let f = fixture().await;
|
||||
let stranger = leaf(&ca(), GATEWAY_NAME, ExtendedKeyUsagePurpose::ClientAuth);
|
||||
assert!(get_with(f.addr, &f.ca, Some(&stranger)).await.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reload_picks_up_a_rotated_pair_and_survives_a_broken_one() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let ca = ca();
|
||||
let first = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth);
|
||||
let second = leaf(&ca, SERVER_NAME, ExtendedKeyUsagePurpose::ServerAuth);
|
||||
let (cert, key) = (dir.path().join("c.pem"), dir.path().join("k.pem"));
|
||||
let put = |l: &Leaf, secs: u64| {
|
||||
std::fs::write(&cert, &l.cert).unwrap();
|
||||
std::fs::write(&key, &l.key).unwrap();
|
||||
let t = SystemTime::UNIX_EPOCH + Duration::from_secs(secs);
|
||||
for p in [&cert, &key] {
|
||||
std::fs::File::options()
|
||||
.write(true)
|
||||
.open(p)
|
||||
.unwrap()
|
||||
.set_modified(t)
|
||||
.unwrap();
|
||||
}
|
||||
};
|
||||
put(&first, 1_000_000);
|
||||
let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider());
|
||||
let r = ReloadingCert::load(&cert, &key, provider).unwrap();
|
||||
let before = r.current();
|
||||
assert!(
|
||||
Arc::ptr_eq(&before, &r.current()),
|
||||
"unchanged files are not re-read"
|
||||
);
|
||||
|
||||
put(&second, 2_000_000);
|
||||
let after = r.current();
|
||||
assert!(!Arc::ptr_eq(&before, &after), "a rotated pair is served");
|
||||
|
||||
std::fs::write(&cert, "not a certificate").unwrap();
|
||||
assert!(
|
||||
Arc::ptr_eq(&after, &r.current()),
|
||||
"a broken pair keeps the last good one"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_is_all_or_nothing() {
|
||||
let env = |pairs: &'static [(&'static str, &'static str)]| {
|
||||
move |n: &str| {
|
||||
pairs
|
||||
.iter()
|
||||
.find(|(k, _)| *k == n)
|
||||
.map(|(_, v)| (*v).to_owned())
|
||||
}
|
||||
};
|
||||
assert_eq!(TcpTlsConfig::from_lookup(env(&[])).unwrap(), None);
|
||||
assert!(
|
||||
TcpTlsConfig::from_lookup(env(&[(CERT_ENV, "/c")])).is_err(),
|
||||
"a stray variable without the listen address is not 'off'"
|
||||
);
|
||||
assert!(
|
||||
TcpTlsConfig::from_lookup(env(&[(LISTEN_ENV, "0.0.0.0:8443")])).is_err(),
|
||||
"a listen address without the TLS material is not a listener"
|
||||
);
|
||||
let full = TcpTlsConfig::from_lookup(env(&[
|
||||
(LISTEN_ENV, "0.0.0.0:8443"),
|
||||
(CERT_ENV, "/c"),
|
||||
(KEY_ENV, "/k"),
|
||||
(CLIENT_CA_ENV, "/ca"),
|
||||
(CLIENT_NAME_ENV, GATEWAY_NAME),
|
||||
]))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(full.listen.port(), 8443);
|
||||
assert_eq!(full.client_name, GATEWAY_NAME);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue