From dff662c6b390cd466034cf0871258a1f1ae1f54a Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 13 Nov 2026 06:23:42 +0100 Subject: [PATCH] refuse tls hellos that carry encrypted client hello before squid sees them --- docs/4-implementation/networking/network.md | 87 +- hive-priv/README.md | 4 +- hive-priv/src/egress.rs | 46 +- hive-priv/src/egress_ech.rs | 1116 +++++++++++++++++++ hive-priv/src/main.rs | 8 +- nix/host-modules/hive-egress.nix | 145 ++- nix/host-modules/hive-squid.nix | 4 +- nix/module-eval/core-toggle.nix | 69 +- nix/vm-tests/hive-squid.nix | 99 +- 9 files changed, 1536 insertions(+), 42 deletions(-) create mode 100644 hive-priv/src/egress_ech.rs diff --git a/docs/4-implementation/networking/network.md b/docs/4-implementation/networking/network.md index 99e9878c6..2c69fd7dd 100644 --- a/docs/4-implementation/networking/network.md +++ b/docs/4-implementation/networking/network.md @@ -517,10 +517,11 @@ credential=present|absent unrestricted_canary=… unrestricted_status=…` line A program that ignores the proxy variables still gets squid's policy for its web traffic. In the `inet hive_egress` table (`nix/host-modules/hive-egress.nix`) the `intercept` chain, at `prerouting`, redirects IPv4 TCP port 80 to -`:3129` and port 443 to `:3130`, the two intercept listeners -that `hive-squid.nix` opens. Squid reads the original destination from -conntrack. The chain exists where the host runs squid -(`services.hyperhive.network.egress.intercept.enable`). +`:3129`, an intercept listener of squid, and port 443 to +`:3131`, the ClientHello relay (see _ECH_ below), which sends the +connection on to squid's other intercept listener, `:3130`. Squid reads +the original destination from conntrack. The chain exists where the host runs +squid (`services.hyperhive.network.egress.intercept.enable`). - **Who gets redirected.** Every flow from the bridge to port 80 or 443, except a flow to an address that stays on the bridge, to one of the host's own @@ -570,25 +571,79 @@ conntrack. The chain exists where the host runs squid - a ClientHello with no SNI, and a URL with an IP literal, which sends none: the host of the `CONNECT` stays an address, and the IP-literal denies refuse it; - - bytes on port 443 that aren't a ClientHello, and on port 80 that aren't an - HTTP request. + - bytes on port 80 that aren't an HTTP request. Bytes on port 443 that aren't + a ClientHello never reach squid (see _ECH_ below). +- **ECH.** Squid 7.6 can't see the `encrypted_client_hello` extension, only the + public name in the outer hello. An agent that holds an ECH configuration could + send a hello with an allowed outer name, a destination address that name + resolves to, and a hidden inner name that the same address serves. Port 443 + therefore doesn't go to squid directly. `hive-priv egress-ech-relay` + (`hive-egress-ech-relay.service`, `egress.intercept.refuseEch`, on by default) + listens on `:3131`. It reads the first TLS records of each redirected + connection, up to four records and 16 KiB of ClientHello within 10 seconds, and + closes the connection when the ClientHello of a `restricted` agent carries the + extension (type `0xfe0d`, and the draft types `0xfe08` to `0xfe0c`, `0xff02` + and `0xffce`). The relay only observes a `log` agent, as everywhere in `log` + mode: its hello goes on like a clean one, and the relay logs it (see below). + It closes a connection too, for any agent, when the bytes aren't a well-formed + ClientHello, when they don't arrive in time, or when the connection wasn't + redirected. It also closes a connection from a source that already has 256 + open, and any connection beyond 2048 in all, so one agent can't use up the + relay. Otherwise it connects to the destination the agent dialed and sends the bytes + on, and squid sees an ordinary intercepted connection: + - the relay connects from an address in `198.18.0.0/15`, which the unit routes + to `lo`, and an `output` rule DNATs its marked connections (mark + `0x45434800`) to squid's port 3130, so squid reads the original destination + from conntrack as before; + - the source address is `198.18.0.0` plus the low 17 bits of the agent's + address, and `egress-whois` maps it back to the lease it came from, so the + pool needs a bridge subnet of /15 or smaller (an assertion checks) and + squid's log shows `198.18.x.y` as the client address of an intercepted HTTPS + flow. The user column still names the agent. The `input` chain drops a pool + address that arrives from the bridge; + - the relay logs to the journal of its unit, at most one line a second per + source address, with the agent named the way squid names it (`-` when no + lease and binding says). A refusal is + `hive-egress-ech-refused agent= dst=
`; an ECH hello of a `log` + agent, which goes on, is + `hive-egress-ech-would-refuse agent= dst=
`; a hello that isn't + well formed is + `hive-egress-hello-refused agent= dst=
reason=`. Squid logs + nothing for a refused flow; + - the mode comes from `agent-allow.conf`, the include `hive-c0re` renders: an + agent is in `log` mode when it holds the line + `acl agent_log_ proxy_auth `. Squid applies the same file, so the + relay and squid can't disagree about an agent's mode, and the file changes + when the operator changes the mode, with no second list to keep in step. A + missing or unreadable file, or an agent with no lease, counts as + `restricted`, and the relay refuses the hello; + - while the relay is down the redirect reaches a closed port, so HTTPS from + `restricted` and `log` agents fails. The unit restarts after 5 seconds. - Squid 7.6 can't see the `encrypted_client_hello` extension, only the public - name in the outer hello. An agent that holds an ECH configuration can send a - hello with an allowed outer name, a destination address that name resolves - to, and a hidden inner name that the same address serves. Squid lets that - flow through, so ECH isn't refused here. + GREASE: a client with no ECH configuration, Chrome and Firefox by default, + sends the extension anyway with random contents, and the protocol makes the two + look the same on the wire. The relay can't tell them apart, so it treats both + as ECH: a `restricted` agent's HTTPS fails when the client ignores the proxy + variables, and a `log` agent's goes on with a would-refuse line. Through the + proxy port (`HTTPS_PROXY`, a `CONNECT`) it works as before: that path has no + interception. A host where that costs more than the gap does can set + `egress.intercept.refuseEch = false`, which sends port 443 straight to squid + and leaves ECH unchecked. The relay only sees the hello of a connection that + the redirect catches. It doesn't touch UDP (QUIC), IPv6 or other ports. - **Reading the log.** Each intercepted HTTPS connection logs twice: first `NONE_NONE/000 … CONNECT
:443 `, before the ClientHello, then the `CONNECT :443 ` line with `TCP_TUNNEL/200` or `TCP_DENIED/403` - (`NONE_NONE/409` when the SNI and the address disagree). An intercepted HTTP - request logs `GET http:/// `. A tunnel to an intercepted + (`NONE_NONE/409` when the SNI and the address disagree). The client address of + an intercepted HTTPS line is the relay's `198.18.x.y` (see _ECH_ above), and an + intercepted HTTP request logs `GET http:/// ` with the agent's own + address. A tunnel to an intercepted destination ends in `ORIGINAL_DST/
`. `cache.log` notes an ACL that met a flow squid refused before its request. -- **Rollout.** The redirect, squid's configuration and `hive-priv` ship in one - build. A hive that still runs an older build has no redirect, and its agents go - direct as before. The agents that use the proxy variables see no change. +- **Rollout.** The redirect, the relay, squid's configuration and `hive-priv` ship + in one build. A hive that still runs an older build has no redirect, and its + agents go direct as before. The agents that use the proxy variables see no + change. ## Container isolation diff --git a/hive-priv/README.md b/hive-priv/README.md index 0b96b24ab..8a1aaa2e6 100644 --- a/hive-priv/README.md +++ b/hive-priv/README.md @@ -38,7 +38,9 @@ not the whole daemon-shared crate. `storage` (btrfs subvolumes, quotas, snapshots, disk measurement), `files` (pause marker, socket dir, legacy tmpfiles) and `infra` (gateway nginx, forge admin, infra containers). `egress` holds four sub-commands that never -touch the socket, `egress-load`, `egress-pin`, `egress-lease` and `egress-whois`; the +touch the socket, `egress-load`, `egress-pin`, `egress-lease` and `egress-whois`, and +`egress_ech` a fifth, `egress-ech-relay`, the long-running ClientHello check in front of +squid's HTTPS intercept port; the `hive-egress` unit, udev, a timer and dnsmasq run the first three to fill the sets of the anti-spoofing table and the interception exemptions, and squid runs the last to name the agent of an intercepted flow (see diff --git a/hive-priv/src/egress.rs b/hive-priv/src/egress.rs index 6adfc5971..7d3e88c13 100644 --- a/hive-priv/src/egress.rs +++ b/hive-priv/src/egress.rs @@ -38,6 +38,8 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use anyhow::{Context as _, Result, bail}; +use crate::egress_ech::same_host; + /// The sub-command squid runs to learn which agent holds a source address. pub(crate) const WHOIS: &str = "egress-whois"; @@ -344,7 +346,9 @@ fn unrestricted_names() -> Vec { /// The agent whose container holds the IPv4 address `ip`, as dnsmasq's lease /// file and the bindings of the ports say: the lease names a MAC, the MAC is -/// bound to one port, and that port's machine is `h-`. `None` when any +/// bound to one port, and that port's machine is `h-`. An address in the +/// pool of the ECH relay (`egress_ech`) stands for the leased address it was +/// derived from. `None` when any /// step is missing or ambiguous (two leases for the address under different /// MACs, a MAC bound twice, a machine that is no agent, a name that is no /// identifier): a flow whose agent cannot be told is refused, never guessed. @@ -357,7 +361,7 @@ pub(crate) fn agent_of(ip: &str, leases: &str, pins: &[Pin], now: u64) -> Option 0 => true, t => t > now, }; - (live && addr.parse::().ok()? == ip).then(|| Mac::parse(mac)) + (live && same_host(ip, addr.parse::().ok()?)).then(|| Mac::parse(mac)) }); let mac = macs.next()??; if macs.any(|other| other != Some(mac)) { @@ -373,20 +377,25 @@ pub(crate) fn agent_of(ip: &str, leases: &str, pins: &[Pin], now: u64) -> Option Some(agent.to_owned()) } +/// [`agent_of`] on the files as they are now, which are read for every call. +pub(crate) fn agent_of_now(ip: &str) -> Option { + let leasefile = std::env::var("HIVE_EGRESS_LEASEFILE").unwrap_or_else(|_| LEASEFILE.into()); + let pins = parse_state(&std::fs::read_to_string(state_path()).unwrap_or_default()); + let leases = std::fs::read_to_string(leasefile).unwrap_or_default(); + agent_of(ip, &leases, &pins, now_secs()) +} + /// `egress-whois`: squid's external ACL helper. Each line on stdin starts with /// a source address; the answer is `OK user=` or `ERR`. The two files /// are read for every line, since squid caches the answers itself. fn whois() -> Result<()> { use std::io::{BufRead as _, Write as _}; - let leasefile = std::env::var("HIVE_EGRESS_LEASEFILE").unwrap_or_else(|_| LEASEFILE.into()); let stdin = std::io::stdin(); let mut stdout = std::io::stdout().lock(); for line in stdin.lock().lines() { let line = line.context("read stdin")?; let ip = line.split_whitespace().next().unwrap_or_default(); - let pins = parse_state(&std::fs::read_to_string(state_path()).unwrap_or_default()); - let leases = std::fs::read_to_string(&leasefile).unwrap_or_default(); - match agent_of(ip, &leases, &pins, now_secs()) { + match agent_of_now(ip) { Some(agent) => writeln!(stdout, "OK user={agent}"), None => writeln!(stdout, "ERR"), } @@ -1117,6 +1126,31 @@ mod tests { ); } + #[test] + fn a_relay_pool_address_names_the_agent_it_was_derived_from() { + let pins = [ + pin("vb-h-atlas", 10, 1, "h-atlas"), + pin("vb-h-uma", 11, 2, "h-uma"), + ]; + let leases = lease_line(NOW + 60, 1, "10.42.0.5") + &lease_line(NOW + 60, 2, "10.42.0.6"); + assert_eq!( + agent_of("198.18.0.5", &leases, &pins, NOW).as_deref(), + Some("atlas") + ); + assert_eq!( + agent_of("198.18.0.6", &leases, &pins, NOW).as_deref(), + Some("uma") + ); + // No lease behind the pool address, and an address of the pool that + // is itself leased: neither names anyone. + assert_eq!(agent_of("198.18.0.7", &leases, &pins, NOW), None); + let leased_pool = lease_line(NOW + 60, 1, "198.18.0.9"); + assert_eq!(agent_of("198.18.0.9", &leased_pool, &pins, NOW), None); + // Two leases that share the pool address are ambiguous. + let twins = lease_line(NOW + 60, 1, "10.42.0.5") + &lease_line(NOW + 60, 2, "10.44.0.5"); + assert_eq!(agent_of("198.18.0.5", &twins, &pins, NOW), None); + } + #[test] fn an_address_nobody_can_be_told_to_hold_has_no_agent() { let pins = [ diff --git a/hive-priv/src/egress_ech.rs b/hive-priv/src/egress_ech.rs new file mode 100644 index 000000000..044dd3164 --- /dev/null +++ b/hive-priv/src/egress_ech.rs @@ -0,0 +1,1116 @@ +//! `egress-ech-relay `: the TLS `ClientHello` check in +//! front of squid's HTTPS intercept port (`nix/host-modules/hive-egress.nix`). +//! +//! Squid 7.6 cannot see the `encrypted_client_hello` extension, so the nft +//! redirect for TCP port 443 ends here. The relay reads the first TLS records of +//! each connection and closes it when the `ClientHello` carries the extension +//! or cannot be parsed. Otherwise it connects to the destination the agent +//! dialed and sends the bytes on. That second connection has to look like the +//! agent's to squid, which reads the destination from conntrack and names the +//! agent from the source address: +//! +//! - it goes to the original destination (`SO_ORIGINAL_DST` of the accepted +//! socket) and carries the packet mark; an nft `output` rule DNATs marked +//! packets to squid's HTTPS intercept port; +//! - it is bound to a source address in [`POOL`], derived from the agent's +//! ([`pool_source`]). The pool is a `local` route on `lo`, so squid's replies +//! come back over the loopback; the agent's own address would send them out +//! over the bridge. [`agent_of`] maps a pool address back to the agent. +//! +//! A connection the relay cannot place (no original destination, no complete +//! hello within [`HELLO_TIMEOUT`], too many connections open) is closed. +//! +//! GREASE: a client with no ECH configuration (Chrome and Firefox by default) +//! sends the extension anyway with random contents, and the wire cannot tell +//! that from a real one. Every `encrypted_client_hello` extension counts as ECH. +//! A `log` agent is observed, not refused: its ECH hello goes on, with a +//! `hive-egress-ech-would-refuse` line. +//! +//! [`agent_of`]: crate::egress::agent_of + +use std::collections::HashMap; +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; +use std::os::fd::AsRawFd as _; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use anyhow::{Context as _, Result, bail}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpSocket, TcpStream}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +/// The sub-command that runs the relay. +pub(crate) const RELAY: &str = "egress-ech-relay"; + +/// The pool of source addresses the relay connects from: `198.18.0.0/15`, which +/// RFC 2544 sets aside for benchmarking, so no network routes it. The hive-egress +/// unit adds it to the `local` table of `lo`. +const POOL: u32 = 0xC612_0000; +/// The part of a pool address that is the agent's: the low 17 bits. The bridge +/// subnet must be no larger than a /15, which the module asserts. +const HOST_BITS: u32 = 0x0001_FFFF; + +/// The source address the relay connects from for an agent at `agent`. +pub(crate) fn pool_source(agent: Ipv4Addr) -> Ipv4Addr { + Ipv4Addr::from_bits(POOL | (agent.to_bits() & HOST_BITS)) +} + +/// Whether `ip` is in the pool. +pub(crate) fn in_pool(ip: Ipv4Addr) -> bool { + ip.to_bits() & !HOST_BITS == POOL +} + +/// Whether `lease` is the address a connection from `source` was made for: +/// the same address, or a pool address built from it. +pub(crate) fn same_host(source: Ipv4Addr, lease: Ipv4Addr) -> bool { + if in_pool(source) { + !in_pool(lease) && pool_source(lease) == source + } else { + source == lease + } +} + +/// Most TLS records a `ClientHello` may be spread over. +const MAX_RECORDS: usize = 4; +/// Largest TLS record a peer may send (RFC 8446, 5.1). +const MAX_RECORD: usize = 16384; +/// Largest `ClientHello` the relay reads. Real ones are a few kilobytes. +const MAX_HELLO: usize = MAX_RECORD; +/// Time a peer has to send its whole `ClientHello`. +const HELLO_TIMEOUT: Duration = Duration::from_secs(10); +/// Time the relay waits for the destination to accept. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// Connections open at once; further ones are closed. +const MAX_CONNECTIONS: usize = 2048; +/// Connections open at once from one source address, so that one agent cannot +/// use up [`MAX_CONNECTIONS`]. A browser keeps a few dozen open; 256 leaves an +/// agent room for a busy crawler and lets eight sources fill the relay. +const MAX_PER_SOURCE: usize = 256; +/// Where the squid include that `hive-c0re` renders names the `log` agents, set +/// by the unit. +const AGENT_ALLOW_ENV: &str = "HIVE_EGRESS_AGENT_ALLOW"; +/// Least time between two log lines for one source address. +const LOG_EVERY: Duration = Duration::from_secs(1); +/// Source addresses the log limiter remembers. +const LOG_TRACKED: usize = 1024; + +/// `SO_ORIGINAL_DST` of ``. +const SO_ORIGINAL_DST: libc::c_int = 80; + +/// The extension types that carry an encrypted `ClientHello`: RFC 9849's +/// `encrypted_client_hello`, and the codepoints earlier drafts of it used +/// (`encrypted_server_name` of the ESNI drafts, the draft codepoints +/// `0xfe08`-`0xfe0c` and `0xff02`). +fn is_ech(extension: u16) -> bool { + matches!(extension, 0xfe0d | 0xfe08..=0xfe0c | 0xff02 | 0xffce) +} + +/// What the first bytes of a connection say. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Verdict { + /// The `ClientHello` is not complete yet. + Incomplete, + /// A well-formed `ClientHello` with no ECH extension. + Clean, + /// A well-formed `ClientHello` with an ECH extension. + Ech, + /// Not a `ClientHello` the relay accepts, and why. + Malformed(&'static str), +} + +fn u16_at(b: &[u8], at: usize) -> Option { + Some(usize::from(u16::from_be_bytes([ + *b.get(at)?, + *b.get(at + 1)?, + ]))) +} + +fn u24_at(b: &[u8], at: usize) -> Option { + Some((usize::from(*b.get(at)?) << 16) | u16_at(b, at + 1)?) +} + +/// Look at the bytes a connection has sent so far, from its first byte. +/// +/// The handshake message is reassembled from the records; there may be at most +/// [`MAX_RECORDS`] of them. Everything the parse cannot account for is +/// [`Verdict::Malformed`]: a record that is not a handshake record, a first +/// message that is not a `ClientHello`, bytes after the hello, a length that +/// overruns its container or leaves bytes over. +pub(crate) fn inspect(stream: &[u8]) -> Verdict { + let mut message = Vec::new(); + let mut at = 0; + for _ in 0..MAX_RECORDS { + let Some(header) = stream.get(at..at + 5) else { + return Verdict::Incomplete; + }; + if header[0] != 22 || header[1] != 3 || header[2] > 4 { + return Verdict::Malformed("not a TLS handshake record"); + } + let len = usize::from(u16::from_be_bytes([header[3], header[4]])); + if len == 0 || len > MAX_RECORD { + return Verdict::Malformed("record length out of range"); + } + let Some(body) = stream.get(at + 5..at + 5 + len) else { + return Verdict::Incomplete; + }; + message.extend_from_slice(body); + at += 5 + len; + if message.len() >= 4 { + if message[0] != 1 { + return Verdict::Malformed("first handshake message is not a ClientHello"); + } + let want = 4 + u24_at(&message, 1).unwrap_or(0); + if want > MAX_HELLO { + return Verdict::Malformed("ClientHello too large"); + } + if message.len() == want { + return hello(&message[4..]); + } + if message.len() > want { + return Verdict::Malformed("bytes after the ClientHello"); + } + } + } + Verdict::Malformed("ClientHello spans too many records") +} + +/// The body of a `ClientHello` (RFC 8446, 4.1.2). +fn hello(body: &[u8]) -> Verdict { + const BAD: Verdict = Verdict::Malformed("ClientHello is malformed"); + if body.first() != Some(&3) { + return BAD; + } + // legacy_version (2), random (32), then the session id. + let Some(&session) = body.get(34) else { + return BAD; + }; + if session > 32 { + return BAD; + } + let mut at = 35 + usize::from(session); + let Some(suites) = u16_at(body, at) else { + return BAD; + }; + if suites < 2 || suites % 2 != 0 { + return BAD; + } + at += 2 + suites; + let Some(&compression) = body.get(at) else { + return BAD; + }; + if compression < 1 { + return BAD; + } + at += 1 + usize::from(compression); + if at == body.len() { + // TLS 1.2 and older allow a hello with no extensions. + return Verdict::Clean; + } + let Some(total) = u16_at(body, at) else { + return BAD; + }; + at += 2; + if body.len() - at != total { + return BAD; + } + let mut ech = false; + while at < body.len() { + let (Some(kind), Some(len)) = (u16_at(body, at), u16_at(body, at + 2)) else { + return BAD; + }; + at += 4; + if at + len > body.len() { + return BAD; + } + at += len; + ech |= u16::try_from(kind).is_ok_and(is_ech); + } + if ech { Verdict::Ech } else { Verdict::Clean } +} + +/// Lets a source address log once per [`LOG_EVERY`]. An address the limiter +/// forgets (it tracks [`LOG_TRACKED`]) logs again, so the memory stays bounded. +#[derive(Default)] +struct LogLimit { + last: HashMap, +} + +impl LogLimit { + fn allow(&mut self, who: Ipv4Addr, now: Instant) -> bool { + if self.last.len() >= LOG_TRACKED && !self.last.contains_key(&who) { + self.last + .retain(|_, at| now.duration_since(*at) < LOG_EVERY); + if self.last.len() >= LOG_TRACKED { + self.last.clear(); + } + } + match self.last.get(&who) { + Some(at) if now.duration_since(*at) < LOG_EVERY => false, + _ => { + self.last.insert(who, now); + true + } + } + } +} + +/// The line a refusal logs: the agent that made the connection, as `egress-whois` +/// names it, and the address it dialed. +fn refusal_line(what: &str, agent: Option<&str>, dst: Ipv4Addr, reason: Option<&str>) -> String { + let agent = agent.unwrap_or("-"); + match reason { + Some(reason) => format!("{what} agent={agent} dst={dst} reason={reason}"), + None => format!("{what} agent={agent} dst={dst}"), + } +} + +/// What the relay shares between connections. +struct Shared { + mark: u32, + per_source: usize, + agent_allow: Option, + limit: Mutex, + open: Mutex>, +} + +/// What the relay does with a connection whose hello is not clean. +#[derive(Debug, PartialEq, Eq)] +enum Action { + /// Send it on to squid. + Forward, + /// Send it on to squid, and log what a restricted agent would have got. + ForwardLogged, + Refuse, +} + +/// A `log` agent is observed, not refused (it is allowed what it is not +/// listed for, and recorded as `would_deny`): its ECH hello goes on. A hello +/// that does not parse is refused for every agent. +fn decide(verdict: &Verdict, log_agent: bool) -> Action { + match verdict { + Verdict::Clean => Action::Forward, + Verdict::Ech if log_agent => Action::ForwardLogged, + Verdict::Ech | Verdict::Malformed(_) | Verdict::Incomplete => Action::Refuse, + } +} + +/// Whether `allow` (the squid include `hive-c0re` renders) puts `agent` in `log` +/// mode. Squid reads the same file to decide the same thing, so the relay and +/// squid cannot disagree about a mode; a missing or unreadable file lists no +/// `log` agent. +fn is_log_agent(allow: &str, agent: &str) -> bool { + let line = format!("acl agent_log_{agent} proxy_auth {agent}"); + allow.lines().any(|l| l.trim() == line) +} + +impl Shared { + fn new(mark: u32, per_source: usize, agent_allow: Option) -> Self { + Self { + mark, + per_source, + agent_allow, + limit: Mutex::default(), + open: Mutex::default(), + } + } + + /// The agent behind `peer` as `egress-whois` names it, and whether it is in + /// `log` mode. Reads files; called off the runtime's threads. + fn who(&self, peer: Ipv4Addr) -> (Option, bool) { + let agent = crate::egress::agent_of_now(&peer.to_string()); + let log = match (&agent, &self.agent_allow) { + (Some(agent), Some(path)) => { + is_log_agent(&std::fs::read_to_string(path).unwrap_or_default(), agent) + } + _ => false, + }; + (agent, log) + } + + /// Log what happened to a connection whose hello was not clean. + fn report( + &self, + peer: Ipv4Addr, + dst: Ipv4Addr, + verdict: &Verdict, + action: &Action, + agent: Option<&str>, + ) { + let allowed = self + .limit + .lock() + .is_ok_and(|mut limit| limit.allow(peer, Instant::now())); + if !allowed { + return; + } + let line = match (verdict, action) { + (Verdict::Ech, Action::ForwardLogged) => { + refusal_line("hive-egress-ech-would-refuse", agent, dst, None) + } + (Verdict::Ech, Action::Refuse) => { + refusal_line("hive-egress-ech-refused", agent, dst, None) + } + (Verdict::Malformed(why), Action::Refuse) => { + refusal_line("hive-egress-hello-refused", agent, dst, Some(why)) + } + _ => return, + }; + eprintln!("{line}"); + } +} + +/// One slot of the connections a source has open, released when dropped. +struct SourceSlot { + shared: Arc, + source: Ipv4Addr, +} + +impl SourceSlot { + /// `None` when `source` already has its share open. + fn take(shared: &Arc, source: Ipv4Addr) -> Option { + let mut open = shared.open.lock().ok()?; + let count = open.entry(source).or_insert(0); + if *count >= shared.per_source { + return None; + } + *count += 1; + Some(Self { + shared: Arc::clone(shared), + source, + }) + } +} + +impl Drop for SourceSlot { + fn drop(&mut self) { + let Ok(mut open) = self.shared.open.lock() else { + return; + }; + if let Some(count) = open.get_mut(&self.source) { + *count -= 1; + if *count == 0 { + open.remove(&self.source); + } + } + } +} + +/// The room a connection holds for as long as it is open. +struct Admission { + _global: OwnedSemaphorePermit, + _source: SourceSlot, +} + +/// Room for a connection from `source`, if both the relay and the source have +/// some left. +fn admit(shared: &Arc, open: &Arc, source: Ipv4Addr) -> Option { + let global = Arc::clone(open).try_acquire_owned().ok()?; + let slot = SourceSlot::take(shared, source)?; + Some(Admission { + _global: global, + _source: slot, + }) +} + +/// The address the peer of `stream` dialed, from the conntrack entry of the +/// redirect. Fails for a connection that was not redirected. +fn original_dst(stream: &TcpStream) -> std::io::Result { + // SAFETY: `sockaddr_in` is plain data for which all zeroes is valid, and + // `getsockopt` writes at most `len` bytes into it. + let (rc, addr) = unsafe { + let mut addr: libc::sockaddr_in = std::mem::zeroed(); + let mut len = libc::socklen_t::try_from(size_of::()).unwrap_or(16); + let rc = libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_IP, + SO_ORIGINAL_DST, + (&raw mut addr).cast(), + &raw mut len, + ); + (rc, addr) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(SocketAddrV4::new( + Ipv4Addr::from_bits(u32::from_be(addr.sin_addr.s_addr)), + u16::from_be(addr.sin_port), + )) +} + +fn set_mark(socket: &TcpSocket, mark: u32) -> std::io::Result<()> { + // SAFETY: `setsockopt` reads `size_of::()` bytes from a live `u32`. + let rc = unsafe { + libc::setsockopt( + socket.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_MARK, + (&raw const mark).cast(), + libc::socklen_t::try_from(size_of::()).unwrap_or(4), + ) + }; + if rc == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +/// Read until [`inspect`] has an answer. `None` is a peer that closed, errored +/// or was too slow before it had one. +async fn read_hello(client: &mut TcpStream, limit: Duration) -> Option<(Vec, Verdict)> { + let read = async { + let mut seen = Vec::new(); + let mut chunk = [0_u8; 4096]; + loop { + let verdict = inspect(&seen); + if verdict != Verdict::Incomplete { + return Some((seen, verdict)); + } + if seen.len() > MAX_RECORDS * (MAX_RECORD + 5) { + return None; + } + match client.read(&mut chunk).await { + Ok(0) | Err(_) => return None, + Ok(n) => seen.extend_from_slice(&chunk[..n]), + } + } + }; + tokio::time::timeout(limit, read).await.ok().flatten() +} + +async fn upstream(shared: &Shared, peer: Ipv4Addr, dst: SocketAddrV4) -> Result { + let socket = TcpSocket::new_v4().context("socket")?; + set_mark(&socket, shared.mark).context("set the packet mark")?; + socket + .bind(SocketAddr::V4(SocketAddrV4::new(pool_source(peer), 0))) + .context("bind the pool address")?; + tokio::time::timeout(CONNECT_TIMEOUT, socket.connect(SocketAddr::V4(dst))) + .await + .context("connect timed out")? + .context("connect") +} + +async fn handle(shared: Arc, mut client: TcpStream, peer: SocketAddr) { + let SocketAddr::V4(peer) = peer else { + return; + }; + let Ok(dst) = original_dst(&client) else { + return; + }; + let Some((seen, verdict)) = read_hello(&mut client, HELLO_TIMEOUT).await else { + return; + }; + let action = if verdict == Verdict::Clean { + Action::Forward + } else { + let looked = { + let shared = Arc::clone(&shared); + let source = *peer.ip(); + tokio::task::spawn_blocking(move || shared.who(source)).await + }; + let (agent, log) = looked.unwrap_or((None, false)); + let action = decide(&verdict, log); + shared.report(*peer.ip(), *dst.ip(), &verdict, &action, agent.as_deref()); + action + }; + if action == Action::Refuse { + return; + } + let Ok(mut server) = upstream(&shared, *peer.ip(), dst).await else { + return; + }; + if server.write_all(&seen).await.is_err() { + return; + } + let _ = tokio::io::copy_bidirectional(&mut client, &mut server).await; +} + +async fn serve(listen: SocketAddr, mark: u32, agent_allow: Option) -> Result<()> { + let listener = TcpListener::bind(listen) + .await + .with_context(|| format!("listen on {listen}"))?; + let shared = Arc::new(Shared::new(mark, MAX_PER_SOURCE, agent_allow)); + let open = Arc::new(Semaphore::new(MAX_CONNECTIONS)); + loop { + let (client, peer) = match listener.accept().await { + Ok(accepted) => accepted, + Err(e) => { + eprintln!("{RELAY}: accept: {e}"); + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + }; + let SocketAddr::V4(source) = peer else { + continue; + }; + let Some(admission) = admit(&shared, &open, *source.ip()) else { + continue; + }; + let shared = Arc::clone(&shared); + tokio::spawn(async move { + handle(shared, client, peer).await; + drop(admission); + }); + } +} + +/// `0x…` or decimal. +fn parse_mark(text: &str) -> Option { + match text.strip_prefix("0x") { + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => text.parse().ok(), + } +} + +async fn start(args: &[String]) -> Result<()> { + let [listen, mark] = args else { + bail!("usage: {RELAY} "); + }; + let listen: SocketAddr = listen.parse().context("listen address")?; + let mark = parse_mark(mark).context("mark is a number")?; + serve(listen, mark, std::env::var(AGENT_ALLOW_ENV).ok()).await +} + +/// Run `egress-ech-relay` when `args` name it; `None` otherwise. Returns the +/// exit code. +pub(crate) async fn run(args: &[String]) -> Option { + let (name, rest) = args.split_first()?; + if name != RELAY { + return None; + } + Some(match start(rest).await { + Ok(()) => 0, + Err(e) => { + eprintln!("{RELAY}: {e:#}"); + 1 + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The pieces of a `ClientHello`, with the values a real client sends. + struct Hello { + session: Vec, + extensions: Vec<(u16, Vec)>, + } + + fn sni(name: &str) -> (u16, Vec) { + let mut data = Vec::new(); + data.extend_from_slice(&u16::try_from(name.len() + 3).unwrap().to_be_bytes()); + data.push(0); + data.extend_from_slice(&u16::try_from(name.len()).unwrap().to_be_bytes()); + data.extend_from_slice(name.as_bytes()); + (0, data) + } + + /// The extensions of a hello from OpenSSL 3: SNI, groups, signature + /// algorithms, supported versions, a key share. + fn plain() -> Hello { + Hello { + session: vec![7; 32], + extensions: vec![ + sni("allowed.example"), + (10, vec![0, 4, 0, 29, 0, 23]), + (13, vec![0, 4, 8, 4, 4, 3]), + (43, vec![2, 3, 4]), + (51, [&[0_u8, 36, 0, 29, 0, 32][..], &[9; 32][..]].concat()), + ], + } + } + + /// An ECH extension as a client sends it: `outer`, a KDF and AEAD, a config + /// id, a 32 byte `enc` and a payload. GREASE has the same layout. + fn ech_extension(kind: u16) -> (u16, Vec) { + let mut data = vec![0, 0, 1, 0, 1, 0x42, 0, 32]; + data.extend_from_slice(&[5; 32]); + data.extend_from_slice(&[0, 144]); + data.extend_from_slice(&[6; 144]); + (kind, data) + } + + impl Hello { + fn body(&self) -> Vec { + let mut body = vec![3, 3]; + body.extend_from_slice(&[1; 32]); + body.push(u8::try_from(self.session.len()).unwrap()); + body.extend_from_slice(&self.session); + body.extend_from_slice(&[0, 4, 0x13, 0x01, 0x13, 0x02]); + body.extend_from_slice(&[1, 0]); + let mut extensions = Vec::new(); + for (kind, data) in &self.extensions { + extensions.extend_from_slice(&kind.to_be_bytes()); + extensions.extend_from_slice(&u16::try_from(data.len()).unwrap().to_be_bytes()); + extensions.extend_from_slice(data); + } + body.extend_from_slice(&u16::try_from(extensions.len()).unwrap().to_be_bytes()); + body.extend_from_slice(&extensions); + body + } + + /// The handshake message: type, 24 bit length, body. + fn message(&self) -> Vec { + let body = self.body(); + let mut message = vec![1, 0]; + message.extend_from_slice(&u16::try_from(body.len()).unwrap().to_be_bytes()); + message.extend_from_slice(&body); + message + } + + /// One record holding the whole message. + fn record(&self) -> Vec { + records(&self.message(), usize::MAX) + } + } + + /// `message` in records of at most `size` bytes. + fn records(message: &[u8], size: usize) -> Vec { + let mut out = Vec::new(); + for part in message.chunks(size.min(message.len().max(1))) { + out.extend_from_slice(&[22, 3, 1]); + out.extend_from_slice(&u16::try_from(part.len()).unwrap().to_be_bytes()); + out.extend_from_slice(part); + } + out + } + + #[test] + fn a_hello_without_ech_is_clean() { + assert_eq!(inspect(&plain().record()), Verdict::Clean); + } + + #[test] + fn a_hello_with_the_ech_extension_is_ech_wherever_it_sits() { + for at in [0, 2, 5] { + let mut hello = plain(); + hello.extensions.insert(at, ech_extension(0xfe0d)); + assert_eq!(inspect(&hello.record()), Verdict::Ech, "at {at}"); + } + } + + #[test] + fn the_draft_codepoints_are_ech_and_their_neighbours_are_not() { + for (kind, expected) in [ + (0xfe0d, Verdict::Ech), + (0xff02, Verdict::Ech), + (0xffce, Verdict::Ech), + (0xfe08, Verdict::Ech), + (0xfe0c, Verdict::Ech), + (0xfe07, Verdict::Clean), + (0xfe0e, Verdict::Clean), + (0xff01, Verdict::Clean), + (0xfd00, Verdict::Clean), + ] { + let mut hello = plain(); + hello.extensions.push(ech_extension(kind)); + assert_eq!(inspect(&hello.record()), expected, "{kind:#x}"); + } + } + + #[test] + fn grease_ech_is_ech() { + // What a client with no configuration sends: the same extension, a + // random config id and random contents, and an empty session id. + let mut hello = plain(); + hello.session = Vec::new(); + let mut grease = vec![0, 0, 1, 0, 1, 0xf3, 0, 32]; + grease.extend_from_slice(&[0xa5; 32]); + grease.extend_from_slice(&[0, 176]); + grease.extend_from_slice(&[0x5a; 176]); + hello.extensions.push((0xfe0d, grease)); + assert_eq!(inspect(&hello.record()), Verdict::Ech); + } + + #[test] + fn a_hello_with_no_extensions_is_clean() { + let mut hello = plain(); + hello.extensions.clear(); + let mut body = hello.body(); + body.truncate(body.len() - 2); + let mut message = vec![1, 0]; + message.extend_from_slice(&u16::try_from(body.len()).unwrap().to_be_bytes()); + message.extend_from_slice(&body); + assert_eq!(inspect(&records(&message, usize::MAX)), Verdict::Clean); + } + + #[test] + fn a_hello_is_incomplete_until_its_last_byte() { + let mut hello = plain(); + hello.extensions.push(ech_extension(0xfe0d)); + let record = hello.record(); + for cut in 0..record.len() { + assert_eq!(inspect(&record[..cut]), Verdict::Incomplete, "cut at {cut}"); + } + assert_eq!(inspect(&record), Verdict::Ech); + } + + #[test] + fn a_hello_in_several_records_is_read_whole() { + let mut hello = plain(); + hello.extensions.push(ech_extension(0xfe0d)); + let message = hello.message(); + // Three records; the extension is in the last. + let split = records(&message, message.len().div_ceil(3)); + assert_eq!(inspect(&split), Verdict::Ech); + let mut clean = plain(); + clean.extensions.push((0x15, vec![0; 300])); + let message = clean.message(); + assert_eq!( + inspect(&records(&message, message.len().div_ceil(3))), + Verdict::Clean + ); + // A record that ends inside the handshake header. + assert_eq!( + inspect(&records(&message, 2)), + Verdict::Malformed("ClientHello spans too many records") + ); + } + + #[test] + fn a_hello_over_too_many_records_is_refused() { + let hello = plain(); + let message = hello.message(); + assert_eq!( + inspect(&records(&message, message.len().div_ceil(MAX_RECORDS))), + Verdict::Clean + ); + assert_eq!( + inspect(&records(&message, message.len().div_ceil(MAX_RECORDS + 1))), + Verdict::Malformed("ClientHello spans too many records") + ); + } + + #[test] + fn what_is_not_a_hello_is_refused() { + let cases: [(&str, Vec); 6] = [ + ("http", b"GET / HTTP/1.1\r\n\r\n".to_vec()), + ("application data", vec![23, 3, 3, 0, 4, 1, 2, 3, 4]), + ("alert", vec![21, 3, 3, 0, 2, 1, 0]), + ("sslv2", vec![0x80, 0x2e, 1, 0, 2, 0, 0, 0, 0]), + ("empty record", vec![22, 3, 1, 0, 0]), + ("server hello", vec![22, 3, 3, 0, 4, 2, 0, 0, 0]), + ]; + for (name, bytes) in cases { + assert!(matches!(inspect(&bytes), Verdict::Malformed(_)), "{name}"); + } + } + + #[test] + fn a_record_header_of_another_protocol_is_refused() { + // A well-formed hello behind a header that is not TLS 1.x handshake. + let good = plain().record(); + assert_eq!(inspect(&good), Verdict::Clean); + for (at, value) in [(0, 23_u8), (1, 2), (1, 4), (2, 5)] { + let mut bent = good.clone(); + bent[at] = value; + assert_eq!( + inspect(&bent), + Verdict::Malformed("not a TLS handshake record"), + "byte {at} = {value}" + ); + } + // The record versions real clients send. + for minor in [1_u8, 2, 3] { + let mut ok = good.clone(); + ok[2] = minor; + assert_eq!(inspect(&ok), Verdict::Clean, "TLS 1.{}", minor - 1); + } + } + + #[test] + fn a_record_over_the_largest_is_refused() { + let mut bytes = vec![22, 3, 1]; + bytes.extend_from_slice(&u16::try_from(MAX_RECORD + 1).unwrap().to_be_bytes()); + assert_eq!( + inspect(&bytes), + Verdict::Malformed("record length out of range") + ); + } + + #[test] + fn a_hello_over_the_largest_is_refused() { + let mut hello = plain(); + hello.extensions.push((0x15, vec![0; MAX_HELLO])); + let message = hello.message(); + assert_eq!( + inspect(&records(&message, MAX_RECORD)), + Verdict::Malformed("ClientHello too large") + ); + } + + #[test] + fn a_hello_whose_lengths_do_not_add_up_is_refused() { + let hello = plain(); + let extensions: usize = hello.extensions.iter().map(|(_, d)| 4 + d.len()).sum(); + // The two bytes before the extensions are their total length. + let total_at = hello.message().len() - extensions - 2; + for delta in [1_u8, 255] { + let mut bent = hello.message(); + bent[total_at + 1] = bent[total_at + 1].wrapping_add(delta); + assert_eq!( + inspect(&records(&bent, usize::MAX)), + Verdict::Malformed("ClientHello is malformed"), + "total off by {delta}" + ); + } + // The session id claims more than 32 bytes. + let mut bent = hello.message(); + bent[4 + 34] = 33; + assert_eq!( + inspect(&records(&bent, usize::MAX)), + Verdict::Malformed("ClientHello is malformed") + ); + // The cipher suites list is odd. + let mut bent = hello.message(); + bent[4 + 35 + 32 + 1] = 3; + assert!(matches!( + inspect(&records(&bent, usize::MAX)), + Verdict::Malformed(_) + )); + } + + #[test] + fn an_extension_that_overruns_the_hello_is_refused_even_before_an_ech_one() { + let mut hello = plain(); + hello.extensions.insert(0, (0x15, vec![0; 4])); + hello.extensions.push(ech_extension(0xfe0d)); + let mut message = hello.message(); + // The first extension's length now claims more than the rest holds. + let first = message.len() + - hello + .extensions + .iter() + .map(|(_, d)| 4 + d.len()) + .sum::(); + message[first + 3] = 0xff; + assert!(matches!( + inspect(&records(&message, usize::MAX)), + Verdict::Malformed(_) + )); + } + + #[test] + fn bytes_after_the_hello_are_refused() { + let mut message = plain().message(); + message.extend_from_slice(&[20, 0, 0, 0]); + assert_eq!( + inspect(&records(&message, usize::MAX)), + Verdict::Malformed("bytes after the ClientHello") + ); + } + + #[test] + fn pool_addresses_map_back_to_the_agent_that_made_the_flow() { + let agent = Ipv4Addr::new(10, 42, 0, 37); + let source = pool_source(agent); + assert_eq!(source, Ipv4Addr::new(198, 18, 0, 37)); + assert!(in_pool(source)); + assert!(!in_pool(agent)); + assert!(same_host(source, agent)); + // A different host of the subnet, and a lease that is itself in the pool. + assert!(!same_host(source, Ipv4Addr::new(10, 42, 0, 38))); + assert!(!same_host(source, source)); + // An address outside the pool is only itself. + assert!(same_host(agent, agent)); + assert!(!same_host(agent, Ipv4Addr::new(10, 42, 0, 38))); + // A /16 bridge keeps its two host bytes; the /15 pool has room for 17 bits. + assert_eq!( + pool_source(Ipv4Addr::new(10, 42, 1, 200)), + Ipv4Addr::new(198, 18, 1, 200) + ); + assert_eq!( + pool_source(Ipv4Addr::new(10, 43, 255, 254)), + Ipv4Addr::new(198, 19, 255, 254) + ); + assert!(!in_pool(Ipv4Addr::new(198, 20, 0, 1))); + assert!(!in_pool(Ipv4Addr::new(198, 17, 255, 255))); + } + + #[test] + fn a_source_logs_once_a_second() { + let t0 = Instant::now(); + let a = Ipv4Addr::new(10, 42, 0, 3); + let b = Ipv4Addr::new(10, 42, 0, 4); + let mut limit = LogLimit::default(); + assert!(limit.allow(a, t0)); + assert!(!limit.allow(a, t0 + Duration::from_millis(900))); + assert!(limit.allow(b, t0 + Duration::from_millis(900))); + assert!(limit.allow(a, t0 + Duration::from_millis(1001))); + } + + #[test] + fn the_log_limiter_stays_bounded() { + let t0 = Instant::now(); + let mut limit = LogLimit::default(); + for n in 0..u32::try_from(LOG_TRACKED * 3).unwrap() { + limit.allow(Ipv4Addr::from_bits(0x0A00_0000 + n), t0); + } + assert!(limit.last.len() <= LOG_TRACKED); + } + + #[test] + fn refusal_lines_name_the_agent_and_the_address() { + let dst = Ipv4Addr::new(203, 0, 113, 2); + assert_eq!( + refusal_line("hive-egress-ech-refused", Some("atlas"), dst, None), + "hive-egress-ech-refused agent=atlas dst=203.0.113.2" + ); + assert_eq!( + refusal_line("hive-egress-hello-refused", None, dst, Some("x")), + "hive-egress-hello-refused agent=- dst=203.0.113.2 reason=x" + ); + } + + #[test] + fn marks_are_decimal_or_hex() { + assert_eq!(parse_mark("0x6563"), Some(0x6563)); + assert_eq!(parse_mark("25955"), Some(25955)); + assert_eq!(parse_mark("x"), None); + } + + /// Both ends of a loopback connection. + async fn pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let near = TcpStream::connect(listener.local_addr().unwrap()); + let (far, accepted) = tokio::join!(near, listener.accept()); + (far.unwrap(), accepted.unwrap().0) + } + + #[tokio::test] + async fn a_hello_that_arrives_in_pieces_is_read_whole() { + let mut hello = plain(); + hello.extensions.push(ech_extension(0xfe0d)); + let bytes = hello.record(); + let (mut peer, mut server) = pair().await; + let send = tokio::spawn(async move { + for piece in bytes.chunks(37) { + peer.write_all(piece).await.unwrap(); + tokio::time::sleep(Duration::from_millis(2)).await; + } + peer + }); + let (seen, verdict) = read_hello(&mut server, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(verdict, Verdict::Ech); + assert_eq!(seen, hello.record()); + send.await.unwrap(); + } + + #[tokio::test] + async fn a_peer_that_stops_or_closes_early_gets_no_verdict() { + let record = plain().record(); + let (mut peer, mut server) = pair().await; + peer.write_all(&record[..40]).await.unwrap(); + // Still open and silent: the limit ends the wait. + let waited = tokio::time::timeout( + Duration::from_secs(5), + read_hello(&mut server, Duration::from_millis(200)), + ) + .await; + assert_eq!(waited.ok(), Some(None), "the limit did not end the read"); + // Closed with the hello cut off. + drop(peer); + assert!( + read_hello(&mut server, Duration::from_secs(5)) + .await + .is_none() + ); + // Control: the whole hello, sent at once, is read. + let (mut peer, mut server) = pair().await; + peer.write_all(&record).await.unwrap(); + let (_, verdict) = read_hello(&mut server, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(verdict, Verdict::Clean); + } + + #[test] + fn a_log_agent_is_observed_and_every_other_agent_refused() { + let ech = Verdict::Ech; + assert_eq!(decide(&Verdict::Clean, false), Action::Forward); + assert_eq!(decide(&Verdict::Clean, true), Action::Forward); + assert_eq!(decide(&ech, false), Action::Refuse); + assert_eq!(decide(&ech, true), Action::ForwardLogged); + // A hello that does not parse is refused for a log agent too. + let bad = Verdict::Malformed("x"); + assert_eq!(decide(&bad, true), Action::Refuse); + assert_eq!(decide(&bad, false), Action::Refuse); + } + + #[test] + fn the_log_mode_is_what_the_squid_include_says() { + let allow = "# source: swarm\n\ + acl agent_allow_atlas proxy_auth atlas\n\ + acl agent_log_logan proxy_auth logan\n\ + acl agent_log_logan_ix ext_user logan\n\ + http_access allow !icpt agent_log_logan !allowed_dst log_mark\n"; + assert!(is_log_agent(allow, "logan")); + // Control for the refusals: the same text names a restricted agent + // without a log rule, an agent whose name extends a log agent's, and + // the other ACL of the log agent. + assert!(!is_log_agent(allow, "atlas")); + assert!(!is_log_agent(allow, "log")); + assert!(!is_log_agent(allow, "logan2")); + assert!(!is_log_agent(allow, "logan_ix")); + assert!(!is_log_agent("", "logan")); + // The ACL must name the agent and nobody besides. + assert!(!is_log_agent( + "acl agent_log_logan proxy_auth logan other\n", + "logan" + )); + } + + fn shared(per_source: usize) -> Arc { + Arc::new(Shared::new(1, per_source, None)) + } + + #[test] + fn a_source_at_its_cap_is_refused_while_another_still_gets_through() { + let relay = shared(2); + let open = Arc::new(Semaphore::new(100)); + let (a, b) = (Ipv4Addr::new(10, 42, 0, 3), Ipv4Addr::new(10, 42, 0, 4)); + let first = admit(&relay, &open, a); + let second = admit(&relay, &open, a); + assert!(first.is_some() && second.is_some(), "a is under its cap"); + assert!(admit(&relay, &open, a).is_none(), "a is at its cap"); + assert!(admit(&relay, &open, b).is_some(), "b is not"); + // The slot is released when the connection ends, and not before. + drop(first); + let third = admit(&relay, &open, a); + assert!(third.is_some(), "a slot of a came free"); + assert!(admit(&relay, &open, a).is_none(), "a is at its cap again"); + drop((second, third)); + assert!( + relay.open.lock().unwrap().get(&a).is_none(), + "nothing is kept for a closed source" + ); + } + + #[test] + fn the_relay_as_a_whole_is_capped_whatever_the_source() { + let relay = shared(10); + let open = Arc::new(Semaphore::new(2)); + let (a, b, c) = ( + Ipv4Addr::new(10, 42, 0, 3), + Ipv4Addr::new(10, 42, 0, 4), + Ipv4Addr::new(10, 42, 0, 5), + ); + let held = (admit(&relay, &open, a), admit(&relay, &open, b)); + assert!(held.0.is_some() && held.1.is_some()); + assert!(admit(&relay, &open, c).is_none(), "the relay is full"); + // A refusal for the relay's cap does not leave a slot taken for c. + assert!(relay.open.lock().unwrap().get(&c).is_none()); + drop(held); + assert!(admit(&relay, &open, c).is_some()); + } + + #[tokio::test] + async fn other_sub_commands_are_not_ours() { + assert_eq!(run(&["egress-whois".to_owned()]).await, None); + assert_eq!(run(&[]).await, None); + } +} diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index fe6d004e4..52381289f 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -23,7 +23,8 @@ //! turns ephemeral, with [`salvage_io`] for its space, reflink and lock calls), //! [`prestart`] (the check of a container's root that runs before each start, //! as a sub-command), [`egress`] (the bridge anti-spoofing bindings, also -//! sub-commands), [`storage`] (btrfs subvolumes, quotas, snapshots, disk +//! sub-commands), [`egress_ech`] (the `ClientHello` check in front of squid's +//! HTTPS intercept port), [`storage`] (btrfs subvolumes, quotas, snapshots, disk //! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles), //! [`infra`] (gateway nginx, infra containers) and [`closure`] (the swarm-signature //! check before an apply, and the cache-facts refresh). Helpers more than one family uses are in [`shared`]. @@ -32,6 +33,7 @@ mod closure; mod container; mod container_config; mod egress; +mod egress_ech; mod files; mod infra; mod prestart; @@ -60,6 +62,10 @@ async fn main() -> Result<()> { if let Some(code) = egress::run(&args) { std::process::exit(code); } + // `egress-ech-relay` is a long-running unit of its own, not a request either. + if let Some(code) = egress_ech::run(&args).await { + std::process::exit(code); + } tracing_subscriber::fmt() .with_env_filter( diff --git a/nix/host-modules/hive-egress.nix b/nix/host-modules/hive-egress.nix index 096fb07ad..117989f63 100644 --- a/nix/host-modules/hive-egress.nix +++ b/nix/host-modules/hive-egress.nix @@ -13,10 +13,10 @@ # the host routes from the agent bridge to anywhere else is logged and # counted, never blocked. `intercept` (`egress.intercept.enable`, set by # hive-squid.nix): the web traffic of the bridge's agents is redirected into -# squid's intercept ports. The traffic of the ports in the set `direct_mac` is -# left alone: the service containers and the unrestricted agents. `hive-priv -# egress-pin` and `egress-load` fill that set from the bindings the bridge -# table uses, so a MAC in no binding is intercepted, and squid refuses the flow. +# squid's intercept ports (port 443 through the ClientHello relay). The ports in +# `direct_mac` are left alone: the service containers and the unrestricted +# agents. `hive-priv egress-pin` and `egress-load` fill that set from the bridge +# table's bindings, so a MAC in no binding is intercepted, and squid refuses it. # # The `bridge` table checks the source address of every frame a port of the # agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because @@ -42,6 +42,19 @@ let # The mesh interface (swarm-wireguard.nix): inter-hive traffic is not egress. meshInterface = "wg-hive"; + # With `refuseEch`, the redirect of port 443 ends at `hive-priv + # egress-ech-relay`, which refuses a ClientHello that carries the ECH + # extension and sends the rest on to squid's HTTPS intercept port. The relay + # connects to the address the agent dialed, bound to a source address of the + # pool and carrying the mark; the `output` chain DNATs those connections to + # squid. `egress_ech.rs` derives the pool address from the agent's and + # `egress-whois` maps it back, so the pool is the one defined there. + intercept = cfg.egress.intercept; + refuseEch = intercept.enable && intercept.refuseEch; + relayPool = "198.18.0.0/15"; + relayMark = "0x45434800"; + relayCommand = "${hivePriv} egress-ech-relay ${cfg.bridgeIp}:${toString intercept.relayPort} ${relayMark}"; + # The sub-commands of hive-priv that fill the bridge table's sets. The path is # the one every platform container's unit uses (hive-priv.nix points it at the # current build). @@ -85,9 +98,9 @@ let # the interface the route to the destination leaves by: the bridge (another # container), `lo` (an address of the host: `fib daddr type local`) and the # mesh interface are not egress. The redirect rewrites the destination to - # the bridge address and the port of squid's intercept listener, which reads - # the original destination from conntrack. IPv6 is left alone: the agent - # bridge has no IPv6 addressing. + # the bridge address and the port of squid's intercept listener (for port 443 + # with `refuseEch`, of the relay), which reads the original destination from + # conntrack. IPv6 is left alone: the agent bridge has no IPv6 addressing. interceptChain = '' chain intercept { type nat hook prerouting priority dstnat; policy accept; @@ -95,7 +108,16 @@ let iifname "${br}" meta nfproto ipv4 fib daddr type local return iifname "${br}" meta nfproto ipv4 fib daddr oifname { "${br}", "${meshInterface}" } return iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString cfg.egress.intercept.httpPort} - iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${toString cfg.egress.intercept.httpsPort} + iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${ + toString (if refuseEch then intercept.relayPort else intercept.httpsPort) + } + } + '' + + lib.optionalString refuseEch '' + + chain intercept_relay { + type nat hook output priority dstnat; policy accept; + meta nfproto ipv4 ip saddr ${relayPool} meta mark ${relayMark} tcp dport 443 counter dnat ip to ${cfg.bridgeIp}:${toString intercept.httpsPort} } ''; @@ -206,6 +228,7 @@ let type filter hook input priority filter - 10; policy accept; iifname "${br}" ip daddr 127.0.0.0/8 counter drop iifname "${br}" ip6 daddr ::1 counter drop + ${lib.optionalString refuseEch ''iifname "${br}" ip saddr ${relayPool} counter drop''} } set direct_mac { type ether_addr; } ${lib.optionalString cfg.egress.intercept.enable interceptChain} @@ -295,8 +318,55 @@ in type = lib.types.port; default = 3130; description = '' - The port on the bridge address where squid listens for the redirected - TCP port 443 traffic. + The port on the bridge address where squid listens for TCP port 443 + traffic: what the ClientHello relay sends on, or with `refuseEch` off + the redirected traffic itself. + ''; + }; + + relayPort = lib.mkOption { + type = lib.types.port; + default = 3131; + description = '' + The port on the bridge address where `hive-priv egress-ech-relay` + listens for the redirected TCP port 443 traffic (`refuseEch`). + ''; + }; + + refuseEch = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Refuse a TLS ClientHello that carries the `encrypted_client_hello` + extension on the redirected port 443 traffic. Squid 7.6 sees only the + outer name of such a hello, so without this an agent that holds an ECH + configuration could dial an allowed name and reach another site behind + the same address. + + `hive-priv egress-ech-relay` reads the first TLS record of each + redirected connection and, if the hello is clean, sends the connection + on to squid. The extension of a client with no configuration (GREASE, + which Chrome and Firefox send by default) looks the same as a real one on + the wire, so it is refused too: those clients fail on the redirected + path and work through the proxy variables. An agent in `log` mode is + observed instead: its hello goes on and the relay logs + `hive-egress-ech-would-refuse`. Turn this off to send port 443 + straight to squid, with ECH unchecked. + ''; + }; + + agentAllowFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + The squid include `hive-c0re` renders with each agent's own rules, in + which an agent in `log` mode has an `acl agent_log_ proxy_auth + ` line. The ClientHello relay (`refuseEch`) reads it to tell a + `log` agent, whose ECH hello is let through and logged as + `hive-egress-ech-would-refuse`, from a restricted one, whose hello is + refused. Squid reads the same file for the same split. `null` or an + unreadable file lists no `log` agent, so every agent's ECH hello is + refused. ''; }; @@ -349,6 +419,14 @@ in Use "log" or "accept". ''; } + { + assertion = !refuseEch || lib.toInt (lib.last (lib.splitString "/" cfg.networkCidr)) >= 15; + message = '' + services.hyperhive.network.bridgePrefixLength must be 15 or more while + egress.intercept.refuseEch is on: the relay names a flow's source with + the low 17 bits of the agent's address. + ''; + } { assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys); message = '' @@ -360,6 +438,53 @@ in } ]; + services.hyperhive.network.exposeHostPorts = lib.mkIf refuseEch [ intercept.relayPort ]; + + # The relay is not part of the tables: with it down, the redirect of port 443 + # reaches a closed port and the agent's connection is refused. + systemd.services.hive-egress-ech-relay = lib.mkIf refuseEch { + description = "refuse TLS hellos with encrypted client hello on the way to squid's HTTPS intercept port"; + wantedBy = [ "multi-user.target" ]; + wants = [ "hive-egress.service" ]; + after = [ + "hive-egress.service" + "systemd-tmpfiles-setup.service" + "systemd-tmpfiles-resetup.service" + ]; + environment = lib.optionalAttrs (intercept.agentAllowFile != null) { + HIVE_EGRESS_AGENT_ALLOW = toString intercept.agentAllowFile; + }; + serviceConfig = { + # The pool is local, so the relay can bind to a source address in it. + ExecStartPre = "+${pkgs.iproute2}/bin/ip route replace local ${relayPool} dev lo"; + ExecStart = relayCommand; + ExecStopPost = "+-${pkgs.iproute2}/bin/ip route del local ${relayPool} dev lo"; + # The bridge address may not be assigned yet at first start. + Restart = "on-failure"; + RestartSec = 5; + # The packet mark needs CAP_NET_ADMIN. The include is `squid`'s group's. + DynamicUser = true; + SupplementaryGroups = lib.optional (intercept.agentAllowFile != null) "squid"; + AmbientCapabilities = "CAP_NET_ADMIN"; + CapabilityBoundingSet = "CAP_NET_ADMIN"; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + RestrictAddressFamilies = "AF_INET"; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + LockPersonality = true; + SystemCallArchitectures = "native"; + }; + }; + systemd.services.hive-egress = { description = "hive nftables tables (agent bridge egress log and source check, host loopback drop)"; wantedBy = [ "multi-user.target" ]; diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index d88210195..f301c7fa6 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -25,6 +25,8 @@ # `hive_agent`); a source it cannot name is refused. The listed-names and deny # rules are the same ones, matched on the identity from that helper. HTTPS is # peeked at for its SNI and spliced: squid checks the name and never decrypts. +# Squid cannot see an ECH extension, so the flows to port 443 reach it through +# `hive-priv egress-ech-relay`, which refuses a hello that carries one. { pkgs, lib, @@ -551,7 +553,7 @@ in ]; services.hyperhive.network.egress.intercept = { enable = lib.mkDefault true; - inherit (cfg) unrestrictedFile; + inherit (cfg) unrestrictedFile agentAllowFile; }; services.hyperhive.network.enable = lib.mkDefault true; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 105ca2b13..435bc034d 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -88,7 +88,10 @@ let # Every hive defaults the agents' queue address to the queue's name, so the # only hive without one is a hive told to have none. Also the host still # setting the four removed host options; no case on its queue reads them. + # It is also the one controller host with `egress.intercept.refuseEch` off, + # which the egress cases read. noAgentQueue = hiveWithController { + network.egress.intercept.refuseEch = false; deploy.hive-controller.queue.agentNatsUrl = null; c0re.operatorPronouns = "they/them"; c0re.contextWindowTokens.sonnet = 900000; @@ -1700,7 +1703,9 @@ let lib.splitString "table bridge hive_egress {" bare.services.hyperhive.network.egress.ruleset ); chain = lib.head ( - lib.splitString "set log_v4" (lib.last (lib.splitString "chain intercept {" inet)) + lib.splitString "chain intercept_relay {" ( + lib.head (lib.splitString "set log_v4" (lib.last (lib.splitString "chain intercept {" inet))) + ) ); rules = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" chain)); idx = pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules; @@ -1714,12 +1719,72 @@ let && idx ''fib daddr oifname { "hive-br0", "wg-hive" } return'' < idx "tcp dport 80" && lib.length redirects == 2 && lib.hasInfix "tcp dport 80 counter redirect to :3129" (lib.head redirects) - && lib.hasInfix "tcp dport 443 counter redirect to :3130" (lib.last redirects) + # Port 443 goes to the ClientHello relay, not to squid. + && lib.hasInfix "tcp dport 443 counter redirect to :3131" (lib.last redirects) # Only IPv4 that comes in on the bridge is looked at. && lib.all (l: lib.hasPrefix ''iifname "hive-br0" meta nfproto ipv4 '' l) (lib.drop 1 rules) && lib.hasInfix "set direct_mac { type ether_addr; }" without && !(lib.hasInfix "chain intercept" without); } + { + # With the relay on, port 443 reaches squid through `hive-priv + # egress-ech-relay`: its connections (from the pool, carrying the mark) + # are DNATed to squid's HTTPS port, the bridge may not send from the + # pool, and the unit routes the pool to `lo`. `noAgentQueue` has the + # relay off: the redirect then goes straight to squid and none of the + # rest is rendered. Its intercept chain is the presence control. + name = "the ClientHello relay sits between the redirect and squid, and refuseEch off removes it"; + ok = + let + ruleset = m: m.services.hyperhive.network.egress.ruleset; + unit = bare.systemd.services.hive-egress-ech-relay; + off = noAgentQueue; + in + lib.hasInfix "chain intercept_relay {" (ruleset bare) + && lib.hasInfix "type nat hook output priority dstnat; policy accept;" (ruleset bare) + && lib.hasInfix "meta nfproto ipv4 ip saddr 198.18.0.0/15 meta mark 0x45434800 tcp dport 443 counter dnat ip to 10.42.0.1:3130" ( + ruleset bare + ) + && lib.hasInfix ''iifname "hive-br0" ip saddr 198.18.0.0/15 counter drop'' (ruleset bare) + && lib.hasInfix "egress-ech-relay 10.42.0.1:3131 0x45434800" unit.serviceConfig.ExecStart + && lib.hasInfix "ip route replace local 198.18.0.0/15 dev lo" unit.serviceConfig.ExecStartPre + && lib.elem "multi-user.target" unit.wantedBy + # The relay reads the include in which squid finds the `log` agents. + && + (unit.environment.HIVE_EGRESS_AGENT_ALLOW or null) + == bare.services.hyperhive.deploy.hive-controller.squid.agentAllowFile + && unit.serviceConfig.SupplementaryGroups == [ "squid" ] + && lib.elem 3131 bare.services.hyperhive.network.exposeHostPorts + && lib.hasInfix "chain intercept {" (ruleset off) + && lib.hasInfix "tcp dport 443 counter redirect to :3130" (ruleset off) + && lib.elem 3130 off.services.hyperhive.network.exposeHostPorts + && !(lib.hasInfix "intercept_relay" (ruleset off)) + && !(lib.hasInfix "198.18.0.0/15" (ruleset off)) + && !(off.systemd.services ? hive-egress-ech-relay) + && !(lib.elem 3131 off.services.hyperhive.network.exposeHostPorts); + } + { + # The relay names a flow's source with the low 17 bits of the agent's + # address, which is unique only in a subnet of /15 or smaller. The + # boundary is read at /15 (accepted) and /14 (refused); `bare`, a /24, is + # the control that the lookup finds nothing otherwise. + name = "the ClientHello relay is refused on a bridge subnet larger than a /15"; + ok = + let + refused = + m: + lib.any ( + a: !a.assertion && lib.hasInfix "bridgePrefixLength must be 15 or more" a.message + ) m.assertions; + in + refused (hiveWithController { + network.bridgePrefixLength = 14; + }) + && !(refused (hiveWithController { + network.bridgePrefixLength = 15; + })) + && !(refused bare); + } { # Squid listens on the two intercept ports, peeks and splices and never # decrypts, names the agent of a flow with the helper, refuses a source diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 426bb1737..abda20011 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -216,6 +216,7 @@ pkgs.testers.runNixOSTest { "L+ /run/hive-rootfs-check - - - - ${pkgs.writeShellScript "hive-priv-stub" '' [ "$1" = egress-load ] && exec ${pkgs.nftables}/bin/nft -f "$2" [ "$1" = egress-whois ] && exec ${hivePriv}/bin/hive-priv "$@" + [ "$1" = egress-ech-relay ] && exec ${hivePriv}/bin/hive-priv "$@" exit 0 ''}" ]; @@ -319,7 +320,10 @@ pkgs.testers.runNixOSTest { "agentonly.t.local" "api.t.local" ]; - environment.systemPackages = [ pkgs.openssl ]; + environment.systemPackages = [ + pkgs.openssl + pkgs.python3 + ]; networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ { address = "192.168.1.3"; @@ -957,7 +961,7 @@ pkgs.testers.runNixOSTest { def agent_lines(): # Log lines of requests from the agent node's addresses. The self-check # connects from the bridge address, so its lines are not counted. - return len(re.findall(r" 192\.168\.1\.[347] [A-Z_]+/\d+ ", journal())) + return len(re.findall(r" (?:192\.168\.1|198\.18\.0)\.[347] [A-Z_]+/\d+ ", journal())) seeded = proxy.succeed(f"cat {ALLOWED}") bind("atlas", "logan") @@ -980,7 +984,7 @@ pkgs.testers.runNixOSTest { with checked("a restricted agent's allowed name is tunnelled without the proxy variables, and the log names the agent"): wait_icpt("https://forge.t.local/", "200") assert re.search( - r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ORIGINAL_DST/203\.0\.113\.2 ", journal() + r"198\.18\.0\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ORIGINAL_DST/203\.0\.113\.2 ", journal() ), "the HTTPS flow, as atlas" assert icpt("http://forge.t.local/") == "200", "plain HTTP" assert re.search(r"192\.168\.1\.3 TCP_\w+/200 .* GET http://forge\.t\.local/ atlas ", journal()), "the HTTP flow, as atlas" @@ -992,7 +996,7 @@ pkgs.testers.runNixOSTest { with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"): assert icpt("https://other.t.local/") == "000", "HTTPS" - assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal()), "logged as atlas" + assert re.search(r"198\.18\.0\.3 TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal()), "logged as atlas" assert icpt("http://other.t.local/") == "403", "plain HTTP" assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas" # Control: the same address reaches the listed name, so the refusal is the policy's. @@ -1027,7 +1031,7 @@ pkgs.testers.runNixOSTest { with checked("a source the bindings do not name is refused, and so is every source when they are gone"): assert icpt("https://forge.t.local/", NOBODY_IP) == "000", "no lease for the address" - assert re.search(r"192\.168\.1\.7 TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 - ", journal()), "logged without a user" + assert re.search(r"198\.18\.0\.7 TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 - ", journal()), "logged without a user" assert icpt("http://forge.t.local/", NOBODY_IP) == "403" assert icpt("https://forge.t.local/") == "200", "control: the bound address, the same name" # A hive whose pin file is missing: the lease names a MAC, no port holds it. @@ -1056,6 +1060,90 @@ pkgs.testers.runNixOSTest { ) assert out.strip() == "", "bytes that are no ClientHello get no answer" + # The ClientHello a client sends, built by hand: SNI, groups, signature + # algorithms and supported versions, and with `ech` the encrypted_client_hello + # extension (0xfe0d) with the layout a real one has. It prints how the + # connection ended: the first byte of what came back, or `closed`. + agent.succeed( + "cat > /tmp/hello.py <<'PY'\n" + "import os, socket, struct, sys\n" + "mode, name, dst, src = sys.argv[1:5]\n" + "def ext(kind, data):\n" + " return struct.pack('>HH', kind, len(data)) + data\n" + "host = name.encode()\n" + "exts = ext(0, struct.pack('>HBH', len(host) + 3, 0, len(host)) + host)\n" + "exts += ext(10, struct.pack('>HHH', 4, 29, 23)) + ext(13, struct.pack('>HHH', 4, 0x0804, 0x0403))\n" + "exts += ext(43, bytes([2, 3, 4])) + ext(51, struct.pack('>H', 0))\n" + "if mode == 'ech':\n" + " exts += ext(0xFE0D, b'\\x00' + struct.pack('>HHBH', 1, 1, 7, 32) + os.urandom(32) + struct.pack('>H', 144) + os.urandom(144))\n" + "body = b'\\x03\\x03' + os.urandom(32) + b'\\x20' + os.urandom(32) + struct.pack('>H', 2) + b'\\x13\\x01' + b'\\x01\\x00'\n" + "body += struct.pack('>H', len(exts)) + exts\n" + "hello = b'\\x01' + struct.pack('>I', len(body))[1:] + body\n" + "s = socket.socket()\n" + "s.settimeout(10)\n" + "s.bind((src, 0))\n" + "s.connect((dst, 443))\n" + "s.sendall(b'\\x16\\x03\\x01' + struct.pack('>H', len(hello)) + hello)\n" + "try:\n" + " first = s.recv(1)\n" + "except OSError:\n" + " first = b'''\n" + "print(first.hex() or 'closed')\n" + "PY" + ) + + def send_hello(mode, src=ATLAS_IP): + return agent.succeed(f"python3 /tmp/hello.py {mode} forge.t.local 203.0.113.2 {src}").strip() + + with checked("a ClientHello with the ECH extension is refused for a restricted agent and observed for a log agent, and the same hello without it is let through"): + seen = agent_lines() + # Control: the origin answers a hello without the extension (a + # handshake record, 0x16), and squid logs the connection as atlas. + assert send_hello("plain") == "16", "the origin answers the hello without ECH" + time.sleep(2) + assert agent_lines() > seen, "squid saw the connection" + seen = agent_lines() + assert send_hello("ech") == "closed", "a hello with an ECH extension gets no answer" + proxy.wait_until_succeeds( + "journalctl -u hive-egress-ech-relay.service -o cat | " + "grep -E '^hive-egress-ech-refused agent=atlas dst=203\\.0\\.113\\.2$'" + ) + time.sleep(2) + assert agent_lines() == seen, "squid saw nothing of it" + # A `log` agent is observed, not refused: its ECH hello goes on to squid + # and the origin, and the relay logs what it would have done. The mode is + # what the squid include says (`acl agent_log_logan proxy_auth logan`). + seen = agent_lines() + assert send_hello("ech", LOGAN_IP) == "16", "logan's hello with ECH is let through" + proxy.wait_until_succeeds( + "journalctl -u hive-egress-ech-relay.service -o cat | " + "grep -E '^hive-egress-ech-would-refuse agent=logan dst=203\\.0\\.113\\.2$'" + ) + time.sleep(2) + assert agent_lines() > seen, "squid saw logan's connection" + assert "hive-egress-ech-refused agent=logan" not in proxy.succeed( + "journalctl -u hive-egress-ech-relay.service -o cat --no-pager" + ), "logan was not refused" + assert "ech-would-refuse agent=atlas" not in proxy.succeed( + "journalctl -u hive-egress-ech-relay.service -o cat --no-pager" + ), "control: the restricted agent got the refusal, not the observation" + assert send_hello("plain", LOGAN_IP) == "16", "control: logan's hello without ECH" + # A source the bindings do not name is refused too, logged without a name. + assert send_hello("ech", NOBODY_IP) == "closed" + proxy.wait_until_succeeds( + "journalctl -u hive-egress-ech-relay.service -o cat | " + "grep -E '^hive-egress-ech-refused agent=- dst=203\\.0\\.113\\.2$'" + ) + # Bytes that are no ClientHello: refused by the relay, with a line of their own. + out = agent.succeed( + "timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true" + ) + assert out.strip() == "", "bytes that are no ClientHello get no answer" + proxy.wait_until_succeeds( + "journalctl -u hive-egress-ech-relay.service -o cat | " + "grep -E '^hive-egress-hello-refused agent=atlas dst=203\\.0\\.113\\.2 reason=not a TLS handshake record$'" + ) + with checked("the traffic that stays on the bridge or is the unrestricted agents' is not intercepted"): seen = agent_lines() # The gateway on the bridge address, in the stand-in's port 443. @@ -1068,6 +1156,7 @@ pkgs.testers.runNixOSTest { assert icpt("https://other.t.local/") == "200", "an unlisted name, directly" assert icpt("https://203.0.113.2/") == "200", "an IP literal, directly" assert agent_lines() == seen, "squid saw nothing of it" + assert send_hello("ech") == "16", "a hello with ECH, directly: the relay is not in the path" proxy.wait_until_succeeds( "journalctl -k -o cat --no-pager | grep -E " "'hive-egress-would-drop(-unmetered)? IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=443 '"