Watch
0
0
Fork
You've already forked hyperhive
0

refuse tls hellos that carry encrypted client hello before squid sees them

This commit is contained in:
atlas 2026-11-13 06:23:42 +01:00 • committed by mara
commit dff662c6b3
4 changed files with 1537 additions and 43 deletions

View file

@ -13,10 +13,10 @@
# the host routes from the agent bridge to anywhere else is logged and
# counted, never blocked. `intercept` (`egress.intercept.enable`, set by
# hive-squid.nix): the web traffic of the bridge's agents is redirected into
# squid's intercept ports. The traffic of the ports in the set `direct_mac` is
# left alone: the service containers and the unrestricted agents. `hive-priv
# egress-pin` and `egress-load` fill that set from the bindings the bridge
# table uses, so a MAC in no binding is intercepted, and squid refuses the flow.
# squid's intercept ports (port 443 through the ClientHello relay). The ports in
# `direct_mac` are left alone: the service containers and the unrestricted
# agents. `hive-priv egress-pin` and `egress-load` fill that set from the bridge
# table's bindings, so a MAC in no binding is intercepted, and squid refuses it.
#
# The `bridge` table checks the source address of every frame a port of the
# agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because
@ -42,6 +42,19 @@ let
# The mesh interface (swarm-wireguard.nix): inter-hive traffic is not egress.
meshInterface = "wg-hive";
# With `refuseEch`, the redirect of port 443 ends at `hive-priv
# egress-ech-relay`, which refuses a ClientHello that carries the ECH
# extension and sends the rest on to squid's HTTPS intercept port. The relay
# connects to the address the agent dialed, bound to a source address of the
# pool and carrying the mark; the `output` chain DNATs those connections to
# squid. `egress_ech.rs` derives the pool address from the agent's and
# `egress-whois` maps it back, so the pool is the one defined there.
intercept = cfg.egress.intercept;
refuseEch = intercept.enable && intercept.refuseEch;
relayPool = "198.18.0.0/15";
relayMark = "0x45434800";
relayCommand = "${hivePriv} egress-ech-relay ${cfg.bridgeIp}:${toString intercept.relayPort} ${relayMark}";
# The sub-commands of hive-priv that fill the bridge table's sets. The path is
# the one every platform container's unit uses (hive-priv.nix points it at the
# current build).
@ -85,9 +98,9 @@ let
# the interface the route to the destination leaves by: the bridge (another
# container), `lo` (an address of the host: `fib daddr type local`) and the
# mesh interface are not egress. The redirect rewrites the destination to
# the bridge address and the port of squid's intercept listener, which reads
# the original destination from conntrack. IPv6 is left alone: the agent
# bridge has no IPv6 addressing.
# the bridge address and the port of squid's intercept listener (for port 443
# with `refuseEch`, of the relay), which reads the original destination from
# conntrack. IPv6 is left alone: the agent bridge has no IPv6 addressing.
interceptChain = ''
chain intercept {
type nat hook prerouting priority dstnat; policy accept;
@ -95,7 +108,16 @@ let
iifname "${br}" meta nfproto ipv4 fib daddr type local return
iifname "${br}" meta nfproto ipv4 fib daddr oifname { "${br}", "${meshInterface}" } return
iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString cfg.egress.intercept.httpPort}
iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${toString cfg.egress.intercept.httpsPort}
iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${
toString (if refuseEch then intercept.relayPort else intercept.httpsPort)
}
}
''
+ lib.optionalString refuseEch ''
chain intercept_relay {
type nat hook output priority dstnat; policy accept;
meta nfproto ipv4 ip saddr ${relayPool} meta mark ${relayMark} tcp dport 443 counter dnat ip to ${cfg.bridgeIp}:${toString intercept.httpsPort}
}
'';
@ -206,6 +228,7 @@ let
type filter hook input priority filter - 10; policy accept;
iifname "${br}" ip daddr 127.0.0.0/8 counter drop
iifname "${br}" ip6 daddr ::1 counter drop
${lib.optionalString refuseEch ''iifname "${br}" ip saddr ${relayPool} counter drop''}
}
set direct_mac { type ether_addr; }
${lib.optionalString cfg.egress.intercept.enable interceptChain}
@ -295,8 +318,55 @@ in
type = lib.types.port;
default = 3130;
description = ''
The port on the bridge address where squid listens for the redirected
TCP port 443 traffic.
The port on the bridge address where squid listens for TCP port 443
traffic: what the ClientHello relay sends on, or with `refuseEch` off
the redirected traffic itself.
'';
};
relayPort = lib.mkOption {
type = lib.types.port;
default = 3131;
description = ''
The port on the bridge address where `hive-priv egress-ech-relay`
listens for the redirected TCP port 443 traffic (`refuseEch`).
'';
};
refuseEch = lib.mkOption {
type = lib.types.bool;
default = true;
description = ''
Refuse a TLS ClientHello that carries the `encrypted_client_hello`
extension on the redirected port 443 traffic. Squid 7.6 sees only the
outer name of such a hello, so without this an agent that holds an ECH
configuration could dial an allowed name and reach another site behind
the same address.
`hive-priv egress-ech-relay` reads the first TLS record of each
redirected connection and, if the hello is clean, sends the connection
on to squid. The extension of a client with no configuration (GREASE,
which Chrome and Firefox send by default) looks the same as a real one on
the wire, so it is refused too: those clients fail on the redirected
path and work through the proxy variables. An agent in `log` mode is
observed instead: its hello goes on and the relay logs
`hive-egress-ech-would-refuse`. Turn this off to send port 443
straight to squid, with ECH unchecked.
'';
};
agentAllowFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The squid include `hive-c0re` renders with each agent's own rules, in
which an agent in `log` mode has an `acl agent_log_<name> proxy_auth
<name>` line. The ClientHello relay (`refuseEch`) reads it to tell a
`log` agent, whose ECH hello is let through and logged as
`hive-egress-ech-would-refuse`, from a restricted one, whose hello is
refused. Squid reads the same file for the same split. `null` or an
unreadable file lists no `log` agent, so every agent's ECH hello is
refused.
'';
};
@ -349,6 +419,14 @@ in
Use "log" or "accept".
'';
}
{
assertion = !refuseEch || lib.toInt (lib.last (lib.splitString "/" cfg.networkCidr)) >= 15;
message = ''
services.hyperhive.network.bridgePrefixLength must be 15 or more while
egress.intercept.refuseEch is on: the relay names a flow's source with
the low 17 bits of the agent's address.
'';
}
{
assertion = !(lib.any (k: isOn (config.boot.kernel.sysctl.${k} or 0)) routeLocalnetKeys);
message = ''
@ -360,6 +438,53 @@ in
}
];
services.hyperhive.network.exposeHostPorts = lib.mkIf refuseEch [ intercept.relayPort ];
# The relay is not part of the tables: with it down, the redirect of port 443
# reaches a closed port and the agent's connection is refused.
systemd.services.hive-egress-ech-relay = lib.mkIf refuseEch {
description = "refuse TLS hellos with encrypted client hello on the way to squid's HTTPS intercept port";
wantedBy = [ "multi-user.target" ];
wants = [ "hive-egress.service" ];
after = [
"hive-egress.service"
"systemd-tmpfiles-setup.service"
"systemd-tmpfiles-resetup.service"
];
environment = lib.optionalAttrs (intercept.agentAllowFile != null) {
HIVE_EGRESS_AGENT_ALLOW = toString intercept.agentAllowFile;
};
serviceConfig = {
# The pool is local, so the relay can bind to a source address in it.
ExecStartPre = "+${pkgs.iproute2}/bin/ip route replace local ${relayPool} dev lo";
ExecStart = relayCommand;
ExecStopPost = "+-${pkgs.iproute2}/bin/ip route del local ${relayPool} dev lo";
# The bridge address may not be assigned yet at first start.
Restart = "on-failure";
RestartSec = 5;
# The packet mark needs CAP_NET_ADMIN. The include is `squid`'s group's.
DynamicUser = true;
SupplementaryGroups = lib.optional (intercept.agentAllowFile != null) "squid";
AmbientCapabilities = "CAP_NET_ADMIN";
CapabilityBoundingSet = "CAP_NET_ADMIN";
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectKernelLogs = true;
ProtectControlGroups = true;
RestrictAddressFamilies = "AF_INET";
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
LockPersonality = true;
SystemCallArchitectures = "native";
};
};
systemd.services.hive-egress = {
description = "hive nftables tables (agent bridge egress log and source check, host loopback drop)";
wantedBy = [ "multi-user.target" ];

View file

@ -25,6 +25,8 @@
# `hive_agent`); a source it cannot name is refused. The listed-names and deny
# rules are the same ones, matched on the identity from that helper. HTTPS is
# peeked at for its SNI and spliced: squid checks the name and never decrypts.
# Squid cannot see an ECH extension, so the flows to port 443 reach it through
# `hive-priv egress-ech-relay`, which refuses a hello that carries one.
{
pkgs,
lib,
@ -551,7 +553,7 @@ in
];
services.hyperhive.network.egress.intercept = {
enable = lib.mkDefault true;
inherit (cfg) unrestrictedFile;
inherit (cfg) unrestrictedFile agentAllowFile;
};
services.hyperhive.network.enable = lib.mkDefault true;

View file

@ -88,7 +88,10 @@ let
# Every hive defaults the agents' queue address to the queue's name, so the
# only hive without one is a hive told to have none. Also the host still
# setting the four removed host options; no case on its queue reads them.
# It is also the one controller host with `egress.intercept.refuseEch` off,
# which the egress cases read.
noAgentQueue = hiveWithController {
network.egress.intercept.refuseEch = false;
deploy.hive-controller.queue.agentNatsUrl = null;
c0re.operatorPronouns = "they/them";
c0re.contextWindowTokens.sonnet = 900000;
@ -1700,7 +1703,9 @@ let
lib.splitString "table bridge hive_egress {" bare.services.hyperhive.network.egress.ruleset
);
chain = lib.head (
lib.splitString "set log_v4" (lib.last (lib.splitString "chain intercept {" inet))
lib.splitString "chain intercept_relay {" (
lib.head (lib.splitString "set log_v4" (lib.last (lib.splitString "chain intercept {" inet)))
)
);
rules = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" chain));
idx = pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules;
@ -1714,12 +1719,72 @@ let
&& idx ''fib daddr oifname { "hive-br0", "wg-hive" } return'' < idx "tcp dport 80"
&& lib.length redirects == 2
&& lib.hasInfix "tcp dport 80 counter redirect to :3129" (lib.head redirects)
&& lib.hasInfix "tcp dport 443 counter redirect to :3130" (lib.last redirects)
# Port 443 goes to the ClientHello relay, not to squid.
&& lib.hasInfix "tcp dport 443 counter redirect to :3131" (lib.last redirects)
# Only IPv4 that comes in on the bridge is looked at.
&& lib.all (l: lib.hasPrefix ''iifname "hive-br0" meta nfproto ipv4 '' l) (lib.drop 1 rules)
&& lib.hasInfix "set direct_mac { type ether_addr; }" without
&& !(lib.hasInfix "chain intercept" without);
}
{
# With the relay on, port 443 reaches squid through `hive-priv
# egress-ech-relay`: its connections (from the pool, carrying the mark)
# are DNATed to squid's HTTPS port, the bridge may not send from the
# pool, and the unit routes the pool to `lo`. `noAgentQueue` has the
# relay off: the redirect then goes straight to squid and none of the
# rest is rendered. Its intercept chain is the presence control.
name = "the ClientHello relay sits between the redirect and squid, and refuseEch off removes it";
ok =
let
ruleset = m: m.services.hyperhive.network.egress.ruleset;
unit = bare.systemd.services.hive-egress-ech-relay;
off = noAgentQueue;
in
lib.hasInfix "chain intercept_relay {" (ruleset bare)
&& lib.hasInfix "type nat hook output priority dstnat; policy accept;" (ruleset bare)
&& lib.hasInfix "meta nfproto ipv4 ip saddr 198.18.0.0/15 meta mark 0x45434800 tcp dport 443 counter dnat ip to 10.42.0.1:3130" (
ruleset bare
)
&& lib.hasInfix ''iifname "hive-br0" ip saddr 198.18.0.0/15 counter drop'' (ruleset bare)
&& lib.hasInfix "egress-ech-relay 10.42.0.1:3131 0x45434800" unit.serviceConfig.ExecStart
&& lib.hasInfix "ip route replace local 198.18.0.0/15 dev lo" unit.serviceConfig.ExecStartPre
&& lib.elem "multi-user.target" unit.wantedBy
# The relay reads the include in which squid finds the `log` agents.
&&
(unit.environment.HIVE_EGRESS_AGENT_ALLOW or null)
== bare.services.hyperhive.deploy.hive-controller.squid.agentAllowFile
&& unit.serviceConfig.SupplementaryGroups == [ "squid" ]
&& lib.elem 3131 bare.services.hyperhive.network.exposeHostPorts
&& lib.hasInfix "chain intercept {" (ruleset off)
&& lib.hasInfix "tcp dport 443 counter redirect to :3130" (ruleset off)
&& lib.elem 3130 off.services.hyperhive.network.exposeHostPorts
&& !(lib.hasInfix "intercept_relay" (ruleset off))
&& !(lib.hasInfix "198.18.0.0/15" (ruleset off))
&& !(off.systemd.services ? hive-egress-ech-relay)
&& !(lib.elem 3131 off.services.hyperhive.network.exposeHostPorts);
}
{
# The relay names a flow's source with the low 17 bits of the agent's
# address, which is unique only in a subnet of /15 or smaller. The
# boundary is read at /15 (accepted) and /14 (refused); `bare`, a /24, is
# the control that the lookup finds nothing otherwise.
name = "the ClientHello relay is refused on a bridge subnet larger than a /15";
ok =
let
refused =
m:
lib.any (
a: !a.assertion && lib.hasInfix "bridgePrefixLength must be 15 or more" a.message
) m.assertions;
in
refused (hiveWithController {
network.bridgePrefixLength = 14;
})
&& !(refused (hiveWithController {
network.bridgePrefixLength = 15;
}))
&& !(refused bare);
}
{
# Squid listens on the two intercept ports, peeks and splices and never
# decrypts, names the agent of a flow with the helper, refuses a source

View file

@ -216,6 +216,7 @@ pkgs.testers.runNixOSTest {
"L+ /run/hive-rootfs-check - - - - ${pkgs.writeShellScript "hive-priv-stub" ''
[ "$1" = egress-load ] && exec ${pkgs.nftables}/bin/nft -f "$2"
[ "$1" = egress-whois ] && exec ${hivePriv}/bin/hive-priv "$@"
[ "$1" = egress-ech-relay ] && exec ${hivePriv}/bin/hive-priv "$@"
exit 0
''}"
];
@ -319,7 +320,10 @@ pkgs.testers.runNixOSTest {
"agentonly.t.local"
"api.t.local"
];
environment.systemPackages = [ pkgs.openssl ];
environment.systemPackages = [
pkgs.openssl
pkgs.python3
];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = "192.168.1.3";
@ -957,7 +961,7 @@ pkgs.testers.runNixOSTest {
def agent_lines():
# Log lines of requests from the agent node's addresses. The self-check
# connects from the bridge address, so its lines are not counted.
return len(re.findall(r" 192\.168\.1\.[347] [A-Z_]+/\d+ ", journal()))
return len(re.findall(r" (?:192\.168\.1|198\.18\.0)\.[347] [A-Z_]+/\d+ ", journal()))
seeded = proxy.succeed(f"cat {ALLOWED}")
bind("atlas", "logan")
@ -980,7 +984,7 @@ pkgs.testers.runNixOSTest {
with checked("a restricted agent's allowed name is tunnelled without the proxy variables, and the log names the agent"):
wait_icpt("https://forge.t.local/", "200")
assert re.search(
r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ORIGINAL_DST/203\.0\.113\.2 ", journal()
r"198\.18\.0\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ORIGINAL_DST/203\.0\.113\.2 ", journal()
), "the HTTPS flow, as atlas"
assert icpt("http://forge.t.local/") == "200", "plain HTTP"
assert re.search(r"192\.168\.1\.3 TCP_\w+/200 .* GET http://forge\.t\.local/ atlas ", journal()), "the HTTP flow, as atlas"
@ -992,7 +996,7 @@ pkgs.testers.runNixOSTest {
with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"):
assert icpt("https://other.t.local/") == "000", "HTTPS"
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal()), "logged as atlas"
assert re.search(r"198\.18\.0\.3 TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal()), "logged as atlas"
assert icpt("http://other.t.local/") == "403", "plain HTTP"
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas"
# Control: the same address reaches the listed name, so the refusal is the policy's.
@ -1027,7 +1031,7 @@ pkgs.testers.runNixOSTest {
with checked("a source the bindings do not name is refused, and so is every source when they are gone"):
assert icpt("https://forge.t.local/", NOBODY_IP) == "000", "no lease for the address"
assert re.search(r"192\.168\.1\.7 TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 - ", journal()), "logged without a user"
assert re.search(r"198\.18\.0\.7 TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 - ", journal()), "logged without a user"
assert icpt("http://forge.t.local/", NOBODY_IP) == "403"
assert icpt("https://forge.t.local/") == "200", "control: the bound address, the same name"
# A hive whose pin file is missing: the lease names a MAC, no port holds it.
@ -1056,6 +1060,90 @@ pkgs.testers.runNixOSTest {
)
assert out.strip() == "", "bytes that are no ClientHello get no answer"
# The ClientHello a client sends, built by hand: SNI, groups, signature
# algorithms and supported versions, and with `ech` the encrypted_client_hello
# extension (0xfe0d) with the layout a real one has. It prints how the
# connection ended: the first byte of what came back, or `closed`.
agent.succeed(
"cat > /tmp/hello.py <<'PY'\n"
"import os, socket, struct, sys\n"
"mode, name, dst, src = sys.argv[1:5]\n"
"def ext(kind, data):\n"
" return struct.pack('>HH', kind, len(data)) + data\n"
"host = name.encode()\n"
"exts = ext(0, struct.pack('>HBH', len(host) + 3, 0, len(host)) + host)\n"
"exts += ext(10, struct.pack('>HHH', 4, 29, 23)) + ext(13, struct.pack('>HHH', 4, 0x0804, 0x0403))\n"
"exts += ext(43, bytes([2, 3, 4])) + ext(51, struct.pack('>H', 0))\n"
"if mode == 'ech':\n"
" exts += ext(0xFE0D, b'\\x00' + struct.pack('>HHBH', 1, 1, 7, 32) + os.urandom(32) + struct.pack('>H', 144) + os.urandom(144))\n"
"body = b'\\x03\\x03' + os.urandom(32) + b'\\x20' + os.urandom(32) + struct.pack('>H', 2) + b'\\x13\\x01' + b'\\x01\\x00'\n"
"body += struct.pack('>H', len(exts)) + exts\n"
"hello = b'\\x01' + struct.pack('>I', len(body))[1:] + body\n"
"s = socket.socket()\n"
"s.settimeout(10)\n"
"s.bind((src, 0))\n"
"s.connect((dst, 443))\n"
"s.sendall(b'\\x16\\x03\\x01' + struct.pack('>H', len(hello)) + hello)\n"
"try:\n"
" first = s.recv(1)\n"
"except OSError:\n"
" first = b'''\n"
"print(first.hex() or 'closed')\n"
"PY"
)
def send_hello(mode, src=ATLAS_IP):
return agent.succeed(f"python3 /tmp/hello.py {mode} forge.t.local 203.0.113.2 {src}").strip()
with checked("a ClientHello with the ECH extension is refused for a restricted agent and observed for a log agent, and the same hello without it is let through"):
seen = agent_lines()
# Control: the origin answers a hello without the extension (a
# handshake record, 0x16), and squid logs the connection as atlas.
assert send_hello("plain") == "16", "the origin answers the hello without ECH"
time.sleep(2)
assert agent_lines() > seen, "squid saw the connection"
seen = agent_lines()
assert send_hello("ech") == "closed", "a hello with an ECH extension gets no answer"
proxy.wait_until_succeeds(
"journalctl -u hive-egress-ech-relay.service -o cat | "
"grep -E '^hive-egress-ech-refused agent=atlas dst=203\\.0\\.113\\.2$'"
)
time.sleep(2)
assert agent_lines() == seen, "squid saw nothing of it"
# A `log` agent is observed, not refused: its ECH hello goes on to squid
# and the origin, and the relay logs what it would have done. The mode is
# what the squid include says (`acl agent_log_logan proxy_auth logan`).
seen = agent_lines()
assert send_hello("ech", LOGAN_IP) == "16", "logan's hello with ECH is let through"
proxy.wait_until_succeeds(
"journalctl -u hive-egress-ech-relay.service -o cat | "
"grep -E '^hive-egress-ech-would-refuse agent=logan dst=203\\.0\\.113\\.2$'"
)
time.sleep(2)
assert agent_lines() > seen, "squid saw logan's connection"
assert "hive-egress-ech-refused agent=logan" not in proxy.succeed(
"journalctl -u hive-egress-ech-relay.service -o cat --no-pager"
), "logan was not refused"
assert "ech-would-refuse agent=atlas" not in proxy.succeed(
"journalctl -u hive-egress-ech-relay.service -o cat --no-pager"
), "control: the restricted agent got the refusal, not the observation"
assert send_hello("plain", LOGAN_IP) == "16", "control: logan's hello without ECH"
# A source the bindings do not name is refused too, logged without a name.
assert send_hello("ech", NOBODY_IP) == "closed"
proxy.wait_until_succeeds(
"journalctl -u hive-egress-ech-relay.service -o cat | "
"grep -E '^hive-egress-ech-refused agent=- dst=203\\.0\\.113\\.2$'"
)
# Bytes that are no ClientHello: refused by the relay, with a line of their own.
out = agent.succeed(
"timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true"
)
assert out.strip() == "", "bytes that are no ClientHello get no answer"
proxy.wait_until_succeeds(
"journalctl -u hive-egress-ech-relay.service -o cat | "
"grep -E '^hive-egress-hello-refused agent=atlas dst=203\\.0\\.113\\.2 reason=not a TLS handshake record$'"
)
with checked("the traffic that stays on the bridge or is the unrestricted agents' is not intercepted"):
seen = agent_lines()
# The gateway on the bridge address, in the stand-in's port 443.
@ -1068,6 +1156,7 @@ pkgs.testers.runNixOSTest {
assert icpt("https://other.t.local/") == "200", "an unlisted name, directly"
assert icpt("https://203.0.113.2/") == "200", "an IP literal, directly"
assert agent_lines() == seen, "squid saw nothing of it"
assert send_hello("ech") == "16", "a hello with ECH, directly: the relay is not in the path"
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
"'hive-egress-would-drop(-unmetered)? IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=443 '"