mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

squid intercepts restricted and log agents' web traffic, named from the bound source address

Browse source
This commit is contained in:
atlas 2026-11-13 05:17:01 +01:00 • committed by mara
commit 712dcf2b25
1 changed files with 1005 additions and 154 deletions
Download patch file Download diff file Expand all files Collapse all files

262
nix/vm-tests/hive-squid.nix
View file

@ -12,8 +12,10 @@
# It also covers ../host-modules/hive-egress.nix: `proxy` loads the real module, so the
# agent's dials to the host's loopback, and its direct connects to the uplink
# (`proxy` forwards and masquerades like the hive host), run against the table
# the module installs.
{ pkgs }:
# the module installs. The same table redirects the agents' web traffic into
# squid's intercept ports; the test writes the bindings and leases that
# `hive-priv egress-whois` reads, standing in for `egress-pin` and dnsmasq.
{ pkgs, hivePriv }:
let
bridgeIp = "192.168.1.1";
@ -105,6 +107,14 @@ let
prefixLength = 64;
};
# The certificate `web` answers TLS with. Nothing trusts it: squid splices,
# so the agent's curl sees the origin's own and runs with `-k`.
webCert = pkgs.runCommand "hive-squid-web-cert" { nativeBuildInputs = [ pkgs.openssl ]; } ''
mkdir $out
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
-subj /CN=web.t.local -days 3650 -keyout $out/key.pem -out $out/cert.pem
'';
# Plain TCP on 443: a CONNECT is answered before any TLS, which is all the
# test reads.
listen443 = addrs: {
@ -199,10 +209,13 @@ pkgs.testers.runNixOSTest {
networking.firewall.interfaces.eth1.allowedTCPPorts = [ 443 ];
# hive-priv.nix links the binary the hive-egress unit loads its tables with;
# `eth1` is no bridge here, so a stub that only loads the ruleset (no
# bindings, no leases) stands in.
# bindings, no leases) stands in, and hands `egress-whois` to the real
# binary.
systemd.tmpfiles.rules = [
"d /var/lib/dnsmasq 0755 root root - -"
"L+ /run/hive-rootfs-check - - - - ${pkgs.writeShellScript "hive-priv-stub" ''
[ "$1" = egress-load ] && exec ${pkgs.nftables}/bin/nft -f "$2"
[ "$1" = egress-whois ] && exec ${hivePriv}/bin/hive-priv "$@"
exit 0
''}"
];
@ -231,12 +244,37 @@ pkgs.testers.runNixOSTest {
web =
{ lib, ... }:
{
imports = [
(listen443 [
"0.0.0.0"
"[::]"
])
];
# TLS on 443 for the intercepted flows (a CONNECT through the proxy is
# answered before any TLS), plain HTTP on 80, and a plain listener on
# 8443 that the proxy never intercepts.
services.nginx = {
enable = true;
virtualHosts.default = {
listen = [
{
addr = "0.0.0.0";
port = 443;
ssl = true;
}
{
addr = "[::]";
port = 443;
ssl = true;
}
{
addr = "0.0.0.0";
port = 80;
}
{
addr = "0.0.0.0";
port = 8443;
}
];
sslCertificate = "${webCert}/cert.pem";
sslCertificateKey = "${webCert}/key.pem";
locations."/".return = "200 web-ok";
};
};
virtualisation.vlans = [ 2 ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce (
[
@ -244,6 +282,12 @@ pkgs.testers.runNixOSTest {
address = "203.0.113.2";
prefixLength = 24;
}
# Another address of the same host: where a flow goes whose SNI
# names the first.
{
address = "203.0.113.9";
prefixLength = 24;
}
]
++ privateAddrs 2
);
@ -255,15 +299,27 @@ pkgs.testers.runNixOSTest {
via = "fd00:99::1";
}
];
networking.firewall.allowedTCPPorts = [ 443 ];
networking.firewall.allowedTCPPorts = [
80
443
8443
];
};
agent =
{ lib, ... }:
{ lib, pkgs, ... }:
{
virtualisation.vlans = [ 1 ];
# The bridge listener, by the name the proxy resolves to the same address.
networking.hosts.${bridgeIp} = [ "bridge.t.local" ];
# The names the proxy resolves, for the flows that skip the proxy variables.
networking.hosts."203.0.113.2" = [
"forge.t.local"
"other.t.local"
"agentonly.t.local"
"api.t.local"
];
environment.systemPackages = [ pkgs.openssl ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = "192.168.1.3";
@ -637,7 +693,9 @@ pkgs.testers.runNixOSTest {
"# source: swarm\n"
"acl agent_allow_atlas proxy_auth atlas\n"
"acl agent_allow_atlas_dst dstdomain agentonly.t.local priv10.t.local\n"
"http_access allow agent_allow_atlas agent_allow_atlas_dst\n",
"acl agent_allow_atlas_ix ext_user atlas\n"
"http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n"
"http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n",
)
wait_status("swarm.t.local", RUTH, "200")
wait_status("agentonly.t.local", ATLAS, "200")
@ -659,7 +717,9 @@ pkgs.testers.runNixOSTest {
"# source: swarm\n"
"acl log_mark annotate_transaction would_deny=1\n"
"acl agent_log_ruth proxy_auth ruth\n"
"http_access allow agent_log_ruth !allowed_dst log_mark\n",
"acl agent_log_ruth_ix ext_user ruth\n"
"http_access allow !icpt agent_log_ruth !allowed_dst log_mark\n"
"http_access allow icpt agent_log_ruth_ix !allowed_dst log_mark\n",
)
wait_status("other.t.local", RUTH, "200")
proxy.wait_until_succeeds(
@ -694,7 +754,9 @@ pkgs.testers.runNixOSTest {
"# source: swarm\n"
"acl log_mark annotate_transaction would_deny=1\n"
"acl agent_log_ruth proxy_auth ruth\n"
"http_access allow agent_log_ruth !allowed_dst log_mark\n",
"acl agent_log_ruth_ix ext_user ruth\n"
"http_access allow !icpt agent_log_ruth !allowed_dst log_mark\n"
"http_access allow icpt agent_log_ruth_ix !allowed_dst log_mark\n",
)
wait_status("other.t.local", RUTH, "200")
write_deny(
@ -703,7 +765,9 @@ pkgs.testers.runNixOSTest {
"http_access deny deny_swarm_dst deny_hit_mark\n"
"acl deny_agent_atlas_dst dstdomain api.t.local\n"
"acl deny_agent_atlas proxy_auth atlas\n"
"http_access deny deny_agent_atlas_dst deny_agent_atlas deny_hit_mark\n"
"acl deny_agent_atlas_ix ext_user atlas\n"
"http_access deny deny_agent_atlas_dst !icpt deny_agent_atlas deny_hit_mark\n"
"http_access deny deny_agent_atlas_dst icpt deny_agent_atlas_ix deny_hit_mark\n"
)
wait_status("forge.t.local", ATLAS, "403")
assert connect("forge.t.local", RUTH) == "403", "a listed name, log mode"
@ -748,8 +812,8 @@ pkgs.testers.runNixOSTest {
assert connect("forge.t.local", ATLAS) == "200", "the file appeared"
assert connect("other.t.local", UMA) == "200", "listed, and now logged in"
def dial(addr):
return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'")[0] == 0
def dial(addr, port=443):
return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/{port}'")[0] == 0
def dropped():
# Packets the table's v4 rule has dropped so far.
@ -796,10 +860,11 @@ pkgs.testers.runNixOSTest {
# first: none of them is forwarded traffic, so none may be logged.
assert connect("forge.t.local", ATLAS) == "200", "through squid"
assert dial("${bridgeIp}"), "the gateway on the bridge IP"
assert dial("203.0.113.2"), "a direct connect to an outside IP works"
# Port 8443: a flow to 80 or 443 is redirected into squid and never forwarded.
assert dial("203.0.113.2", 8443), "a direct connect to an outside IP works"
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=443 '"
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=8443 '"
)
agent.succeed("ping -c1 -W2 203.0.113.2")
proxy.wait_until_succeeds(
@ -850,5 +915,164 @@ pkgs.testers.runNixOSTest {
out = proxy.succeed("nft list chain inet hive_egress forward")
found = re.search(r'oifname != "eth1" counter packets (\d+) bytes \d+$', out, re.M)
assert found and int(found.group(1)) >= 100, out
# ---- The agents' web traffic that skips the proxy variables ----
# The test stands in for `egress-pin` and dnsmasq's lease script: it writes
# the pin file and the lease file that `hive-priv egress-whois` reads, and
# the exemption set that `egress-pin` fills on a hive. The agent node holds
# three addresses, one per agent, and the exemption is by its MAC.
AGENT_MAC = agent.succeed("cat /sys/class/net/eth1/address").strip()
ATLAS_IP, LOGAN_IP, NOBODY_IP = "192.168.1.3", "192.168.1.4", "192.168.1.7"
BOUND = {"atlas": (ATLAS_IP, AGENT_MAC), "logan": (LOGAN_IP, "02:00:00:00:00:04")}
agent.succeed(f"ip addr add {LOGAN_IP}/24 dev eth1; ip addr add {NOBODY_IP}/24 dev eth1")
def bind(*names):
pins = "".join(f"vb-{n} {i + 2} {BOUND[n][1]} h-{n}\n" for i, n in enumerate(names))
leases = "".join(f"0 {BOUND[n][1]} {BOUND[n][0]} {n} *\n" for n in names)
proxy.succeed(
f"printf '%s' {shlex.quote(pins)} > /run/hive-egress-pins "
f"&& printf '%s' {shlex.quote(leases)} > /var/lib/dnsmasq/dnsmasq.leases"
)
def icpt(url, src=ATLAS_IP, extra=""):
# The status of the answer to a request that carries no proxy variables;
# 000 when the connection was closed or the TLS handshake failed.
return agent.succeed(
f"curl -sS -k -m 15 --interface {src} {extra} -o /dev/null -w '%{{http_code}}' {url} 2>/dev/null || true"
).strip()
def wait_icpt(url, want, src=ATLAS_IP, extra=""):
# The reload that follows a list write is asynchronous.
got = None
for _ in range(30):
got = icpt(url, src, extra)
if got == want:
return
time.sleep(1)
raise AssertionError(f"{url} from {src}: wanted {want}, last got {got}")
def journal():
return proxy.succeed("journalctl -u squid.service -o cat --no-pager")
def agent_lines():
# Log lines of requests from the agent node's addresses. The self-check
# connects from the bridge address, so its lines are not counted.
return len(re.findall(r" 192\.168\.1\.[347] [A-Z_]+/\d+ ", journal()))
seeded = proxy.succeed(f"cat {ALLOWED}")
bind("atlas", "logan")
# What hive-c0re renders: `atlas` has a name of its own, `logan` is in log mode.
write_allow(
seeded,
"# source: swarm\n"
"acl agent_allow_atlas proxy_auth atlas\n"
"acl agent_allow_atlas_ix ext_user atlas\n"
"acl agent_allow_atlas_dst dstdomain agentonly.t.local\n"
"http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n"
"http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n"
"acl log_mark annotate_transaction would_deny=1\n"
"acl agent_log_logan proxy_auth logan\n"
"acl agent_log_logan_ix ext_user logan\n"
"http_access allow !icpt agent_log_logan !allowed_dst log_mark\n"
"http_access allow icpt agent_log_logan_ix !allowed_dst log_mark\n",
)
with checked("a restricted agent's allowed name is tunnelled without the proxy variables, and the log names the agent"):
wait_icpt("https://forge.t.local/", "200")
assert re.search(
r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ORIGINAL_DST/203\.0\.113\.2 ", journal()
), "the HTTPS flow, as atlas"
assert icpt("http://forge.t.local/") == "200", "plain HTTP"
assert re.search(r"192\.168\.1\.3 TCP_\w+/200 .* GET http://forge\.t\.local/ atlas ", journal()), "the HTTP flow, as atlas"
assert icpt("https://agentonly.t.local/") == "200", "a name of atlas alone"
assert re.search(r"CONNECT agentonly\.t\.local:443 atlas ", journal()), "as atlas"
# The proxy-aware path is the same as before, and is not given the address's identity.
assert connect("forge.t.local", ATLAS) == "200", "a credential through the proxy port"
assert connect("forge.t.local") == "407", "no credential: the address does not stand in for one"
with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"):
assert icpt("https://other.t.local/") == "000", "HTTPS"
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal()), "logged as atlas"
assert icpt("http://other.t.local/") == "403", "plain HTTP"
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas"
# Control: the same address reaches the listed name, so the refusal is the policy's.
assert icpt("https://forge.t.local/") == "200"
with checked("the swarm's deny list refuses an intercepted name that is otherwise allowed, and logs egress_denied"):
assert icpt("https://api.t.local/") == "200", "control: a base-list name"
write_deny(
"acl deny_hit_mark annotate_transaction egress_deny=1\n"
"acl deny_swarm_dst dstdomain api.t.local\n"
"http_access deny deny_swarm_dst deny_hit_mark\n"
)
wait_icpt("https://api.t.local/", "000")
proxy.wait_until_succeeds(
"journalctl -u squid.service -o cat | grep -E "
"'egress_denied user=atlas method=CONNECT host=api\\.t\\.local port=443 result=TCP_DENIED/403'"
)
assert icpt("https://forge.t.local/") == "200", "control: another name is unaffected"
with checked("a log-mode agent's unlisted name is allowed and marked would_deny, and the deny list still wins"):
assert icpt("https://other.t.local/", LOGAN_IP) == "200"
proxy.wait_until_succeeds(
"journalctl -u squid.service -o cat | grep -E "
"'would_deny user=logan method=CONNECT host=other\\.t\\.local port=443 result=TCP_TUNNEL/200'"
)
assert icpt("https://forge.t.local/", LOGAN_IP) == "200", "a listed name"
assert not re.search(r"would_deny user=logan .* host=forge\.t\.local", journal()), "a listed name is not marked"
assert icpt("https://api.t.local/", LOGAN_IP) == "000", "the deny list refuses it in log mode too"
assert icpt("https://other.t.local/") == "000", "control: atlas, unmarked and refused"
write_deny("")
wait_icpt("https://api.t.local/", "200")
with checked("a source the bindings do not name is refused, and so is every source when they are gone"):
assert icpt("https://forge.t.local/", NOBODY_IP) == "000", "no lease for the address"
assert re.search(r"192\.168\.1\.7 TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 - ", journal()), "logged without a user"
assert icpt("http://forge.t.local/", NOBODY_IP) == "403"
assert icpt("https://forge.t.local/") == "200", "control: the bound address, the same name"
# A hive whose pin file is missing: the lease names a MAC, no port holds it.
proxy.succeed("rm /run/hive-egress-pins")
time.sleep(12) # the helper's answers are kept for 10 seconds
assert icpt("https://forge.t.local/") == "000", "no pin file"
bind("atlas", "logan")
wait_icpt("https://forge.t.local/", "200")
with checked("an SNI that the destination does not serve, a missing SNI and an IP literal are refused"):
# Control: the right SNI at the right address is the 200 above. The
# same name sent to another address of the origin is not.
assert icpt("https://forge.t.local/", extra="--resolve forge.t.local:443:203.0.113.9") == "000"
assert re.search(r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ", journal()), "squid answered 409"
assert icpt("http://forge.t.local/", extra="-H 'Host: forge.t.local' --resolve forge.t.local:80:203.0.113.9") == "409", "the Host header too"
hello = "echo | timeout 10 openssl s_client -bind 192.168.1.3 -connect 203.0.113.2:443"
out = agent.succeed(f"{hello} -servername forge.t.local 2>&1 || true")
assert "BEGIN CERTIFICATE" in out, "control: with an SNI the origin's certificate comes back"
out = agent.succeed(f"{hello} -noservername 2>&1 || true")
assert "BEGIN CERTIFICATE" not in out, "no SNI, no certificate"
assert icpt("https://203.0.113.2/") == "000", "an IP literal sends no SNI"
assert re.search(r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ", journal()), "logged as atlas"
assert icpt("http://203.0.113.2/") == "403", "an IP literal as the Host"
out = agent.succeed(
"timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true"
)
assert out.strip() == "", "bytes that are no ClientHello get no answer"
with checked("the traffic that stays on the bridge or is the unrestricted agents' is not intercepted"):
seen = agent_lines()
# The gateway on the bridge address, in the stand-in's port 443.
assert icpt("http://${bridgeIp}:443/") != "000", "the host's own listener answers"
assert agent_lines() == seen, "squid saw nothing of it"
# Unrestricted agents and service containers are the MACs in `direct_mac`.
assert icpt("https://other.t.local/") == "000", "control: an unlisted name is refused"
proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
seen = agent_lines()
assert icpt("https://other.t.local/") == "200", "an unlisted name, directly"
assert icpt("https://203.0.113.2/") == "200", "an IP literal, directly"
assert agent_lines() == seen, "squid saw nothing of it"
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
"'hive-egress-would-drop(-unmetered)? IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=443 '"
)
proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
assert icpt("https://other.t.local/") == "000", "intercepted again"
'';
}