mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity
hyperhive/nix/vm-tests/hive-squid.nix
atlas bd79b58e78 log the agent bridge's forwarded flows in the hive_egress table 
A forward chain in the hive-owned nft table logs each new flow from the agent bridge to the uplink (v4 and v6) and counts every packet. It never blocks. The mesh interface is skipped. Logging is rate limited per source address; a flow that limit cannot log gets one line from a shared limit with its own prefix. services.hyperhive.network.egress.forward selects accept, log (default) or drop; drop is refused at evaluation until the interception it needs exists.
2026-11-12 09:20:51 +01:00

843 lines
41 KiB
Nix
Raw Blame History

# `vmTests.<system>.hive-squid`: runs ../host-modules/hive-squid.nix in a VM, which
# the module-eval suite can only read, and checks that an agent with the proxy
# variables of nix/agent-modules/egress-proxy.nix reaches squid and skips it
# where it should. Three nodes on two networks:
#
# agent (vlan 1) --> proxy (vlan 1 = the stand-in bridge, vlan 2 = uplink) --> web (vlan 2)
#
# `proxy` imports the squid module against stub options for the few hive
# options it reads, so no hive-c0re build is pulled in; the module-eval suite
# covers the real option wiring.
#
# It also covers ../host-modules/hive-egress.nix: `proxy` loads the real module, so the
# agent's dials to the host's loopback, and its direct connects to the uplink
# (`proxy` forwards and masquerades like the hive host), run against the table
# the module installs.
{ pkgs }:
let
bridgeIp = "192.168.1.1";
hiveStubs =
{ lib, config, ... }:
{
options.services.hyperhive = {
deploy.hive-controller.enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
network = {
bridgeIp = lib.mkOption {
type = lib.types.str;
default = bridgeIp;
};
networkCidr = lib.mkOption {
type = lib.types.str;
default = "192.168.1.0/24";
};
exposeHostPorts = lib.mkOption {
type = lib.types.listOf lib.types.port;
default = [ ];
};
enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
bridgeName = lib.mkOption {
type = lib.types.str;
default = "eth1";
};
};
otel = {
scrapeTargets = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
};
journalUnits = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
};
};
};
# The user hive-c0re runs as, which owns the users file squid reads.
config.users.users.hive-core = {
isSystemUser = true;
group = "hive-core";
};
config.users.groups.hive-core = { };
# The firewall hole hive-network.nix opens for `exposeHostPorts`.
config.networking.firewall.interfaces.eth1.allowedTCPPorts =
config.services.hyperhive.network.exposeHostPorts;
};
# The uplink is a public range. The private ranges squid must refuse are
# extra subnets on the same link, so the proxy can reach a listener in each:
# a name that resolves there is refused only by squid's own rules.
privateNets = [
{
name = "priv10.t.local";
net = "10.99.0";
}
{
name = "priv172.t.local";
net = "172.16.99";
}
{
name = "priv192.t.local";
net = "192.168.99";
}
{
name = "cgnat.t.local";
net = "100.64.99";
}
];
privateAddrs =
host:
map (p: {
address = "${p.net}.${toString host}";
prefixLength = 24;
}) privateNets;
ulaAddr = host: {
address = "fd00:99::${toString host}";
prefixLength = 64;
};
bridgeUla = host: {
address = "fd00:98::${toString host}";
prefixLength = 64;
};
# Plain TCP on 443: a CONNECT is answered before any TLS, which is all the
# test reads.
listen443 = addrs: {
services.nginx = {
enable = true;
virtualHosts.default.listen = map (addr: {
inherit addr;
port = 443;
}) addrs;
};
};
in
pkgs.testers.runNixOSTest {
name = "hive-squid";
nodes = {
proxy =
{ lib, pkgs, ... }:
{
imports = [
hiveStubs
../host-modules/hive-squid.nix
../host-modules/hive-egress.nix
# Listeners a leaked destination would reach: loopback (`::` lands on `::1`) and the bridge.
(listen443 [
"127.0.0.1"
# An address the agent can route to the proxy: the agent's own lo owns 127.0.0.1 itself.
"127.0.0.5"
"[::1]"
bridgeIp
])
];
virtualisation.vlans = [
1
2
];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = bridgeIp;
prefixLength = 24;
}
];
networking.interfaces.eth2.ipv4.addresses = lib.mkForce (
[
{
address = "203.0.113.1";
prefixLength = 24;
}
]
++ privateAddrs 1
);
networking.interfaces.eth2.ipv6.addresses = [ (ulaAddr 1) ];
networking.interfaces.eth1.ipv6.addresses = [ (bridgeUla 1) ];
# What hive-network.nix sets for the agent bridge: forwarding, and NAT
# for everything that leaves it. The agent's direct path out exists.
networking.nat = {
enable = true;
internalInterfaces = [ "eth1" ];
};
boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = 1;
# Names the proxy resolves, each a different destination class.
networking.hosts = {
# `other.t.local` and the self-check's deny canary resolve, are
# reachable and are on no allow-list. The canary is in the hosts file
# because the destination denies look every name up first, and the VM
# has no resolver to answer for an unknown one before the client gives up.
"203.0.113.2" = [
"forge.t.local"
"cache.t.local"
"api.t.local"
"other.t.local"
"swarm.t.local"
"agentonly.t.local"
"deny-canary.invalid"
];
"127.0.0.1" = [ "loop.t.local" ];
${bridgeIp} = [
"bridge.t.local"
"basebridge.t.local"
];
"169.254.169.254" = [ "meta.t.local" ];
"fd00:99::2" = [ "ula.t.local" ];
"::" = [ "any6.t.local" ];
}
// lib.listToAttrs (
map (p: {
name = "${p.net}.2";
value = [ p.name ];
}) privateNets
);
# The gateway: nginx answers on the bridge, which agents reach directly.
networking.firewall.interfaces.eth1.allowedTCPPorts = [ 443 ];
# Renders the users file the way hive-c0re does.
environment.systemPackages = [
pkgs.mkpasswd
pkgs.nftables
];
services.hyperhive.deploy.hive-controller.squid.selfcheck.unrestrictedCanary = "other.t.local";
services.hyperhive.deploy.hive-controller.squid.selfcheck.canary = "cache.t.local";
services.hyperhive.deploy.hive-controller.squid.baseAllowedDomains = [
"api.t.local"
"basebridge.t.local"
];
services.hyperhive.deploy.hive-controller.squid.allowedDomains = [
"forge.t.local"
"loop.t.local"
"bridge.t.local"
"meta.t.local"
"ula.t.local"
"any6.t.local"
]
++ map (p: p.name) privateNets;
};
web =
{ lib, ... }:
{
imports = [
(listen443 [
"0.0.0.0"
"[::]"
])
];
virtualisation.vlans = [ 2 ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce (
[
{
address = "203.0.113.2";
prefixLength = 24;
}
]
++ privateAddrs 2
);
networking.interfaces.eth1.ipv6.addresses = [ (ulaAddr 2) ];
networking.interfaces.eth1.ipv6.routes = [
{
address = "fd00:98::";
prefixLength = 64;
via = "fd00:99::1";
}
];
networking.firewall.allowedTCPPorts = [ 443 ];
};
agent =
{ lib, ... }:
{
virtualisation.vlans = [ 1 ];
# The bridge listener, by the name the proxy resolves to the same address.
networking.hosts.${bridgeIp} = [ "bridge.t.local" ];
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
{
address = "192.168.1.3";
prefixLength = 24;
}
];
networking.interfaces.eth1.ipv4.routes = [
{
address = "203.0.113.0";
prefixLength = 24;
via = bridgeIp;
}
{
address = "198.51.100.0";
prefixLength = 24;
via = bridgeIp;
}
];
networking.interfaces.eth1.ipv6.addresses = [ (bridgeUla 3) ];
networking.interfaces.eth1.ipv6.routes = [
{
address = "fd00:99::";
prefixLength = 64;
via = "fd00:98::1";
}
];
};
};
testScript = ''
import re
import secrets
import shlex
import time
from contextlib import contextmanager
# Drawn fresh each run. The users file holds only their hashes, as
# hive-c0re renders it. `atlas` is a restricted agent and `uma` an
# unrestricted one; the two self-check users share one secret.
PW = {
u: secrets.token_urlsafe(24)
for u in ("atlas", "uma", "selfcheck.squid", "ruth")
}
PW["selfcheck-unrestricted.squid"] = PW["selfcheck.squid"]
ATLAS = "atlas:" + PW["atlas"]
UMA = "uma:" + PW["uma"]
SELFCHECK = "selfcheck.squid:" + PW["selfcheck.squid"]
SELFCHECK_U = "selfcheck-unrestricted.squid:" + PW["selfcheck.squid"]
RUTH = "ruth:" + PW["ruth"]
USERS = "/var/lib/hive-squid/users"
LIST = "/var/lib/hive-squid/unrestricted"
ALLOWED = "/var/lib/hive-squid/allowed"
AGENT_ALLOW = "/var/lib/hive-squid/agent-allow.conf"
DENY = "/var/lib/hive-squid/deny.conf"
UNRESTRICTED = ["uma", "selfcheck-unrestricted.squid"]
start_all()
proxy.wait_for_unit("squid.service")
proxy.wait_for_unit("squid-exporter.service")
proxy.wait_for_unit("nginx.service")
web.wait_for_unit("nginx.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
# `start_all` boots the agent beside the proxy; its network is up only once
# it has reached multi-user.target and can open the proxy's port.
agent.wait_for_unit("multi-user.target")
agent.wait_until_succeeds("timeout 3 bash -c 'echo > /dev/tcp/${bridgeIp}/3128'")
PRIVATE = ["priv10.t.local", "priv172.t.local", "priv192.t.local", "cgnat.t.local", "ula.t.local", "any6.t.local"]
SEEN = []
def connect(host, cred=None):
# The status squid answered the CONNECT with. 000 means no answer, which
# no assertion expects: squid closes its listener while a reload re-reads
# its configuration, so 000 is retried.
auth = f"-U '{cred}' " if cred else ""
got = "000"
for _ in range(15):
got = agent.succeed(
"curl -sS -m 15 -p -x http://${bridgeIp}:3128 " + auth + "-o /dev/null "
f"-w '%{{http_connect}}' https://{host}/ 2>/dev/null || true"
).strip()
if got != "000":
break
time.sleep(1)
SEEN.append(f"{host} as {cred.split(':')[0] if cred else '-'}: {got}")
return got
@contextmanager
def checked(name):
# A subtest that prints what the proxy and the agent were doing when it fails.
with subtest(name):
try:
yield
except BaseException:
print("recent connects:\n" + "\n".join(SEEN[-12:]))
for vm, cmd in (
(proxy, "systemctl status squid.service squid-unrestricted-reload.service squid-selfcheck.service --no-pager -n 0"),
(proxy, "journalctl -u squid.service -o cat --no-pager | tail -n 40"),
(proxy, "ss -ltn"),
(agent, "ip -brief address; ip route; systemctl is-system-running"),
):
print(f"--- {vm.name}: {cmd}\n" + vm.execute(cmd)[1])
raise
def write_users(*creds):
# What hive-c0re does: hashes only, written aside and renamed into
# place, in squid's group. The sleep keeps two writes in different
# seconds, which is how the helper tells the file changed.
lines = " ".join(
f"printf '%s:%s\\n' {u} \"$(mkpasswd -m sha-512 -s <<< {p})\";"
for u, p in (c.split(":", 1) for c in creds)
)
proxy.succeed(
f"({lines}) > {USERS}.new && chown hive-core:squid {USERS}.new "
f"&& chmod 640 {USERS}.new && mv {USERS}.new {USERS} && sleep 1"
)
def write_list(*names):
# What hive-c0re does for the list. Squid reads it on a reload, which
# squid-unrestricted-reload does when the file changes.
body = "printf '%s\\n' " + " ".join(names) if names else ":"
proxy.succeed(
f"({body}) > {LIST}.new && chown hive-core:squid {LIST}.new "
f"&& chmod 640 {LIST}.new && mv {LIST}.new {LIST}"
)
def write_allow(allowed, agent_conf):
# What hive-c0re does for its two files: written aside and renamed into
# place, in squid's group. Squid reads them on a reload, which
# squid-unrestricted-reload does when either changes.
for path, body in ((ALLOWED, allowed), (AGENT_ALLOW, agent_conf)):
proxy.succeed(
f"printf '%s' {shlex.quote(body)} > {path}.new && chown hive-core:squid {path}.new "
f"&& chmod 640 {path}.new && mv {path}.new {path}"
)
def write_deny(body):
# What hive-c0re does for the deny list: written aside and renamed into
# place, in squid's group; the reload is squid-unrestricted-reload's.
proxy.succeed(
f"printf '%s' {shlex.quote(body)} > {DENY}.new && chown hive-core:squid {DENY}.new "
f"&& chmod 640 {DENY}.new && mv {DENY}.new {DENY}"
)
def wait_status(host, cred, want):
# The reload that follows a list write is asynchronous.
got = None
for _ in range(30):
got = connect(host, cred)
if got == want:
return
time.sleep(1)
raise AssertionError(f"{host} as {cred.split(':')[0]}: wanted {want}, last got {got}")
with checked("the private listeners answer the proxy itself, so a refusal below is squid's"):
for addr in ["10.99.0.2", "172.16.99.2", "192.168.99.2", "100.64.99.2", "fd00:99::2"]:
proxy.wait_until_succeeds(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'")
assert connect("forge.t.local", None) == "407", "control: the public name is challenged, not refused"
with checked("squid starts on the empty users file and list and refuses every login"):
assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640"
assert proxy.succeed(f"stat -c '%U:%G %a' {LIST}").strip() == "hive-core:squid 640"
assert proxy.succeed("stat -c %a /var/lib/hive-squid").strip() == "2750"
assert proxy.succeed(f"stat -c %s {USERS}").strip() == "0"
assert proxy.succeed(f"stat -c %s {LIST}").strip() == "0"
assert proxy.succeed(f"stat -c '%U:%G %a' {ALLOWED}").strip() == "hive-core:squid 640"
assert proxy.succeed(f"stat -c '%U:%G %a' {AGENT_ALLOW}").strip() == "hive-core:squid 640"
assert proxy.succeed(f"stat -c %s {AGENT_ALLOW}").strip() == "0"
assert proxy.succeed(f"stat -c '%U:%G %a' {DENY}").strip() == "hive-core:squid 640"
assert proxy.succeed(f"stat -c %s {DENY}").strip() == "0", "no deny list, no rule"
# The allow-list file starts as the hive's own allowedDomains.
proxy.succeed(f"grep -Fx forge.t.local {ALLOWED}")
assert connect("forge.t.local", ATLAS) == "407", "no such user"
assert connect("other.t.local", UMA) == "407", "no such user, unlisted name"
with checked("the users file holds hashes, never a password"):
write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U)
users = proxy.succeed(f"cat {USERS}")
rows = [l.split(":", 1) for l in users.splitlines()]
assert sorted(r[0] for r in rows) == sorted(
["atlas", "uma", "selfcheck.squid", "selfcheck-unrestricted.squid"]
), users
assert all(re.fullmatch(r"\$6\$\S+", r[1]) for r in rows), users
assert not any(p in users for p in PW.values()), users
with checked("an allowed name is tunnelled with a credential and challenged without one"):
assert connect("forge.t.local", ATLAS) == "200", "allow canary"
assert connect("cache.t.local", ATLAS) == "200", "the self-check canary, allowed on top of the list"
assert connect("forge.t.local") == "407", "no credential"
assert connect("forge.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password"
assert connect("forge.t.local", "nobody:" + PW["atlas"]) == "407", "unknown user"
# DES would accept the first eight characters; the hash must not.
assert connect("forge.t.local", "atlas:" + PW["atlas"][:8]) == "407", "a prefix of the password"
with checked("a base-list name is open to a restricted user, with a credential, and nothing else is"):
assert connect("api.t.local", ATLAS) == "200", "restricted user, base name"
assert connect("api.t.local") == "407", "no credential"
assert connect("api.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password"
assert connect("other.t.local", ATLAS) == "403", "control: an unlisted name is still refused"
assert connect("basebridge.t.local", ATLAS) == "403", "a base name that resolves to the bridge"
with checked("with an empty list every user is restricted"):
assert connect("other.t.local", UMA) == "403", "uma, not listed yet"
assert connect("other.t.local", ATLAS) == "403"
assert connect("forge.t.local", UMA) == "200", "control: uma still reaches a listed name"
with checked("a listed user reaches an unlisted name and the others are refused"):
write_list(*UNRESTRICTED)
wait_status("other.t.local", UMA, "200")
assert connect("other.t.local", ATLAS) == "403", "restricted"
assert connect("forge.t.local", ATLAS) == "200", "restricted, listed name"
assert connect("forge.t.local", UMA) == "200", "unrestricted, listed name"
assert connect("other.t.local") == "407", "no credential"
assert connect("other.t.local", "uma:" + secrets.token_urlsafe(24)) == "407", "wrong password"
assert connect("deny-canary.invalid", ATLAS) == "403", "restricted, an unlisted name"
with checked("neither kind of user reaches a host-local or private destination, or another port"):
for cred in (ATLAS, UMA):
who = cred.split(":")[0]
assert connect("loop.t.local", cred) == "403", f"{who}: loopback"
assert connect("bridge.t.local", cred) == "403", f"{who}: bridge address"
assert connect("meta.t.local", cred) == "403", f"{who}: link-local"
assert connect("127.0.0.1", cred) == "403", f"{who}: loopback literal"
assert connect("203.0.113.2", cred) == "403", f"{who}: IP literal"
for name in PRIVATE:
assert connect(name, cred) == "403", f"{who}: {name}, a listed name that resolves to a private address"
assert connect("other.t.local:22", cred) == "403", f"{who}: port 22"
assert connect("other.t.local:80", cred) == "403", f"{who}: CONNECT to port 80"
with checked("an agent cannot read cachemgr"):
agent.succeed(
"test $(curl -s -m 15 -o /dev/null -w '%{http_code}' "
"http://${bridgeIp}:3128/squid-internal-mgr/counters) = 403"
)
# The variables an agent's `proxy.env` carries (nix/agent-modules/egress-proxy.nix),
# as a curl on the agent reads them.
def agent_env(cred, no_proxy):
url = f"http://{cred}@${bridgeIp}:3128"
pairs = {
"HTTPS_PROXY": url, "HTTP_PROXY": url, "https_proxy": url, "http_proxy": url,
"NO_PROXY": no_proxy, "no_proxy": no_proxy,
}
return " ".join(f"{k}='{v}'" for k, v in pairs.items())
def agent_get(cred, no_proxy, url):
# The status of the answer, whoever gave it; 000 when none came.
return agent.succeed(
f"env {agent_env(cred, no_proxy)} curl -sS -m 15 -o /dev/null -w '%{{http_code}}' {url} 2>/dev/null || true"
).strip()
def squid_saw(pattern):
return re.search(pattern, proxy.succeed("journalctl -u squid.service -o cat"))
with checked("an agent with the proxy variables reaches an allowed name through squid and a swarm-internal name directly"):
NO_PROXY = "localhost,127.0.0.1,::1,bridge.t.local,${bridgeIp}"
# Plain HTTP to the listeners' port 443, so the answer carries a status.
assert agent_get(ATLAS, NO_PROXY, "http://forge.t.local:443/") not in ("000", "403", "407")
assert squid_saw(r"192\.168\.1\.3 TCP_\w+/\d+ .* GET http://forge\.t\.local:443/ atlas "), "through squid, as atlas"
assert agent_get(ATLAS, NO_PROXY, "http://bridge.t.local:443/") not in ("000", "403", "407")
assert not squid_saw(r"GET http://bridge\.t\.local:443/ "), "a swarm-internal name never reached squid"
# Control: without the exemption the same request goes to squid, which refuses the bridge.
assert agent_get(ATLAS, "localhost", "http://bridge.t.local:443/") == "403"
assert squid_saw(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://bridge\.t\.local:443/ "), "control: squid refuses it"
# CONNECT takes the same path: HTTPS_PROXY names the proxy for https URLs.
assert agent.succeed(
f"env {agent_env(ATLAS, NO_PROXY)} curl -sS -m 15 -o /dev/null -w '%{{http_connect}}' https://forge.t.local/ 2>/dev/null || true"
).strip() == "200"
with checked("an agent without the proxy variables goes direct"):
assert agent.succeed(
"curl -sS -m 15 -o /dev/null -w '%{http_code}' http://bridge.t.local:443/ 2>/dev/null || true"
).strip() not in ("000", "403", "407")
with checked("the self-check passes with its credential"):
proxy.succeed("install -d -m 0700 /run/credstore")
proxy.succeed(f"printf '%s\\n' {PW['selfcheck.squid']} > /run/credstore/hive-squid.selfcheck")
proxy.succeed("chmod 400 /run/credstore/hive-squid.selfcheck")
proxy.succeed("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'squid-selfcheck ok=true allow_canary=cache.t.local "
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403 "
"anon_status=407 credential=present "
"unrestricted_canary=other.t.local unrestricted_status=200'"
)
with checked("the self-check fails, and says why, when the unrestricted user is not listed"):
write_list()
wait_status("other.t.local", UMA, "403")
proxy.fail("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'squid-selfcheck ok=false allow_canary=cache.t.local "
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403 "
"anon_status=407 credential=present "
"unrestricted_canary=other.t.local unrestricted_status=403'"
)
write_list(*UNRESTRICTED)
wait_status("other.t.local", UMA, "200")
with checked("the self-check fails, and says why, without a credential or with the wrong one"):
proxy.succeed("mv /run/credstore/hive-squid.selfcheck /run/credstore/kept")
proxy.fail("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'squid-selfcheck ok=false allow_canary=cache.t.local "
"allow_status=407 deny_canary=deny-canary.invalid deny_status=407 "
"anon_status=407 credential=absent "
"unrestricted_canary=other.t.local unrestricted_status=407'"
)
proxy.succeed(f"printf '%s\\n' {secrets.token_urlsafe(24)} > /run/credstore/hive-squid.selfcheck")
proxy.fail("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'allow_status=407 deny_canary=deny-canary.invalid deny_status=407 "
"anon_status=407 credential=present "
"unrestricted_canary=other.t.local unrestricted_status=407'"
)
proxy.succeed("mv -f /run/credstore/kept /run/credstore/hive-squid.selfcheck")
with checked("squid's access log reaches the journal under squid.service, with the user"):
journal = proxy.succeed("journalctl -u squid.service -o cat")
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT other\.t\.local:443 uma ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/407 .* CONNECT forge\.t\.local:443 - ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443 ", journal), journal
# Exporter scrapes are cachemgr requests and stay out of the log.
assert "squid-internal-mgr" not in journal, journal
with checked("the exporter serves squid's counters on loopback"):
metrics = proxy.succeed("curl -sf http://127.0.0.1:9301/metrics")
print("\n".join(l for l in metrics.splitlines() if l.startswith(("squid_up", "squid_client_http"))))
assert 'squid_up{host="${bridgeIp}"} 1' in metrics
m = re.search(r"^squid_client_http_requests_total (\S+)$", metrics, re.M)
assert m and float(m.group(1)) >= 9, metrics
m = re.search(r"^squid_client_http_errors_total (\S+)$", metrics, re.M)
assert m and float(m.group(1)) >= 1, metrics
proxy.fail("curl -sf -m 5 http://${bridgeIp}:9301/metrics")
with checked("squid survives a reload and keeps the list"):
proxy.succeed("systemctl reload squid.service")
proxy.wait_for_unit("squid.service")
assert connect("forge.t.local", ATLAS) == "200"
assert connect("other.t.local", UMA) == "200"
assert connect("other.t.local", ATLAS) == "403"
with checked("a user added while squid runs can log in, with no reload, and is restricted"):
write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U, RUTH)
assert connect("forge.t.local", RUTH) == "200"
assert connect("forge.t.local", ATLAS) == "200"
assert connect("other.t.local", RUTH) == "403"
with checked("a user the list gains is unrestricted after the reload and loses it again"):
write_list(*UNRESTRICTED, "ruth")
wait_status("other.t.local", RUTH, "200")
assert connect("loop.t.local", RUTH) == "403", "still no loopback"
write_list(*UNRESTRICTED)
wait_status("other.t.local", RUTH, "403")
assert connect("other.t.local", UMA) == "200", "control: uma is still listed"
with checked("the swarm's allow-list adds to the hive's names, opens one to every restricted user, and a per-agent rule opens one to that agent alone"):
seeded = proxy.succeed(f"cat {ALLOWED}")
assert connect("swarm.t.local", RUTH) == "403", "control: not allowed yet"
assert connect("agentonly.t.local", ATLAS) == "403", "control: not allowed yet"
assert connect("forge.t.local", RUTH) == "200", "control: the hive's own name"
write_allow(
"# source: swarm\n" + seeded + "swarm.t.local\n",
"# source: swarm\n"
"acl agent_allow_atlas proxy_auth atlas\n"
"acl agent_allow_atlas_dst dstdomain agentonly.t.local priv10.t.local\n"
"http_access allow agent_allow_atlas agent_allow_atlas_dst\n",
)
wait_status("swarm.t.local", RUTH, "200")
wait_status("agentonly.t.local", ATLAS, "200")
assert connect("swarm.t.local", ATLAS) == "200", "swarm-wide, another user"
assert connect("agentonly.t.local", RUTH) == "403", "the per-agent name, another restricted user"
assert connect("agentonly.t.local") == "407", "no credential"
assert connect("forge.t.local", RUTH) == "200", "the hive's own name stays"
assert connect("priv10.t.local", ATLAS) == "403", "a per-agent name that resolves privately"
assert connect("agentonly.t.local", UMA) == "200", "control: unrestricted reaches it anyway"
write_allow(seeded, "")
wait_status("swarm.t.local", RUTH, "403")
wait_status("agentonly.t.local", ATLAS, "403")
assert connect("forge.t.local", RUTH) == "200", "the hive's own name after the swarm list is gone"
with checked("a log-mode agent reaches an unlisted name and is logged as would_deny, and nothing else changes"):
# What hive-c0re renders for `ruth` in log mode, after the agent rules.
write_allow(
seeded,
"# source: swarm\n"
"acl log_mark annotate_transaction would_deny=1\n"
"acl agent_log_ruth proxy_auth ruth\n"
"http_access allow agent_log_ruth !allowed_dst log_mark\n",
)
wait_status("other.t.local", RUTH, "200")
proxy.wait_until_succeeds(
"journalctl -u squid.service -o cat | grep -E "
"'would_deny user=ruth method=CONNECT host=other\\.t\\.local port=443 result='"
)
assert connect("forge.t.local", RUTH) == "200", "a listed name"
assert connect("other.t.local", ATLAS) == "403", "control: a restricted agent is still refused"
assert connect("other.t.local") == "407", "no credential"
assert connect("other.t.local", "ruth:" + secrets.token_urlsafe(24)) == "407", "wrong password"
assert connect("deny-canary.invalid", SELFCHECK) == "403", "the self-check's deny canary"
for name in ["loop.t.local", "bridge.t.local", "meta.t.local", "127.0.0.1", "203.0.113.2", "priv10.t.local", "other.t.local:22", "other.t.local:80"]:
assert connect(name, RUTH) == "403", f"log mode still refuses {name}"
assert squid_saw(r"would_deny user=ruth method=CONNECT host=other\.t\.local port=443 "), "the unlisted name is marked"
assert not squid_saw(r"would_deny user=ruth .* host=forge\.t\.local"), "a listed name is not marked"
assert not squid_saw(r"would_deny user=atlas "), "a restricted agent is not marked"
assert not squid_saw(r"would_deny .* host=loop\.t\.local"), "a refused request is not marked"
write_allow(seeded, "")
wait_status("other.t.local", RUTH, "403")
with checked("the swarm's deny list is refused before any allow, in every mode, with or without a credential, and logged as egress_denied"):
# What hive-c0re renders: `forge.t.local` and `other.t.local` for every
# agent, `api.t.local` (a base-list name) for atlas alone. `ruth` is in
# log mode, so her rule follows. The names are allowed or reachable
# without the list, which the controls establish first.
assert connect("forge.t.local", ATLAS) == "200", "control: allowed by the hive"
assert connect("forge.t.local", UMA) == "200", "control: allowed, unrestricted"
assert connect("other.t.local", UMA) == "200", "control: unrestricted reaches it"
assert connect("api.t.local", ATLAS) == "200", "control: base list"
write_allow(
seeded,
"# source: swarm\n"
"acl log_mark annotate_transaction would_deny=1\n"
"acl agent_log_ruth proxy_auth ruth\n"
"http_access allow agent_log_ruth !allowed_dst log_mark\n",
)
wait_status("other.t.local", RUTH, "200")
write_deny(
"acl deny_hit_mark annotate_transaction egress_deny=1\n"
"acl deny_swarm_dst dstdomain forge.t.local other.t.local\n"
"http_access deny deny_swarm_dst deny_hit_mark\n"
"acl deny_agent_atlas_dst dstdomain api.t.local\n"
"acl deny_agent_atlas proxy_auth atlas\n"
"http_access deny deny_agent_atlas_dst deny_agent_atlas deny_hit_mark\n"
)
wait_status("forge.t.local", ATLAS, "403")
assert connect("forge.t.local", RUTH) == "403", "a listed name, log mode"
assert connect("forge.t.local", UMA) == "403", "a listed name, unrestricted"
assert connect("other.t.local", UMA) == "403", "unrestricted"
assert connect("other.t.local", RUTH) == "403", "log mode"
assert connect("forge.t.local") == "403", "no credential: refused, not challenged"
assert connect("api.t.local", ATLAS) == "403", "a base-list name, denied for atlas"
assert connect("api.t.local", RUTH) == "200", "control: another agent keeps the base name"
assert connect("api.t.local") == "407", "control: a per-agent deny does not answer a client it cannot name"
proxy.wait_until_succeeds(
"journalctl -u squid.service -o cat | grep -E "
"'egress_denied user=atlas method=CONNECT host=api\\.t\\.local port=443 result=TCP_DENIED/403'"
)
assert squid_saw(r"egress_denied user=- method=CONNECT host=forge\.t\.local port=443 "), "the anonymous client's hit"
assert not squid_saw(r"egress_denied .* host=loop\.t\.local"), "another refusal is not a deny-list hit"
assert not squid_saw(r"egress_denied user=ruth .* host=api\.t\.local"), "another agent's name is not a hit"
write_deny("")
write_allow(seeded, "")
wait_status("forge.t.local", ATLAS, "200")
assert connect("api.t.local", ATLAS) == "200", "the list emptied"
assert connect("other.t.local", UMA) == "200", "the list emptied, unrestricted"
with checked("squid starts on a missing list, treats every user as restricted, and recovers when it appears"):
proxy.succeed(f"rm {LIST}")
proxy.succeed("systemctl restart squid.service")
proxy.wait_for_unit("squid.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
assert connect("forge.t.local", UMA) == "200", "logins work"
assert connect("other.t.local", UMA) == "403", "no list, no unrestricted user"
write_list(*UNRESTRICTED)
wait_status("other.t.local", UMA, "200")
with checked("squid starts on a missing users file, refuses logins, and recovers when it appears"):
proxy.succeed(f"rm {USERS}")
proxy.succeed("systemctl restart squid.service")
proxy.wait_for_unit("squid.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
assert connect("forge.t.local", ATLAS) == "407", "no file, no login"
assert connect("other.t.local", UMA) == "407", "no file, no login, even when listed"
write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U)
assert connect("forge.t.local", ATLAS) == "200", "the file appeared"
assert connect("other.t.local", UMA) == "200", "listed, and now logged in"
def dial(addr):
return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'")[0] == 0
def dropped():
# Packets the table's v4 rule has dropped so far.
out = proxy.succeed("nft list chain inet hive_egress input")
found = re.search(r"ip daddr 127\.0\.0\.0/8 counter packets (\d+)", out)
assert found, out
return int(found.group(1))
with checked("the agent bridge cannot reach the host's loopback, and the table is what stops it"):
assert dial("${bridgeIp}"), "control: a listener on the bridge IP answers"
# The agent has a route for an address the proxy owns on lo, and both
# ends accept loopback addresses on the link, so only the table is left
# between the dial and the listener.
for vm, dev in ((agent, "eth1"), (proxy, "eth1")):
vm.succeed(
"sysctl -w net.ipv4.conf.all.route_localnet=1 net.ipv4.conf.all.rp_filter=0 "
f"net.ipv4.conf.{dev}.route_localnet=1 net.ipv4.conf.{dev}.rp_filter=0"
)
agent.succeed("ip route add 127.0.0.5/32 via ${bridgeIp} table local")
proxy.succeed("nft delete table inet hive_egress")
assert dial("127.0.0.5"), "control: with the table gone the loopback listener answers"
proxy.succeed("systemctl reload hive-egress.service")
before = dropped()
assert not dial("127.0.0.5"), "the table drops the dial"
assert dropped() > before, "the drop rule counted it"
assert dial("${bridgeIp}"), "the bridge IP still answers"
# A firewall reload rewrites the NixOS chains and leaves this table alone.
proxy.succeed("systemctl reload firewall.service")
assert not dial("127.0.0.5"), "still dropped after a firewall reload"
assert dial("${bridgeIp}"), "the bridge IP still answers after the reload"
# With the kernel default back, the dial is refused with no table either.
proxy.succeed("sysctl -w net.ipv4.conf.all.route_localnet=0 net.ipv4.conf.eth1.route_localnet=0")
proxy.succeed("nft delete table inet hive_egress")
assert not dial("127.0.0.5"), "route_localnet=0 refuses it alone"
proxy.succeed("systemctl reload hive-egress.service")
def would_drop(pattern):
# The forward chain's log lines in the kernel log that match `pattern`.
out = proxy.succeed("journalctl -k -o cat --no-pager | grep hive-egress-would-drop || true")
return [line for line in out.splitlines() if re.search(pattern, line)]
with checked("a direct connect from the agent bridge still works in log mode, and the flow is logged with the agent's address"):
# Through squid, to the gateway on the bridge and out of the proxy itself
# first: none of them is forwarded traffic, so none may be logged.
assert connect("forge.t.local", ATLAS) == "200", "through squid"
assert dial("${bridgeIp}"), "the gateway on the bridge IP"
assert dial("203.0.113.2"), "a direct connect to an outside IP works"
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=443 '"
)
agent.succeed("ping -c1 -W2 203.0.113.2")
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=ICMP '"
)
# Control: the same journal holds no line for the traffic that is not forwarded.
assert not would_drop(r"DST=192\.168\.1\.1 "), "the bridge IP is the host's own address"
assert not would_drop(r"DPT=3128 "), "squid's port is the host's own address"
assert not would_drop(r"SRC=203\.0\.113\.1 "), "squid's own connections leave the host and are not forwarded"
out = proxy.succeed("nft list chain inet hive_egress forward")
found = re.search(r'oifname != "eth1" counter packets (\d+) bytes \d+$', out, re.M)
assert found and int(found.group(1)) > 0, out
with checked("the same holds over IPv6"):
agent.wait_until_succeeds("ping -6 -c1 -W2 fd00:98::1")
assert dial("fd00:99::2"), "a direct v6 connect works"
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=fd00:0*98:(0+:)*0*3 DST=fd00:0*99:(0+:)*0*2 .*PROTO=TCP .*DPT=443 '"
)
with checked("a flow to the mesh interface is not logged and one to the uplink is"):
proxy.succeed("ip link add wg-hive type dummy && ip link set wg-hive up")
proxy.succeed("ip route add 198.51.100.0/24 dev wg-hive")
agent.execute("echo x > /dev/udp/198.51.100.7/9; echo x > /dev/udp/203.0.113.77/9")
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E "
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*DST=203\\.0\\.113\\.77 '"
)
assert not would_drop(r"DST=198\.51\.100\.7 "), "the mesh interface is not egress"
with checked("one busy agent address is cut to a burst plus a trickle, the rest still leave a trace, and every flow is logged once at most"):
agent.execute("for i in $(seq 1 100); do echo x > /dev/udp/203.0.113.78/$((3000 + i)); done")
proxy.wait_until_succeeds(
"journalctl -k -o cat --no-pager | grep -E 'hive-egress-would-drop-unmetered .*DST=203\\.0\\.113\\.78 '"
)
time.sleep(3)
lines = would_drop(r"DST=203\.0\.113\.78 ")
metered = [line for line in lines if "hive-egress-would-drop IN=" in line]
unmetered = [line for line in lines if "hive-egress-would-drop-unmetered IN=" in line]
assert 1 <= len(metered) <= 30, f"100 new flows: {len(metered)} per-source lines"
assert 1 <= len(unmetered) <= 80, f"100 new flows: {len(unmetered)} unmetered lines"
ports = [re.search(r"DPT=(\d+) ", line).group(1) for line in lines]
assert len(ports) == len(set(ports)), "a flow was logged twice"
out = proxy.succeed("nft list chain inet hive_egress forward")
found = re.search(r'oifname != "eth1" counter packets (\d+) bytes \d+$', out, re.M)
assert found and int(found.group(1)) >= 100, out
'';
}
View git blame Copy permalink