egress.forward defaults to reject while egress.intercept.enable is on (a hive controller host turns it on) and to log otherwise, because reject without the intercept would refuse all web traffic and an assertion already forbids it. log stays an explicit choice. The hive-squid VM test runs its log-mode half on the base node, so the base pins forward = log at mkDefault priority and the reject specialisation overrides it. A module-eval case covers the default on a controller host, on a host without the intercept, and an explicit log.
1314 lines
68 KiB
Nix
1314 lines
68 KiB
Nix
# `vmTests.<system>.hive-squid`: runs ../host-modules/hive-squid.nix in a VM, which
|
|
# the module-eval suite can only read, and checks that an agent with the proxy
|
|
# variables of nix/agent-modules/egress-proxy.nix reaches squid and skips it
|
|
# where it should. Three nodes on two networks:
|
|
#
|
|
# agent (vlan 1) --> proxy (vlan 1 = the stand-in bridge, vlan 2 = uplink) --> web (vlan 2)
|
|
#
|
|
# `proxy` imports the squid module against stub options for the few hive
|
|
# options it reads, so no hive-c0re build is pulled in; the module-eval suite
|
|
# covers the real option wiring.
|
|
#
|
|
# It also covers ../host-modules/hive-egress.nix: `proxy` loads the real module, so the
|
|
# agent's dials to the host's loopback, and its direct connects to the uplink
|
|
# (`proxy` forwards and masquerades like the hive host), run against the table
|
|
# the module installs. The same table redirects the agents' web traffic into
|
|
# squid's intercept ports; the test writes the bindings and leases that
|
|
# `hive-priv egress-whois` reads, standing in for `egress-pin` and dnsmasq.
|
|
{ pkgs, hivePriv }:
|
|
let
|
|
bridgeIp = "192.168.1.1";
|
|
|
|
hiveStubs =
|
|
{ lib, config, ... }:
|
|
{
|
|
options.services.hyperhive = {
|
|
deploy.hive-controller.enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
};
|
|
network = {
|
|
bridgeIp = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = bridgeIp;
|
|
};
|
|
networkCidr = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "192.168.1.0/24";
|
|
};
|
|
exposeHostPorts = lib.mkOption {
|
|
type = lib.types.listOf lib.types.port;
|
|
default = [ ];
|
|
};
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
};
|
|
bridgeName = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "eth1";
|
|
};
|
|
};
|
|
otel = {
|
|
scrapeTargets = lib.mkOption {
|
|
type = lib.types.attrsOf lib.types.str;
|
|
default = { };
|
|
};
|
|
journalUnits = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
};
|
|
};
|
|
};
|
|
# The user hive-c0re runs as, which owns the users file squid reads.
|
|
config.users.users.hive-core = {
|
|
isSystemUser = true;
|
|
group = "hive-core";
|
|
};
|
|
config.users.groups.hive-core = { };
|
|
# The firewall hole hive-network.nix opens for `exposeHostPorts`.
|
|
config.networking.firewall.interfaces.eth1.allowedTCPPorts =
|
|
config.services.hyperhive.network.exposeHostPorts;
|
|
};
|
|
|
|
# The uplink is a public range. The private ranges squid must refuse are
|
|
# extra subnets on the same link, so the proxy can reach a listener in each:
|
|
# a name that resolves there is refused only by squid's own rules.
|
|
privateNets = [
|
|
{
|
|
name = "priv10.t.local";
|
|
net = "10.99.0";
|
|
}
|
|
{
|
|
name = "priv172.t.local";
|
|
net = "172.16.99";
|
|
}
|
|
{
|
|
name = "priv192.t.local";
|
|
net = "192.168.99";
|
|
}
|
|
{
|
|
name = "cgnat.t.local";
|
|
net = "100.64.99";
|
|
}
|
|
];
|
|
privateAddrs =
|
|
host:
|
|
map (p: {
|
|
address = "${p.net}.${toString host}";
|
|
prefixLength = 24;
|
|
}) privateNets;
|
|
ulaAddr = host: {
|
|
address = "fd00:99::${toString host}";
|
|
prefixLength = 64;
|
|
};
|
|
bridgeUla = host: {
|
|
address = "fd00:98::${toString host}";
|
|
prefixLength = 64;
|
|
};
|
|
|
|
# The certificate `web` answers TLS with. Nothing trusts it: squid splices,
|
|
# so the agent's curl sees the origin's own and runs with `-k`.
|
|
webCert = pkgs.runCommand "hive-squid-web-cert" { nativeBuildInputs = [ pkgs.openssl ]; } ''
|
|
mkdir $out
|
|
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
|
|
-subj /CN=web.t.local -days 3650 -keyout $out/key.pem -out $out/cert.pem
|
|
'';
|
|
|
|
# Plain TCP on 443: a CONNECT is answered before any TLS, which is all the
|
|
# test reads.
|
|
listen443 = addrs: {
|
|
services.nginx = {
|
|
enable = true;
|
|
virtualHosts.default.listen = map (addr: {
|
|
inherit addr;
|
|
port = 443;
|
|
}) addrs;
|
|
};
|
|
};
|
|
in
|
|
pkgs.testers.runNixOSTest {
|
|
name = "hive-squid";
|
|
|
|
# Without this the driver allows an hour, the runner's whole job limit.
|
|
globalTimeout = 1800;
|
|
|
|
nodes = {
|
|
proxy =
|
|
{ lib, pkgs, ... }:
|
|
{
|
|
imports = [
|
|
hiveStubs
|
|
../host-modules/hive-squid.nix
|
|
../host-modules/hive-egress.nix
|
|
# Listeners a leaked destination would reach: loopback (`::` lands on `::1`) and the bridge.
|
|
(listen443 [
|
|
"127.0.0.1"
|
|
# An address the agent can route to the proxy: the agent's own lo owns 127.0.0.1 itself.
|
|
"127.0.0.5"
|
|
"[::1]"
|
|
bridgeIp
|
|
])
|
|
];
|
|
virtualisation.vlans = [
|
|
1
|
|
2
|
|
];
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
|
|
{
|
|
address = bridgeIp;
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
networking.interfaces.eth2.ipv4.addresses = lib.mkForce (
|
|
[
|
|
{
|
|
address = "203.0.113.1";
|
|
prefixLength = 24;
|
|
}
|
|
]
|
|
++ privateAddrs 1
|
|
);
|
|
networking.interfaces.eth2.ipv6.addresses = [ (ulaAddr 1) ];
|
|
networking.interfaces.eth1.ipv6.addresses = [ (bridgeUla 1) ];
|
|
# What hive-network.nix sets for the agent bridge: forwarding, and NAT
|
|
# for everything that leaves it. The agent's direct path out exists.
|
|
networking.nat = {
|
|
enable = true;
|
|
internalInterfaces = [ "eth1" ];
|
|
};
|
|
boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = 1;
|
|
# Names the proxy resolves, each a different destination class.
|
|
networking.hosts = {
|
|
# `other.t.local` and the self-check's deny canary resolve, are
|
|
# reachable and are on no allow-list. The canary is in the hosts file
|
|
# because the destination denies look every name up first, and the VM
|
|
# has no resolver to answer for an unknown one before the client gives up.
|
|
"203.0.113.2" = [
|
|
"forge.t.local"
|
|
"cache.t.local"
|
|
"api.t.local"
|
|
"other.t.local"
|
|
"swarm.t.local"
|
|
"agentonly.t.local"
|
|
"deny-canary.invalid"
|
|
];
|
|
"127.0.0.1" = [ "loop.t.local" ];
|
|
${bridgeIp} = [
|
|
"bridge.t.local"
|
|
"basebridge.t.local"
|
|
];
|
|
"169.254.169.254" = [ "meta.t.local" ];
|
|
"fd00:99::2" = [ "ula.t.local" ];
|
|
"::" = [ "any6.t.local" ];
|
|
}
|
|
// lib.listToAttrs (
|
|
map (p: {
|
|
name = "${p.net}.2";
|
|
value = [ p.name ];
|
|
}) privateNets
|
|
);
|
|
# The gateway: nginx answers on the bridge, which agents reach directly.
|
|
networking.firewall.interfaces.eth1.allowedTCPPorts = [ 443 ];
|
|
# hive-priv.nix links the binary the hive-egress unit loads its tables with;
|
|
# `eth1` is no bridge here, so a stub that only loads the ruleset (no
|
|
# bindings, no leases) stands in, and hands `egress-whois` to the real
|
|
# binary.
|
|
systemd.tmpfiles.rules = [
|
|
"d /var/lib/dnsmasq 0755 root root - -"
|
|
"L+ /run/hive-rootfs-check - - - - ${pkgs.writeShellScript "hive-priv-stub" ''
|
|
[ "$1" = egress-load ] && exec ${pkgs.nftables}/bin/nft -f "$2"
|
|
[ "$1" = egress-whois ] && exec ${hivePriv}/bin/hive-priv "$@"
|
|
[ "$1" = egress-ech-relay ] && exec ${hivePriv}/bin/hive-priv "$@"
|
|
exit 0
|
|
''}"
|
|
];
|
|
# Renders the users file the way hive-c0re does.
|
|
environment.systemPackages = [
|
|
pkgs.mkpasswd
|
|
pkgs.nftables
|
|
];
|
|
# The log-mode half of the test runs on this node, and the reject-mode
|
|
# half switches to the specialisation at its end. The forward mode is
|
|
# fixed when the node evaluates; the switch reloads `hive-egress` with
|
|
# the other ruleset. The default is `reject` while the intercept is on,
|
|
# so the base pins `log`, at a priority the specialisation overrides.
|
|
services.hyperhive.deploy.hive-controller.egress.forward = lib.mkDefault "log";
|
|
specialisation.reject.configuration.services.hyperhive.deploy.hive-controller.egress.forward =
|
|
"reject";
|
|
services.hyperhive.deploy.hive-controller.squid.selfcheck.unrestrictedCanary = "other.t.local";
|
|
services.hyperhive.deploy.hive-controller.squid.selfcheck.canary = "cache.t.local";
|
|
services.hyperhive.deploy.hive-controller.squid.baseAllowedDomains = [
|
|
"api.t.local"
|
|
"basebridge.t.local"
|
|
];
|
|
services.hyperhive.deploy.hive-controller.squid.allowedDomains = [
|
|
"forge.t.local"
|
|
"loop.t.local"
|
|
"bridge.t.local"
|
|
"meta.t.local"
|
|
"ula.t.local"
|
|
"any6.t.local"
|
|
]
|
|
++ map (p: p.name) privateNets;
|
|
};
|
|
|
|
web =
|
|
{ lib, ... }:
|
|
{
|
|
# TLS on 443 for the intercepted flows (a CONNECT through the proxy is
|
|
# answered before any TLS), plain HTTP on 80, and a plain listener on
|
|
# 8443 that the proxy never intercepts.
|
|
services.nginx = {
|
|
enable = true;
|
|
virtualHosts.default = {
|
|
listen = [
|
|
{
|
|
addr = "0.0.0.0";
|
|
port = 443;
|
|
ssl = true;
|
|
}
|
|
{
|
|
addr = "[::]";
|
|
port = 443;
|
|
ssl = true;
|
|
}
|
|
{
|
|
addr = "0.0.0.0";
|
|
port = 80;
|
|
}
|
|
{
|
|
addr = "0.0.0.0";
|
|
port = 8443;
|
|
}
|
|
];
|
|
# nginx writes `ssl_certificate` only for a host with `addSSL`; the
|
|
# `listen` entries above already say which ports are TLS and which plain.
|
|
addSSL = true;
|
|
sslCertificate = "${webCert}/cert.pem";
|
|
sslCertificateKey = "${webCert}/key.pem";
|
|
locations."/".return = "200 web-ok";
|
|
};
|
|
};
|
|
virtualisation.vlans = [ 2 ];
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce (
|
|
[
|
|
{
|
|
address = "203.0.113.2";
|
|
prefixLength = 24;
|
|
}
|
|
# Another address of the same host: where a flow goes whose SNI
|
|
# names the first.
|
|
{
|
|
address = "203.0.113.9";
|
|
prefixLength = 24;
|
|
}
|
|
]
|
|
++ privateAddrs 2
|
|
);
|
|
networking.interfaces.eth1.ipv6.addresses = [ (ulaAddr 2) ];
|
|
networking.interfaces.eth1.ipv6.routes = [
|
|
{
|
|
address = "fd00:98::";
|
|
prefixLength = 64;
|
|
via = "fd00:99::1";
|
|
}
|
|
];
|
|
networking.firewall.allowedTCPPorts = [
|
|
80
|
|
443
|
|
8443
|
|
];
|
|
};
|
|
|
|
agent =
|
|
{ lib, pkgs, ... }:
|
|
{
|
|
virtualisation.vlans = [ 1 ];
|
|
# The bridge listener, by the name the proxy resolves to the same address.
|
|
networking.hosts.${bridgeIp} = [ "bridge.t.local" ];
|
|
# The names the proxy resolves, for the flows that skip the proxy variables.
|
|
networking.hosts."203.0.113.2" = [
|
|
"forge.t.local"
|
|
"other.t.local"
|
|
"agentonly.t.local"
|
|
"api.t.local"
|
|
];
|
|
environment.systemPackages = [
|
|
pkgs.openssl
|
|
pkgs.python3
|
|
];
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
|
|
{
|
|
address = "192.168.1.3";
|
|
prefixLength = 24;
|
|
}
|
|
];
|
|
networking.interfaces.eth1.ipv4.routes = [
|
|
{
|
|
address = "203.0.113.0";
|
|
prefixLength = 24;
|
|
via = bridgeIp;
|
|
}
|
|
{
|
|
address = "198.51.100.0";
|
|
prefixLength = 24;
|
|
via = bridgeIp;
|
|
}
|
|
];
|
|
networking.interfaces.eth1.ipv6.addresses = [ (bridgeUla 3) ];
|
|
networking.interfaces.eth1.ipv6.routes = [
|
|
{
|
|
address = "fd00:99::";
|
|
prefixLength = 64;
|
|
via = "fd00:98::1";
|
|
}
|
|
];
|
|
};
|
|
};
|
|
|
|
testScript = ''
|
|
import ipaddress
|
|
import re
|
|
import secrets
|
|
import shlex
|
|
import time
|
|
from contextlib import contextmanager
|
|
|
|
# Drawn fresh each run. The users file holds only their hashes, as
|
|
# hive-c0re renders it. `atlas` is a restricted agent and `uma` an
|
|
# unrestricted one; the two self-check users share one secret.
|
|
PW = {
|
|
u: secrets.token_urlsafe(24)
|
|
for u in ("atlas", "uma", "selfcheck.squid", "ruth")
|
|
}
|
|
PW["selfcheck-unrestricted.squid"] = PW["selfcheck.squid"]
|
|
ATLAS = "atlas:" + PW["atlas"]
|
|
UMA = "uma:" + PW["uma"]
|
|
SELFCHECK = "selfcheck.squid:" + PW["selfcheck.squid"]
|
|
SELFCHECK_U = "selfcheck-unrestricted.squid:" + PW["selfcheck.squid"]
|
|
RUTH = "ruth:" + PW["ruth"]
|
|
USERS = "/var/lib/hive-squid/users"
|
|
LIST = "/var/lib/hive-squid/unrestricted"
|
|
ALLOWED = "/var/lib/hive-squid/allowed"
|
|
AGENT_ALLOW = "/var/lib/hive-squid/agent-allow.conf"
|
|
DENY = "/var/lib/hive-squid/deny.conf"
|
|
UNRESTRICTED = ["uma", "selfcheck-unrestricted.squid"]
|
|
|
|
start_all()
|
|
proxy.wait_for_unit("squid.service")
|
|
proxy.wait_for_unit("squid-exporter.service")
|
|
proxy.wait_for_unit("nginx.service")
|
|
web.wait_for_unit("nginx.service")
|
|
proxy.wait_for_open_port(3128, "${bridgeIp}")
|
|
# `start_all` boots the agent beside the proxy; its network is up only once
|
|
# it has reached multi-user.target and can open the proxy's port.
|
|
agent.wait_for_unit("multi-user.target")
|
|
agent.wait_until_succeeds("timeout 3 bash -c 'echo > /dev/tcp/${bridgeIp}/3128'")
|
|
|
|
PRIVATE = ["priv10.t.local", "priv172.t.local", "priv192.t.local", "cgnat.t.local", "ula.t.local", "any6.t.local"]
|
|
|
|
SEEN = []
|
|
|
|
def connect(host, cred=None):
|
|
# The status squid answered the CONNECT with. 000 means no answer, which
|
|
# no assertion expects: squid closes its listener while a reload re-reads
|
|
# its configuration, so 000 is retried.
|
|
auth = f"-U '{cred}' " if cred else ""
|
|
got = "000"
|
|
for _ in range(15):
|
|
got = agent.succeed(
|
|
"curl -sS -m 15 -p -x http://${bridgeIp}:3128 " + auth + "-o /dev/null "
|
|
f"-w '%{{http_connect}}' https://{host}/ 2>/dev/null || true"
|
|
).strip()
|
|
if got != "000":
|
|
break
|
|
time.sleep(1)
|
|
SEEN.append(f"{host} as {cred.split(':')[0] if cred else '-'}: {got}")
|
|
return got
|
|
|
|
@contextmanager
|
|
def checked(name):
|
|
# A subtest that prints what the proxy and the agent were doing when it fails.
|
|
with subtest(name):
|
|
try:
|
|
yield
|
|
except BaseException:
|
|
print("recent connects:\n" + "\n".join(SEEN[-12:]))
|
|
for vm, cmd in (
|
|
(proxy, "systemctl status squid.service squid-unrestricted-reload.service squid-selfcheck.service --no-pager -n 0"),
|
|
(proxy, "journalctl -u squid.service -o cat --no-pager | tail -n 40"),
|
|
(proxy, "ss -ltn"),
|
|
(agent, "ip -brief address; ip route; systemctl is-system-running"),
|
|
):
|
|
print(f"--- {vm.name}: {cmd}\n" + vm.execute(cmd)[1])
|
|
raise
|
|
|
|
def journal():
|
|
return proxy.succeed("journalctl -u squid.service -o cat --no-pager")
|
|
|
|
def squid_reconfigs():
|
|
# How many reconfigurations squid has begun, and whether its intercept
|
|
# listeners are open again since the last one.
|
|
parts = journal().split("Reconfiguring Squid Cache")
|
|
return len(parts) - 1, "Accepting NAT intercepted SSL bumped HTTPS Socket connections" in parts[-1]
|
|
|
|
def wait_reconfigured(before):
|
|
# squid closes its listeners while it reconfigures, which begins a moment
|
|
# after squid-unrestricted-reload starts and can outlast the unit. Returns
|
|
# once a reconfiguration beyond the `before` count has finished, the unit
|
|
# is idle and the listeners accept again.
|
|
state = None
|
|
for _ in range(60):
|
|
count, reopened = squid_reconfigs()
|
|
idle = proxy.succeed("systemctl show -p ActiveState --value squid-unrestricted-reload.service").strip()
|
|
listening = proxy.succeed("ss -Hltn")
|
|
state = (count, before, reopened, idle)
|
|
if (
|
|
count > before
|
|
and reopened
|
|
and idle == "inactive"
|
|
and all(f"${bridgeIp}:{port} " in listening for port in (3128, 3129, 3130))
|
|
):
|
|
return
|
|
time.sleep(0.5)
|
|
raise AssertionError(f"squid did not apply the write: (reconfigurations, before, reopened, reload unit) = {state}")
|
|
|
|
def write_and_wait(cmd):
|
|
# Runs a command that writes a list squid reads on a reload, and returns
|
|
# once squid has applied it.
|
|
before = squid_reconfigs()[0]
|
|
proxy.succeed(cmd)
|
|
wait_reconfigured(before)
|
|
|
|
def write_users(*creds):
|
|
# What hive-c0re does: hashes only, written aside and renamed into
|
|
# place, in squid's group. The sleep keeps two writes in different
|
|
# seconds, which is how the helper tells the file changed.
|
|
lines = " ".join(
|
|
f"printf '%s:%s\\n' {u} \"$(mkpasswd -m sha-512 -s <<< {p})\";"
|
|
for u, p in (c.split(":", 1) for c in creds)
|
|
)
|
|
proxy.succeed(
|
|
f"({lines}) > {USERS}.new && chown hive-core:squid {USERS}.new "
|
|
f"&& chmod 640 {USERS}.new && mv {USERS}.new {USERS} && sleep 1"
|
|
)
|
|
|
|
def write_list(*names):
|
|
# What hive-c0re does for the list. Squid reads it on a reload, which
|
|
# squid-unrestricted-reload does when the file changes.
|
|
body = "printf '%s\\n' " + " ".join(names) if names else ":"
|
|
write_and_wait(
|
|
f"({body}) > {LIST}.new && chown hive-core:squid {LIST}.new "
|
|
f"&& chmod 640 {LIST}.new && mv {LIST}.new {LIST}"
|
|
)
|
|
|
|
def write_allow(allowed, agent_conf):
|
|
# What hive-c0re does for its two files: written aside and renamed into
|
|
# place, in squid's group. Squid reads them on a reload, which
|
|
# squid-unrestricted-reload does when either changes.
|
|
write_and_wait(
|
|
" && ".join(
|
|
f"printf '%s' {shlex.quote(body)} > {path}.new && chown hive-core:squid {path}.new "
|
|
f"&& chmod 640 {path}.new && mv {path}.new {path}"
|
|
for path, body in ((ALLOWED, allowed), (AGENT_ALLOW, agent_conf))
|
|
)
|
|
)
|
|
|
|
def write_deny(body):
|
|
# What hive-c0re does for the deny list: written aside and renamed into
|
|
# place, in squid's group; the reload is squid-unrestricted-reload's.
|
|
write_and_wait(
|
|
f"printf '%s' {shlex.quote(body)} > {DENY}.new && chown hive-core:squid {DENY}.new "
|
|
f"&& chmod 640 {DENY}.new && mv {DENY}.new {DENY}"
|
|
)
|
|
|
|
def wait_status(host, cred, want):
|
|
# The reload that follows a list write is asynchronous.
|
|
got = None
|
|
for _ in range(30):
|
|
got = connect(host, cred)
|
|
if got == want:
|
|
return
|
|
time.sleep(1)
|
|
raise AssertionError(f"{host} as {cred.split(':')[0]}: wanted {want}, last got {got}")
|
|
|
|
with checked("the private listeners answer the proxy itself, so a refusal below is squid's"):
|
|
for addr in ["10.99.0.2", "172.16.99.2", "192.168.99.2", "100.64.99.2", "fd00:99::2"]:
|
|
proxy.wait_until_succeeds(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'")
|
|
assert connect("forge.t.local", None) == "407", "control: the public name is challenged, not refused"
|
|
|
|
with checked("squid starts on the empty users file and list and refuses every login"):
|
|
assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640"
|
|
assert proxy.succeed(f"stat -c '%U:%G %a' {LIST}").strip() == "hive-core:squid 640"
|
|
assert proxy.succeed("stat -c %a /var/lib/hive-squid").strip() == "2750"
|
|
assert proxy.succeed(f"stat -c %s {USERS}").strip() == "0"
|
|
assert proxy.succeed(f"stat -c %s {LIST}").strip() == "0"
|
|
assert proxy.succeed(f"stat -c '%U:%G %a' {ALLOWED}").strip() == "hive-core:squid 640"
|
|
assert proxy.succeed(f"stat -c '%U:%G %a' {AGENT_ALLOW}").strip() == "hive-core:squid 640"
|
|
assert proxy.succeed(f"stat -c %s {AGENT_ALLOW}").strip() == "0"
|
|
assert proxy.succeed(f"stat -c '%U:%G %a' {DENY}").strip() == "hive-core:squid 640"
|
|
assert proxy.succeed(f"stat -c %s {DENY}").strip() == "0", "no deny list, no rule"
|
|
# The allow-list file starts as the hive's own allowedDomains.
|
|
proxy.succeed(f"grep -Fx forge.t.local {ALLOWED}")
|
|
assert connect("forge.t.local", ATLAS) == "407", "no such user"
|
|
assert connect("other.t.local", UMA) == "407", "no such user, unlisted name"
|
|
|
|
with checked("the users file holds hashes, never a password"):
|
|
write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U)
|
|
users = proxy.succeed(f"cat {USERS}")
|
|
rows = [l.split(":", 1) for l in users.splitlines()]
|
|
assert sorted(r[0] for r in rows) == sorted(
|
|
["atlas", "uma", "selfcheck.squid", "selfcheck-unrestricted.squid"]
|
|
), users
|
|
assert all(re.fullmatch(r"\$6\$\S+", r[1]) for r in rows), users
|
|
assert not any(p in users for p in PW.values()), users
|
|
|
|
with checked("an allowed name is tunnelled with a credential and challenged without one"):
|
|
assert connect("forge.t.local", ATLAS) == "200", "allow canary"
|
|
assert connect("cache.t.local", ATLAS) == "200", "the self-check canary, allowed on top of the list"
|
|
assert connect("forge.t.local") == "407", "no credential"
|
|
assert connect("forge.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password"
|
|
assert connect("forge.t.local", "nobody:" + PW["atlas"]) == "407", "unknown user"
|
|
# DES would accept the first eight characters; the hash must not.
|
|
assert connect("forge.t.local", "atlas:" + PW["atlas"][:8]) == "407", "a prefix of the password"
|
|
|
|
with checked("a base-list name is open to a restricted user, with a credential, and nothing else is"):
|
|
assert connect("api.t.local", ATLAS) == "200", "restricted user, base name"
|
|
assert connect("api.t.local") == "407", "no credential"
|
|
assert connect("api.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password"
|
|
assert connect("other.t.local", ATLAS) == "403", "control: an unlisted name is still refused"
|
|
assert connect("basebridge.t.local", ATLAS) == "403", "a base name that resolves to the bridge"
|
|
|
|
with checked("with an empty list every user is restricted"):
|
|
assert connect("other.t.local", UMA) == "403", "uma, not listed yet"
|
|
assert connect("other.t.local", ATLAS) == "403"
|
|
assert connect("forge.t.local", UMA) == "200", "control: uma still reaches a listed name"
|
|
|
|
with checked("a listed user reaches an unlisted name and the others are refused"):
|
|
write_list(*UNRESTRICTED)
|
|
wait_status("other.t.local", UMA, "200")
|
|
assert connect("other.t.local", ATLAS) == "403", "restricted"
|
|
assert connect("forge.t.local", ATLAS) == "200", "restricted, listed name"
|
|
assert connect("forge.t.local", UMA) == "200", "unrestricted, listed name"
|
|
assert connect("other.t.local") == "407", "no credential"
|
|
assert connect("other.t.local", "uma:" + secrets.token_urlsafe(24)) == "407", "wrong password"
|
|
assert connect("deny-canary.invalid", ATLAS) == "403", "restricted, an unlisted name"
|
|
|
|
with checked("neither kind of user reaches a host-local or private destination, or another port"):
|
|
for cred in (ATLAS, UMA):
|
|
who = cred.split(":")[0]
|
|
assert connect("loop.t.local", cred) == "403", f"{who}: loopback"
|
|
assert connect("bridge.t.local", cred) == "403", f"{who}: bridge address"
|
|
assert connect("meta.t.local", cred) == "403", f"{who}: link-local"
|
|
assert connect("127.0.0.1", cred) == "403", f"{who}: loopback literal"
|
|
assert connect("203.0.113.2", cred) == "403", f"{who}: IP literal"
|
|
for name in PRIVATE:
|
|
assert connect(name, cred) == "403", f"{who}: {name}, a listed name that resolves to a private address"
|
|
assert connect("other.t.local:22", cred) == "403", f"{who}: port 22"
|
|
assert connect("other.t.local:80", cred) == "403", f"{who}: CONNECT to port 80"
|
|
|
|
with checked("an agent cannot read cachemgr"):
|
|
agent.succeed(
|
|
"test $(curl -s -m 15 -o /dev/null -w '%{http_code}' "
|
|
"http://${bridgeIp}:3128/squid-internal-mgr/counters) = 403"
|
|
)
|
|
|
|
# The variables an agent's `proxy.env` carries (nix/agent-modules/egress-proxy.nix),
|
|
# as a curl on the agent reads them.
|
|
def agent_env(cred, no_proxy):
|
|
url = f"http://{cred}@${bridgeIp}:3128"
|
|
pairs = {
|
|
"HTTPS_PROXY": url, "HTTP_PROXY": url, "https_proxy": url, "http_proxy": url,
|
|
"NO_PROXY": no_proxy, "no_proxy": no_proxy,
|
|
}
|
|
return " ".join(f"{k}='{v}'" for k, v in pairs.items())
|
|
|
|
def agent_get(cred, no_proxy, url):
|
|
# The status of the answer, whoever gave it; 000 when none came.
|
|
return agent.succeed(
|
|
f"env {agent_env(cred, no_proxy)} curl -sS -m 15 -o /dev/null -w '%{{http_code}}' {url} 2>/dev/null || true"
|
|
).strip()
|
|
|
|
def squid_saw(pattern):
|
|
return re.search(pattern, proxy.succeed("journalctl -u squid.service -o cat"))
|
|
|
|
with checked("an agent with the proxy variables reaches an allowed name through squid and a swarm-internal name directly"):
|
|
NO_PROXY = "localhost,127.0.0.1,::1,bridge.t.local,${bridgeIp}"
|
|
# Plain HTTP to the listeners' port 443, so the answer carries a status.
|
|
assert agent_get(ATLAS, NO_PROXY, "http://forge.t.local:443/") not in ("000", "403", "407")
|
|
assert squid_saw(r"192\.168\.1\.3 TCP_\w+/\d+ .* GET http://forge\.t\.local:443/ atlas "), "through squid, as atlas"
|
|
assert agent_get(ATLAS, NO_PROXY, "http://bridge.t.local:443/") not in ("000", "403", "407")
|
|
assert not squid_saw(r"GET http://bridge\.t\.local:443/ "), "a swarm-internal name never reached squid"
|
|
# Control: without the exemption the same request goes to squid, which refuses the bridge.
|
|
assert agent_get(ATLAS, "localhost", "http://bridge.t.local:443/") == "403"
|
|
assert squid_saw(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://bridge\.t\.local:443/ "), "control: squid refuses it"
|
|
# CONNECT takes the same path: HTTPS_PROXY names the proxy for https URLs.
|
|
assert agent.succeed(
|
|
f"env {agent_env(ATLAS, NO_PROXY)} curl -sS -m 15 -o /dev/null -w '%{{http_connect}}' https://forge.t.local/ 2>/dev/null || true"
|
|
).strip() == "200"
|
|
|
|
with checked("an agent without the proxy variables goes direct"):
|
|
assert agent.succeed(
|
|
"curl -sS -m 15 -o /dev/null -w '%{http_code}' http://bridge.t.local:443/ 2>/dev/null || true"
|
|
).strip() not in ("000", "403", "407")
|
|
|
|
with checked("the self-check passes with its credential"):
|
|
proxy.succeed("install -d -m 0700 /run/credstore")
|
|
proxy.succeed(f"printf '%s\\n' {PW['selfcheck.squid']} > /run/credstore/hive-squid.selfcheck")
|
|
proxy.succeed("chmod 400 /run/credstore/hive-squid.selfcheck")
|
|
proxy.succeed("systemctl start squid-selfcheck.service")
|
|
proxy.succeed(
|
|
"journalctl -u squid-selfcheck.service -o cat | "
|
|
"grep -F 'squid-selfcheck ok=true allow_canary=cache.t.local "
|
|
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403 "
|
|
"anon_status=407 credential=present "
|
|
"unrestricted_canary=other.t.local unrestricted_status=200'"
|
|
)
|
|
|
|
with checked("the self-check fails, and says why, when the unrestricted user is not listed"):
|
|
write_list()
|
|
wait_status("other.t.local", UMA, "403")
|
|
proxy.fail("systemctl start squid-selfcheck.service")
|
|
proxy.succeed(
|
|
"journalctl -u squid-selfcheck.service -o cat | "
|
|
"grep -F 'squid-selfcheck ok=false allow_canary=cache.t.local "
|
|
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403 "
|
|
"anon_status=407 credential=present "
|
|
"unrestricted_canary=other.t.local unrestricted_status=403'"
|
|
)
|
|
write_list(*UNRESTRICTED)
|
|
wait_status("other.t.local", UMA, "200")
|
|
|
|
with checked("the self-check fails, and says why, without a credential or with the wrong one"):
|
|
proxy.succeed("mv /run/credstore/hive-squid.selfcheck /run/credstore/kept")
|
|
proxy.fail("systemctl start squid-selfcheck.service")
|
|
proxy.succeed(
|
|
"journalctl -u squid-selfcheck.service -o cat | "
|
|
"grep -F 'squid-selfcheck ok=false allow_canary=cache.t.local "
|
|
"allow_status=407 deny_canary=deny-canary.invalid deny_status=407 "
|
|
"anon_status=407 credential=absent "
|
|
"unrestricted_canary=other.t.local unrestricted_status=407'"
|
|
)
|
|
proxy.succeed(f"printf '%s\\n' {secrets.token_urlsafe(24)} > /run/credstore/hive-squid.selfcheck")
|
|
proxy.fail("systemctl start squid-selfcheck.service")
|
|
proxy.succeed(
|
|
"journalctl -u squid-selfcheck.service -o cat | "
|
|
"grep -F 'allow_status=407 deny_canary=deny-canary.invalid deny_status=407 "
|
|
"anon_status=407 credential=present "
|
|
"unrestricted_canary=other.t.local unrestricted_status=407'"
|
|
)
|
|
proxy.succeed("mv -f /run/credstore/kept /run/credstore/hive-squid.selfcheck")
|
|
|
|
with checked("squid's access log reaches the journal under squid.service, with the user"):
|
|
access_log = proxy.succeed("journalctl -u squid.service -o cat")
|
|
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ", access_log), access_log
|
|
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT other\.t\.local:443 uma ", access_log), access_log
|
|
assert re.search(r"192\.168\.1\.3 TCP_DENIED/407 .* CONNECT forge\.t\.local:443 - ", access_log), access_log
|
|
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", access_log), access_log
|
|
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443 ", access_log), access_log
|
|
# Exporter scrapes are cachemgr requests and stay out of the log.
|
|
assert "squid-internal-mgr" not in access_log, access_log
|
|
|
|
with checked("the exporter serves squid's counters on loopback"):
|
|
metrics = proxy.succeed("curl -sf http://127.0.0.1:9301/metrics")
|
|
print("\n".join(l for l in metrics.splitlines() if l.startswith(("squid_up", "squid_client_http"))))
|
|
assert 'squid_up{host="${bridgeIp}"} 1' in metrics
|
|
m = re.search(r"^squid_client_http_requests_total (\S+)$", metrics, re.M)
|
|
assert m and float(m.group(1)) >= 9, metrics
|
|
m = re.search(r"^squid_client_http_errors_total (\S+)$", metrics, re.M)
|
|
assert m and float(m.group(1)) >= 1, metrics
|
|
proxy.fail("curl -sf -m 5 http://${bridgeIp}:9301/metrics")
|
|
|
|
with checked("squid survives a reload and keeps the list"):
|
|
proxy.succeed("systemctl reload squid.service")
|
|
proxy.wait_for_unit("squid.service")
|
|
assert connect("forge.t.local", ATLAS) == "200"
|
|
assert connect("other.t.local", UMA) == "200"
|
|
assert connect("other.t.local", ATLAS) == "403"
|
|
|
|
with checked("a user added while squid runs can log in, with no reload, and is restricted"):
|
|
write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U, RUTH)
|
|
assert connect("forge.t.local", RUTH) == "200"
|
|
assert connect("forge.t.local", ATLAS) == "200"
|
|
assert connect("other.t.local", RUTH) == "403"
|
|
|
|
with checked("a user the list gains is unrestricted after the reload and loses it again"):
|
|
write_list(*UNRESTRICTED, "ruth")
|
|
wait_status("other.t.local", RUTH, "200")
|
|
assert connect("loop.t.local", RUTH) == "403", "still no loopback"
|
|
write_list(*UNRESTRICTED)
|
|
wait_status("other.t.local", RUTH, "403")
|
|
assert connect("other.t.local", UMA) == "200", "control: uma is still listed"
|
|
|
|
with checked("the swarm's allow-list adds to the hive's names, opens one to every restricted user, and a per-agent rule opens one to that agent alone"):
|
|
seeded = proxy.succeed(f"cat {ALLOWED}")
|
|
assert connect("swarm.t.local", RUTH) == "403", "control: not allowed yet"
|
|
assert connect("agentonly.t.local", ATLAS) == "403", "control: not allowed yet"
|
|
assert connect("forge.t.local", RUTH) == "200", "control: the hive's own name"
|
|
write_allow(
|
|
"# source: swarm\n" + seeded + "swarm.t.local\n",
|
|
"# source: swarm\n"
|
|
"acl agent_allow_atlas proxy_auth atlas\n"
|
|
"acl agent_allow_atlas_dst dstdomain agentonly.t.local priv10.t.local\n"
|
|
"acl agent_allow_atlas_ix ext_user atlas\n"
|
|
"http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n"
|
|
"http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n",
|
|
)
|
|
wait_status("swarm.t.local", RUTH, "200")
|
|
wait_status("agentonly.t.local", ATLAS, "200")
|
|
assert connect("swarm.t.local", ATLAS) == "200", "swarm-wide, another user"
|
|
assert connect("agentonly.t.local", RUTH) == "403", "the per-agent name, another restricted user"
|
|
assert connect("agentonly.t.local") == "407", "no credential"
|
|
assert connect("forge.t.local", RUTH) == "200", "the hive's own name stays"
|
|
assert connect("priv10.t.local", ATLAS) == "403", "a per-agent name that resolves privately"
|
|
assert connect("agentonly.t.local", UMA) == "200", "control: unrestricted reaches it anyway"
|
|
write_allow(seeded, "")
|
|
wait_status("swarm.t.local", RUTH, "403")
|
|
wait_status("agentonly.t.local", ATLAS, "403")
|
|
assert connect("forge.t.local", RUTH) == "200", "the hive's own name after the swarm list is gone"
|
|
|
|
with checked("a log-mode agent reaches an unlisted name and is logged as would_deny, and nothing else changes"):
|
|
# What hive-c0re renders for `ruth` in log mode, after the agent rules.
|
|
write_allow(
|
|
seeded,
|
|
"# source: swarm\n"
|
|
"acl log_mark annotate_transaction would_deny=1\n"
|
|
"acl agent_log_ruth proxy_auth ruth\n"
|
|
"acl agent_log_ruth_ix ext_user ruth\n"
|
|
"http_access allow !icpt agent_log_ruth !allowed_dst log_mark\n"
|
|
"http_access allow icpt agent_log_ruth_ix !allowed_dst log_mark\n",
|
|
)
|
|
wait_status("other.t.local", RUTH, "200")
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -u squid.service -o cat | grep -E "
|
|
"'would_deny user=ruth method=CONNECT host=other\\.t\\.local port=443 result='"
|
|
)
|
|
assert connect("forge.t.local", RUTH) == "200", "a listed name"
|
|
assert connect("other.t.local", ATLAS) == "403", "control: a restricted agent is still refused"
|
|
assert connect("other.t.local") == "407", "no credential"
|
|
assert connect("other.t.local", "ruth:" + secrets.token_urlsafe(24)) == "407", "wrong password"
|
|
assert connect("deny-canary.invalid", SELFCHECK) == "403", "the self-check's deny canary"
|
|
for name in ["loop.t.local", "bridge.t.local", "meta.t.local", "127.0.0.1", "203.0.113.2", "priv10.t.local", "other.t.local:22", "other.t.local:80"]:
|
|
assert connect(name, RUTH) == "403", f"log mode still refuses {name}"
|
|
assert squid_saw(r"would_deny user=ruth method=CONNECT host=other\.t\.local port=443 "), "the unlisted name is marked"
|
|
assert not squid_saw(r"would_deny user=ruth .* host=forge\.t\.local"), "a listed name is not marked"
|
|
assert not squid_saw(r"would_deny user=atlas "), "a restricted agent is not marked"
|
|
assert not squid_saw(r"would_deny .* host=loop\.t\.local"), "a refused request is not marked"
|
|
write_allow(seeded, "")
|
|
wait_status("other.t.local", RUTH, "403")
|
|
|
|
with checked("the swarm's deny list is refused before any allow, in every mode, with or without a credential, and logged as egress_denied"):
|
|
# What hive-c0re renders: `forge.t.local` and `other.t.local` for every
|
|
# agent, `api.t.local` (a base-list name) for atlas alone. `ruth` is in
|
|
# log mode, so her rule follows. The names are allowed or reachable
|
|
# without the list, which the controls establish first.
|
|
assert connect("forge.t.local", ATLAS) == "200", "control: allowed by the hive"
|
|
assert connect("forge.t.local", UMA) == "200", "control: allowed, unrestricted"
|
|
assert connect("other.t.local", UMA) == "200", "control: unrestricted reaches it"
|
|
assert connect("api.t.local", ATLAS) == "200", "control: base list"
|
|
write_allow(
|
|
seeded,
|
|
"# source: swarm\n"
|
|
"acl log_mark annotate_transaction would_deny=1\n"
|
|
"acl agent_log_ruth proxy_auth ruth\n"
|
|
"acl agent_log_ruth_ix ext_user ruth\n"
|
|
"http_access allow !icpt agent_log_ruth !allowed_dst log_mark\n"
|
|
"http_access allow icpt agent_log_ruth_ix !allowed_dst log_mark\n",
|
|
)
|
|
wait_status("other.t.local", RUTH, "200")
|
|
write_deny(
|
|
"acl deny_hit_mark annotate_transaction egress_deny=1\n"
|
|
"acl deny_swarm_dst dstdomain forge.t.local other.t.local\n"
|
|
"http_access deny deny_swarm_dst deny_hit_mark\n"
|
|
"acl deny_agent_atlas_dst dstdomain api.t.local\n"
|
|
"acl deny_agent_atlas proxy_auth atlas\n"
|
|
"acl deny_agent_atlas_ix ext_user atlas\n"
|
|
"http_access deny deny_agent_atlas_dst !icpt deny_agent_atlas deny_hit_mark\n"
|
|
"http_access deny deny_agent_atlas_dst icpt deny_agent_atlas_ix deny_hit_mark\n"
|
|
)
|
|
wait_status("forge.t.local", ATLAS, "403")
|
|
assert connect("forge.t.local", RUTH) == "403", "a listed name, log mode"
|
|
assert connect("forge.t.local", UMA) == "403", "a listed name, unrestricted"
|
|
assert connect("other.t.local", UMA) == "403", "unrestricted"
|
|
assert connect("other.t.local", RUTH) == "403", "log mode"
|
|
assert connect("forge.t.local") == "403", "no credential: refused, not challenged"
|
|
assert connect("api.t.local", ATLAS) == "403", "a base-list name, denied for atlas"
|
|
assert connect("api.t.local", RUTH) == "200", "control: another agent keeps the base name"
|
|
assert connect("api.t.local") == "407", "control: a per-agent deny does not answer a client it cannot name"
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -u squid.service -o cat | grep -E "
|
|
"'egress_denied user=atlas method=CONNECT host=api\\.t\\.local port=443 result=TCP_DENIED/403'"
|
|
)
|
|
assert squid_saw(r"egress_denied user=- method=CONNECT host=forge\.t\.local port=443 "), "the anonymous client's hit"
|
|
assert not squid_saw(r"egress_denied .* host=loop\.t\.local"), "another refusal is not a deny-list hit"
|
|
assert not squid_saw(r"egress_denied user=ruth .* host=api\.t\.local"), "another agent's name is not a hit"
|
|
write_deny("")
|
|
write_allow(seeded, "")
|
|
wait_status("forge.t.local", ATLAS, "200")
|
|
assert connect("api.t.local", ATLAS) == "200", "the list emptied"
|
|
assert connect("other.t.local", UMA) == "200", "the list emptied, unrestricted"
|
|
|
|
with checked("squid starts on a missing list, treats every user as restricted, and recovers when it appears"):
|
|
proxy.succeed(f"rm {LIST}")
|
|
proxy.succeed("systemctl restart squid.service")
|
|
proxy.wait_for_unit("squid.service")
|
|
proxy.wait_for_open_port(3128, "${bridgeIp}")
|
|
assert connect("forge.t.local", UMA) == "200", "logins work"
|
|
assert connect("other.t.local", UMA) == "403", "no list, no unrestricted user"
|
|
write_list(*UNRESTRICTED)
|
|
wait_status("other.t.local", UMA, "200")
|
|
|
|
with checked("squid starts on a missing users file, refuses logins, and recovers when it appears"):
|
|
proxy.succeed(f"rm {USERS}")
|
|
proxy.succeed("systemctl restart squid.service")
|
|
proxy.wait_for_unit("squid.service")
|
|
proxy.wait_for_open_port(3128, "${bridgeIp}")
|
|
assert connect("forge.t.local", ATLAS) == "407", "no file, no login"
|
|
assert connect("other.t.local", UMA) == "407", "no file, no login, even when listed"
|
|
write_users(ATLAS, UMA, SELFCHECK, SELFCHECK_U)
|
|
assert connect("forge.t.local", ATLAS) == "200", "the file appeared"
|
|
assert connect("other.t.local", UMA) == "200", "listed, and now logged in"
|
|
|
|
def dial(addr, port=443):
|
|
return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/{port}'")[0] == 0
|
|
|
|
def dropped():
|
|
# Packets the table's v4 rule has dropped so far.
|
|
out = proxy.succeed("nft list chain inet hive_egress input")
|
|
found = re.search(r"ip daddr 127\.0\.0\.0/8 counter packets (\d+)", out)
|
|
assert found, out
|
|
return int(found.group(1))
|
|
|
|
with checked("the agent bridge cannot reach the host's loopback, and the table is what stops it"):
|
|
assert dial("${bridgeIp}"), "control: a listener on the bridge IP answers"
|
|
# The agent has a route for an address the proxy owns on lo, and both
|
|
# ends accept loopback addresses on the link, so only the table is left
|
|
# between the dial and the listener.
|
|
for vm, dev in ((agent, "eth1"), (proxy, "eth1")):
|
|
vm.succeed(
|
|
"sysctl -w net.ipv4.conf.all.route_localnet=1 net.ipv4.conf.all.rp_filter=0 "
|
|
f"net.ipv4.conf.{dev}.route_localnet=1 net.ipv4.conf.{dev}.rp_filter=0"
|
|
)
|
|
agent.succeed("ip route add 127.0.0.5/32 via ${bridgeIp} table local")
|
|
proxy.succeed("nft delete table inet hive_egress")
|
|
assert dial("127.0.0.5"), "control: with the table gone the loopback listener answers"
|
|
proxy.succeed("systemctl reload hive-egress.service")
|
|
before = dropped()
|
|
assert not dial("127.0.0.5"), "the table drops the dial"
|
|
assert dropped() > before, "the drop rule counted it"
|
|
assert dial("${bridgeIp}"), "the bridge IP still answers"
|
|
# A firewall reload rewrites the NixOS chains and leaves this table alone.
|
|
proxy.succeed("systemctl reload firewall.service")
|
|
assert not dial("127.0.0.5"), "still dropped after a firewall reload"
|
|
assert dial("${bridgeIp}"), "the bridge IP still answers after the reload"
|
|
# With the kernel default back, the dial is refused with no table either.
|
|
proxy.succeed("sysctl -w net.ipv4.conf.all.route_localnet=0 net.ipv4.conf.eth1.route_localnet=0")
|
|
proxy.succeed("nft delete table inet hive_egress")
|
|
assert not dial("127.0.0.5"), "route_localnet=0 refuses it alone"
|
|
proxy.succeed("systemctl reload hive-egress.service")
|
|
|
|
def would_drop(pattern):
|
|
# The forward chain's log lines in the kernel log that match `pattern`.
|
|
out = proxy.succeed("journalctl -k -o cat --no-pager | grep hive-egress-would-drop || true")
|
|
return [line for line in out.splitlines() if re.search(pattern, line)]
|
|
|
|
with checked("a direct connect from the agent bridge still works in log mode, and the flow is logged with the agent's address"):
|
|
# Through squid, to the gateway on the bridge and out of the proxy itself
|
|
# first: none of them is forwarded traffic, so none may be logged.
|
|
assert connect("forge.t.local", ATLAS) == "200", "through squid"
|
|
assert dial("${bridgeIp}"), "the gateway on the bridge IP"
|
|
# Port 8443: a flow to 80 or 443 is redirected into squid and never forwarded.
|
|
assert dial("203.0.113.2", 8443), "a direct connect to an outside IP works"
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -k -o cat --no-pager | grep -E "
|
|
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=8443 '"
|
|
)
|
|
agent.succeed("ping -c1 -W2 203.0.113.2")
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -k -o cat --no-pager | grep -E "
|
|
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=ICMP '"
|
|
)
|
|
# Control: the same journal holds no line for the traffic that is not forwarded.
|
|
assert not would_drop(r"DST=192\.168\.1\.1 "), "the bridge IP is the host's own address"
|
|
assert not would_drop(r"DPT=3128 "), "squid's port is the host's own address"
|
|
assert not would_drop(r"SRC=203\.0\.113\.1 "), "squid's own connections leave the host and are not forwarded"
|
|
out = proxy.succeed("nft list chain inet hive_egress forward")
|
|
found = re.search(r'oifname != "eth1" counter packets (\d+) bytes \d+$', out, re.M)
|
|
assert found and int(found.group(1)) > 0, out
|
|
|
|
with checked("the same holds over IPv6"):
|
|
agent.wait_until_succeeds("ping -6 -c1 -W2 fd00:98::1")
|
|
assert dial("fd00:99::2"), "a direct v6 connect works"
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -k -o cat --no-pager | grep -E "
|
|
"'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=fd00:0*98:(0+:)*0*3 DST=fd00:0*99:(0+:)*0*2 .*PROTO=TCP .*DPT=443 '"
|
|
)
|
|
|
|
with checked("one busy agent address is cut to a burst plus a trickle, the rest still leave a trace, and every flow is logged once at most"):
|
|
agent.execute("for i in $(seq 1 100); do echo x > /dev/udp/203.0.113.78/$((3000 + i)); done")
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -k -o cat --no-pager | grep -E 'hive-egress-would-drop-unmetered .*DST=203\\.0\\.113\\.78 '"
|
|
)
|
|
time.sleep(3)
|
|
lines = would_drop(r"DST=203\.0\.113\.78 ")
|
|
metered = [line for line in lines if "hive-egress-would-drop IN=" in line]
|
|
unmetered = [line for line in lines if "hive-egress-would-drop-unmetered IN=" in line]
|
|
assert 1 <= len(metered) <= 30, f"100 new flows: {len(metered)} per-source lines"
|
|
assert 1 <= len(unmetered) <= 80, f"100 new flows: {len(unmetered)} unmetered lines"
|
|
dpts = [re.search(r"DPT=(\d+) ", line) for line in lines]
|
|
assert all(dpts), f"a logged line has no destination port: {lines}"
|
|
ports = [found.group(1) for found in dpts if found]
|
|
assert len(ports) == len(set(ports)), "a flow was logged twice"
|
|
out = proxy.succeed("nft list chain inet hive_egress forward")
|
|
found = re.search(r'oifname != "eth1" counter packets (\d+) bytes \d+$', out, re.M)
|
|
assert found and int(found.group(1)) >= 100, out
|
|
|
|
# ---- The agents' web traffic that skips the proxy variables ----
|
|
# The test stands in for `egress-pin` and dnsmasq's lease script: it writes
|
|
# the pin file and the lease file that `hive-priv egress-whois` reads, and
|
|
# the exemption set that `egress-pin` fills on a hive. The agent node holds
|
|
# three addresses, one per agent, and the exemption is by its MAC.
|
|
AGENT_MAC = agent.succeed("cat /sys/class/net/eth1/address").strip()
|
|
ATLAS_IP, LOGAN_IP, NOBODY_IP = "192.168.1.3", "192.168.1.4", "192.168.1.7"
|
|
# squid logs an intercepted HTTPS flow from the relay's source address, the pool
|
|
# address of the agent (`pool_source` in hive-priv/src/egress_ech.rs): 198.18.0.0/15
|
|
# with the low 17 bits of the agent's address.
|
|
def relay_source(ip):
|
|
return str(ipaddress.IPv4Address(0xC6120000 | (int(ipaddress.IPv4Address(ip)) & 0x1FFFF)))
|
|
|
|
AGENT_SOURCES = "|".join(re.escape(a) for ip in (ATLAS_IP, LOGAN_IP, NOBODY_IP) for a in (ip, relay_source(ip)))
|
|
BOUND = {"atlas": (ATLAS_IP, AGENT_MAC), "logan": (LOGAN_IP, "02:00:00:00:00:04")}
|
|
agent.succeed(f"ip addr add {LOGAN_IP}/24 dev eth1; ip addr add {NOBODY_IP}/24 dev eth1")
|
|
|
|
def bind(*names):
|
|
pins = "".join(f"vb-{n} {i + 2} {BOUND[n][1]} h-{n}\n" for i, n in enumerate(names))
|
|
leases = "".join(f"0 {BOUND[n][1]} {BOUND[n][0]} {n} *\n" for n in names)
|
|
proxy.succeed(
|
|
f"printf '%s' {shlex.quote(pins)} > /run/hive-egress-pins "
|
|
f"&& printf '%s' {shlex.quote(leases)} > /var/lib/dnsmasq/dnsmasq.leases"
|
|
)
|
|
|
|
def icpt(url, src=ATLAS_IP, extra=""):
|
|
# The status of the answer to a request that carries no proxy variables;
|
|
# 000 when the connection was closed or the TLS handshake failed.
|
|
return agent.succeed(
|
|
f"curl -sS -k -m 15 --interface {src} {extra} -o /dev/null -w '%{{http_code}}' {url} 2>/dev/null || true"
|
|
).strip()
|
|
|
|
def wait_icpt(url, want, src=ATLAS_IP, extra=""):
|
|
# The reload that follows a list write is asynchronous.
|
|
got = None
|
|
for _ in range(30):
|
|
got = icpt(url, src, extra)
|
|
if got == want:
|
|
return
|
|
time.sleep(1)
|
|
raise AssertionError(f"{url} from {src}: wanted {want}, last got {got}")
|
|
|
|
def refused(url, logged, src=ATLAS_IP, extra="", want=("000",)):
|
|
# A probe the proxy refuses without a status of its own. A closed
|
|
# listener gives the same 000, so the refusal counts only when squid also
|
|
# logs a new line that matches `logged` for it.
|
|
before = len(re.findall(logged, journal(), re.M))
|
|
got = icpt(url, src, extra)
|
|
assert got in want, f"{url} from {src}: wanted one of {want}, got {got}"
|
|
for _ in range(20):
|
|
if len(re.findall(logged, journal(), re.M)) > before:
|
|
return
|
|
time.sleep(0.5)
|
|
raise AssertionError(f"{url} from {src} got {got}, but squid logged no new line matching {logged}")
|
|
|
|
def agent_lines():
|
|
# Log lines of requests from the agent node's addresses. The self-check
|
|
# connects from the bridge address, so its lines are not counted.
|
|
return len(re.findall(rf" (?:{AGENT_SOURCES}) [A-Z_]+/\d+ ", journal()))
|
|
|
|
seeded = proxy.succeed(f"cat {ALLOWED}")
|
|
bind("atlas", "logan")
|
|
# What hive-c0re renders: `atlas` has a name of its own, `logan` is in log mode.
|
|
write_allow(
|
|
seeded,
|
|
"# source: swarm\n"
|
|
"acl agent_allow_atlas proxy_auth atlas\n"
|
|
"acl agent_allow_atlas_ix ext_user atlas\n"
|
|
"acl agent_allow_atlas_dst dstdomain agentonly.t.local\n"
|
|
"http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n"
|
|
"http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n"
|
|
"acl log_mark annotate_transaction would_deny=1\n"
|
|
"acl agent_log_logan proxy_auth logan\n"
|
|
"acl agent_log_logan_ix ext_user logan\n"
|
|
"http_access allow !icpt agent_log_logan !allowed_dst log_mark\n"
|
|
"http_access allow icpt agent_log_logan_ix !allowed_dst log_mark\n",
|
|
)
|
|
|
|
with checked("a restricted agent's allowed name is tunnelled without the proxy variables, and the log names the agent"):
|
|
wait_icpt("https://forge.t.local/", "200")
|
|
assert re.search(
|
|
re.escape(relay_source(ATLAS_IP)) + r" TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ORIGINAL_DST/203\.0\.113\.2 ",
|
|
journal(),
|
|
), "the HTTPS flow, as atlas"
|
|
assert icpt("http://forge.t.local/") == "200", "plain HTTP"
|
|
assert re.search(r"192\.168\.1\.3 TCP_\w+/200 .* GET http://forge\.t\.local/ atlas ", journal()), "the HTTP flow, as atlas"
|
|
assert icpt("https://agentonly.t.local/") == "200", "a name of atlas alone"
|
|
assert re.search(r"CONNECT agentonly\.t\.local:443 atlas ", journal()), "as atlas"
|
|
# The proxy-aware path is the same as before, and is not given the address's identity.
|
|
assert connect("forge.t.local", ATLAS) == "200", "a credential through the proxy port"
|
|
assert connect("forge.t.local") == "407", "no credential: the address does not stand in for one"
|
|
|
|
with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"):
|
|
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
|
assert icpt("http://other.t.local/") == "403", "plain HTTP"
|
|
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas"
|
|
# Control: the same address reaches the listed name, so the refusal is the policy's.
|
|
assert icpt("https://forge.t.local/") == "200"
|
|
|
|
with checked("the swarm's deny list refuses an intercepted name that is otherwise allowed, and logs egress_denied"):
|
|
assert icpt("https://api.t.local/") == "200", "control: a base-list name"
|
|
write_deny(
|
|
"acl deny_hit_mark annotate_transaction egress_deny=1\n"
|
|
"acl deny_swarm_dst dstdomain api.t.local\n"
|
|
"http_access deny deny_swarm_dst deny_hit_mark\n"
|
|
)
|
|
refused("https://api.t.local/", r"egress_denied user=atlas method=CONNECT host=api\.t\.local port=443 result=TCP_DENIED/403")
|
|
assert icpt("https://forge.t.local/") == "200", "control: another name is unaffected"
|
|
|
|
with checked("a log-mode agent's unlisted name is allowed and marked would_deny, and the deny list still wins"):
|
|
assert icpt("https://other.t.local/", LOGAN_IP) == "200"
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -u squid.service -o cat | grep -E "
|
|
"'would_deny user=logan method=CONNECT host=other\\.t\\.local port=443 result=TCP_TUNNEL/200'"
|
|
)
|
|
assert icpt("https://forge.t.local/", LOGAN_IP) == "200", "a listed name"
|
|
assert not re.search(r"would_deny user=logan .* host=forge\.t\.local", journal()), "a listed name is not marked"
|
|
refused("https://api.t.local/", r"egress_denied user=logan method=CONNECT host=api\.t\.local port=443 result=TCP_DENIED/403", LOGAN_IP)
|
|
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
|
write_deny("")
|
|
wait_icpt("https://api.t.local/", "200")
|
|
|
|
def forwarded(ip):
|
|
# Log lines of requests from `ip`, or its relay source, that squid sent on
|
|
# to an origin. A refusal has no hierarchy entry: `HIER_NONE/-`.
|
|
srcs = f"{re.escape(ip)}|{re.escape(relay_source(ip))}"
|
|
return re.findall(rf"^.* (?:{srcs}) \S+ \S+ \S+ \S+ \S+ (?!HIER_NONE/)\S+/\S+ .*$", journal(), re.M)
|
|
|
|
with checked("a source the bindings do not name is refused, and so is every source when they are gone"):
|
|
# `http_access deny icpt !icpt_agent` meets the CONNECT squid builds from the
|
|
# destination address, before the peek, so no SNI is read and nothing is
|
|
# sent to the origin. Squid then answers the request on the bumped
|
|
# connection with its own 403 page. A refusal after the peek, such as a
|
|
# bound agent's unlisted name, closes the connection instead: that is the 000.
|
|
refused(
|
|
"https://forge.t.local/",
|
|
re.escape(relay_source(NOBODY_IP)) + r" TCP_DENIED/\d{3} .* CONNECT 203\.0\.113\.2:443 - HIER_NONE/- ",
|
|
NOBODY_IP,
|
|
want=("000", "403"),
|
|
)
|
|
assert icpt("http://forge.t.local/", NOBODY_IP) == "403"
|
|
assert not forwarded(NOBODY_IP), "nothing of a source without a lease reached an origin"
|
|
assert forwarded(ATLAS_IP), "control: the same lookup finds the bound address's tunnelled requests"
|
|
assert icpt("https://forge.t.local/") == "200", "control: the bound address, the same name"
|
|
# A hive whose pin file is missing: the lease names a MAC, no port holds it.
|
|
proxy.succeed("rm /run/hive-egress-pins")
|
|
time.sleep(12) # the helper's answers are kept for 10 seconds
|
|
sent_on = len(forwarded(ATLAS_IP))
|
|
refused(
|
|
"https://forge.t.local/",
|
|
re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/\d{3} .* HIER_NONE/- ",
|
|
want=("000", "403"),
|
|
)
|
|
assert len(forwarded(ATLAS_IP)) == sent_on, "no pin file, nothing sent on"
|
|
bind("atlas", "logan")
|
|
wait_icpt("https://forge.t.local/", "200")
|
|
|
|
with checked("an SNI that the destination does not serve, a missing SNI and an IP literal are refused"):
|
|
# Control: the right SNI at the right address is the 200 above. The
|
|
# same name sent to another address of the origin is not.
|
|
refused(
|
|
"https://forge.t.local/",
|
|
r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ",
|
|
extra="--resolve forge.t.local:443:203.0.113.9",
|
|
)
|
|
assert icpt("http://forge.t.local/", extra="-H 'Host: forge.t.local' --resolve forge.t.local:80:203.0.113.9") == "409", "the Host header too"
|
|
hello = "echo | timeout 10 openssl s_client -bind 192.168.1.3 -connect 203.0.113.2:443"
|
|
out = agent.succeed(f"{hello} -servername forge.t.local 2>&1 || true")
|
|
assert "BEGIN CERTIFICATE" in out, "control: with an SNI the origin's certificate comes back"
|
|
out = agent.succeed(f"{hello} -noservername 2>&1 || true")
|
|
assert "BEGIN CERTIFICATE" not in out, "no SNI, no certificate"
|
|
refused("https://203.0.113.2/", r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ")
|
|
assert icpt("http://203.0.113.2/") == "403", "an IP literal as the Host"
|
|
out = agent.succeed(
|
|
"timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true"
|
|
)
|
|
assert out.strip() == "", "bytes that are no ClientHello get no answer"
|
|
|
|
# The ClientHello a client sends, built by hand: SNI, groups, signature
|
|
# algorithms and supported versions, and with `ech` the encrypted_client_hello
|
|
# extension (0xfe0d) with the layout a real one has. It prints how the
|
|
# connection ended: the first byte of what came back, or `closed`.
|
|
agent.succeed(
|
|
"cat > /tmp/hello.py <<'PY'\n"
|
|
"import os, socket, struct, sys\n"
|
|
"mode, name, dst, src = sys.argv[1:5]\n"
|
|
"def ext(kind, data):\n"
|
|
" return struct.pack('>HH', kind, len(data)) + data\n"
|
|
"host = name.encode()\n"
|
|
"exts = ext(0, struct.pack('>HBH', len(host) + 3, 0, len(host)) + host)\n"
|
|
"exts += ext(10, struct.pack('>HHH', 4, 29, 23)) + ext(13, struct.pack('>HHH', 4, 0x0804, 0x0403))\n"
|
|
"exts += ext(43, bytes([2, 3, 4])) + ext(51, struct.pack('>H', 0))\n"
|
|
"if mode == 'ech':\n"
|
|
" exts += ext(0xFE0D, b'\\x00' + struct.pack('>HHBH', 1, 1, 7, 32) + os.urandom(32) + struct.pack('>H', 144) + os.urandom(144))\n"
|
|
"body = b'\\x03\\x03' + os.urandom(32) + b'\\x20' + os.urandom(32) + struct.pack('>H', 2) + b'\\x13\\x01' + b'\\x01\\x00'\n"
|
|
"body += struct.pack('>H', len(exts)) + exts\n"
|
|
"hello = b'\\x01' + struct.pack('>I', len(body))[1:] + body\n"
|
|
"s = socket.socket()\n"
|
|
"s.settimeout(10)\n"
|
|
"s.bind((src, 0))\n"
|
|
"s.connect((dst, 443))\n"
|
|
"s.sendall(b'\\x16\\x03\\x01' + struct.pack('>H', len(hello)) + hello)\n"
|
|
"try:\n"
|
|
" first = s.recv(1)\n"
|
|
"except OSError:\n"
|
|
" first = b'''\n"
|
|
"print(first.hex() or 'closed')\n"
|
|
"PY"
|
|
)
|
|
|
|
def send_hello(mode, src=ATLAS_IP):
|
|
return agent.succeed(f"python3 /tmp/hello.py {mode} forge.t.local 203.0.113.2 {src}").strip()
|
|
|
|
with checked("a ClientHello with the ECH extension is refused for a restricted agent and observed for a log agent, and the same hello without it is let through"):
|
|
seen = agent_lines()
|
|
# Control: the origin answers a hello without the extension (a
|
|
# handshake record, 0x16), and squid logs the connection as atlas.
|
|
assert send_hello("plain") == "16", "the origin answers the hello without ECH"
|
|
time.sleep(2)
|
|
assert agent_lines() > seen, "squid saw the connection"
|
|
seen = agent_lines()
|
|
assert send_hello("ech") == "closed", "a hello with an ECH extension gets no answer"
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -u hive-egress-ech-relay.service -o cat | "
|
|
"grep -E '^hive-egress-ech-refused agent=atlas dst=203\\.0\\.113\\.2$'"
|
|
)
|
|
time.sleep(2)
|
|
assert agent_lines() == seen, "squid saw nothing of it"
|
|
# A `log` agent is observed, not refused: its ECH hello goes on to squid
|
|
# and the origin, and the relay logs what it would have done. The mode is
|
|
# what the squid include says (`acl agent_log_logan proxy_auth logan`).
|
|
seen = agent_lines()
|
|
assert send_hello("ech", LOGAN_IP) == "16", "logan's hello with ECH is let through"
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -u hive-egress-ech-relay.service -o cat | "
|
|
"grep -E '^hive-egress-ech-would-refuse agent=logan dst=203\\.0\\.113\\.2$'"
|
|
)
|
|
time.sleep(2)
|
|
assert agent_lines() > seen, "squid saw logan's connection"
|
|
assert "hive-egress-ech-refused agent=logan" not in proxy.succeed(
|
|
"journalctl -u hive-egress-ech-relay.service -o cat --no-pager"
|
|
), "logan was not refused"
|
|
assert "ech-would-refuse agent=atlas" not in proxy.succeed(
|
|
"journalctl -u hive-egress-ech-relay.service -o cat --no-pager"
|
|
), "control: the restricted agent got the refusal, not the observation"
|
|
assert send_hello("plain", LOGAN_IP) == "16", "control: logan's hello without ECH"
|
|
# A source the bindings do not name is refused too, logged without a name.
|
|
assert send_hello("ech", NOBODY_IP) == "closed"
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -u hive-egress-ech-relay.service -o cat | "
|
|
"grep -E '^hive-egress-ech-refused agent=- dst=203\\.0\\.113\\.2$'"
|
|
)
|
|
# Bytes that are no ClientHello: refused by the relay, with a line of their own.
|
|
out = agent.succeed(
|
|
"timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true"
|
|
)
|
|
assert out.strip() == "", "bytes that are no ClientHello get no answer"
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -u hive-egress-ech-relay.service -o cat | "
|
|
"grep -E '^hive-egress-hello-refused agent=atlas dst=203\\.0\\.113\\.2 reason=not a TLS handshake record$'"
|
|
)
|
|
|
|
with checked("the traffic that stays on the bridge or is the unrestricted agents' is not intercepted"):
|
|
seen = agent_lines()
|
|
# The gateway on the bridge address, in the stand-in's port 443.
|
|
assert icpt("http://${bridgeIp}:443/") != "000", "the host's own listener answers"
|
|
assert agent_lines() == seen, "squid saw nothing of it"
|
|
# Unrestricted agents and service containers are the MACs in `direct_mac`.
|
|
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
|
proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
|
seen = agent_lines()
|
|
assert icpt("https://other.t.local/") == "200", "an unlisted name, directly"
|
|
assert icpt("https://203.0.113.2/") == "200", "an IP literal, directly"
|
|
assert agent_lines() == seen, "squid saw nothing of it"
|
|
assert send_hello("ech") == "16", "a hello with ECH, directly: the relay is not in the path"
|
|
proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
|
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
|
# Control: a new flow of the same MAC, now off the set, is logged, and the
|
|
# kernel log is in order, so a line of the exempt flows would be before it.
|
|
agent.execute("echo x > /dev/udp/203.0.113.79/9")
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -k -o cat --no-pager | grep -E "
|
|
"'hive-egress-would-drop(-unmetered)? IN=eth1 OUT=eth2 .*DST=203\\.0\\.113\\.79 '"
|
|
)
|
|
assert not would_drop(r"DST=203\.0\.113\.2 .*DPT=443 "), "the exempt MAC's flows leave no line"
|
|
|
|
# ---- Reject mode ----
|
|
# The node switches to its `reject` specialisation, which reloads the
|
|
# tables with the refusing forward chain. Nothing is in `direct_mac` yet.
|
|
def wait_reject_line(pattern):
|
|
proxy.wait_until_succeeds(
|
|
"journalctl -k -o cat --no-pager | grep -E "
|
|
+ shlex.quote(f"hive-egress-reject(-unmetered)? IN=eth1 OUT=eth2 .*{pattern}")
|
|
)
|
|
|
|
def dial_status(addr, port):
|
|
# 0 connected, 1 refused, 124 no answer in 3 seconds.
|
|
return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/{port}'")[0]
|
|
|
|
def udp_status(addr):
|
|
# A connected UDP socket reports the ICMP error of the first datagram
|
|
# on the second write: non-zero when the host refused it. Nothing
|
|
# answers a datagram that is let through, so that case is 0.
|
|
return agent.execute(f"bash -c 'exec 3<>/dev/udp/{addr}/9; echo x >&3; sleep 1; echo y >&3'")[0]
|
|
|
|
with checked("reject mode: web still goes through squid and everything else from a restricted agent is refused"):
|
|
proxy.succeed("/run/current-system/specialisation/reject/bin/switch-to-configuration test")
|
|
out = proxy.succeed("nft list chain inet hive_egress forward")
|
|
assert "tcp reset" in out and "admin-prohibited" in out, out
|
|
assert icpt("https://forge.t.local/") == "200", "control: web still goes through squid"
|
|
assert dial_status("203.0.113.2", 8443) == 1, "TCP 8443 is reset at once, not left to time out"
|
|
wait_reject_line(r"SRC=192\.168\.1\.3 DST=203\.0\.113\.2 .*PROTO=TCP .*DPT=8443 ")
|
|
status, out = agent.execute("ping -c1 -W3 203.0.113.2")
|
|
assert status != 0 and "filtered" in out.lower(), f"ICMP is refused with admin-prohibited: {out}"
|
|
wait_reject_line(r"SRC=192\.168\.1\.3 DST=203\.0\.113\.2 .*PROTO=ICMP ")
|
|
assert udp_status("203.0.113.77") != 0, "a datagram is refused with an ICMP error"
|
|
wait_reject_line(r"DST=203\.0\.113\.77 .*PROTO=UDP ")
|
|
out = proxy.succeed("nft list chain inet hive_egress forward")
|
|
found = re.search(r'oifname != "eth1" counter packets (\d+) bytes \d+$', out, re.M)
|
|
assert found and int(found.group(1)) > 0, out
|
|
|
|
with checked("reject mode: a MAC in direct_mac passes and is refused again once it leaves"):
|
|
proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
|
assert dial_status("203.0.113.2", 8443) == 0, "an exempt MAC (a service container's stand-in) reaches TCP 8443"
|
|
agent.succeed("ping -c1 -W3 203.0.113.2")
|
|
assert udp_status("203.0.113.77") == 0, "control: an exempt MAC's datagram is not refused"
|
|
proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
|
assert dial_status("203.0.113.2", 8443) == 1, "refused again"
|
|
|
|
# `hive-priv egress-pin` fills `agent_allow` on a hive host; here the
|
|
# elements are added by hand, as for `direct_mac`.
|
|
with checked("reject mode: an agent_allow element passes that MAC to that address and port only, and is refused again once it leaves"):
|
|
proxy.succeed(f"nft add element inet hive_egress agent_allow '{{ {AGENT_MAC} . 203.0.113.2 . 8443 }}'")
|
|
assert dial_status("203.0.113.2", 8443) == 0, "the allowed address and port"
|
|
# Another address of the same host, on the same port, is not in the set.
|
|
assert dial_status("203.0.113.9", 8443) == 1, "another address, same port"
|
|
wait_reject_line(r"SRC=192\.168\.1\.3 DST=203\.0\.113\.9 .*PROTO=TCP .*DPT=8443 ")
|
|
# Only TCP to the port is let through.
|
|
status, out = agent.execute("ping -c1 -W3 203.0.113.2")
|
|
assert status != 0, f"ICMP to the allowed address is still refused: {out}"
|
|
assert udp_status("203.0.113.2") != 0, "a datagram to the allowed address and port number is refused"
|
|
# An element of another MAC allows nothing for this one.
|
|
proxy.succeed("nft add element inet hive_egress agent_allow '{ 02:00:00:00:00:99 . 203.0.113.9 . 8443 }'")
|
|
assert dial_status("203.0.113.9", 8443) == 1, "another MAC's element"
|
|
proxy.succeed(f"nft delete element inet hive_egress agent_allow '{{ {AGENT_MAC} . 203.0.113.2 . 8443 }}'")
|
|
assert dial_status("203.0.113.2", 8443) == 1, "refused again"
|
|
'';
|
|
}
|