hive-squid vm test: wait for squid's reload before probing, and count a refusal only with its log line
This commit is contained in:
parent
5747c4c325
commit
16260108ee
1 changed files with 78 additions and 28 deletions
|
|
@ -438,6 +438,43 @@ pkgs.testers.runNixOSTest {
|
||||||
print(f"--- {vm.name}: {cmd}\n" + vm.execute(cmd)[1])
|
print(f"--- {vm.name}: {cmd}\n" + vm.execute(cmd)[1])
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
def journal():
|
||||||
|
return proxy.succeed("journalctl -u squid.service -o cat --no-pager")
|
||||||
|
|
||||||
|
def squid_reconfigs():
|
||||||
|
# How many reconfigurations squid has begun, and whether its intercept
|
||||||
|
# listeners are open again since the last one.
|
||||||
|
parts = journal().split("Reconfiguring Squid Cache")
|
||||||
|
return len(parts) - 1, "Accepting NAT intercepted SSL bumped HTTPS Socket connections" in parts[-1]
|
||||||
|
|
||||||
|
def wait_reconfigured(before):
|
||||||
|
# squid closes its listeners while it reconfigures, which begins a moment
|
||||||
|
# after squid-unrestricted-reload starts and can outlast the unit. Returns
|
||||||
|
# once a reconfiguration beyond the `before` count has finished, the unit
|
||||||
|
# is idle and the listeners accept again.
|
||||||
|
state = None
|
||||||
|
for _ in range(60):
|
||||||
|
count, reopened = squid_reconfigs()
|
||||||
|
idle = proxy.succeed("systemctl show -p ActiveState --value squid-unrestricted-reload.service").strip()
|
||||||
|
listening = proxy.succeed("ss -Hltn")
|
||||||
|
state = (count, before, reopened, idle)
|
||||||
|
if (
|
||||||
|
count > before
|
||||||
|
and reopened
|
||||||
|
and idle == "inactive"
|
||||||
|
and all(f"${bridgeIp}:{port} " in listening for port in (3128, 3129, 3130))
|
||||||
|
):
|
||||||
|
return
|
||||||
|
time.sleep(0.5)
|
||||||
|
raise AssertionError(f"squid did not apply the write: (reconfigurations, before, reopened, reload unit) = {state}")
|
||||||
|
|
||||||
|
def write_and_wait(cmd):
|
||||||
|
# Runs a command that writes a list squid reads on a reload, and returns
|
||||||
|
# once squid has applied it.
|
||||||
|
before = squid_reconfigs()[0]
|
||||||
|
proxy.succeed(cmd)
|
||||||
|
wait_reconfigured(before)
|
||||||
|
|
||||||
def write_users(*creds):
|
def write_users(*creds):
|
||||||
# What hive-c0re does: hashes only, written aside and renamed into
|
# What hive-c0re does: hashes only, written aside and renamed into
|
||||||
# place, in squid's group. The sleep keeps two writes in different
|
# place, in squid's group. The sleep keeps two writes in different
|
||||||
|
|
@ -455,7 +492,7 @@ pkgs.testers.runNixOSTest {
|
||||||
# What hive-c0re does for the list. Squid reads it on a reload, which
|
# What hive-c0re does for the list. Squid reads it on a reload, which
|
||||||
# squid-unrestricted-reload does when the file changes.
|
# squid-unrestricted-reload does when the file changes.
|
||||||
body = "printf '%s\\n' " + " ".join(names) if names else ":"
|
body = "printf '%s\\n' " + " ".join(names) if names else ":"
|
||||||
proxy.succeed(
|
write_and_wait(
|
||||||
f"({body}) > {LIST}.new && chown hive-core:squid {LIST}.new "
|
f"({body}) > {LIST}.new && chown hive-core:squid {LIST}.new "
|
||||||
f"&& chmod 640 {LIST}.new && mv {LIST}.new {LIST}"
|
f"&& chmod 640 {LIST}.new && mv {LIST}.new {LIST}"
|
||||||
)
|
)
|
||||||
|
|
@ -464,16 +501,18 @@ pkgs.testers.runNixOSTest {
|
||||||
# What hive-c0re does for its two files: written aside and renamed into
|
# What hive-c0re does for its two files: written aside and renamed into
|
||||||
# place, in squid's group. Squid reads them on a reload, which
|
# place, in squid's group. Squid reads them on a reload, which
|
||||||
# squid-unrestricted-reload does when either changes.
|
# squid-unrestricted-reload does when either changes.
|
||||||
for path, body in ((ALLOWED, allowed), (AGENT_ALLOW, agent_conf)):
|
write_and_wait(
|
||||||
proxy.succeed(
|
" && ".join(
|
||||||
f"printf '%s' {shlex.quote(body)} > {path}.new && chown hive-core:squid {path}.new "
|
f"printf '%s' {shlex.quote(body)} > {path}.new && chown hive-core:squid {path}.new "
|
||||||
f"&& chmod 640 {path}.new && mv {path}.new {path}"
|
f"&& chmod 640 {path}.new && mv {path}.new {path}"
|
||||||
|
for path, body in ((ALLOWED, allowed), (AGENT_ALLOW, agent_conf))
|
||||||
)
|
)
|
||||||
|
)
|
||||||
|
|
||||||
def write_deny(body):
|
def write_deny(body):
|
||||||
# What hive-c0re does for the deny list: written aside and renamed into
|
# What hive-c0re does for the deny list: written aside and renamed into
|
||||||
# place, in squid's group; the reload is squid-unrestricted-reload's.
|
# place, in squid's group; the reload is squid-unrestricted-reload's.
|
||||||
proxy.succeed(
|
write_and_wait(
|
||||||
f"printf '%s' {shlex.quote(body)} > {DENY}.new && chown hive-core:squid {DENY}.new "
|
f"printf '%s' {shlex.quote(body)} > {DENY}.new && chown hive-core:squid {DENY}.new "
|
||||||
f"&& chmod 640 {DENY}.new && mv {DENY}.new {DENY}"
|
f"&& chmod 640 {DENY}.new && mv {DENY}.new {DENY}"
|
||||||
)
|
)
|
||||||
|
|
@ -961,8 +1000,18 @@ pkgs.testers.runNixOSTest {
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
raise AssertionError(f"{url} from {src}: wanted {want}, last got {got}")
|
raise AssertionError(f"{url} from {src}: wanted {want}, last got {got}")
|
||||||
|
|
||||||
def journal():
|
def refused(url, logged, src=ATLAS_IP, extra="", want=("000",)):
|
||||||
return proxy.succeed("journalctl -u squid.service -o cat --no-pager")
|
# A probe the proxy refuses without a status of its own. A closed
|
||||||
|
# listener gives the same 000, so the refusal counts only when squid also
|
||||||
|
# logs a new line that matches `logged` for it.
|
||||||
|
before = len(re.findall(logged, journal(), re.M))
|
||||||
|
got = icpt(url, src, extra)
|
||||||
|
assert got in want, f"{url} from {src}: wanted one of {want}, got {got}"
|
||||||
|
for _ in range(20):
|
||||||
|
if len(re.findall(logged, journal(), re.M)) > before:
|
||||||
|
return
|
||||||
|
time.sleep(0.5)
|
||||||
|
raise AssertionError(f"{url} from {src} got {got}, but squid logged no new line matching {logged}")
|
||||||
|
|
||||||
def agent_lines():
|
def agent_lines():
|
||||||
# Log lines of requests from the agent node's addresses. The self-check
|
# Log lines of requests from the agent node's addresses. The self-check
|
||||||
|
|
@ -1002,10 +1051,7 @@ pkgs.testers.runNixOSTest {
|
||||||
assert connect("forge.t.local") == "407", "no credential: the address does not stand in for one"
|
assert connect("forge.t.local") == "407", "no credential: the address does not stand in for one"
|
||||||
|
|
||||||
with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"):
|
with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"):
|
||||||
assert icpt("https://other.t.local/") == "000", "HTTPS"
|
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
||||||
assert re.search(
|
|
||||||
re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal()
|
|
||||||
), "logged as atlas"
|
|
||||||
assert icpt("http://other.t.local/") == "403", "plain HTTP"
|
assert icpt("http://other.t.local/") == "403", "plain HTTP"
|
||||||
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas"
|
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas"
|
||||||
# Control: the same address reaches the listed name, so the refusal is the policy's.
|
# Control: the same address reaches the listed name, so the refusal is the policy's.
|
||||||
|
|
@ -1018,11 +1064,7 @@ pkgs.testers.runNixOSTest {
|
||||||
"acl deny_swarm_dst dstdomain api.t.local\n"
|
"acl deny_swarm_dst dstdomain api.t.local\n"
|
||||||
"http_access deny deny_swarm_dst deny_hit_mark\n"
|
"http_access deny deny_swarm_dst deny_hit_mark\n"
|
||||||
)
|
)
|
||||||
wait_icpt("https://api.t.local/", "000")
|
refused("https://api.t.local/", r"egress_denied user=atlas method=CONNECT host=api\.t\.local port=443 result=TCP_DENIED/403")
|
||||||
proxy.wait_until_succeeds(
|
|
||||||
"journalctl -u squid.service -o cat | grep -E "
|
|
||||||
"'egress_denied user=atlas method=CONNECT host=api\\.t\\.local port=443 result=TCP_DENIED/403'"
|
|
||||||
)
|
|
||||||
assert icpt("https://forge.t.local/") == "200", "control: another name is unaffected"
|
assert icpt("https://forge.t.local/") == "200", "control: another name is unaffected"
|
||||||
|
|
||||||
with checked("a log-mode agent's unlisted name is allowed and marked would_deny, and the deny list still wins"):
|
with checked("a log-mode agent's unlisted name is allowed and marked would_deny, and the deny list still wins"):
|
||||||
|
|
@ -1033,8 +1075,8 @@ pkgs.testers.runNixOSTest {
|
||||||
)
|
)
|
||||||
assert icpt("https://forge.t.local/", LOGAN_IP) == "200", "a listed name"
|
assert icpt("https://forge.t.local/", LOGAN_IP) == "200", "a listed name"
|
||||||
assert not re.search(r"would_deny user=logan .* host=forge\.t\.local", journal()), "a listed name is not marked"
|
assert not re.search(r"would_deny user=logan .* host=forge\.t\.local", journal()), "a listed name is not marked"
|
||||||
assert icpt("https://api.t.local/", LOGAN_IP) == "000", "the deny list refuses it in log mode too"
|
refused("https://api.t.local/", r"egress_denied user=logan method=CONNECT host=api\.t\.local port=443 result=TCP_DENIED/403", LOGAN_IP)
|
||||||
assert icpt("https://other.t.local/") == "000", "control: atlas, unmarked and refused"
|
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
||||||
write_deny("")
|
write_deny("")
|
||||||
wait_icpt("https://api.t.local/", "200")
|
wait_icpt("https://api.t.local/", "200")
|
||||||
|
|
||||||
|
|
@ -1050,10 +1092,12 @@ pkgs.testers.runNixOSTest {
|
||||||
# sent to the origin. Squid then answers the request on the bumped
|
# sent to the origin. Squid then answers the request on the bumped
|
||||||
# connection with its own 403 page. A refusal after the peek, such as a
|
# connection with its own 403 page. A refusal after the peek, such as a
|
||||||
# bound agent's unlisted name, closes the connection instead: that is the 000.
|
# bound agent's unlisted name, closes the connection instead: that is the 000.
|
||||||
assert icpt("https://forge.t.local/", NOBODY_IP) in ("000", "403"), "no lease for the address"
|
refused(
|
||||||
assert re.search(
|
"https://forge.t.local/",
|
||||||
re.escape(relay_source(NOBODY_IP)) + r" TCP_DENIED/\d{3} .* CONNECT 203\.0\.113\.2:443 - HIER_NONE/- ", journal()
|
re.escape(relay_source(NOBODY_IP)) + r" TCP_DENIED/\d{3} .* CONNECT 203\.0\.113\.2:443 - HIER_NONE/- ",
|
||||||
), "logged without a user"
|
NOBODY_IP,
|
||||||
|
want=("000", "403"),
|
||||||
|
)
|
||||||
assert icpt("http://forge.t.local/", NOBODY_IP) == "403"
|
assert icpt("http://forge.t.local/", NOBODY_IP) == "403"
|
||||||
assert not forwarded(NOBODY_IP), "nothing of a source without a lease reached an origin"
|
assert not forwarded(NOBODY_IP), "nothing of a source without a lease reached an origin"
|
||||||
assert forwarded(ATLAS_IP), "control: the same lookup finds the bound address's tunnelled requests"
|
assert forwarded(ATLAS_IP), "control: the same lookup finds the bound address's tunnelled requests"
|
||||||
|
|
@ -1062,7 +1106,11 @@ pkgs.testers.runNixOSTest {
|
||||||
proxy.succeed("rm /run/hive-egress-pins")
|
proxy.succeed("rm /run/hive-egress-pins")
|
||||||
time.sleep(12) # the helper's answers are kept for 10 seconds
|
time.sleep(12) # the helper's answers are kept for 10 seconds
|
||||||
sent_on = len(forwarded(ATLAS_IP))
|
sent_on = len(forwarded(ATLAS_IP))
|
||||||
assert icpt("https://forge.t.local/") in ("000", "403"), "no pin file"
|
refused(
|
||||||
|
"https://forge.t.local/",
|
||||||
|
re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/\d{3} .* HIER_NONE/- ",
|
||||||
|
want=("000", "403"),
|
||||||
|
)
|
||||||
assert len(forwarded(ATLAS_IP)) == sent_on, "no pin file, nothing sent on"
|
assert len(forwarded(ATLAS_IP)) == sent_on, "no pin file, nothing sent on"
|
||||||
bind("atlas", "logan")
|
bind("atlas", "logan")
|
||||||
wait_icpt("https://forge.t.local/", "200")
|
wait_icpt("https://forge.t.local/", "200")
|
||||||
|
|
@ -1070,16 +1118,18 @@ pkgs.testers.runNixOSTest {
|
||||||
with checked("an SNI that the destination does not serve, a missing SNI and an IP literal are refused"):
|
with checked("an SNI that the destination does not serve, a missing SNI and an IP literal are refused"):
|
||||||
# Control: the right SNI at the right address is the 200 above. The
|
# Control: the right SNI at the right address is the 200 above. The
|
||||||
# same name sent to another address of the origin is not.
|
# same name sent to another address of the origin is not.
|
||||||
assert icpt("https://forge.t.local/", extra="--resolve forge.t.local:443:203.0.113.9") == "000"
|
refused(
|
||||||
assert re.search(r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ", journal()), "squid answered 409"
|
"https://forge.t.local/",
|
||||||
|
r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ",
|
||||||
|
extra="--resolve forge.t.local:443:203.0.113.9",
|
||||||
|
)
|
||||||
assert icpt("http://forge.t.local/", extra="-H 'Host: forge.t.local' --resolve forge.t.local:80:203.0.113.9") == "409", "the Host header too"
|
assert icpt("http://forge.t.local/", extra="-H 'Host: forge.t.local' --resolve forge.t.local:80:203.0.113.9") == "409", "the Host header too"
|
||||||
hello = "echo | timeout 10 openssl s_client -bind 192.168.1.3 -connect 203.0.113.2:443"
|
hello = "echo | timeout 10 openssl s_client -bind 192.168.1.3 -connect 203.0.113.2:443"
|
||||||
out = agent.succeed(f"{hello} -servername forge.t.local 2>&1 || true")
|
out = agent.succeed(f"{hello} -servername forge.t.local 2>&1 || true")
|
||||||
assert "BEGIN CERTIFICATE" in out, "control: with an SNI the origin's certificate comes back"
|
assert "BEGIN CERTIFICATE" in out, "control: with an SNI the origin's certificate comes back"
|
||||||
out = agent.succeed(f"{hello} -noservername 2>&1 || true")
|
out = agent.succeed(f"{hello} -noservername 2>&1 || true")
|
||||||
assert "BEGIN CERTIFICATE" not in out, "no SNI, no certificate"
|
assert "BEGIN CERTIFICATE" not in out, "no SNI, no certificate"
|
||||||
assert icpt("https://203.0.113.2/") == "000", "an IP literal sends no SNI"
|
refused("https://203.0.113.2/", r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ")
|
||||||
assert re.search(r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ", journal()), "logged as atlas"
|
|
||||||
assert icpt("http://203.0.113.2/") == "403", "an IP literal as the Host"
|
assert icpt("http://203.0.113.2/") == "403", "an IP literal as the Host"
|
||||||
out = agent.succeed(
|
out = agent.succeed(
|
||||||
"timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true"
|
"timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true"
|
||||||
|
|
@ -1176,7 +1226,7 @@ pkgs.testers.runNixOSTest {
|
||||||
assert icpt("http://${bridgeIp}:443/") != "000", "the host's own listener answers"
|
assert icpt("http://${bridgeIp}:443/") != "000", "the host's own listener answers"
|
||||||
assert agent_lines() == seen, "squid saw nothing of it"
|
assert agent_lines() == seen, "squid saw nothing of it"
|
||||||
# Unrestricted agents and service containers are the MACs in `direct_mac`.
|
# Unrestricted agents and service containers are the MACs in `direct_mac`.
|
||||||
assert icpt("https://other.t.local/") == "000", "control: an unlisted name is refused"
|
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
||||||
proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
||||||
seen = agent_lines()
|
seen = agent_lines()
|
||||||
assert icpt("https://other.t.local/") == "200", "an unlisted name, directly"
|
assert icpt("https://other.t.local/") == "200", "an unlisted name, directly"
|
||||||
|
|
@ -1184,7 +1234,7 @@ pkgs.testers.runNixOSTest {
|
||||||
assert agent_lines() == seen, "squid saw nothing of it"
|
assert agent_lines() == seen, "squid saw nothing of it"
|
||||||
assert send_hello("ech") == "16", "a hello with ECH, directly: the relay is not in the path"
|
assert send_hello("ech") == "16", "a hello with ECH, directly: the relay is not in the path"
|
||||||
proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
||||||
assert icpt("https://other.t.local/") == "000", "intercepted again"
|
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
||||||
# Control: a new flow of the same MAC, now off the set, is logged, and the
|
# Control: a new flow of the same MAC, now off the set, is logged, and the
|
||||||
# kernel log is in order, so a line of the exempt flows would be before it.
|
# kernel log is in order, so a line of the exempt flows would be before it.
|
||||||
agent.execute("echo x > /dev/udp/203.0.113.79/9")
|
agent.execute("echo x > /dev/udp/203.0.113.79/9")
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue