hive-squid vm test: wait for squid's reload before probing, and count a refusal only with its log line
This commit is contained in:
parent
5747c4c325
commit
16260108ee
1 changed files with 78 additions and 28 deletions
|
|
@ -438,6 +438,43 @@ pkgs.testers.runNixOSTest {
|
|||
print(f"--- {vm.name}: {cmd}\n" + vm.execute(cmd)[1])
|
||||
raise
|
||||
|
||||
def journal():
|
||||
return proxy.succeed("journalctl -u squid.service -o cat --no-pager")
|
||||
|
||||
def squid_reconfigs():
|
||||
# How many reconfigurations squid has begun, and whether its intercept
|
||||
# listeners are open again since the last one.
|
||||
parts = journal().split("Reconfiguring Squid Cache")
|
||||
return len(parts) - 1, "Accepting NAT intercepted SSL bumped HTTPS Socket connections" in parts[-1]
|
||||
|
||||
def wait_reconfigured(before):
|
||||
# squid closes its listeners while it reconfigures, which begins a moment
|
||||
# after squid-unrestricted-reload starts and can outlast the unit. Returns
|
||||
# once a reconfiguration beyond the `before` count has finished, the unit
|
||||
# is idle and the listeners accept again.
|
||||
state = None
|
||||
for _ in range(60):
|
||||
count, reopened = squid_reconfigs()
|
||||
idle = proxy.succeed("systemctl show -p ActiveState --value squid-unrestricted-reload.service").strip()
|
||||
listening = proxy.succeed("ss -Hltn")
|
||||
state = (count, before, reopened, idle)
|
||||
if (
|
||||
count > before
|
||||
and reopened
|
||||
and idle == "inactive"
|
||||
and all(f"${bridgeIp}:{port} " in listening for port in (3128, 3129, 3130))
|
||||
):
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise AssertionError(f"squid did not apply the write: (reconfigurations, before, reopened, reload unit) = {state}")
|
||||
|
||||
def write_and_wait(cmd):
|
||||
# Runs a command that writes a list squid reads on a reload, and returns
|
||||
# once squid has applied it.
|
||||
before = squid_reconfigs()[0]
|
||||
proxy.succeed(cmd)
|
||||
wait_reconfigured(before)
|
||||
|
||||
def write_users(*creds):
|
||||
# What hive-c0re does: hashes only, written aside and renamed into
|
||||
# place, in squid's group. The sleep keeps two writes in different
|
||||
|
|
@ -455,7 +492,7 @@ pkgs.testers.runNixOSTest {
|
|||
# What hive-c0re does for the list. Squid reads it on a reload, which
|
||||
# squid-unrestricted-reload does when the file changes.
|
||||
body = "printf '%s\\n' " + " ".join(names) if names else ":"
|
||||
proxy.succeed(
|
||||
write_and_wait(
|
||||
f"({body}) > {LIST}.new && chown hive-core:squid {LIST}.new "
|
||||
f"&& chmod 640 {LIST}.new && mv {LIST}.new {LIST}"
|
||||
)
|
||||
|
|
@ -464,16 +501,18 @@ pkgs.testers.runNixOSTest {
|
|||
# What hive-c0re does for its two files: written aside and renamed into
|
||||
# place, in squid's group. Squid reads them on a reload, which
|
||||
# squid-unrestricted-reload does when either changes.
|
||||
for path, body in ((ALLOWED, allowed), (AGENT_ALLOW, agent_conf)):
|
||||
proxy.succeed(
|
||||
write_and_wait(
|
||||
" && ".join(
|
||||
f"printf '%s' {shlex.quote(body)} > {path}.new && chown hive-core:squid {path}.new "
|
||||
f"&& chmod 640 {path}.new && mv {path}.new {path}"
|
||||
for path, body in ((ALLOWED, allowed), (AGENT_ALLOW, agent_conf))
|
||||
)
|
||||
)
|
||||
|
||||
def write_deny(body):
|
||||
# What hive-c0re does for the deny list: written aside and renamed into
|
||||
# place, in squid's group; the reload is squid-unrestricted-reload's.
|
||||
proxy.succeed(
|
||||
write_and_wait(
|
||||
f"printf '%s' {shlex.quote(body)} > {DENY}.new && chown hive-core:squid {DENY}.new "
|
||||
f"&& chmod 640 {DENY}.new && mv {DENY}.new {DENY}"
|
||||
)
|
||||
|
|
@ -961,8 +1000,18 @@ pkgs.testers.runNixOSTest {
|
|||
time.sleep(1)
|
||||
raise AssertionError(f"{url} from {src}: wanted {want}, last got {got}")
|
||||
|
||||
def journal():
|
||||
return proxy.succeed("journalctl -u squid.service -o cat --no-pager")
|
||||
def refused(url, logged, src=ATLAS_IP, extra="", want=("000",)):
|
||||
# A probe the proxy refuses without a status of its own. A closed
|
||||
# listener gives the same 000, so the refusal counts only when squid also
|
||||
# logs a new line that matches `logged` for it.
|
||||
before = len(re.findall(logged, journal(), re.M))
|
||||
got = icpt(url, src, extra)
|
||||
assert got in want, f"{url} from {src}: wanted one of {want}, got {got}"
|
||||
for _ in range(20):
|
||||
if len(re.findall(logged, journal(), re.M)) > before:
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise AssertionError(f"{url} from {src} got {got}, but squid logged no new line matching {logged}")
|
||||
|
||||
def agent_lines():
|
||||
# Log lines of requests from the agent node's addresses. The self-check
|
||||
|
|
@ -1002,10 +1051,7 @@ pkgs.testers.runNixOSTest {
|
|||
assert connect("forge.t.local") == "407", "no credential: the address does not stand in for one"
|
||||
|
||||
with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"):
|
||||
assert icpt("https://other.t.local/") == "000", "HTTPS"
|
||||
assert re.search(
|
||||
re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal()
|
||||
), "logged as atlas"
|
||||
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
||||
assert icpt("http://other.t.local/") == "403", "plain HTTP"
|
||||
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas"
|
||||
# Control: the same address reaches the listed name, so the refusal is the policy's.
|
||||
|
|
@ -1018,11 +1064,7 @@ pkgs.testers.runNixOSTest {
|
|||
"acl deny_swarm_dst dstdomain api.t.local\n"
|
||||
"http_access deny deny_swarm_dst deny_hit_mark\n"
|
||||
)
|
||||
wait_icpt("https://api.t.local/", "000")
|
||||
proxy.wait_until_succeeds(
|
||||
"journalctl -u squid.service -o cat | grep -E "
|
||||
"'egress_denied user=atlas method=CONNECT host=api\\.t\\.local port=443 result=TCP_DENIED/403'"
|
||||
)
|
||||
refused("https://api.t.local/", r"egress_denied user=atlas method=CONNECT host=api\.t\.local port=443 result=TCP_DENIED/403")
|
||||
assert icpt("https://forge.t.local/") == "200", "control: another name is unaffected"
|
||||
|
||||
with checked("a log-mode agent's unlisted name is allowed and marked would_deny, and the deny list still wins"):
|
||||
|
|
@ -1033,8 +1075,8 @@ pkgs.testers.runNixOSTest {
|
|||
)
|
||||
assert icpt("https://forge.t.local/", LOGAN_IP) == "200", "a listed name"
|
||||
assert not re.search(r"would_deny user=logan .* host=forge\.t\.local", journal()), "a listed name is not marked"
|
||||
assert icpt("https://api.t.local/", LOGAN_IP) == "000", "the deny list refuses it in log mode too"
|
||||
assert icpt("https://other.t.local/") == "000", "control: atlas, unmarked and refused"
|
||||
refused("https://api.t.local/", r"egress_denied user=logan method=CONNECT host=api\.t\.local port=443 result=TCP_DENIED/403", LOGAN_IP)
|
||||
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
||||
write_deny("")
|
||||
wait_icpt("https://api.t.local/", "200")
|
||||
|
||||
|
|
@ -1050,10 +1092,12 @@ pkgs.testers.runNixOSTest {
|
|||
# sent to the origin. Squid then answers the request on the bumped
|
||||
# connection with its own 403 page. A refusal after the peek, such as a
|
||||
# bound agent's unlisted name, closes the connection instead: that is the 000.
|
||||
assert icpt("https://forge.t.local/", NOBODY_IP) in ("000", "403"), "no lease for the address"
|
||||
assert re.search(
|
||||
re.escape(relay_source(NOBODY_IP)) + r" TCP_DENIED/\d{3} .* CONNECT 203\.0\.113\.2:443 - HIER_NONE/- ", journal()
|
||||
), "logged without a user"
|
||||
refused(
|
||||
"https://forge.t.local/",
|
||||
re.escape(relay_source(NOBODY_IP)) + r" TCP_DENIED/\d{3} .* CONNECT 203\.0\.113\.2:443 - HIER_NONE/- ",
|
||||
NOBODY_IP,
|
||||
want=("000", "403"),
|
||||
)
|
||||
assert icpt("http://forge.t.local/", NOBODY_IP) == "403"
|
||||
assert not forwarded(NOBODY_IP), "nothing of a source without a lease reached an origin"
|
||||
assert forwarded(ATLAS_IP), "control: the same lookup finds the bound address's tunnelled requests"
|
||||
|
|
@ -1062,7 +1106,11 @@ pkgs.testers.runNixOSTest {
|
|||
proxy.succeed("rm /run/hive-egress-pins")
|
||||
time.sleep(12) # the helper's answers are kept for 10 seconds
|
||||
sent_on = len(forwarded(ATLAS_IP))
|
||||
assert icpt("https://forge.t.local/") in ("000", "403"), "no pin file"
|
||||
refused(
|
||||
"https://forge.t.local/",
|
||||
re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/\d{3} .* HIER_NONE/- ",
|
||||
want=("000", "403"),
|
||||
)
|
||||
assert len(forwarded(ATLAS_IP)) == sent_on, "no pin file, nothing sent on"
|
||||
bind("atlas", "logan")
|
||||
wait_icpt("https://forge.t.local/", "200")
|
||||
|
|
@ -1070,16 +1118,18 @@ pkgs.testers.runNixOSTest {
|
|||
with checked("an SNI that the destination does not serve, a missing SNI and an IP literal are refused"):
|
||||
# Control: the right SNI at the right address is the 200 above. The
|
||||
# same name sent to another address of the origin is not.
|
||||
assert icpt("https://forge.t.local/", extra="--resolve forge.t.local:443:203.0.113.9") == "000"
|
||||
assert re.search(r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ", journal()), "squid answered 409"
|
||||
refused(
|
||||
"https://forge.t.local/",
|
||||
r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ",
|
||||
extra="--resolve forge.t.local:443:203.0.113.9",
|
||||
)
|
||||
assert icpt("http://forge.t.local/", extra="-H 'Host: forge.t.local' --resolve forge.t.local:80:203.0.113.9") == "409", "the Host header too"
|
||||
hello = "echo | timeout 10 openssl s_client -bind 192.168.1.3 -connect 203.0.113.2:443"
|
||||
out = agent.succeed(f"{hello} -servername forge.t.local 2>&1 || true")
|
||||
assert "BEGIN CERTIFICATE" in out, "control: with an SNI the origin's certificate comes back"
|
||||
out = agent.succeed(f"{hello} -noservername 2>&1 || true")
|
||||
assert "BEGIN CERTIFICATE" not in out, "no SNI, no certificate"
|
||||
assert icpt("https://203.0.113.2/") == "000", "an IP literal sends no SNI"
|
||||
assert re.search(r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ", journal()), "logged as atlas"
|
||||
refused("https://203.0.113.2/", r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ")
|
||||
assert icpt("http://203.0.113.2/") == "403", "an IP literal as the Host"
|
||||
out = agent.succeed(
|
||||
"timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true"
|
||||
|
|
@ -1176,7 +1226,7 @@ pkgs.testers.runNixOSTest {
|
|||
assert icpt("http://${bridgeIp}:443/") != "000", "the host's own listener answers"
|
||||
assert agent_lines() == seen, "squid saw nothing of it"
|
||||
# Unrestricted agents and service containers are the MACs in `direct_mac`.
|
||||
assert icpt("https://other.t.local/") == "000", "control: an unlisted name is refused"
|
||||
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
||||
proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
||||
seen = agent_lines()
|
||||
assert icpt("https://other.t.local/") == "200", "an unlisted name, directly"
|
||||
|
|
@ -1184,7 +1234,7 @@ pkgs.testers.runNixOSTest {
|
|||
assert agent_lines() == seen, "squid saw nothing of it"
|
||||
assert send_hello("ech") == "16", "a hello with ECH, directly: the relay is not in the path"
|
||||
proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
|
||||
assert icpt("https://other.t.local/") == "000", "intercepted again"
|
||||
refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
|
||||
# Control: a new flow of the same MAC, now off the set, is logged, and the
|
||||
# kernel log is in order, so a line of the exempt flows would be before it.
|
||||
agent.execute("echo x > /dev/udp/203.0.113.79/9")
|
||||
|
|
|
|||
Loading…
Reference in a new issue