Watch
0
0
Fork
You've already forked hyperhive
0

hive-squid vm test: wait for squid's reload before probing, and count a refusal only with its log line

This commit is contained in:
atlas 2026-11-19 14:36:55 +01:00 • committed by mara
commit 16260108ee

View file

@ -438,6 +438,43 @@ pkgs.testers.runNixOSTest {
print(f"--- {vm.name}: {cmd}\n" + vm.execute(cmd)[1]) print(f"--- {vm.name}: {cmd}\n" + vm.execute(cmd)[1])
raise raise
def journal():
return proxy.succeed("journalctl -u squid.service -o cat --no-pager")
def squid_reconfigs():
# How many reconfigurations squid has begun, and whether its intercept
# listeners are open again since the last one.
parts = journal().split("Reconfiguring Squid Cache")
return len(parts) - 1, "Accepting NAT intercepted SSL bumped HTTPS Socket connections" in parts[-1]
def wait_reconfigured(before):
# squid closes its listeners while it reconfigures, which begins a moment
# after squid-unrestricted-reload starts and can outlast the unit. Returns
# once a reconfiguration beyond the `before` count has finished, the unit
# is idle and the listeners accept again.
state = None
for _ in range(60):
count, reopened = squid_reconfigs()
idle = proxy.succeed("systemctl show -p ActiveState --value squid-unrestricted-reload.service").strip()
listening = proxy.succeed("ss -Hltn")
state = (count, before, reopened, idle)
if (
count > before
and reopened
and idle == "inactive"
and all(f"${bridgeIp}:{port} " in listening for port in (3128, 3129, 3130))
):
return
time.sleep(0.5)
raise AssertionError(f"squid did not apply the write: (reconfigurations, before, reopened, reload unit) = {state}")
def write_and_wait(cmd):
# Runs a command that writes a list squid reads on a reload, and returns
# once squid has applied it.
before = squid_reconfigs()[0]
proxy.succeed(cmd)
wait_reconfigured(before)
def write_users(*creds): def write_users(*creds):
# What hive-c0re does: hashes only, written aside and renamed into # What hive-c0re does: hashes only, written aside and renamed into
# place, in squid's group. The sleep keeps two writes in different # place, in squid's group. The sleep keeps two writes in different
@ -455,7 +492,7 @@ pkgs.testers.runNixOSTest {
# What hive-c0re does for the list. Squid reads it on a reload, which # What hive-c0re does for the list. Squid reads it on a reload, which
# squid-unrestricted-reload does when the file changes. # squid-unrestricted-reload does when the file changes.
body = "printf '%s\\n' " + " ".join(names) if names else ":" body = "printf '%s\\n' " + " ".join(names) if names else ":"
proxy.succeed( write_and_wait(
f"({body}) > {LIST}.new && chown hive-core:squid {LIST}.new " f"({body}) > {LIST}.new && chown hive-core:squid {LIST}.new "
f"&& chmod 640 {LIST}.new && mv {LIST}.new {LIST}" f"&& chmod 640 {LIST}.new && mv {LIST}.new {LIST}"
) )
@ -464,16 +501,18 @@ pkgs.testers.runNixOSTest {
# What hive-c0re does for its two files: written aside and renamed into # What hive-c0re does for its two files: written aside and renamed into
# place, in squid's group. Squid reads them on a reload, which # place, in squid's group. Squid reads them on a reload, which
# squid-unrestricted-reload does when either changes. # squid-unrestricted-reload does when either changes.
for path, body in ((ALLOWED, allowed), (AGENT_ALLOW, agent_conf)): write_and_wait(
proxy.succeed( " && ".join(
f"printf '%s' {shlex.quote(body)} > {path}.new && chown hive-core:squid {path}.new " f"printf '%s' {shlex.quote(body)} > {path}.new && chown hive-core:squid {path}.new "
f"&& chmod 640 {path}.new && mv {path}.new {path}" f"&& chmod 640 {path}.new && mv {path}.new {path}"
for path, body in ((ALLOWED, allowed), (AGENT_ALLOW, agent_conf))
) )
)
def write_deny(body): def write_deny(body):
# What hive-c0re does for the deny list: written aside and renamed into # What hive-c0re does for the deny list: written aside and renamed into
# place, in squid's group; the reload is squid-unrestricted-reload's. # place, in squid's group; the reload is squid-unrestricted-reload's.
proxy.succeed( write_and_wait(
f"printf '%s' {shlex.quote(body)} > {DENY}.new && chown hive-core:squid {DENY}.new " f"printf '%s' {shlex.quote(body)} > {DENY}.new && chown hive-core:squid {DENY}.new "
f"&& chmod 640 {DENY}.new && mv {DENY}.new {DENY}" f"&& chmod 640 {DENY}.new && mv {DENY}.new {DENY}"
) )
@ -961,8 +1000,18 @@ pkgs.testers.runNixOSTest {
time.sleep(1) time.sleep(1)
raise AssertionError(f"{url} from {src}: wanted {want}, last got {got}") raise AssertionError(f"{url} from {src}: wanted {want}, last got {got}")
def journal(): def refused(url, logged, src=ATLAS_IP, extra="", want=("000",)):
return proxy.succeed("journalctl -u squid.service -o cat --no-pager") # A probe the proxy refuses without a status of its own. A closed
# listener gives the same 000, so the refusal counts only when squid also
# logs a new line that matches `logged` for it.
before = len(re.findall(logged, journal(), re.M))
got = icpt(url, src, extra)
assert got in want, f"{url} from {src}: wanted one of {want}, got {got}"
for _ in range(20):
if len(re.findall(logged, journal(), re.M)) > before:
return
time.sleep(0.5)
raise AssertionError(f"{url} from {src} got {got}, but squid logged no new line matching {logged}")
def agent_lines(): def agent_lines():
# Log lines of requests from the agent node's addresses. The self-check # Log lines of requests from the agent node's addresses. The self-check
@ -1002,10 +1051,7 @@ pkgs.testers.runNixOSTest {
assert connect("forge.t.local") == "407", "no credential: the address does not stand in for one" assert connect("forge.t.local") == "407", "no credential: the address does not stand in for one"
with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"): with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"):
assert icpt("https://other.t.local/") == "000", "HTTPS" refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
assert re.search(
re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal()
), "logged as atlas"
assert icpt("http://other.t.local/") == "403", "plain HTTP" assert icpt("http://other.t.local/") == "403", "plain HTTP"
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas" assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas"
# Control: the same address reaches the listed name, so the refusal is the policy's. # Control: the same address reaches the listed name, so the refusal is the policy's.
@ -1018,11 +1064,7 @@ pkgs.testers.runNixOSTest {
"acl deny_swarm_dst dstdomain api.t.local\n" "acl deny_swarm_dst dstdomain api.t.local\n"
"http_access deny deny_swarm_dst deny_hit_mark\n" "http_access deny deny_swarm_dst deny_hit_mark\n"
) )
wait_icpt("https://api.t.local/", "000") refused("https://api.t.local/", r"egress_denied user=atlas method=CONNECT host=api\.t\.local port=443 result=TCP_DENIED/403")
proxy.wait_until_succeeds(
"journalctl -u squid.service -o cat | grep -E "
"'egress_denied user=atlas method=CONNECT host=api\\.t\\.local port=443 result=TCP_DENIED/403'"
)
assert icpt("https://forge.t.local/") == "200", "control: another name is unaffected" assert icpt("https://forge.t.local/") == "200", "control: another name is unaffected"
with checked("a log-mode agent's unlisted name is allowed and marked would_deny, and the deny list still wins"): with checked("a log-mode agent's unlisted name is allowed and marked would_deny, and the deny list still wins"):
@ -1033,8 +1075,8 @@ pkgs.testers.runNixOSTest {
) )
assert icpt("https://forge.t.local/", LOGAN_IP) == "200", "a listed name" assert icpt("https://forge.t.local/", LOGAN_IP) == "200", "a listed name"
assert not re.search(r"would_deny user=logan .* host=forge\.t\.local", journal()), "a listed name is not marked" assert not re.search(r"would_deny user=logan .* host=forge\.t\.local", journal()), "a listed name is not marked"
assert icpt("https://api.t.local/", LOGAN_IP) == "000", "the deny list refuses it in log mode too" refused("https://api.t.local/", r"egress_denied user=logan method=CONNECT host=api\.t\.local port=443 result=TCP_DENIED/403", LOGAN_IP)
assert icpt("https://other.t.local/") == "000", "control: atlas, unmarked and refused" refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
write_deny("") write_deny("")
wait_icpt("https://api.t.local/", "200") wait_icpt("https://api.t.local/", "200")
@ -1050,10 +1092,12 @@ pkgs.testers.runNixOSTest {
# sent to the origin. Squid then answers the request on the bumped # sent to the origin. Squid then answers the request on the bumped
# connection with its own 403 page. A refusal after the peek, such as a # connection with its own 403 page. A refusal after the peek, such as a
# bound agent's unlisted name, closes the connection instead: that is the 000. # bound agent's unlisted name, closes the connection instead: that is the 000.
assert icpt("https://forge.t.local/", NOBODY_IP) in ("000", "403"), "no lease for the address" refused(
assert re.search( "https://forge.t.local/",
re.escape(relay_source(NOBODY_IP)) + r" TCP_DENIED/\d{3} .* CONNECT 203\.0\.113\.2:443 - HIER_NONE/- ", journal() re.escape(relay_source(NOBODY_IP)) + r" TCP_DENIED/\d{3} .* CONNECT 203\.0\.113\.2:443 - HIER_NONE/- ",
), "logged without a user" NOBODY_IP,
want=("000", "403"),
)
assert icpt("http://forge.t.local/", NOBODY_IP) == "403" assert icpt("http://forge.t.local/", NOBODY_IP) == "403"
assert not forwarded(NOBODY_IP), "nothing of a source without a lease reached an origin" assert not forwarded(NOBODY_IP), "nothing of a source without a lease reached an origin"
assert forwarded(ATLAS_IP), "control: the same lookup finds the bound address's tunnelled requests" assert forwarded(ATLAS_IP), "control: the same lookup finds the bound address's tunnelled requests"
@ -1062,7 +1106,11 @@ pkgs.testers.runNixOSTest {
proxy.succeed("rm /run/hive-egress-pins") proxy.succeed("rm /run/hive-egress-pins")
time.sleep(12) # the helper's answers are kept for 10 seconds time.sleep(12) # the helper's answers are kept for 10 seconds
sent_on = len(forwarded(ATLAS_IP)) sent_on = len(forwarded(ATLAS_IP))
assert icpt("https://forge.t.local/") in ("000", "403"), "no pin file" refused(
"https://forge.t.local/",
re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/\d{3} .* HIER_NONE/- ",
want=("000", "403"),
)
assert len(forwarded(ATLAS_IP)) == sent_on, "no pin file, nothing sent on" assert len(forwarded(ATLAS_IP)) == sent_on, "no pin file, nothing sent on"
bind("atlas", "logan") bind("atlas", "logan")
wait_icpt("https://forge.t.local/", "200") wait_icpt("https://forge.t.local/", "200")
@ -1070,16 +1118,18 @@ pkgs.testers.runNixOSTest {
with checked("an SNI that the destination does not serve, a missing SNI and an IP literal are refused"): with checked("an SNI that the destination does not serve, a missing SNI and an IP literal are refused"):
# Control: the right SNI at the right address is the 200 above. The # Control: the right SNI at the right address is the 200 above. The
# same name sent to another address of the origin is not. # same name sent to another address of the origin is not.
assert icpt("https://forge.t.local/", extra="--resolve forge.t.local:443:203.0.113.9") == "000" refused(
assert re.search(r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ", journal()), "squid answered 409" "https://forge.t.local/",
r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ",
extra="--resolve forge.t.local:443:203.0.113.9",
)
assert icpt("http://forge.t.local/", extra="-H 'Host: forge.t.local' --resolve forge.t.local:80:203.0.113.9") == "409", "the Host header too" assert icpt("http://forge.t.local/", extra="-H 'Host: forge.t.local' --resolve forge.t.local:80:203.0.113.9") == "409", "the Host header too"
hello = "echo | timeout 10 openssl s_client -bind 192.168.1.3 -connect 203.0.113.2:443" hello = "echo | timeout 10 openssl s_client -bind 192.168.1.3 -connect 203.0.113.2:443"
out = agent.succeed(f"{hello} -servername forge.t.local 2>&1 || true") out = agent.succeed(f"{hello} -servername forge.t.local 2>&1 || true")
assert "BEGIN CERTIFICATE" in out, "control: with an SNI the origin's certificate comes back" assert "BEGIN CERTIFICATE" in out, "control: with an SNI the origin's certificate comes back"
out = agent.succeed(f"{hello} -noservername 2>&1 || true") out = agent.succeed(f"{hello} -noservername 2>&1 || true")
assert "BEGIN CERTIFICATE" not in out, "no SNI, no certificate" assert "BEGIN CERTIFICATE" not in out, "no SNI, no certificate"
assert icpt("https://203.0.113.2/") == "000", "an IP literal sends no SNI" refused("https://203.0.113.2/", r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ")
assert re.search(r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ", journal()), "logged as atlas"
assert icpt("http://203.0.113.2/") == "403", "an IP literal as the Host" assert icpt("http://203.0.113.2/") == "403", "an IP literal as the Host"
out = agent.succeed( out = agent.succeed(
"timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true" "timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true"
@ -1176,7 +1226,7 @@ pkgs.testers.runNixOSTest {
assert icpt("http://${bridgeIp}:443/") != "000", "the host's own listener answers" assert icpt("http://${bridgeIp}:443/") != "000", "the host's own listener answers"
assert agent_lines() == seen, "squid saw nothing of it" assert agent_lines() == seen, "squid saw nothing of it"
# Unrestricted agents and service containers are the MACs in `direct_mac`. # Unrestricted agents and service containers are the MACs in `direct_mac`.
assert icpt("https://other.t.local/") == "000", "control: an unlisted name is refused" refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'") proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
seen = agent_lines() seen = agent_lines()
assert icpt("https://other.t.local/") == "200", "an unlisted name, directly" assert icpt("https://other.t.local/") == "200", "an unlisted name, directly"
@ -1184,7 +1234,7 @@ pkgs.testers.runNixOSTest {
assert agent_lines() == seen, "squid saw nothing of it" assert agent_lines() == seen, "squid saw nothing of it"
assert send_hello("ech") == "16", "a hello with ECH, directly: the relay is not in the path" assert send_hello("ech") == "16", "a hello with ECH, directly: the relay is not in the path"
proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'") proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'")
assert icpt("https://other.t.local/") == "000", "intercepted again" refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ")
# Control: a new flow of the same MAC, now off the set, is logged, and the # Control: a new flow of the same MAC, now off the set, is logged, and the
# kernel log is in order, so a line of the exempt flows would be before it. # kernel log is in order, so a line of the exempt flows would be before it.
agent.execute("echo x > /dev/udp/203.0.113.79/9") agent.execute("echo x > /dev/udp/203.0.113.79/9")