From 16260108eefcff5b39138f4d7e3970fa63a68a35 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 19 Nov 2026 14:36:55 +0100 Subject: [PATCH] hive-squid vm test: wait for squid's reload before probing, and count a refusal only with its log line --- nix/vm-tests/hive-squid.nix | 106 ++++++++++++++++++++++++++---------- 1 file changed, 78 insertions(+), 28 deletions(-) diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 47c3a5310..7f592e4db 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -438,6 +438,43 @@ pkgs.testers.runNixOSTest { print(f"--- {vm.name}: {cmd}\n" + vm.execute(cmd)[1]) raise + def journal(): + return proxy.succeed("journalctl -u squid.service -o cat --no-pager") + + def squid_reconfigs(): + # How many reconfigurations squid has begun, and whether its intercept + # listeners are open again since the last one. + parts = journal().split("Reconfiguring Squid Cache") + return len(parts) - 1, "Accepting NAT intercepted SSL bumped HTTPS Socket connections" in parts[-1] + + def wait_reconfigured(before): + # squid closes its listeners while it reconfigures, which begins a moment + # after squid-unrestricted-reload starts and can outlast the unit. Returns + # once a reconfiguration beyond the `before` count has finished, the unit + # is idle and the listeners accept again. + state = None + for _ in range(60): + count, reopened = squid_reconfigs() + idle = proxy.succeed("systemctl show -p ActiveState --value squid-unrestricted-reload.service").strip() + listening = proxy.succeed("ss -Hltn") + state = (count, before, reopened, idle) + if ( + count > before + and reopened + and idle == "inactive" + and all(f"${bridgeIp}:{port} " in listening for port in (3128, 3129, 3130)) + ): + return + time.sleep(0.5) + raise AssertionError(f"squid did not apply the write: (reconfigurations, before, reopened, reload unit) = {state}") + + def write_and_wait(cmd): + # Runs a command that writes a list squid reads on a reload, and returns + # once squid has applied it. + before = squid_reconfigs()[0] + proxy.succeed(cmd) + wait_reconfigured(before) + def write_users(*creds): # What hive-c0re does: hashes only, written aside and renamed into # place, in squid's group. The sleep keeps two writes in different @@ -455,7 +492,7 @@ pkgs.testers.runNixOSTest { # What hive-c0re does for the list. Squid reads it on a reload, which # squid-unrestricted-reload does when the file changes. body = "printf '%s\\n' " + " ".join(names) if names else ":" - proxy.succeed( + write_and_wait( f"({body}) > {LIST}.new && chown hive-core:squid {LIST}.new " f"&& chmod 640 {LIST}.new && mv {LIST}.new {LIST}" ) @@ -464,16 +501,18 @@ pkgs.testers.runNixOSTest { # What hive-c0re does for its two files: written aside and renamed into # place, in squid's group. Squid reads them on a reload, which # squid-unrestricted-reload does when either changes. - for path, body in ((ALLOWED, allowed), (AGENT_ALLOW, agent_conf)): - proxy.succeed( + write_and_wait( + " && ".join( f"printf '%s' {shlex.quote(body)} > {path}.new && chown hive-core:squid {path}.new " f"&& chmod 640 {path}.new && mv {path}.new {path}" + for path, body in ((ALLOWED, allowed), (AGENT_ALLOW, agent_conf)) ) + ) def write_deny(body): # What hive-c0re does for the deny list: written aside and renamed into # place, in squid's group; the reload is squid-unrestricted-reload's. - proxy.succeed( + write_and_wait( f"printf '%s' {shlex.quote(body)} > {DENY}.new && chown hive-core:squid {DENY}.new " f"&& chmod 640 {DENY}.new && mv {DENY}.new {DENY}" ) @@ -961,8 +1000,18 @@ pkgs.testers.runNixOSTest { time.sleep(1) raise AssertionError(f"{url} from {src}: wanted {want}, last got {got}") - def journal(): - return proxy.succeed("journalctl -u squid.service -o cat --no-pager") + def refused(url, logged, src=ATLAS_IP, extra="", want=("000",)): + # A probe the proxy refuses without a status of its own. A closed + # listener gives the same 000, so the refusal counts only when squid also + # logs a new line that matches `logged` for it. + before = len(re.findall(logged, journal(), re.M)) + got = icpt(url, src, extra) + assert got in want, f"{url} from {src}: wanted one of {want}, got {got}" + for _ in range(20): + if len(re.findall(logged, journal(), re.M)) > before: + return + time.sleep(0.5) + raise AssertionError(f"{url} from {src} got {got}, but squid logged no new line matching {logged}") def agent_lines(): # Log lines of requests from the agent node's addresses. The self-check @@ -1002,10 +1051,7 @@ pkgs.testers.runNixOSTest { assert connect("forge.t.local") == "407", "no credential: the address does not stand in for one" with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"): - assert icpt("https://other.t.local/") == "000", "HTTPS" - assert re.search( - re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal() - ), "logged as atlas" + refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ") assert icpt("http://other.t.local/") == "403", "plain HTTP" assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas" # Control: the same address reaches the listed name, so the refusal is the policy's. @@ -1018,11 +1064,7 @@ pkgs.testers.runNixOSTest { "acl deny_swarm_dst dstdomain api.t.local\n" "http_access deny deny_swarm_dst deny_hit_mark\n" ) - wait_icpt("https://api.t.local/", "000") - proxy.wait_until_succeeds( - "journalctl -u squid.service -o cat | grep -E " - "'egress_denied user=atlas method=CONNECT host=api\\.t\\.local port=443 result=TCP_DENIED/403'" - ) + refused("https://api.t.local/", r"egress_denied user=atlas method=CONNECT host=api\.t\.local port=443 result=TCP_DENIED/403") assert icpt("https://forge.t.local/") == "200", "control: another name is unaffected" with checked("a log-mode agent's unlisted name is allowed and marked would_deny, and the deny list still wins"): @@ -1033,8 +1075,8 @@ pkgs.testers.runNixOSTest { ) assert icpt("https://forge.t.local/", LOGAN_IP) == "200", "a listed name" assert not re.search(r"would_deny user=logan .* host=forge\.t\.local", journal()), "a listed name is not marked" - assert icpt("https://api.t.local/", LOGAN_IP) == "000", "the deny list refuses it in log mode too" - assert icpt("https://other.t.local/") == "000", "control: atlas, unmarked and refused" + refused("https://api.t.local/", r"egress_denied user=logan method=CONNECT host=api\.t\.local port=443 result=TCP_DENIED/403", LOGAN_IP) + refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ") write_deny("") wait_icpt("https://api.t.local/", "200") @@ -1050,10 +1092,12 @@ pkgs.testers.runNixOSTest { # sent to the origin. Squid then answers the request on the bumped # connection with its own 403 page. A refusal after the peek, such as a # bound agent's unlisted name, closes the connection instead: that is the 000. - assert icpt("https://forge.t.local/", NOBODY_IP) in ("000", "403"), "no lease for the address" - assert re.search( - re.escape(relay_source(NOBODY_IP)) + r" TCP_DENIED/\d{3} .* CONNECT 203\.0\.113\.2:443 - HIER_NONE/- ", journal() - ), "logged without a user" + refused( + "https://forge.t.local/", + re.escape(relay_source(NOBODY_IP)) + r" TCP_DENIED/\d{3} .* CONNECT 203\.0\.113\.2:443 - HIER_NONE/- ", + NOBODY_IP, + want=("000", "403"), + ) assert icpt("http://forge.t.local/", NOBODY_IP) == "403" assert not forwarded(NOBODY_IP), "nothing of a source without a lease reached an origin" assert forwarded(ATLAS_IP), "control: the same lookup finds the bound address's tunnelled requests" @@ -1062,7 +1106,11 @@ pkgs.testers.runNixOSTest { proxy.succeed("rm /run/hive-egress-pins") time.sleep(12) # the helper's answers are kept for 10 seconds sent_on = len(forwarded(ATLAS_IP)) - assert icpt("https://forge.t.local/") in ("000", "403"), "no pin file" + refused( + "https://forge.t.local/", + re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/\d{3} .* HIER_NONE/- ", + want=("000", "403"), + ) assert len(forwarded(ATLAS_IP)) == sent_on, "no pin file, nothing sent on" bind("atlas", "logan") wait_icpt("https://forge.t.local/", "200") @@ -1070,16 +1118,18 @@ pkgs.testers.runNixOSTest { with checked("an SNI that the destination does not serve, a missing SNI and an IP literal are refused"): # Control: the right SNI at the right address is the 200 above. The # same name sent to another address of the origin is not. - assert icpt("https://forge.t.local/", extra="--resolve forge.t.local:443:203.0.113.9") == "000" - assert re.search(r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ", journal()), "squid answered 409" + refused( + "https://forge.t.local/", + r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ", + extra="--resolve forge.t.local:443:203.0.113.9", + ) assert icpt("http://forge.t.local/", extra="-H 'Host: forge.t.local' --resolve forge.t.local:80:203.0.113.9") == "409", "the Host header too" hello = "echo | timeout 10 openssl s_client -bind 192.168.1.3 -connect 203.0.113.2:443" out = agent.succeed(f"{hello} -servername forge.t.local 2>&1 || true") assert "BEGIN CERTIFICATE" in out, "control: with an SNI the origin's certificate comes back" out = agent.succeed(f"{hello} -noservername 2>&1 || true") assert "BEGIN CERTIFICATE" not in out, "no SNI, no certificate" - assert icpt("https://203.0.113.2/") == "000", "an IP literal sends no SNI" - assert re.search(r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ", journal()), "logged as atlas" + refused("https://203.0.113.2/", r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ") assert icpt("http://203.0.113.2/") == "403", "an IP literal as the Host" out = agent.succeed( "timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true" @@ -1176,7 +1226,7 @@ pkgs.testers.runNixOSTest { assert icpt("http://${bridgeIp}:443/") != "000", "the host's own listener answers" assert agent_lines() == seen, "squid saw nothing of it" # Unrestricted agents and service containers are the MACs in `direct_mac`. - assert icpt("https://other.t.local/") == "000", "control: an unlisted name is refused" + refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ") proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'") seen = agent_lines() assert icpt("https://other.t.local/") == "200", "an unlisted name, directly" @@ -1184,7 +1234,7 @@ pkgs.testers.runNixOSTest { assert agent_lines() == seen, "squid saw nothing of it" assert send_hello("ech") == "16", "a hello with ECH, directly: the relay is not in the path" proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'") - assert icpt("https://other.t.local/") == "000", "intercepted again" + refused("https://other.t.local/", re.escape(relay_source(ATLAS_IP)) + r" TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ") # Control: a new flow of the same MAC, now off the set, is logged, and the # kernel log is in order, so a line of the exempt flows would be before it. agent.execute("echo x > /dev/udp/203.0.113.79/9")