vm tests: fail on a missing log port, give up early on a broken hive

nix/vm-tests/hive-squid.nix

@@ -843,7 +843,9 @@ pkgs.testers.runNixOSTest {
         unmetered = [line for line in lines if "hive-egress-would-drop-unmetered IN=" in line]
         assert 1 <= len(metered) <= 30, f"100 new flows: {len(metered)} per-source lines"
         assert 1 <= len(unmetered) <= 80, f"100 new flows: {len(unmetered)} unmetered lines"
-        ports = [re.search(r"DPT=(\d+) ", line).group(1) for line in lines]
+        dpts = [re.search(r"DPT=(\d+) ", line) for line in lines]
+        assert all(dpts), f"a logged line has no destination port: {lines}"
+        ports = [found.group(1) for found in dpts if found]
         assert len(ports) == len(set(ports)), "a flow was logged twice"
         out = proxy.succeed("nft list chain inet hive_egress forward")
         found = re.search(r'oifname != "eth1" counter packets (\d+) bytes \d+$', out, re.M)

nix/vm-tests/semi-spread.nix

@@ -304,6 +304,7 @@ pkgs.testers.runNixOSTest (
     testScript = ''
       import json
       import shlex
+      import time
 
       API = "http://localhost"
       SOCKET = "${controllerSocket}"
@@ -354,6 +355,37 @@ pkgs.testers.runNixOSTest (
               diagnose(m, unit, container)
               raise
 
+      HIVE_UNITS = ["hive-priv.socket", "hive-priv.service", "hive-c0re"]
+
+      def hive_fault():
+          # Why the hive cannot reach the state the swarm waits for, or None. The
+          # privileged helper is socket-activated: its socket is active while it
+          # is usable, and failed once the helper's start limit is hit. The status
+          # sweep runs every minute and fails once or twice while the swarm queue
+          # comes up, so only three failures within three minutes count.
+          if hive.execute("systemctl is-active --quiet hive-priv.socket && ! systemctl is-failed --quiet hive-priv.service")[0] != 0:
+              return "hive-priv is not up on the hive"
+          swept = hive.succeed("journalctl -u hive-c0re --since '3 min ago' --no-pager -o cat | grep -c 'agent status: sweep failed' || true").strip()
+          if int(swept) >= 3:
+              return f"hive-c0re's agent-status sweep failed {swept} times in the last 3 minutes"
+          return None
+
+      def settle_hive_agents(command, timeout):
+          # settle on the swarm machine, but give up as soon as the hive is broken
+          # and dump both machines.
+          deadline = time.monotonic() + timeout
+          fault = None
+          while time.monotonic() < deadline:
+              if swarm.execute(command)[0] == 0:
+                  return
+              fault = hive_fault()
+              if fault:
+                  break
+              time.sleep(10)
+          diagnose(hive, HIVE_UNITS)
+          diagnose(swarm, ["swarm-controller"])
+          raise Exception(fault or f"the agents did not settle within {timeout}s")
+
       def store_sealed(timeout):
           # `bao status` exits 2 when it reaches a sealed store and 1 for any
           # address, CA or client-certificate problem.
@@ -627,13 +659,11 @@ pkgs.testers.runNixOSTest (
           create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})
 
       with subtest("assert 4: the agents are deployed on the hive, from the cache on heavyduty, and need a login"):
-          settle(
+          settle_hive_agents(
-              swarm,
               f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status "
               "| jq -e '[.[] | select(.hive == \"" + HIVE + "\" and .freshness == \"fresh\" and .needs_login == true)"
               " | .name] | map(select(. == \"iris\" or . == \"spawn\")) | unique | length == 2'",
               3600,
-              ["swarm-controller"],
           )
           # The closures came over the VPN from the cache: the hive's nix
           # names it as its only extra substituter and holds the toplevels now.