diff --git a/nix/vm-tests/semi-spread.nix b/nix/vm-tests/semi-spread.nix index 37f72ee9f..e4df7c293 100644 --- a/nix/vm-tests/semi-spread.nix +++ b/nix/vm-tests/semi-spread.nix @@ -213,21 +213,63 @@ let return f"hive-c0re's agent-status sweep failed {swept} times in the last 3 minutes" return None - def settle_hive_agents(command, timeout): - # settle on the swarm machine, but give up as soon as the hive is broken - # and dump both machines. + # The controller retries a failed update_meta or build_meta on its next + # 10-minute round; a minute of slack on top. + CONTROLLER_ROUND = 660 + + def controller_events(since): + # The controller's journal since the epoch second `since`, oldest first: + # (seconds, the failed node's kind or "", message). + out = swarm.succeed( + f"journalctl -u swarm-controller --since @{since} -o json --no-pager | jq -c " + "'[(.__REALTIME_TIMESTAMP | tonumber / 1000000), (.F_KIND // \"\"), " + "(.MESSAGE | if type == \"array\" then implode else . end)]'" + ) + return [json.loads(line) for line in out.splitlines() if line.strip()] + + def swarm_fault(since): + # Why the swarm cannot get a pushed build of the agents to the hive, or + # None. A build the builder did not push is final; a failed update_meta or + # build_meta is one only when no later round got past it. + now = int(swarm.succeed("date +%s")) + events = controller_events(since) + for _, _, message in events: + if "did not push it to the cache" in message: + why = heavyduty.execute( + "journalctl -M swarm-builder -u swarm-builder MESSAGE='closure not pushed to the cache' " + "-o json --no-pager | jq -r .F_REASON | tail -n 1" + )[1].strip() + return f"the swarm builder did not push a closure to the cache ({why or 'its journal names no reason'}): {message}" + for kind, success in [("update_meta", "update_meta: done"), ("build_meta", "swarm jobq: deploy requested")]: + failed = [(t, m) for t, k, m in events if k == kind and "node failed" in m] + if not failed: + continue + at, message = failed[-1] + if any(t > at and success in m for t, _, m in events): + continue + if now - at > CONTROLLER_ROUND: + return f"the controller's {kind} failed {int(now - at)}s ago and no later round succeeded: {message[-800:]}" + return None + + def settle_hive_agents(command, timeout, since): + # settle on the swarm machine, but give up as soon as the hive or the + # swarm's build of the agents is broken, and dump the three machines. deadline = time.monotonic() + timeout fault = None while time.monotonic() < deadline: if swarm.execute(command)[0] == 0: return - fault = hive_fault() + fault = hive_fault() or swarm_fault(since) if fault: break time.sleep(10) diagnose(hive, HIVE_UNITS) diagnose(swarm, ["swarm-controller"]) - raise Exception(fault or f"the agents did not settle within {timeout}s") + diagnose(heavyduty, "swarm-builder", "swarm-builder") + if not fault: + deploys = sum("swarm jobq: deploy requested" in m for _, _, m in controller_events(since)) + fault = f"the agents did not settle within {timeout}s; the controller sent {deploys} deploy request(s) since the creates" + raise Exception(fault) def store_sealed(timeout): # `bao status` exits 2 when it reaches a sealed store and 1 for any @@ -493,11 +535,22 @@ let f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" " f"push {FORGE}/hyperhive/hyperhive.git main" ) + # The forge records a push after the push returns. Until it does, the + # repo reads as empty and an agent create's update_meta fails, to be + # retried only on the controller's next 10-minute round. These are the + # two reads that update_meta makes. + swarm.wait_until_succeeds( + f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/repos/hyperhive/hyperhive " + "| jq -e '.empty == false' && " + f"curl --fail --silent --insecure -H @{FORGE_AUTH} -o /dev/null {FORGE}/api/v1/repos/hyperhive/hyperhive/branches/main", + timeout=120, + ) with subtest("setup.md steps 8 and 9: two agents, through the endpoint the swarm UI calls"): swarm.wait_until_succeeds( f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=1800 ) + created = int(swarm.succeed("date +%s")) create_agent({"name": "spawn", "hive": HIVE, "archetype": ARCHETYPE}) create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE}) @@ -506,7 +559,8 @@ let f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status " "| jq -e '[.[] | select(.hive == \"" + HIVE + "\" and .freshness == \"fresh\" and .needs_login == true)" " | .name] | map(select(. == \"iris\" or . == \"spawn\")) | unique | length == 2'", - 3600, + 1200, + created, ) # The closures came over the VPN from the cache: the hive's nix # names it as its only extra substituter and holds the toplevels now.