Watch
0
0
Fork
You've already forked hyperhive
0

semi-spread: wait for the pushed repo, bound the settle, name swarm faults

The test created its agents 0.5 s after pushing hyperhive/hyperhive, while
the forge still read the repo as empty, so the create's update_meta failed
and the next try came on the controller's 10-minute round. It now waits for
the two forge reads update_meta makes.

settle_hive_agents waited 3600 s, the CI job's own limit, so a stall was
killed rather than reported. It now waits 1200 s and also stops on a swarm
fault: a build the builder did not push to the cache, or an update_meta or
build_meta failure no later round got past. Timeouts and faults dump the
swarm builder's journal from heavyduty too.

Refs #6627
This commit is contained in:
atlas 2026-11-19 10:51:15 +01:00 • committed by mara
commit 7c2f7443fb

View file

@ -213,21 +213,63 @@ let
return f"hive-c0re's agent-status sweep failed {swept} times in the last 3 minutes" return f"hive-c0re's agent-status sweep failed {swept} times in the last 3 minutes"
return None return None
def settle_hive_agents(command, timeout): # The controller retries a failed update_meta or build_meta on its next
# settle on the swarm machine, but give up as soon as the hive is broken # 10-minute round; a minute of slack on top.
# and dump both machines. CONTROLLER_ROUND = 660
def controller_events(since):
# The controller's journal since the epoch second `since`, oldest first:
# (seconds, the failed node's kind or "", message).
out = swarm.succeed(
f"journalctl -u swarm-controller --since @{since} -o json --no-pager | jq -c "
"'[(.__REALTIME_TIMESTAMP | tonumber / 1000000), (.F_KIND // \"\"), "
"(.MESSAGE | if type == \"array\" then implode else . end)]'"
)
return [json.loads(line) for line in out.splitlines() if line.strip()]
def swarm_fault(since):
# Why the swarm cannot get a pushed build of the agents to the hive, or
# None. A build the builder did not push is final; a failed update_meta or
# build_meta is one only when no later round got past it.
now = int(swarm.succeed("date +%s"))
events = controller_events(since)
for _, _, message in events:
if "did not push it to the cache" in message:
why = heavyduty.execute(
"journalctl -M swarm-builder -u swarm-builder MESSAGE='closure not pushed to the cache' "
"-o json --no-pager | jq -r .F_REASON | tail -n 1"
)[1].strip()
return f"the swarm builder did not push a closure to the cache ({why or 'its journal names no reason'}): {message}"
for kind, success in [("update_meta", "update_meta: done"), ("build_meta", "swarm jobq: deploy requested")]:
failed = [(t, m) for t, k, m in events if k == kind and "node failed" in m]
if not failed:
continue
at, message = failed[-1]
if any(t > at and success in m for t, _, m in events):
continue
if now - at > CONTROLLER_ROUND:
return f"the controller's {kind} failed {int(now - at)}s ago and no later round succeeded: {message[-800:]}"
return None
def settle_hive_agents(command, timeout, since):
# settle on the swarm machine, but give up as soon as the hive or the
# swarm's build of the agents is broken, and dump the three machines.
deadline = time.monotonic() + timeout deadline = time.monotonic() + timeout
fault = None fault = None
while time.monotonic() < deadline: while time.monotonic() < deadline:
if swarm.execute(command)[0] == 0: if swarm.execute(command)[0] == 0:
return return
fault = hive_fault() fault = hive_fault() or swarm_fault(since)
if fault: if fault:
break break
time.sleep(10) time.sleep(10)
diagnose(hive, HIVE_UNITS) diagnose(hive, HIVE_UNITS)
diagnose(swarm, ["swarm-controller"]) diagnose(swarm, ["swarm-controller"])
raise Exception(fault or f"the agents did not settle within {timeout}s") diagnose(heavyduty, "swarm-builder", "swarm-builder")
if not fault:
deploys = sum("swarm jobq: deploy requested" in m for _, _, m in controller_events(since))
fault = f"the agents did not settle within {timeout}s; the controller sent {deploys} deploy request(s) since the creates"
raise Exception(fault)
def store_sealed(timeout): def store_sealed(timeout):
# `bao status` exits 2 when it reaches a sealed store and 1 for any # `bao status` exits 2 when it reaches a sealed store and 1 for any
@ -493,11 +535,22 @@ let
f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" " f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
f"push {FORGE}/hyperhive/hyperhive.git main" f"push {FORGE}/hyperhive/hyperhive.git main"
) )
# The forge records a push after the push returns. Until it does, the
# repo reads as empty and an agent create's update_meta fails, to be
# retried only on the controller's next 10-minute round. These are the
# two reads that update_meta makes.
swarm.wait_until_succeeds(
f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/repos/hyperhive/hyperhive "
"| jq -e '.empty == false' && "
f"curl --fail --silent --insecure -H @{FORGE_AUTH} -o /dev/null {FORGE}/api/v1/repos/hyperhive/hyperhive/branches/main",
timeout=120,
)
with subtest("setup.md steps 8 and 9: two agents, through the endpoint the swarm UI calls"): with subtest("setup.md steps 8 and 9: two agents, through the endpoint the swarm UI calls"):
swarm.wait_until_succeeds( swarm.wait_until_succeeds(
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=1800 f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=1800
) )
created = int(swarm.succeed("date +%s"))
create_agent({"name": "spawn", "hive": HIVE, "archetype": ARCHETYPE}) create_agent({"name": "spawn", "hive": HIVE, "archetype": ARCHETYPE})
create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE}) create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})
@ -506,7 +559,8 @@ let
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status " f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status "
"| jq -e '[.[] | select(.hive == \"" + HIVE + "\" and .freshness == \"fresh\" and .needs_login == true)" "| jq -e '[.[] | select(.hive == \"" + HIVE + "\" and .freshness == \"fresh\" and .needs_login == true)"
" | .name] | map(select(. == \"iris\" or . == \"spawn\")) | unique | length == 2'", " | .name] | map(select(. == \"iris\" or . == \"spawn\")) | unique | length == 2'",
3600, 1200,
created,
) )
# The closures came over the VPN from the cache: the hive's nix # The closures came over the VPN from the cache: the hive's nix
# names it as its only extra substituter and holds the toplevels now. # names it as its only extra substituter and holds the toplevels now.