semi-spread: wait for the pushed repo, bound the settle, name swarm faults
The test created its agents 0.5 s after pushing hyperhive/hyperhive, while the forge still read the repo as empty, so the create's update_meta failed and the next try came on the controller's 10-minute round. It now waits for the two forge reads update_meta makes. settle_hive_agents waited 3600 s, the CI job's own limit, so a stall was killed rather than reported. It now waits 1200 s and also stops on a swarm fault: a build the builder did not push to the cache, or an update_meta or build_meta failure no later round got past. Timeouts and faults dump the swarm builder's journal from heavyduty too. Refs #6627
This commit is contained in:
parent
70a8ef77a7
commit
7c2f7443fb
1 changed files with 60 additions and 6 deletions
|
|
@ -213,21 +213,63 @@ let
|
||||||
return f"hive-c0re's agent-status sweep failed {swept} times in the last 3 minutes"
|
return f"hive-c0re's agent-status sweep failed {swept} times in the last 3 minutes"
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def settle_hive_agents(command, timeout):
|
# The controller retries a failed update_meta or build_meta on its next
|
||||||
# settle on the swarm machine, but give up as soon as the hive is broken
|
# 10-minute round; a minute of slack on top.
|
||||||
# and dump both machines.
|
CONTROLLER_ROUND = 660
|
||||||
|
|
||||||
|
def controller_events(since):
|
||||||
|
# The controller's journal since the epoch second `since`, oldest first:
|
||||||
|
# (seconds, the failed node's kind or "", message).
|
||||||
|
out = swarm.succeed(
|
||||||
|
f"journalctl -u swarm-controller --since @{since} -o json --no-pager | jq -c "
|
||||||
|
"'[(.__REALTIME_TIMESTAMP | tonumber / 1000000), (.F_KIND // \"\"), "
|
||||||
|
"(.MESSAGE | if type == \"array\" then implode else . end)]'"
|
||||||
|
)
|
||||||
|
return [json.loads(line) for line in out.splitlines() if line.strip()]
|
||||||
|
|
||||||
|
def swarm_fault(since):
|
||||||
|
# Why the swarm cannot get a pushed build of the agents to the hive, or
|
||||||
|
# None. A build the builder did not push is final; a failed update_meta or
|
||||||
|
# build_meta is one only when no later round got past it.
|
||||||
|
now = int(swarm.succeed("date +%s"))
|
||||||
|
events = controller_events(since)
|
||||||
|
for _, _, message in events:
|
||||||
|
if "did not push it to the cache" in message:
|
||||||
|
why = heavyduty.execute(
|
||||||
|
"journalctl -M swarm-builder -u swarm-builder MESSAGE='closure not pushed to the cache' "
|
||||||
|
"-o json --no-pager | jq -r .F_REASON | tail -n 1"
|
||||||
|
)[1].strip()
|
||||||
|
return f"the swarm builder did not push a closure to the cache ({why or 'its journal names no reason'}): {message}"
|
||||||
|
for kind, success in [("update_meta", "update_meta: done"), ("build_meta", "swarm jobq: deploy requested")]:
|
||||||
|
failed = [(t, m) for t, k, m in events if k == kind and "node failed" in m]
|
||||||
|
if not failed:
|
||||||
|
continue
|
||||||
|
at, message = failed[-1]
|
||||||
|
if any(t > at and success in m for t, _, m in events):
|
||||||
|
continue
|
||||||
|
if now - at > CONTROLLER_ROUND:
|
||||||
|
return f"the controller's {kind} failed {int(now - at)}s ago and no later round succeeded: {message[-800:]}"
|
||||||
|
return None
|
||||||
|
|
||||||
|
def settle_hive_agents(command, timeout, since):
|
||||||
|
# settle on the swarm machine, but give up as soon as the hive or the
|
||||||
|
# swarm's build of the agents is broken, and dump the three machines.
|
||||||
deadline = time.monotonic() + timeout
|
deadline = time.monotonic() + timeout
|
||||||
fault = None
|
fault = None
|
||||||
while time.monotonic() < deadline:
|
while time.monotonic() < deadline:
|
||||||
if swarm.execute(command)[0] == 0:
|
if swarm.execute(command)[0] == 0:
|
||||||
return
|
return
|
||||||
fault = hive_fault()
|
fault = hive_fault() or swarm_fault(since)
|
||||||
if fault:
|
if fault:
|
||||||
break
|
break
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
diagnose(hive, HIVE_UNITS)
|
diagnose(hive, HIVE_UNITS)
|
||||||
diagnose(swarm, ["swarm-controller"])
|
diagnose(swarm, ["swarm-controller"])
|
||||||
raise Exception(fault or f"the agents did not settle within {timeout}s")
|
diagnose(heavyduty, "swarm-builder", "swarm-builder")
|
||||||
|
if not fault:
|
||||||
|
deploys = sum("swarm jobq: deploy requested" in m for _, _, m in controller_events(since))
|
||||||
|
fault = f"the agents did not settle within {timeout}s; the controller sent {deploys} deploy request(s) since the creates"
|
||||||
|
raise Exception(fault)
|
||||||
|
|
||||||
def store_sealed(timeout):
|
def store_sealed(timeout):
|
||||||
# `bao status` exits 2 when it reaches a sealed store and 1 for any
|
# `bao status` exits 2 when it reaches a sealed store and 1 for any
|
||||||
|
|
@ -493,11 +535,22 @@ let
|
||||||
f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
|
f"GIT_SSL_NO_VERIFY=1 git -C /root/hyperhive.git -c \"http.extraHeader=$(cat {FORGE_AUTH})\" "
|
||||||
f"push {FORGE}/hyperhive/hyperhive.git main"
|
f"push {FORGE}/hyperhive/hyperhive.git main"
|
||||||
)
|
)
|
||||||
|
# The forge records a push after the push returns. Until it does, the
|
||||||
|
# repo reads as empty and an agent create's update_meta fails, to be
|
||||||
|
# retried only on the controller's next 10-minute round. These are the
|
||||||
|
# two reads that update_meta makes.
|
||||||
|
swarm.wait_until_succeeds(
|
||||||
|
f"curl --fail --silent --insecure -H @{FORGE_AUTH} {FORGE}/api/v1/repos/hyperhive/hyperhive "
|
||||||
|
"| jq -e '.empty == false' && "
|
||||||
|
f"curl --fail --silent --insecure -H @{FORGE_AUTH} -o /dev/null {FORGE}/api/v1/repos/hyperhive/hyperhive/branches/main",
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
|
||||||
with subtest("setup.md steps 8 and 9: two agents, through the endpoint the swarm UI calls"):
|
with subtest("setup.md steps 8 and 9: two agents, through the endpoint the swarm UI calls"):
|
||||||
swarm.wait_until_succeeds(
|
swarm.wait_until_succeeds(
|
||||||
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=1800
|
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/permissions >/dev/null", timeout=1800
|
||||||
)
|
)
|
||||||
|
created = int(swarm.succeed("date +%s"))
|
||||||
create_agent({"name": "spawn", "hive": HIVE, "archetype": ARCHETYPE})
|
create_agent({"name": "spawn", "hive": HIVE, "archetype": ARCHETYPE})
|
||||||
create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})
|
create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})
|
||||||
|
|
||||||
|
|
@ -506,7 +559,8 @@ let
|
||||||
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status "
|
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status "
|
||||||
"| jq -e '[.[] | select(.hive == \"" + HIVE + "\" and .freshness == \"fresh\" and .needs_login == true)"
|
"| jq -e '[.[] | select(.hive == \"" + HIVE + "\" and .freshness == \"fresh\" and .needs_login == true)"
|
||||||
" | .name] | map(select(. == \"iris\" or . == \"spawn\")) | unique | length == 2'",
|
" | .name] | map(select(. == \"iris\" or . == \"spawn\")) | unique | length == 2'",
|
||||||
3600,
|
1200,
|
||||||
|
created,
|
||||||
)
|
)
|
||||||
# The closures came over the VPN from the cache: the hive's nix
|
# The closures came over the VPN from the cache: the hive's nix
|
||||||
# names it as its only extra substituter and holds the toplevels now.
|
# names it as its only extra substituter and holds the toplevels now.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue