mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

vm tests: fail on a missing log port, give up early on a broken hive

Browse source
This commit is contained in:
atlas 2026-11-13 07:39:08 +01:00 • committed by mara
commit df02ff89aa
1 changed files with 36 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

36
nix/vm-tests/semi-spread.nix
View file

@ -304,6 +304,7 @@ pkgs.testers.runNixOSTest (
testScript = ''
import json
import shlex
import time
API = "http://localhost"
SOCKET = "${controllerSocket}"
@ -354,6 +355,37 @@ pkgs.testers.runNixOSTest (
diagnose(m, unit, container)
raise
HIVE_UNITS = ["hive-priv.socket", "hive-priv.service", "hive-c0re"]
def hive_fault():
# Why the hive cannot reach the state the swarm waits for, or None. The
# privileged helper is socket-activated: its socket is active while it
# is usable, and failed once the helper's start limit is hit. The status
# sweep runs every minute and fails once or twice while the swarm queue
# comes up, so only three failures within three minutes count.
if hive.execute("systemctl is-active --quiet hive-priv.socket && ! systemctl is-failed --quiet hive-priv.service")[0] != 0:
return "hive-priv is not up on the hive"
swept = hive.succeed("journalctl -u hive-c0re --since '3 min ago' --no-pager -o cat | grep -c 'agent status: sweep failed' || true").strip()
if int(swept) >= 3:
return f"hive-c0re's agent-status sweep failed {swept} times in the last 3 minutes"
return None
def settle_hive_agents(command, timeout):
# settle on the swarm machine, but give up as soon as the hive is broken
# and dump both machines.
deadline = time.monotonic() + timeout
fault = None
while time.monotonic() < deadline:
if swarm.execute(command)[0] == 0:
return
fault = hive_fault()
if fault:
break
time.sleep(10)
diagnose(hive, HIVE_UNITS)
diagnose(swarm, ["swarm-controller"])
raise Exception(fault or f"the agents did not settle within {timeout}s")
def store_sealed(timeout):
# `bao status` exits 2 when it reaches a sealed store and 1 for any
# address, CA or client-certificate problem.
@ -627,13 +659,11 @@ pkgs.testers.runNixOSTest (
create_agent({"name": "iris", "hive": HIVE, "archetype": ARCHETYPE})
with subtest("assert 4: the agents are deployed on the hive, from the cache on heavyduty, and need a login"):
settle(
swarm,
settle_hive_agents(
f"curl --fail --silent --unix-socket {SOCKET} {API}/api/agents/status "
"| jq -e '[.[] | select(.hive == \"" + HIVE + "\" and .freshness == \"fresh\" and .needs_login == true)"
" | .name] | map(select(. == \"iris\" or . == \"spawn\")) | unique | length == 2'",
3600,
["swarm-controller"],
)
# The closures came over the VPN from the cache: the hive's nix
# names it as its only extra substituter and holds the toplevels now.