Watch
0
0
Fork
You've already forked hyperhive
0

semi-spread vm test: a controller journal entry with no message cannot crash the settle loop

This commit is contained in:
atlas 2026-11-19 13:52:02 +01:00 • committed by mara
commit 0370076463

View file

@ -219,13 +219,22 @@ let
def controller_events(since): def controller_events(since):
# The controller's journal since the epoch second `since`, oldest first: # The controller's journal since the epoch second `since`, oldest first:
# (seconds, the failed node's kind or "", message). # (seconds, the failed node's kind or "", message). An entry whose
# MESSAGE is null or whose fields do not parse is dropped or has the
# field coerced to "", so every caller can test `in message`.
out = swarm.succeed( out = swarm.succeed(
f"journalctl -u swarm-controller --since @{since} -o json --no-pager | jq -c " f"journalctl -u swarm-controller --since @{since} -o json --no-pager | jq -c "
"'[(.__REALTIME_TIMESTAMP | tonumber / 1000000), (.F_KIND // \"\"), " "'try [(.__REALTIME_TIMESTAMP | tonumber / 1000000), (.F_KIND // \"\"), "
"(.MESSAGE | if type == \"array\" then implode else . end)]'" "(.MESSAGE | if type == \"array\" then implode else . end)] catch empty'"
) )
return [json.loads(line) for line in out.splitlines() if line.strip()] events = []
for line in out.splitlines():
try:
at, kind, message = json.loads(line)
events.append((float(at), kind if isinstance(kind, str) else "", message if isinstance(message, str) else ""))
except (ValueError, TypeError):
continue
return events
def swarm_fault(since): def swarm_fault(since):
# Why the swarm cannot get a pushed build of the agents to the hive, or # Why the swarm cannot get a pushed build of the agents to the hive, or
@ -259,7 +268,13 @@ let
while time.monotonic() < deadline: while time.monotonic() < deadline:
if swarm.execute(command)[0] == 0: if swarm.execute(command)[0] == 0:
return return
fault = hive_fault() or swarm_fault(since) # A fault check that itself fails is no verdict: the loop keeps
# waiting for the deadline instead of dying on it.
try:
fault = hive_fault() or swarm_fault(since)
except Exception as e:
print(f"fault check failed: {e!r}")
fault = None
if fault: if fault:
break break
time.sleep(10) time.sleep(10)