semi-spread vm test: a controller journal entry with no message cannot crash the settle loop
This commit is contained in:
parent
9e04477ec7
commit
0370076463
1 changed files with 20 additions and 5 deletions
|
|
@ -219,13 +219,22 @@ let
|
||||||
|
|
||||||
def controller_events(since):
|
def controller_events(since):
|
||||||
# The controller's journal since the epoch second `since`, oldest first:
|
# The controller's journal since the epoch second `since`, oldest first:
|
||||||
# (seconds, the failed node's kind or "", message).
|
# (seconds, the failed node's kind or "", message). An entry whose
|
||||||
|
# MESSAGE is null or whose fields do not parse is dropped or has the
|
||||||
|
# field coerced to "", so every caller can test `in message`.
|
||||||
out = swarm.succeed(
|
out = swarm.succeed(
|
||||||
f"journalctl -u swarm-controller --since @{since} -o json --no-pager | jq -c "
|
f"journalctl -u swarm-controller --since @{since} -o json --no-pager | jq -c "
|
||||||
"'[(.__REALTIME_TIMESTAMP | tonumber / 1000000), (.F_KIND // \"\"), "
|
"'try [(.__REALTIME_TIMESTAMP | tonumber / 1000000), (.F_KIND // \"\"), "
|
||||||
"(.MESSAGE | if type == \"array\" then implode else . end)]'"
|
"(.MESSAGE | if type == \"array\" then implode else . end)] catch empty'"
|
||||||
)
|
)
|
||||||
return [json.loads(line) for line in out.splitlines() if line.strip()]
|
events = []
|
||||||
|
for line in out.splitlines():
|
||||||
|
try:
|
||||||
|
at, kind, message = json.loads(line)
|
||||||
|
events.append((float(at), kind if isinstance(kind, str) else "", message if isinstance(message, str) else ""))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
continue
|
||||||
|
return events
|
||||||
|
|
||||||
def swarm_fault(since):
|
def swarm_fault(since):
|
||||||
# Why the swarm cannot get a pushed build of the agents to the hive, or
|
# Why the swarm cannot get a pushed build of the agents to the hive, or
|
||||||
|
|
@ -259,7 +268,13 @@ let
|
||||||
while time.monotonic() < deadline:
|
while time.monotonic() < deadline:
|
||||||
if swarm.execute(command)[0] == 0:
|
if swarm.execute(command)[0] == 0:
|
||||||
return
|
return
|
||||||
|
# A fault check that itself fails is no verdict: the loop keeps
|
||||||
|
# waiting for the deadline instead of dying on it.
|
||||||
|
try:
|
||||||
fault = hive_fault() or swarm_fault(since)
|
fault = hive_fault() or swarm_fault(since)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"fault check failed: {e!r}")
|
||||||
|
fault = None
|
||||||
if fault:
|
if fault:
|
||||||
break
|
break
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue