semi-spread vm test: a controller journal entry with no message cannot crash the settle loop
This commit is contained in:
parent
9e04477ec7
commit
0370076463
1 changed files with 20 additions and 5 deletions
|
|
@ -219,13 +219,22 @@ let
|
|||
|
||||
def controller_events(since):
|
||||
# The controller's journal since the epoch second `since`, oldest first:
|
||||
# (seconds, the failed node's kind or "", message).
|
||||
# (seconds, the failed node's kind or "", message). An entry whose
|
||||
# MESSAGE is null or whose fields do not parse is dropped or has the
|
||||
# field coerced to "", so every caller can test `in message`.
|
||||
out = swarm.succeed(
|
||||
f"journalctl -u swarm-controller --since @{since} -o json --no-pager | jq -c "
|
||||
"'[(.__REALTIME_TIMESTAMP | tonumber / 1000000), (.F_KIND // \"\"), "
|
||||
"(.MESSAGE | if type == \"array\" then implode else . end)]'"
|
||||
"'try [(.__REALTIME_TIMESTAMP | tonumber / 1000000), (.F_KIND // \"\"), "
|
||||
"(.MESSAGE | if type == \"array\" then implode else . end)] catch empty'"
|
||||
)
|
||||
return [json.loads(line) for line in out.splitlines() if line.strip()]
|
||||
events = []
|
||||
for line in out.splitlines():
|
||||
try:
|
||||
at, kind, message = json.loads(line)
|
||||
events.append((float(at), kind if isinstance(kind, str) else "", message if isinstance(message, str) else ""))
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
return events
|
||||
|
||||
def swarm_fault(since):
|
||||
# Why the swarm cannot get a pushed build of the agents to the hive, or
|
||||
|
|
@ -259,7 +268,13 @@ let
|
|||
while time.monotonic() < deadline:
|
||||
if swarm.execute(command)[0] == 0:
|
||||
return
|
||||
fault = hive_fault() or swarm_fault(since)
|
||||
# A fault check that itself fails is no verdict: the loop keeps
|
||||
# waiting for the deadline instead of dying on it.
|
||||
try:
|
||||
fault = hive_fault() or swarm_fault(since)
|
||||
except Exception as e:
|
||||
print(f"fault check failed: {e!r}")
|
||||
fault = None
|
||||
if fault:
|
||||
break
|
||||
time.sleep(10)
|
||||
|
|
|
|||
Loading…
Reference in a new issue