diff --git a/nix/vm-tests/semi-spread.nix b/nix/vm-tests/semi-spread.nix index e4df7c293..645dd13cc 100644 --- a/nix/vm-tests/semi-spread.nix +++ b/nix/vm-tests/semi-spread.nix @@ -219,13 +219,22 @@ let def controller_events(since): # The controller's journal since the epoch second `since`, oldest first: - # (seconds, the failed node's kind or "", message). + # (seconds, the failed node's kind or "", message). An entry whose + # MESSAGE is null or whose fields do not parse is dropped or has the + # field coerced to "", so every caller can test `in message`. out = swarm.succeed( f"journalctl -u swarm-controller --since @{since} -o json --no-pager | jq -c " - "'[(.__REALTIME_TIMESTAMP | tonumber / 1000000), (.F_KIND // \"\"), " - "(.MESSAGE | if type == \"array\" then implode else . end)]'" + "'try [(.__REALTIME_TIMESTAMP | tonumber / 1000000), (.F_KIND // \"\"), " + "(.MESSAGE | if type == \"array\" then implode else . end)] catch empty'" ) - return [json.loads(line) for line in out.splitlines() if line.strip()] + events = [] + for line in out.splitlines(): + try: + at, kind, message = json.loads(line) + events.append((float(at), kind if isinstance(kind, str) else "", message if isinstance(message, str) else "")) + except (ValueError, TypeError): + continue + return events def swarm_fault(since): # Why the swarm cannot get a pushed build of the agents to the hive, or @@ -259,7 +268,13 @@ let while time.monotonic() < deadline: if swarm.execute(command)[0] == 0: return - fault = hive_fault() or swarm_fault(since) + # A fault check that itself fails is no verdict: the loop keeps + # waiting for the deadline instead of dying on it. + try: + fault = hive_fault() or swarm_fault(since) + except Exception as e: + print(f"fault check failed: {e!r}") + fault = None if fault: break time.sleep(10)