From b9df538940082c2c3bd9e2e368929adef293d4f6 Mon Sep 17 00:00:00 2001 From: iris Date: Wed, 27 May 2026 23:37:21 +0200 Subject: [PATCH] frontend/dashboard: heartbeat + watchdog so a dead SharedWorker self-heals (#515) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Firefox kills "idle" SharedWorkers under memory pressure with no native signal to the client. The page's port silently becomes a no-op and events stop flowing — observable symptom: mara's "dashboard never refreshes; F5 fixes it" (because F5 creates a fresh page that creates a fresh worker). The worker now pings every connected port every 30s. The client tracks last-activity-from-worker on every message arrival (incl. pings, since those carry no URL — bumped before the URL filter in the route handler). A visibility-gated watchdog polls every 15s; if the page is visible AND has active subs AND hasn't heard from the worker in >90s, it presumes the worker dead, logs a console warning, and re-subscribes on a fresh port. Three pings missed before we act, so a normal tab-throttle blip doesn't false-positive. The fresh-port re-subscribe re-uses the bfcache-restore code path (same shape: drop stale listeners, getSharedPort → new SharedWorker, re-attach each route + repost subscribe). Recovery is per-tab — when one tab's watchdog fires and brings up a new worker, other tabs that share the named worker pick it up on their own watchdog cycle. Falls back gracefully on environments without SharedWorker (the existing direct-EventSource path is untouched) and is invisible on the healthy path — pings are 30s apart, no UI surface. --- frontend/packages/dashboard/src/common.js | 75 ++++++++++++++++++- .../packages/dashboard/src/stream-worker.js | 26 +++++++ 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/frontend/packages/dashboard/src/common.js b/frontend/packages/dashboard/src/common.js index 2bc430d8..f18b406d 100644 --- a/frontend/packages/dashboard/src/common.js +++ b/frontend/packages/dashboard/src/common.js @@ -99,6 +99,71 @@ function getSharedPort() { return _sharedPort; } +// #515: detect when the SharedWorker has been killed. Firefox aggressively +// reclaims "idle" SharedWorkers under memory pressure (or just on tab +// lifecycle quirks we don't fully understand), and there's no native +// signal to the client when that happens — `postMessage` on a dead +// port silently no-ops, and the page just stops receiving events. The +// observable symptom is "dashboard never refreshes; F5 fixes it" (a +// fresh page creates a fresh worker), which is mara's report on #515. +// +// Pipeline: +// - The worker pings every connected port every WORKER_PING_INTERVAL_MS. +// - `route` (every message arrival, including pings) bumps +// `_lastWorkerActivityAt`. +// - `startWorkerWatchdog` polls every WORKER_WATCHDOG_INTERVAL_MS. +// If the page is visible AND we have active subs AND we haven't +// heard from the worker in > WORKER_DEAD_THRESHOLD_MS, we presume +// the worker is dead, log a warning, and re-subscribe on a fresh +// port. +// +// Numbers picked so a real Firefox tab-suspend / unsuspend cycle (which +// can pause the watchdog itself) doesn't false-positive: 90s without a +// 30s ping means at least three pings missed. Visibility-gated so a +// backgrounded tab — where Firefox throttles setInterval to 1Hz min and +// our healthcheck wouldn't trigger reliably anyway — doesn't try to +// reconnect uselessly. +const WORKER_DEAD_THRESHOLD_MS = 90_000; +const WORKER_WATCHDOG_INTERVAL_MS = 15_000; +let _lastWorkerActivityAt = 0; +function noteWorkerActivity() { _lastWorkerActivityAt = Date.now(); } +let _watchdogTimer = null; +function startWorkerWatchdog() { + if (_watchdogTimer != null) return; + _watchdogTimer = setInterval(() => { + if (typeof document !== 'undefined' && document.visibilityState !== 'visible') return; + if (!_activeSubs.size) return; + if (!_sharedPort) return; + const sinceLast = Date.now() - _lastWorkerActivityAt; + if (sinceLast < WORKER_DEAD_THRESHOLD_MS) return; + console.warn( + 'hyperhive-stream worker silent for ' + Math.round(sinceLast / 1000) + + 's, presumed dead — re-subscribing on a fresh port'); + rebindOnFreshPort(); + }, WORKER_WATCHDOG_INTERVAL_MS); +} +function rebindOnFreshPort() { + // Drop any listeners attached to the dead port (best-effort; calls + // on a dead port throw, which we swallow). The active-subs registry + // is the source of truth for what we need to re-attach. + if (_sharedPort) { + for (const sub of _activeSubs.values()) { + try { _sharedPort.removeEventListener('message', sub.route); } catch {} + } + } + _sharedPort = null; + const port = getSharedPort(); + if (!port) return; // SharedWorker unsupported / unavailable — nothing to do + for (const [url, sub] of _activeSubs) { + sub.target.readyState = 0; // CONNECTING — worker will (re-)fire 'open' + port.addEventListener('message', sub.route); + try { port.postMessage({ kind: 'subscribe', url }); } catch {} + } + // Reset the activity clock so the watchdog gives the fresh worker + // a full window to settle before re-triggering. + noteWorkerActivity(); +} + // Registry of live subscriptions on this page. Keyed by url so a // second openStream call for the same URL (would only happen on a // hypothetical multi-consumer page) attaches to the existing route @@ -122,6 +187,7 @@ let _lifecycleBound = false; function bindLifecycleOnce() { if (_lifecycleBound) return; _lifecycleBound = true; + startWorkerWatchdog(); window.addEventListener('pagehide', () => { if (!_sharedPort) return; for (const url of _activeSubs.keys()) { @@ -179,8 +245,12 @@ export function openStream(url) { }, }; const route = (e) => { + // #515: any message from the worker is proof of life — note it + // before the URL filter, since heartbeat pings carry no URL. + noteWorkerActivity(); const m = e.data; - if (!m || m.url !== url) return; + if (!m || m.kind === 'ping') return; + if (m.url !== url) return; if (m.kind === 'open') { target.readyState = 1; // OPEN if (target.onopen) { @@ -202,6 +272,9 @@ export function openStream(url) { _activeSubs.set(url, { target, route }); port.addEventListener('message', route); port.postMessage({ kind: 'subscribe', url }); + // #515: seed the activity clock so the watchdog has a baseline; it + // would otherwise compare against 0 (epoch) and trigger immediately. + noteWorkerActivity(); return target; } diff --git a/frontend/packages/dashboard/src/stream-worker.js b/frontend/packages/dashboard/src/stream-worker.js index 3d7af676..06e46c34 100644 --- a/frontend/packages/dashboard/src/stream-worker.js +++ b/frontend/packages/dashboard/src/stream-worker.js @@ -29,6 +29,18 @@ // re-sync after a reconnect gap. // { kind: 'message', url: '...', data: '' } // { kind: 'error', url: '...' } relayed from EventSource.onerror. +// { kind: 'ping' } #515: heartbeat — fired every +// PING_INTERVAL_MS to every +// connected port. The client's +// watchdog uses these as +// proof-of-life; silence past +// ~3× the interval triggers a +// re-subscribe on a fresh port +// (recovers from Firefox killing +// the SharedWorker out from +// under us, which it does under +// memory pressure with no native +// signal to the client). // // Subscriptions are tracked per (port, url): a single port can // subscribe to multiple URLs (today only one is in use but the shape @@ -37,6 +49,19 @@ // don't keep idle streams open. const streams = new Map(); +// All currently-connected ports. Used by the heartbeat tick to fan +// pings out across every tab regardless of which URLs each port is +// subscribed to. Stays disjoint from per-stream `entry.ports` (which +// is URL-scoped); a port may be in `allPorts` without any active +// subscription (e.g. between a tab loading and its first subscribe). +const allPorts = new Set(); +const PING_INTERVAL_MS = 30_000; +setInterval(() => { + for (const port of allPorts) { + try { port.postMessage({ kind: 'ping' }); } + catch { /* port dead — cleanup happens lazily on next subscribe */ } + } +}, PING_INTERVAL_MS); function getOrCreateStream(url) { let entry = streams.get(url); @@ -77,6 +102,7 @@ function unsubscribe(port, url) { self.onconnect = (connectEvent) => { const port = connectEvent.ports[0]; + allPorts.add(port); const subscribedUrls = new Set(); port.onmessage = (e) => { const msg = e.data;