From 7d834c12002f5139098b54811940b1bb5aebd943 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 10 Jul 2026 12:08:02 +0200 Subject: [PATCH] fix(web-probe): retry transient realtime navigation --- config/hwlab-node-lanes.yaml | 2 ++ scripts/src/hwlab-node-lanes.ts | 4 +++ ...web-observe-runner-realtime-source.test.ts | 31 +++++++++++++++++ ...node-web-observe-runner-realtime-source.ts | 34 ++++++++++++++++--- 4 files changed, 67 insertions(+), 4 deletions(-) diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index cd5e2a0b..60fa479c 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -557,6 +557,8 @@ templates: barrierTimeoutMs: 45000 terminalTimeoutMs: 480000 terminalQuietMs: 2000 + navigationMaxAttempts: 4 + navigationRetryDelayMs: 1000 reconnectQuietMs: 3000 forbiddenRequestPaths: - /v1/workbench/sync diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 60d010f4..3b3328e6 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -159,6 +159,8 @@ export interface HwlabRuntimeWebProbeRealtimeFanoutProfileSpec { readonly barrierTimeoutMs: number; readonly terminalTimeoutMs: number; readonly terminalQuietMs: number; + readonly navigationMaxAttempts: number; + readonly navigationRetryDelayMs: number; readonly reconnectQuietMs: number; readonly forbiddenRequestPaths: readonly string[]; readonly requiredEventTypes: Readonly<{ @@ -1324,6 +1326,8 @@ function webProbeRealtimeFanoutProfileConfig(value: unknown, path: string): Hwla barrierTimeoutMs: boundedIntegerField(raw, "barrierTimeoutMs", path, 1_000, 120_000), terminalTimeoutMs: boundedIntegerField(raw, "terminalTimeoutMs", path, 1_000, 600_000), terminalQuietMs: boundedIntegerField(raw, "terminalQuietMs", path, 250, 10_000), + navigationMaxAttempts: boundedIntegerField(raw, "navigationMaxAttempts", path, 1, 10), + navigationRetryDelayMs: boundedIntegerField(raw, "navigationRetryDelayMs", path, 100, 10_000), reconnectQuietMs: boundedIntegerField(raw, "reconnectQuietMs", path, 250, 30_000), forbiddenRequestPaths, requiredEventTypes: { diff --git a/scripts/src/hwlab-node-web-observe-runner-realtime-source.test.ts b/scripts/src/hwlab-node-web-observe-runner-realtime-source.test.ts index e929d157..ae9e1d2c 100644 --- a/scripts/src/hwlab-node-web-observe-runner-realtime-source.test.ts +++ b/scripts/src/hwlab-node-web-observe-runner-realtime-source.test.ts @@ -38,3 +38,34 @@ test("realtime fanout keeps the YAML terminal timeout when durationMs is absent" assert.equal(effectiveProfile(profile, undefined).terminalTimeoutMs, 480_000); assert.equal(effectiveProfile(profile, 5_000).terminalTimeoutMs, 5_000); }); + +test("realtime fanout retries a YAML-bounded transient health navigation", async () => { + const source = nodeWebObserveRunnerRealtimeSource(); + const build = new Function( + "sleep", + "isRetryableNavigationError", + "baseUrl", + `${source}\nreturn realtimeGotoHealth;`, + ) as ( + sleep: (milliseconds: number) => Promise, + isRetryableNavigationError: (message: string) => boolean, + baseUrl: string, + ) => (page: { goto: () => Promise<{ status: () => number }> }, timeoutMs: number, profile: Record, label: string) => Promise>; + const gotoHealth = build( + async () => {}, + (message) => /ERR_NETWORK_CHANGED/u.test(message), + "https://hwlab.example.test", + ); + let calls = 0; + const result = await gotoHealth({ + async goto() { + calls += 1; + if (calls === 1) throw new Error("page.goto: net::ERR_NETWORK_CHANGED"); + return { status: () => 200 }; + }, + }, 45_000, { navigationMaxAttempts: 4, navigationRetryDelayMs: 1_000 }, "subscriber-a"); + + assert.equal(result.ok, true); + assert.equal(calls, 2); + assert.equal((result.attempts as unknown[]).length, 2); +}); diff --git a/scripts/src/hwlab-node-web-observe-runner-realtime-source.ts b/scripts/src/hwlab-node-web-observe-runner-realtime-source.ts index c9ec7f61..0c6fd73f 100644 --- a/scripts/src/hwlab-node-web-observe-runner-realtime-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-realtime-source.ts @@ -72,7 +72,7 @@ async function validateRealtimeFanout(command) { const storageState = await context.storageState(); subscriberA = await realtimeNewSubscriber(storageState, sessionId, "subscriber-a", effective); subscriberB = await realtimeNewSubscriber(storageState, sessionId, "subscriber-b", effective); - debug = await realtimeNewDebugContext(storageState); + debug = await realtimeNewDebugContext(storageState, effective); const connectedA = await realtimeWaitProductConnected(subscriberA, effective.barrierTimeoutMs); const connectedB = await realtimeWaitProductConnected(subscriberB, effective.barrierTimeoutMs); realtimeAssertLiveConnected(connectedA, sessionId, "subscriber-a", effective); @@ -334,6 +334,8 @@ function realtimeFanoutEffectiveProfile(profile, durationMs) { barrierTimeoutMs: boundedInteger(profile.barrierTimeoutMs, 45000, 1000, 120000), terminalTimeoutMs: boundedInteger(terminalTimeoutMs, 480000, 1000, 600000), terminalQuietMs: boundedInteger(profile.terminalQuietMs, 2000, 250, 10000), + navigationMaxAttempts: boundedInteger(profile.navigationMaxAttempts, 4, 1, 10), + navigationRetryDelayMs: boundedInteger(profile.navigationRetryDelayMs, 1000, 100, 10000), reconnectQuietMs: boundedInteger(profile.reconnectQuietMs, 3000, 250, 30000), forbiddenRequestPaths: Array.isArray(profile.forbiddenRequestPaths) ? profile.forbiddenRequestPaths.map(String) : [], requiredEventTypes: profile.requiredEventTypes && typeof profile.requiredEventTypes === "object" ? profile.requiredEventTypes : {}, @@ -344,7 +346,7 @@ function realtimeFanoutEffectiveProfile(profile, durationMs) { async function realtimeNewSubscriber(storageState, sessionId, label, profile) { const subscriberContext = await browser.newContext({ storageState, viewport }); const subscriberPage = await subscriberContext.newPage(); - await subscriberPage.goto(new URL("/health/live", baseUrl).toString(), { waitUntil: "domcontentloaded", timeout: profile.barrierTimeoutMs }); + await realtimeGotoHealth(subscriberPage, profile.barrierTimeoutMs, profile, label); const key = label + "-" + sessionId; await subscriberPage.evaluate(({ key: streamKey, sessionId: scopedSessionId, profile: inputProfile }) => { const root = window.__unideskRealtimeFanoutProducts ??= {}; @@ -399,13 +401,37 @@ async function realtimeNewSubscriber(storageState, sessionId, label, profile) { return { label, key, context: subscriberContext, page: subscriberPage }; } -async function realtimeNewDebugContext(storageState) { +async function realtimeNewDebugContext(storageState, profile) { const debugContext = await browser.newContext({ storageState, viewport }); const debugPage = await debugContext.newPage(); - await debugPage.goto(new URL("/health/live", baseUrl).toString(), { waitUntil: "domcontentloaded", timeout: 30000 }); + await realtimeGotoHealth(debugPage, profile.barrierTimeoutMs, profile, "debug-subscriber"); return { context: debugContext, page: debugPage }; } +async function realtimeGotoHealth(targetPage, timeoutMs, profile, label) { + const attempts = []; + const target = new URL("/health/live", baseUrl).toString(); + for (let attempt = 1; attempt <= profile.navigationMaxAttempts; attempt += 1) { + try { + const response = await targetPage.goto(target, { waitUntil: "domcontentloaded", timeout: timeoutMs }); + const status = response?.status?.() ?? null; + if (status !== null && status >= 400) throw new Error(label + " health navigation returned HTTP " + status); + attempts.push({ attempt, ok: true, status, valuesRedacted: true }); + return { ok: true, attempts, valuesRedacted: true }; + } catch (error) { + const message = String(error?.message || error || "unknown").slice(0, 240); + attempts.push({ attempt, ok: false, message, valuesRedacted: true }); + if (attempt >= profile.navigationMaxAttempts || !isRetryableNavigationError(message)) { + const wrapped = error instanceof Error ? error : new Error(message); + wrapped.details = { ...(wrapped.details || {}), label, attempts, valuesRedacted: true }; + throw wrapped; + } + await sleep(profile.navigationRetryDelayMs * attempt); + } + } + throw new Error(label + " health navigation retry exhausted"); +} + async function realtimeRunTurn(input) { const turn = input.turn; const prompt = input.prompt;