Merge pull request #1689 from pikasTech/fix/2467-realtime-navigation-retry

fix: YAML-first 重试 realtime probe 瞬态导航

config/hwlab-node-lanes.yaml

@@ -557,6 +557,8 @@ templates:
           barrierTimeoutMs: 45000
           terminalTimeoutMs: 480000
           terminalQuietMs: 2000
+          navigationMaxAttempts: 4
+          navigationRetryDelayMs: 1000
           reconnectQuietMs: 3000
           forbiddenRequestPaths:
             - /v1/workbench/sync

scripts/src/hwlab-node-lanes.ts

@@ -159,6 +159,8 @@ export interface HwlabRuntimeWebProbeRealtimeFanoutProfileSpec {
   readonly barrierTimeoutMs: number;
   readonly terminalTimeoutMs: number;
   readonly terminalQuietMs: number;
+  readonly navigationMaxAttempts: number;
+  readonly navigationRetryDelayMs: number;
   readonly reconnectQuietMs: number;
   readonly forbiddenRequestPaths: readonly string[];
   readonly requiredEventTypes: Readonly<{
@@ -1324,6 +1326,8 @@ function webProbeRealtimeFanoutProfileConfig(value: unknown, path: string): Hwla
     barrierTimeoutMs: boundedIntegerField(raw, "barrierTimeoutMs", path, 1_000, 120_000),
     terminalTimeoutMs: boundedIntegerField(raw, "terminalTimeoutMs", path, 1_000, 600_000),
     terminalQuietMs: boundedIntegerField(raw, "terminalQuietMs", path, 250, 10_000),
+    navigationMaxAttempts: boundedIntegerField(raw, "navigationMaxAttempts", path, 1, 10),
+    navigationRetryDelayMs: boundedIntegerField(raw, "navigationRetryDelayMs", path, 100, 10_000),
     reconnectQuietMs: boundedIntegerField(raw, "reconnectQuietMs", path, 250, 30_000),
     forbiddenRequestPaths,
     requiredEventTypes: {

scripts/src/hwlab-node-web-observe-runner-realtime-source.test.ts

@@ -38,3 +38,34 @@ test("realtime fanout keeps the YAML terminal timeout when durationMs is absent"
   assert.equal(effectiveProfile(profile, undefined).terminalTimeoutMs, 480_000);
   assert.equal(effectiveProfile(profile, 5_000).terminalTimeoutMs, 5_000);
 });
+
+test("realtime fanout retries a YAML-bounded transient health navigation", async () => {
+  const source = nodeWebObserveRunnerRealtimeSource();
+  const build = new Function(
+    "sleep",
+    "isRetryableNavigationError",
+    "baseUrl",
+    `${source}\nreturn realtimeGotoHealth;`,
+  ) as (
+    sleep: (milliseconds: number) => Promise<void>,
+    isRetryableNavigationError: (message: string) => boolean,
+    baseUrl: string,
+  ) => (page: { goto: () => Promise<{ status: () => number }> }, timeoutMs: number, profile: Record<string, number>, label: string) => Promise<Record<string, unknown>>;
+  const gotoHealth = build(
+    async () => {},
+    (message) => /ERR_NETWORK_CHANGED/u.test(message),
+    "https://hwlab.example.test",
+  );
+  let calls = 0;
+  const result = await gotoHealth({
+    async goto() {
+      calls += 1;
+      if (calls === 1) throw new Error("page.goto: net::ERR_NETWORK_CHANGED");
+      return { status: () => 200 };
+    },
+  }, 45_000, { navigationMaxAttempts: 4, navigationRetryDelayMs: 1_000 }, "subscriber-a");
+
+  assert.equal(result.ok, true);
+  assert.equal(calls, 2);
+  assert.equal((result.attempts as unknown[]).length, 2);
+});

scripts/src/hwlab-node-web-observe-runner-realtime-source.ts

@@ -72,7 +72,7 @@ async function validateRealtimeFanout(command) {
     const storageState = await context.storageState();
     subscriberA = await realtimeNewSubscriber(storageState, sessionId, "subscriber-a", effective);
     subscriberB = await realtimeNewSubscriber(storageState, sessionId, "subscriber-b", effective);
-    debug = await realtimeNewDebugContext(storageState);
+    debug = await realtimeNewDebugContext(storageState, effective);
     const connectedA = await realtimeWaitProductConnected(subscriberA, effective.barrierTimeoutMs);
     const connectedB = await realtimeWaitProductConnected(subscriberB, effective.barrierTimeoutMs);
     realtimeAssertLiveConnected(connectedA, sessionId, "subscriber-a", effective);
@@ -334,6 +334,8 @@ function realtimeFanoutEffectiveProfile(profile, durationMs) {
     barrierTimeoutMs: boundedInteger(profile.barrierTimeoutMs, 45000, 1000, 120000),
     terminalTimeoutMs: boundedInteger(terminalTimeoutMs, 480000, 1000, 600000),
     terminalQuietMs: boundedInteger(profile.terminalQuietMs, 2000, 250, 10000),
+    navigationMaxAttempts: boundedInteger(profile.navigationMaxAttempts, 4, 1, 10),
+    navigationRetryDelayMs: boundedInteger(profile.navigationRetryDelayMs, 1000, 100, 10000),
     reconnectQuietMs: boundedInteger(profile.reconnectQuietMs, 3000, 250, 30000),
     forbiddenRequestPaths: Array.isArray(profile.forbiddenRequestPaths) ? profile.forbiddenRequestPaths.map(String) : [],
     requiredEventTypes: profile.requiredEventTypes && typeof profile.requiredEventTypes === "object" ? profile.requiredEventTypes : {},
@@ -344,7 +346,7 @@ function realtimeFanoutEffectiveProfile(profile, durationMs) {
 async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
   const subscriberContext = await browser.newContext({ storageState, viewport });
   const subscriberPage = await subscriberContext.newPage();
-  await subscriberPage.goto(new URL("/health/live", baseUrl).toString(), { waitUntil: "domcontentloaded", timeout: profile.barrierTimeoutMs });
+  await realtimeGotoHealth(subscriberPage, profile.barrierTimeoutMs, profile, label);
   const key = label + "-" + sessionId;
   await subscriberPage.evaluate(({ key: streamKey, sessionId: scopedSessionId, profile: inputProfile }) => {
     const root = window.__unideskRealtimeFanoutProducts ??= {};
@@ -399,13 +401,37 @@ async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
   return { label, key, context: subscriberContext, page: subscriberPage };
 }
 
-async function realtimeNewDebugContext(storageState) {
+async function realtimeNewDebugContext(storageState, profile) {
   const debugContext = await browser.newContext({ storageState, viewport });
   const debugPage = await debugContext.newPage();
-  await debugPage.goto(new URL("/health/live", baseUrl).toString(), { waitUntil: "domcontentloaded", timeout: 30000 });
+  await realtimeGotoHealth(debugPage, profile.barrierTimeoutMs, profile, "debug-subscriber");
   return { context: debugContext, page: debugPage };
 }
 
+async function realtimeGotoHealth(targetPage, timeoutMs, profile, label) {
+  const attempts = [];
+  const target = new URL("/health/live", baseUrl).toString();
+  for (let attempt = 1; attempt <= profile.navigationMaxAttempts; attempt += 1) {
+    try {
+      const response = await targetPage.goto(target, { waitUntil: "domcontentloaded", timeout: timeoutMs });
+      const status = response?.status?.() ?? null;
+      if (status !== null && status >= 400) throw new Error(label + " health navigation returned HTTP " + status);
+      attempts.push({ attempt, ok: true, status, valuesRedacted: true });
+      return { ok: true, attempts, valuesRedacted: true };
+    } catch (error) {
+      const message = String(error?.message || error || "unknown").slice(0, 240);
+      attempts.push({ attempt, ok: false, message, valuesRedacted: true });
+      if (attempt >= profile.navigationMaxAttempts || !isRetryableNavigationError(message)) {
+        const wrapped = error instanceof Error ? error : new Error(message);
+        wrapped.details = { ...(wrapped.details || {}), label, attempts, valuesRedacted: true };
+        throw wrapped;
+      }
+      await sleep(profile.navigationRetryDelayMs * attempt);
+    }
+  }
+  throw new Error(label + " health navigation retry exhausted");
+}
+
 async function realtimeRunTurn(input) {
   const turn = input.turn;
   const prompt = input.prompt;