From 07b541e5a59ddcd29f9018c00603bab043345794 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 27 Jun 2026 09:10:48 +0000 Subject: [PATCH] fix: recover observer shell during workbench quick verify --- .../hwlab-node-web-observe-analyzer-source.ts | 172 +++++++++++++++++- .../hwlab-node-web-observe-runner-source.ts | 95 ++++++++-- scripts/src/hwlab-node-web-sentinel-cicd.ts | 10 +- 3 files changed, 255 insertions(+), 22 deletions(-) diff --git a/scripts/src/hwlab-node-web-observe-analyzer-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-source.ts index ae73a72f..98e6c309 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-source.ts @@ -762,6 +762,7 @@ function compactSampleForAnalysis(sample) { observerInitiated: sample.observerInitiated ?? null, url: sample.url ?? null, path: sample.path ?? null, + title: sample.title ?? null, routeSessionId: sample.routeSessionId ?? null, activeSessionId: sample.activeSessionId ?? null, messages: compactDomItems(sample.messages), @@ -770,6 +771,7 @@ function compactSampleForAnalysis(sample) { sessionRail: compactSessionRail(sample.sessionRail), turns: compactDomItems(sample.turns), diagnostics: compactDomItems(sample.diagnostics), + composer: compactComposer(sample.composer), projectManagement: compactProjectManagementSample(sample.projectManagement), pageProvenance: compactSamplePageProvenance(sample.pageProvenance), performance: compactPerformanceItems(sample.performance) @@ -858,6 +860,20 @@ function compactSessionRail(value) { }; } +function compactComposer(value) { + if (!value || typeof value !== "object") return null; + return { + inputPresent: value.inputPresent === true, + inputDisabled: value.inputDisabled === true, + warningPresent: value.warningPresent === true, + submitPresent: value.submitPresent === true, + submitDisabled: value.submitDisabled === true, + submitAction: value.submitAction ?? null, + activeStatus: value.activeStatus ?? null, + valuesRedacted: true + }; +} + function compactLoadingItems(items) { if (!Array.isArray(items)) return []; return items.map((item) => { @@ -954,7 +970,8 @@ function compactPerformanceItems(items) { transferSize: item?.transferSize ?? null, encodedBodySize: item?.encodedBodySize ?? null, decodedBodySize: item?.decodedBodySize ?? null, - nextHopProtocol: item?.nextHopProtocol ?? null + nextHopProtocol: item?.nextHopProtocol ?? null, + responseStatus: Number.isFinite(Number(item?.responseStatus)) ? Number(item.responseStatus) : null })); } @@ -1781,6 +1798,139 @@ function controlledNavigationHydrationCrossPageDiff(row, windows, sampleBySeq) { return isBlankHydrationProjectionSample(control) || isBlankHydrationProjectionSample(observer); } +function crossPageDiffHasWorkbenchAppShellNotReady(row, sampleBySeq) { + const control = sampleBySeq.get(Number(row?.control?.seq)); + const observer = sampleBySeq.get(Number(row?.observer?.seq)); + return workbenchSampleAppShellNotReady(control) || workbenchSampleAppShellNotReady(observer); +} + +function workbenchSampleAppShellNotReady(sample) { + if (!sample || !isWorkbenchPathSample(sample)) return false; + const routeSessionId = stringOrNull(sample.routeSessionId) || workbenchSessionIdFromPath(samplePathname(sample)); + if (!routeSessionId) return false; + const messageCount = Array.isArray(sample.messages) ? sample.messages.length : Number(sample.messageCount ?? 0); + const traceRowCount = Array.isArray(sample.traceRows) ? sample.traceRows.length : Number(sample.traceRowCount ?? 0); + const turnCount = Array.isArray(sample.turns) ? sample.turns.length : Number(sample.turnCount ?? 0); + const loadingCount = Array.isArray(sample.loadings) ? sample.loadings.length : 0; + const diagnosticCount = Array.isArray(sample.diagnostics) ? sample.diagnostics.length : 0; + const railVisibleCount = Number(sample?.sessionRail?.visibleCount ?? 0); + const composer = objectValue(sample.composer); + const composerPresent = composer.inputPresent === true || composer.submitPresent === true; + const provenance = objectValue(sample.pageProvenance); + const hasWorkbenchAssets = Number(provenance.scriptCount ?? 0) > 0 || Number(provenance.stylesheetCount ?? 0) > 0; + return !stringOrNull(sample.activeSessionId) + && Number(messageCount) === 0 + && Number(traceRowCount) === 0 + && Number(turnCount) === 0 + && Number(loadingCount) === 0 + && Number(diagnosticCount) === 0 + && Number(railVisibleCount) === 0 + && !composerPresent + && hasWorkbenchAssets; +} + +function detectWorkbenchAppShellNotReady(samples) { + const rows = (Array.isArray(samples) ? samples : []) + .filter(workbenchSampleAppShellNotReady) + .map((sample) => workbenchAppShellNotReadyRef(sample)); + return annotateWorkbenchAppShellNotReadyTiming(rows); +} + +function annotateWorkbenchAppShellNotReadyTiming(rows) { + const groups = new Map(); + const splitGapMs = Math.max(1000, Number(alertThresholds.crossPageProjectionDivergenceRedMs || alertThresholds.visibleLoadingSlowMs || 10_000)); + for (const row of rows.slice().sort((a, b) => timestampMs(a.ts) - timestampMs(b.ts))) { + const ms = timestampMs(row.ts); + const key = [row.pageRole || "control", row.pageId || "default", row.routeSessionId || row.url || ""].join(":"); + const group = groups.get(key) || []; + let segment = group.at(-1); + if (!segment || !Number.isFinite(ms) || !Number.isFinite(segment.lastMs) || ms - segment.lastMs > splitGapMs) { + segment = { rows: [], firstMs: Number.isFinite(ms) ? ms : null, lastMs: Number.isFinite(ms) ? ms : null }; + group.push(segment); + } + segment.rows.push(row); + if (Number.isFinite(ms)) { + if (segment.firstMs === null || ms < segment.firstMs) segment.firstMs = ms; + if (segment.lastMs === null || ms > segment.lastMs) segment.lastMs = ms; + } + groups.set(key, group); + } + const result = []; + for (const group of groups.values()) { + for (let segmentIndex = 0; segmentIndex < group.length; segmentIndex += 1) { + const segment = group[segmentIndex]; + const observedSpanMs = segment.firstMs === null || segment.lastMs === null ? null : segment.lastMs - segment.firstMs; + for (const row of segment.rows) { + result.push({ + ...row, + segmentIndex, + observedFirstAt: segment.firstMs === null ? null : new Date(segment.firstMs).toISOString(), + observedLastAt: segment.lastMs === null ? null : new Date(segment.lastMs).toISOString(), + observedSpanMs, + }); + } + } + } + return result; +} + +function workbenchAppShellNotReadyRef(sample) { + const provenance = objectValue(sample?.pageProvenance); + const performance = workbenchAssetPerformanceSummary(sample); + return { + ...ref(sample), + title: stringOrNull(sample?.title), + messageCount: Array.isArray(sample?.messages) ? sample.messages.length : 0, + turnCount: Array.isArray(sample?.turns) ? sample.turns.length : 0, + traceRowCount: Array.isArray(sample?.traceRows) ? sample.traceRows.length : 0, + sessionRailVisibleCount: Number(sample?.sessionRail?.visibleCount ?? 0), + composerInputPresent: sample?.composer?.inputPresent === true, + composerSubmitPresent: sample?.composer?.submitPresent === true, + pageProvenance: { + documentReadyState: stringOrNull(provenance.documentReadyState), + pageLoadSeq: numberOrNull(provenance.pageLoadSeq), + timeOrigin: numberOrNull(provenance.timeOrigin), + assetFingerprint: stringOrNull(provenance.assetFingerprint), + scriptCount: numberOrNull(provenance.scriptCount), + stylesheetCount: numberOrNull(provenance.stylesheetCount), + scripts: arrayStrings(provenance.scripts).slice(0, 8), + stylesheets: arrayStrings(provenance.stylesheets).slice(0, 8), + valuesRedacted: true + }, + assetPerformance: performance, + valuesRedacted: true + }; +} + +function workbenchAssetPerformanceSummary(sample) { + const assets = (Array.isArray(sample?.performance) ? sample.performance : []) + .filter((entry) => /^(script|link|css)$/iu.test(String(entry?.initiatorType || "")) || /\.(?:js|css)$/iu.test(String(entry?.name || ""))) + .map((entry) => ({ + path: urlPath(entry?.name), + initiatorType: entry?.initiatorType ?? null, + duration: numberOrNull(entry?.duration), + responseStatus: numberOrNull(entry?.responseStatus), + transferSize: numberOrNull(entry?.transferSize), + encodedBodySize: numberOrNull(entry?.encodedBodySize), + decodedBodySize: numberOrNull(entry?.decodedBodySize), + nextHopProtocol: entry?.nextHopProtocol ?? null, + valuesRedacted: true + })); + const responseStatusCounts = {}; + for (const item of assets) { + const key = item.responseStatus === null ? "unknown" : String(item.responseStatus); + responseStatusCounts[key] = (responseStatusCounts[key] || 0) + 1; + } + return { + assetCount: assets.length, + zeroStatusCount: assets.filter((item) => item.responseStatus === 0).length, + missingStatusCount: assets.filter((item) => item.responseStatus === null).length, + responseStatusCounts, + assets: assets.slice(0, 12), + valuesRedacted: true + }; +} + function objectValue(value) { return value && typeof value === "object" && !Array.isArray(value) ? value : {}; } @@ -2735,14 +2885,32 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN const crossPageTraceVisibilityDiffs = crossPageDiffs.filter((item) => item.diffKind === "trace-visibility"); const crossPageProjectionBudgetMs = alertThresholds.crossPageProjectionDivergenceRedMs; const sampleBySeq = new Map(samples.map((item) => [Number(item?.seq), item]).filter(([seq]) => Number.isFinite(seq))); + const appShellNotReadyRows = detectWorkbenchAppShellNotReady(samples); + const persistentAppShellNotReadyRows = appShellNotReadyRows.filter((item) => Number(item.observedSpanMs ?? 0) > crossPageProjectionBudgetMs); + const transientAppShellNotReadyRows = appShellNotReadyRows.filter((item) => Number(item.observedSpanMs ?? 0) <= crossPageProjectionBudgetMs); + if (persistentAppShellNotReadyRows.length > 0) findings.push({ + id: "workbench-app-shell-not-ready", + severity: "red", + summary: "Workbench route document loaded but the app shell never mounted or hydrated within the configured budget; treat this as page/runtime startup evidence, not a Workbench projection divergence", + count: persistentAppShellNotReadyRows.length, + budgetMs: crossPageProjectionBudgetMs, + samples: persistentAppShellNotReadyRows.slice(0, 20), + rootCause: "workbench_page_app_shell_not_ready", + rootCauseStatus: "confirmed-from-dom-and-asset-provenance", + rootCauseConfidence: "high", + valuesRedacted: true + }); + if (transientAppShellNotReadyRows.length > 0) findings.push({ id: "workbench-app-shell-transient-not-ready", severity: "info", summary: "Workbench route briefly had a document and assets but no mounted app shell; retained as startup context", count: transientAppShellNotReadyRows.length, budgetMs: crossPageProjectionBudgetMs, samples: transientAppShellNotReadyRows.slice(0, 20) }); const timedCrossPageProjectionDiffs = annotateCrossPageDiffTiming(crossPageProjectionDiffs); const controlledNavigationHydrationProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq)); - const evaluatedCrossPageProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => !controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq)); + const appShellNotReadyProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => !controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq) && crossPageDiffHasWorkbenchAppShellNotReady(item, sampleBySeq)); + const evaluatedCrossPageProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => !controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq) && !crossPageDiffHasWorkbenchAppShellNotReady(item, sampleBySeq)); const persistentCrossPageProjectionDiffs = evaluatedCrossPageProjectionDiffs.filter((item) => Number(item.observedSpanMs ?? 0) > crossPageProjectionBudgetMs); const transientCrossPageProjectionDiffs = evaluatedCrossPageProjectionDiffs.filter((item) => Number(item.observedSpanMs ?? 0) <= crossPageProjectionBudgetMs); if (persistentCrossPageProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-divergence", severity: "red", summary: "control and observer pages saw different projection state for the same sampled session beyond the configured budget", count: persistentCrossPageProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: persistentCrossPageProjectionDiffs.slice(0, 20) }); if (transientCrossPageProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-transient-divergence", severity: "info", summary: "control and observer pages briefly differed near a sampled transition; retained as transient evidence but not treated as persistent projection failure", count: transientCrossPageProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: transientCrossPageProjectionDiffs.slice(0, 20) }); if (controlledNavigationHydrationProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-controlled-navigation-hydration", severity: "info", summary: "control and observer pages differed while a non-blocking session-invariance navigation command still had an unhydrated blank page; retained as context but not treated as a red projection blocker", count: controlledNavigationHydrationProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: controlledNavigationHydrationProjectionDiffs.slice(0, 20) }); + if (appShellNotReadyProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-app-shell-not-ready", severity: "info", summary: "cross-page projection differences were explained by a page whose Workbench app shell was not mounted; see workbench-app-shell-not-ready for the blocking root cause", count: appShellNotReadyProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: appShellNotReadyProjectionDiffs.slice(0, 20) }); if (crossPageTraceVisibilityDiffs.length > 0) findings.push({ id: "cross-page-trace-visibility-divergence", severity: "info", summary: "control and observer pages differed only in visible trace row count; this is local disclosure/hydration visibility, not session/message projection divergence", count: crossPageTraceVisibilityDiffs.length, samples: crossPageTraceVisibilityDiffs.slice(0, 20) }); const traceMessageDuplicates = detectTraceMessageDuplication(samples); if (traceMessageDuplicates.length > 0) findings.push({ id: "trace-assistant-message-duplicates-final-response", severity: "amber", summary: "trace-frame rendered duplicate visible assistant final rows; the fixed Final Response renderer summary block is excluded", count: traceMessageDuplicates.length, finalResponseSummaryBlockCounted: false, traceFrameSource: "traceRows-only", samples: traceMessageDuplicates.slice(0, 20) }); diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 01da4208..8a421ba2 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -423,26 +423,89 @@ async function syncObserverPageToControlSession(reason, explicitSessionId = null const targetUrl = new URL(target, baseUrl).toString(); const beforeUrl = pageUrl(observerPage); const beforeSessionId = routeSessionIdFromUrl(beforeUrl); - if (sessionId && beforeSessionId === sessionId && !forceRefresh) return { ok: true, reason, changed: false, observerRoundTrip: false, sessionId, beforeUrl, afterUrl: beforeUrl, pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch }; - observerPageEpoch += 1; - let status = null; - let statusText = null; - const response = await observerPage.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: 45000 }).catch((error) => ({ observerGotoError: errorSummary(error) })); - if (response?.observerGotoError) return { ok: false, reason, changed: false, sessionId: sessionId ?? null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, error: response.observerGotoError, valuesRedacted: true }; - status = typeof response?.status === "function" ? response.status() : null; - statusText = typeof response?.statusText === "function" ? response.statusText() : null; - const readiness = await waitForTargetPageReady(observerPage, targetUrl, { timeoutMs: 15000 }); - if (!readiness.ok) { - lastObserverRefreshAtMs = Date.now(); - return { ok: false, reason, changed: true, observerRoundTrip: forceRefresh, sessionId: sessionId ?? null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, httpStatus: status, statusText, readiness, hydration: null, failureKind: readiness.reason || "observer-target-not-ready", valuesRedacted: true }; + const attempts = []; + if (sessionId && beforeSessionId === sessionId && !forceRefresh) { + const current = await observerSessionReadiness(targetUrl, sessionId, { readinessTimeoutMs: 1000, hydrationTimeoutMs: 1000 }); + if (current.ok) return { ok: true, reason, changed: false, observerRoundTrip: false, sessionId, beforeUrl, afterUrl: beforeUrl, pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, readiness: current.readiness, hydration: current.hydration, valuesRedacted: true }; + attempts.push({ attempt: 0, ok: false, shortCircuitRejected: true, failureKind: current.failureKind, readiness: current.readiness, hydration: current.hydration, beforeUrl, afterUrl: pageUrl(observerPage), valuesRedacted: true }); } - if (!isWorkbenchPathname(safeUrlPath(targetUrl) || "")) { + const maxAttempts = Number.isFinite(Number(options?.maxAttempts)) ? Math.max(1, Number(options.maxAttempts)) : 2; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const attemptBeforeUrl = pageUrl(observerPage); + observerPageEpoch += 1; + let status = null; + let statusText = null; + const response = await observerPage.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: 45000 }).catch((error) => ({ observerGotoError: errorSummary(error) })); + if (response?.observerGotoError) { + attempts.push({ attempt, ok: false, failureKind: navigationFailureKind(response.observerGotoError?.message || response.observerGotoError?.name || "observer-navigation-error"), beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), error: response.observerGotoError, valuesRedacted: true }); + if (attempt < maxAttempts && isRetryableNavigationError(response.observerGotoError?.message || response.observerGotoError?.name || "")) { + await recreateObserverPageForNavigation("observer-goto-error", attempt).catch((resetError) => appendJsonl(files.errors, eventRecord("observer-page-reset-error", { commandId: activeCommandId, attempt, error: errorSummary(resetError) }))); + continue; + } + lastObserverRefreshAtMs = Date.now(); + return { ok: false, reason, changed: false, observerRoundTrip: forceRefresh, sessionId: sessionId ?? null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, error: response.observerGotoError, attempts, valuesRedacted: true }; + } + status = typeof response?.status === "function" ? response.status() : null; + statusText = typeof response?.statusText === "function" ? response.statusText() : null; + const readiness = await waitForTargetPageReady(observerPage, targetUrl, { timeoutMs: 15000 }); + if (!readiness.ok) { + attempts.push({ attempt, ok: false, failureKind: readiness.reason || "observer-target-not-ready", beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), httpStatus: status, statusText, readiness, valuesRedacted: true }); + if (attempt < maxAttempts && observerReadinessRetryable(readiness)) { + await recreateObserverPageForNavigation(readiness.reason || "observer-target-not-ready", attempt).catch((resetError) => appendJsonl(files.errors, eventRecord("observer-page-reset-error", { commandId: activeCommandId, attempt, error: errorSummary(resetError) }))); + continue; + } + lastObserverRefreshAtMs = Date.now(); + return { ok: false, reason, changed: true, observerRoundTrip: forceRefresh, sessionId: sessionId ?? null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, httpStatus: status, statusText, readiness, hydration: null, attempts, failureKind: readiness.reason || "observer-target-not-ready", valuesRedacted: true }; + } + if (!isWorkbenchPathname(safeUrlPath(targetUrl) || "")) { + lastObserverRefreshAtMs = Date.now(); + attempts.push({ attempt, ok: true, beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), httpStatus: status, statusText, readiness, valuesRedacted: true }); + return { ok: true, reason, changed: true, observerRoundTrip: forceRefresh, sessionId: null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, httpStatus: status, statusText, readiness, hydration: null, attempts, valuesRedacted: true }; + } + const hydration = await waitForWorkbenchSessionHydrated(observerPage, sessionId, { timeoutMs: 15000 }); + attempts.push({ attempt, ok: hydration.ok === true, failureKind: hydration.ok === true ? null : hydration.reason || "observer-session-hydration-failed", beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), httpStatus: status, statusText, readiness, hydration, valuesRedacted: true }); + if (hydration.ok === true) { + lastObserverRefreshAtMs = Date.now(); + return { ok: true, reason, changed: true, observerRoundTrip: forceRefresh, sessionId: sessionId ?? null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, httpStatus: status, statusText, readiness, hydration, attempts, failureKind: null, valuesRedacted: true }; + } + if (attempt < maxAttempts && observerHydrationRetryable(hydration)) { + await recreateObserverPageForNavigation(hydration.reason || "observer-session-hydration-failed", attempt).catch((resetError) => appendJsonl(files.errors, eventRecord("observer-page-reset-error", { commandId: activeCommandId, attempt, error: errorSummary(resetError) }))); + continue; + } lastObserverRefreshAtMs = Date.now(); - return { ok: true, reason, changed: true, observerRoundTrip: forceRefresh, sessionId: null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, httpStatus: status, statusText, readiness, hydration: null, valuesRedacted: true }; + return { ok: false, reason, changed: true, observerRoundTrip: forceRefresh, sessionId: sessionId ?? null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, httpStatus: status, statusText, readiness, hydration, attempts, failureKind: hydration.reason || "observer-session-hydration-failed", valuesRedacted: true }; } - const hydration = await waitForWorkbenchSessionHydrated(observerPage, sessionId, { timeoutMs: 15000 }); lastObserverRefreshAtMs = Date.now(); - return { ok: hydration.ok === true, reason, changed: true, observerRoundTrip: forceRefresh, sessionId: sessionId ?? null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, httpStatus: status, statusText, readiness, hydration, failureKind: hydration.ok === true ? null : hydration.reason || "observer-session-hydration-failed", valuesRedacted: true }; + return { ok: false, reason, changed: true, observerRoundTrip: forceRefresh, sessionId: sessionId ?? null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, attempts, failureKind: "observer-sync-retry-exhausted", valuesRedacted: true }; +} + +async function observerSessionReadiness(targetUrl, sessionId, options = {}) { + const readiness = await waitForTargetPageReady(observerPage, targetUrl, { timeoutMs: options.readinessTimeoutMs ?? 1000 }); + if (!readiness.ok) return { ok: false, failureKind: readiness.reason || "observer-target-not-ready", readiness, hydration: null, valuesRedacted: true }; + if (!isWorkbenchPathname(safeUrlPath(targetUrl) || "")) return { ok: true, readiness, hydration: null, valuesRedacted: true }; + const hydration = await waitForWorkbenchSessionHydrated(observerPage, sessionId, { timeoutMs: options.hydrationTimeoutMs ?? 1000 }); + return { ok: hydration.ok === true, failureKind: hydration.ok === true ? null : hydration.reason || "observer-session-hydration-failed", readiness, hydration, valuesRedacted: true }; +} + +function observerReadinessRetryable(readiness) { + const reason = String(readiness?.reason || ""); + const snapshot = readiness?.snapshot || {}; + return /workbench-app-not-ready|observer-target-not-ready/iu.test(reason) + || snapshot.workbenchShellVisible === false + || snapshot.sessionRailPresent === false && snapshot.commandInputPresent === false; +} + +function observerHydrationRetryable(hydration) { + const reason = String(hydration?.reason || ""); + return /observer-session-hydration-timeout|observer-session-hydration-failed|active-session-not-hydrated|route-session-not-hydrated/iu.test(reason); +} + +async function recreateObserverPageForNavigation(reason, attempt) { + const before = pageUrl(observerPage); + if (observerPage && !observerPage.isClosed()) await observerPage.close().catch(() => {}); + observerPage = await context.newPage(); + attachPassiveListeners(observerPage, "observer", observerPageId); + await appendJsonl(files.control, eventRecord("observer-page-recreated", { reason, attempt, beforeUrl: before, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, valuesRedacted: true })); } async function waitForWorkbenchSessionHydrated(targetPage, sessionId, options = {}) { diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 38656d58..2ebedd18 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -3778,14 +3778,16 @@ function quickVerifyPromptWaitScript(stateDir: string, promptIndex: number, time " const promptTraceId = commandTraceId(prompt);", " if (!done || !promptTraceId) return { ok: true, round: promptIndex, status: 'command-pending', commandId: prompt.commandId || null, traceId: promptTraceId || null, finalResponseEmpty: true, commandPhase: prompt.phase || null, traceMissing: !promptTraceId, valuesRedacted: true };", " const segment = segmentFor(samples, prompts, promptIndex - 1);", - " const traceId = promptTraceId || chooseTraceId(segment, prompt);", - " const status = statusFor(segment, traceId);", - " const sampleForTrace = traceId ? segment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || null : null;", + " const controlSegment = segment.filter((sample) => sample.pageRole === 'control');", + " const statusSegment = controlSegment.length > 0 ? controlSegment : segment;", + " const traceId = promptTraceId || chooseTraceId(statusSegment, prompt) || chooseTraceId(segment, prompt);", + " const status = statusFor(statusSegment, traceId);", + " const sampleForTrace = traceId ? statusSegment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || segment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || null : null;", " const lastSample = sampleForTrace || segment.slice(-1)[0] || null;", " const composerSample = segment.filter((sample) => sample.pageRole === 'control').slice(-1)[0] || lastSample;", " const composer = composerSample?.composer || {};", " const composerReadyForTurn = composer.inputPresent === true && composer.inputDisabled !== true && composer.submitPresent === true && composer.warningPresent !== true && composer.submitAction === 'turn';", - " return { ok: true, round: promptIndex, status, traceId, finalResponseEmpty: finalResponseEmpty(segment, traceId), lastSeq: numOrNull(lastSample?.seq), lastTs: lastSample?.ts || null, composerReadyForTurn, composerAction: composer.submitAction || null, source: 'observe-artifact-wait-script', valuesRedacted: true };", + " return { ok: true, round: promptIndex, status, traceId, finalResponseEmpty: finalResponseEmpty(statusSegment, traceId), lastSeq: numOrNull(lastSample?.seq), lastTs: lastSample?.ts || null, composerReadyForTurn, composerAction: composer.submitAction || null, sampleScope: controlSegment.length > 0 ? 'control' : 'all', segmentSampleCount: segment.length, statusSampleCount: statusSegment.length, source: 'observe-artifact-wait-script', valuesRedacted: true };", "}", "function norm(value) { return String(value || '').trim().toLowerCase().replace(/_/gu, '-'); }", "function successful(value) { return ['completed', 'succeeded', 'success'].includes(norm(value)); }",