Codex
566 lines
48 KiB
TypeScript
566 lines
48 KiB
TypeScript
// SPEC: PJ2026-01040111 long-running Workbench observation.
|
||
// Responsibility: Analyzer top-level findings, cross-page projection drilldown, and frontend freeze source fragment.
|
||
|
||
export function nodeWebObserveAnalyzerFindingsSource(): string {
|
||
return String.raw`function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, requestRate, pageProvenance, commandFailures = [], manifest = {}, apiDomLag = null, browserProcess = null, frontendPerformance = null) {
|
||
const findings = [];
|
||
const effectiveApiDomLag = apiDomLag || buildApiDomLagReport(samples, network);
|
||
if (commandFailures.length > 0) findings.push({ id: "observer-command-failed", severity: "red", summary: "observer control commands failed; analyze must surface command failure instead of hiding it in command artifacts", count: commandFailures.length, commands: commandFailures.slice(0, 20) });
|
||
findings.push(...buildFrontendFreezeFindings(errors, control));
|
||
findings.push(...buildBrowserProcessFindings(browserProcess, runtimeAlerts));
|
||
findings.push(...buildFrontendPerformanceFindings(frontendPerformance));
|
||
findings.push(...buildControlledNavigationRootCauseFindings(control, manifest));
|
||
findings.push(...buildSessionInvariantFindings(control, manifest));
|
||
const commandTimes = control
|
||
.filter((item) => item.phase === "completed" || item.phase === "started" || item.type === "observer-periodic-refresh")
|
||
.map((item) => Date.parse(item.ts))
|
||
.filter(Number.isFinite);
|
||
const controlledNavigationWindows = sessionInvariantNavigationWindows(control);
|
||
const routeSessions = new Set(samples.map((item) => item.routeSessionId).filter(Boolean));
|
||
const activeSessions = new Set(samples.map((item) => item.activeSessionId).filter(Boolean));
|
||
const routeSessionUnexpected = sessionChangeSamplesOutsideControlledNavigation(samples, "routeSessionId", controlledNavigationWindows);
|
||
const activeSessionUnexpected = sessionChangeSamplesOutsideControlledNavigation(samples, "activeSessionId", controlledNavigationWindows);
|
||
if (routeSessions.size > 1 && routeSessionUnexpected.length > 0) findings.push({ id: "session-route-changed", severity: "amber", summary: "route session changed outside controlled session-invariance navigation windows", routeSessionCount: routeSessions.size, samples: sampleRefs(routeSessionUnexpected, (item) => item.routeSessionId) });
|
||
if (activeSessions.size > 1 && activeSessionUnexpected.length > 0) findings.push({ id: "active-session-changed", severity: "amber", summary: "active session changed outside controlled session-invariance navigation windows", activeSessionCount: activeSessions.size, samples: sampleRefs(activeSessionUnexpected, (item) => item.activeSessionId) });
|
||
const mismatches = samples.filter((item) => item.routeSessionId && item.activeSessionId && item.routeSessionId !== item.activeSessionId && !sampleInControlledNavigationWindow(item, controlledNavigationWindows));
|
||
if (mismatches.length > 0) findings.push({ id: "route-active-session-mismatch", severity: "red", summary: "routeSessionId and activeSessionId diverged", count: mismatches.length, samples: mismatches.slice(0, 10).map(ref) });
|
||
const uncommandedChanges = [];
|
||
const commandedPromptSeqs = new Set((sampleMetrics?.timeline ?? []).filter((item) => Number(item?.promptIndex) > 0).map((item) => item.seq).filter((seq) => seq !== null && seq !== undefined));
|
||
const previousDigestByPage = new Map();
|
||
for (const sample of samples) {
|
||
const pageKey = samplePageKey(sample);
|
||
const previous = previousDigestByPage.get(pageKey);
|
||
const next = digestSample(sample);
|
||
if (previous && previous.digest !== next && !commandedPromptSeqs.has(sample?.seq) && !nearCommand(sample, commandTimes, alertThresholds.uncommandedStateChangeCommandWindowMs)) {
|
||
uncommandedChanges.push({
|
||
...ref(sample),
|
||
previousSeq: previous.sample?.seq ?? null,
|
||
previousTs: previous.sample?.timestamp ?? previous.sample?.ts ?? null,
|
||
fromDigest: previous.digest,
|
||
toDigest: next,
|
||
fromMessageCount: previous.sample?.messageCount ?? previous.sample?.messages?.length ?? null,
|
||
toMessageCount: sample?.messageCount ?? sample?.messages?.length ?? null,
|
||
fromTraceRowCount: Array.isArray(previous.sample?.traceRows) ? previous.sample.traceRows.length : previous.sample?.traceRowCount ?? null,
|
||
toTraceRowCount: Array.isArray(sample?.traceRows) ? sample.traceRows.length : sample?.traceRowCount ?? null,
|
||
fromPath: previous.sample?.path ?? previous.sample?.urlPath ?? null,
|
||
toPath: sample?.path ?? sample?.urlPath ?? null,
|
||
detail: "visible digest changed without nearby command",
|
||
});
|
||
}
|
||
previousDigestByPage.set(pageKey, { digest: next, sample });
|
||
}
|
||
if (uncommandedChanges.length > 0) findings.push({ id: "uncommanded-visible-state-change", severity: "amber", summary: "visible message/trace digest changed without a nearby command", count: uncommandedChanges.length, samples: uncommandedChanges.slice(0, 20) });
|
||
const finalFlicker = detectFinalFlicker(samples);
|
||
if (finalFlicker.length > 0) findings.push({ id: "final-response-flicker", severity: "red", summary: "message text digest disappeared or switched to diagnostic-like text after non-empty final text", count: finalFlicker.length, samples: finalFlicker.slice(0, 20) });
|
||
const terminalZeroElapsed = detectTerminalZeroElapsed(samples);
|
||
if (terminalZeroElapsed.length > 0) findings.push({ id: "turn-terminal-zero-elapsed", severity: "amber", summary: "terminal Code Agent card displayed 耗时 0 秒; terminal duration issue is a non-blocking timing alert", count: terminalZeroElapsed.length, samples: terminalZeroElapsed.slice(0, 20) });
|
||
const cardTiming = sampleMetrics?.codeAgentCardTiming || {};
|
||
const cardTimingSummary = cardTiming.summary || {};
|
||
if (Number(cardTimingSummary.missingElapsedCount ?? 0) > 0) findings.push({ id: "code-agent-card-elapsed-missing", severity: "amber", summary: "visible Code Agent card did not display total elapsed time; elapsed visibility is a non-blocking timing alert", count: cardTimingSummary.missingElapsedCount, samples: (cardTiming.missingElapsed || []).slice(0, 20) });
|
||
if (Number(cardTimingSummary.missingRecentUpdateCount ?? 0) > 0) findings.push({ id: "code-agent-card-running-recent-update-missing", severity: "amber", summary: "non-terminal Code Agent card did not display 最近更新; recent-update visibility is a non-blocking timing alert", count: cardTimingSummary.missingRecentUpdateCount, samples: (cardTiming.missingRecentUpdate || []).slice(0, 20) });
|
||
const roundCompletion = cardTiming.roundCompletion || {};
|
||
if (Number(cardTimingSummary.durationUnderreportedCount ?? 0) > 0) {
|
||
findings.push({
|
||
id: "code-agent-card-duration-underreported",
|
||
severity: "amber",
|
||
summary: "completed Code Agent card total elapsed is shorter than trace/final-response duration evidence; timing mismatch is a non-blocking alert",
|
||
timingSourceOfTruth: "trace-completion-total-or-final-response-duration",
|
||
timingStatus: timingStatusFromRows(cardTiming.durationUnderreported, "business-turn-completed"),
|
||
timingAlert: true,
|
||
count: cardTimingSummary.durationUnderreportedCount,
|
||
samples: (cardTiming.durationUnderreported || []).slice(0, 20),
|
||
});
|
||
}
|
||
if (Number(cardTimingSummary.durationMismatchCount ?? 0) > 0) {
|
||
findings.push({
|
||
id: "code-agent-card-duration-mismatch",
|
||
severity: "amber",
|
||
summary: "completed Code Agent card total elapsed does not match sealed completion/final-response timing evidence; timing mismatch is a non-blocking alert",
|
||
timingSourceOfTruth: "trace-completion-total-or-final-response-duration",
|
||
timingStatus: timingStatusFromRows(cardTiming.durationMismatches, "business-turn-completed"),
|
||
timingAlert: true,
|
||
count: cardTimingSummary.durationMismatchCount,
|
||
samples: (cardTiming.durationMismatches || []).slice(0, 20),
|
||
});
|
||
}
|
||
const traceOrder = sampleMetrics?.traceOrder || {};
|
||
const traceOrderSummary = traceOrder.summary || {};
|
||
if (Number(traceOrderSummary.orderAnomalyCount ?? 0) > 0) {
|
||
findings.push({
|
||
id: "trace-row-order-nonmonotonic",
|
||
severity: "red",
|
||
summary: "visible trace rows are not monotonic by total time, clock time, or projected sequence in DOM order",
|
||
count: traceOrderSummary.orderAnomalyCount,
|
||
samples: (traceOrder.orderAnomalies || []).slice(0, 20),
|
||
});
|
||
}
|
||
if (Number(traceOrderSummary.completionNotLastCount ?? 0) > 0) {
|
||
findings.push({
|
||
id: "trace-completion-row-not-last",
|
||
severity: "red",
|
||
summary: "visible trace shows a completion row before later trace rows for the same trace",
|
||
count: traceOrderSummary.completionNotLastCount,
|
||
samples: (traceOrder.completionNotLast || []).slice(0, 20),
|
||
});
|
||
}
|
||
if (Number(cardTimingSummary.roundCompletionElapsedMismatchCount ?? 0) > 0) findings.push({ id: "round-completion-elapsed-mismatch", severity: "amber", summary: "Trace row 轮次完成（总耗时 ...） does not match the visible Code Agent card total elapsed time within YAML timing slack; timing mismatch is a non-blocking alert", timingSourceOfTruth: "trace-round-completion-total", timingStatus: timingStatusFromRows(roundCompletion.elapsedMismatches, "business-turn-completed"), timingAlert: true, count: cardTimingSummary.roundCompletionElapsedMismatchCount, toleranceSeconds: cardTimingSummary.elapsedMismatchToleranceSeconds, samples: (roundCompletion.elapsedMismatches || []).slice(0, 20) });
|
||
if (Number(cardTimingSummary.roundCompletionFinalResponseMissingCount ?? 0) > 0) findings.push({ id: "round-completion-final-response-missing", severity: "red", summary: "Trace row showed 轮次完成, but no final response was visible in the Code Agent card afterward", count: cardTimingSummary.roundCompletionFinalResponseMissingCount, samples: (roundCompletion.finalResponseMissing || []).slice(0, 20) });
|
||
if (Number(cardTimingSummary.roundCompletionPostTimingChangeCount ?? 0) > 0) findings.push({ id: "round-completion-post-timing-change", severity: "amber", summary: "After 轮次完成, card total elapsed or 最近更新 continued changing; terminal timing alert is non-blocking", count: cardTimingSummary.roundCompletionPostTimingChangeCount, samples: (roundCompletion.postCompletionTimingChanges || []).slice(0, 20) });
|
||
if (Number(cardTimingSummary.roundCompletionPostRecentUpdateVisibleCount ?? 0) > 0) findings.push({ id: "round-completion-recent-update-still-visible", severity: "info", summary: "最近更新 was still visible after 轮次完成; inspect whether terminal cards should hide activity age or keep it sealed", count: cardTimingSummary.roundCompletionPostRecentUpdateVisibleCount, samples: (roundCompletion.postCompletionRecentUpdateVisible || []).slice(0, 20) });
|
||
const scrollJumps = [];
|
||
for (let i = 1; i < samples.length; i += 1) {
|
||
const prevY = Number(samples[i - 1]?.scroll?.y ?? 0);
|
||
const nextY = Number(samples[i]?.scroll?.y ?? 0);
|
||
if (prevY > alertThresholds.scrollJumpFromY && nextY < alertThresholds.scrollJumpToY && !nearCommand(samples[i], commandTimes, alertThresholds.scrollJumpCommandWindowMs)) scrollJumps.push({ from: ref(samples[i - 1]), to: ref(samples[i]) });
|
||
}
|
||
if (scrollJumps.length > 0) findings.push({ id: "scroll-jump-top", severity: "amber", summary: "scroll position jumped near top without nearby command", count: scrollJumps.length, samples: scrollJumps.slice(0, 10) });
|
||
const traceTerminal = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.some((row) => isTerminalTraceText((row.status || "") + " " + (row.textPreview || ""))));
|
||
const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0);
|
||
if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) });
|
||
const workbenchInPlaceProjectionLag = detectWorkbenchInPlaceProjectionLag(samples, network, control);
|
||
if (workbenchInPlaceProjectionLag.terminalTraceMissing.length > 0) findings.push({
|
||
id: "workbench-terminal-trace-not-hydrated-in-place",
|
||
severity: "red",
|
||
summary: "Workbench rendered a terminal turn/message in-place while the same trace still had no visible run-record trace rows",
|
||
count: workbenchInPlaceProjectionLag.terminalTraceMissing.length,
|
||
samples: workbenchInPlaceProjectionLag.terminalTraceMissing.slice(0, 20),
|
||
rootCause: "workbench_trace_projection_not_hydrated_in_place",
|
||
rootCauseStatus: "confirmed-from-dom-samples",
|
||
rootCauseConfidence: "high",
|
||
valuesRedacted: true
|
||
});
|
||
if (workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.length > 0) findings.push({
|
||
id: "workbench-terminal-api-dom-not-refreshed-in-place",
|
||
severity: "red",
|
||
summary: "Workbench REST returned terminal/final trace evidence but the same in-place page did not render terminal message plus trace rows within the configured budget",
|
||
count: workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.length,
|
||
budgetMs: workbenchInPlaceProjectionLag.terminalApiDomLag.summary.budgetMs,
|
||
windowMs: workbenchInPlaceProjectionLag.terminalApiDomLag.summary.windowMs,
|
||
samples: workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.slice(0, 20),
|
||
rootCause: "workbench_rest_terminal_projection_dom_lag",
|
||
rootCauseStatus: "confirmed-from-network-body-summary-and-dom-samples",
|
||
rootCauseConfidence: "high",
|
||
valuesRedacted: true
|
||
});
|
||
const turnStateTriad = sampleMetrics?.workbenchTurnStateTriad || {};
|
||
const turnStateTriadSummary = turnStateTriad.summary || {};
|
||
const turnStateTriadRows = [
|
||
...(Array.isArray(turnStateTriad.invalidFullTriads) ? turnStateTriad.invalidFullTriads : []),
|
||
...(Array.isArray(turnStateTriad.cardFinalResponseMismatches) ? turnStateTriad.cardFinalResponseMismatches : [])
|
||
];
|
||
if (Number(turnStateTriadSummary.invalidRowCount ?? 0) > 0) {
|
||
const drilldown = turnStateTriad.drilldown ?? buildWorkbenchTurnStateTriadDrilldown(turnStateTriadRows);
|
||
const rootCause = workbenchTriadRootCauseFromDrilldown(drilldown, turnStateTriadSummary);
|
||
findings.push({
|
||
id: "workbench-turn-state-triad-inconsistent",
|
||
severity: "red",
|
||
summary: rootCause.summary,
|
||
count: turnStateTriadSummary.invalidRowCount,
|
||
fullTriadCount: turnStateTriadSummary.fullTriadRowCount,
|
||
invalidFullTriadCount: turnStateTriadSummary.invalidFullTriadCount,
|
||
cardFinalResponseMismatchCount: turnStateTriadSummary.cardFinalResponseMismatchCount,
|
||
legacyCollectorMissingCount: turnStateTriadSummary.collectorMissingRowCount,
|
||
collectorMissingFields: Array.isArray(turnStateTriadSummary.collectorMissingFields) ? turnStateTriadSummary.collectorMissingFields : [],
|
||
dominantMismatchKind: rootCause.dominantMismatchKind,
|
||
allowedTuples: [
|
||
{ railStatus: "completed", cardStatus: "completed", finalResponsePresent: true },
|
||
{ railStatus: "running", cardStatus: "running", finalResponsePresent: false }
|
||
],
|
||
samples: turnStateTriadRows.slice(0, 20),
|
||
drilldown,
|
||
collectorMissingSamples: Array.isArray(turnStateTriad.collectorMissingRows) ? turnStateTriad.collectorMissingRows.slice(0, 10) : [],
|
||
sourceOfTruth: rootCause.sourceOfTruth + "; do not repair via DOM fallback or GET-side state mutation",
|
||
nextAction: rootCause.nextAction,
|
||
rootCause: rootCause.rootCause,
|
||
rootCauseStatus: rootCause.rootCauseStatus,
|
||
rootCauseConfidence: rootCause.rootCauseConfidence,
|
||
valuesRedacted: true
|
||
});
|
||
}
|
||
const promptFailures = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.chatPostOk === false && !promptCommandHasAuthoritativeSubmitSideEffect(control, item)) : [];
|
||
if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat or /v1/agent/chat/steer POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) });
|
||
const promptSteerRounds = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.steerUsed === true) : [];
|
||
if (promptSteerRounds.length > 0) findings.push({ id: "prompt-routed-to-steer", severity: "amber", summary: "sendPrompt was submitted through /v1/agent/chat/steer; verify the previous turn was truly in-flight and not an unsealed terminal failure", count: promptSteerRounds.length, rounds: promptSteerRounds.slice(0, 10) });
|
||
const elapsedZeroResets = Array.isArray(sampleMetrics?.turnTimingElapsedZeroResets) ? sampleMetrics.turnTimingElapsedZeroResets : [];
|
||
if (elapsedZeroResets.length > 0) findings.push({ id: "turn-timing-total-elapsed-zero-reset", severity: "amber", summary: "Code Agent total elapsed jumped from a non-zero value back to 0 seconds; timing reset is a non-blocking alert", timingSourceOfTruth: "dom-card-total-elapsed-sequence", timingStatus: timingStatusFromRows(elapsedZeroResets), timingAlert: true, count: elapsedZeroResets.length, samples: elapsedZeroResets.slice(0, 20) });
|
||
const elapsedDecreases = Array.isArray(sampleMetrics?.turnTimingNonMonotonic)
|
||
? sampleMetrics.turnTimingNonMonotonic.filter((item) => item.metric === "totalElapsedSeconds" && item.anomaly !== "zero-reset")
|
||
: [];
|
||
if (elapsedDecreases.length > 0) findings.push({ id: "turn-timing-total-elapsed-decrease", severity: "amber", summary: "Code Agent total elapsed decreased between adjacent samples; timing decrease is a non-blocking alert", timingSourceOfTruth: "dom-card-total-elapsed-sequence", timingStatus: timingStatusFromRows(elapsedDecreases), timingAlert: true, count: elapsedDecreases.length, samples: elapsedDecreases.slice(0, 20) });
|
||
const elapsedForwardJumps = Array.isArray(sampleMetrics?.turnTimingTotalElapsedForwardJumps) ? sampleMetrics.turnTimingTotalElapsedForwardJumps : [];
|
||
if (elapsedForwardJumps.length > 0) findings.push({ id: "turn-timing-total-elapsed-forward-jump", severity: "amber", summary: "Code Agent total elapsed jumped forward faster than browser sample interval; timing jump is a non-blocking alert", timingSourceOfTruth: "dom-card-total-elapsed-sequence", timingStatus: timingStatusFromRows(elapsedForwardJumps), timingAlert: true, count: elapsedForwardJumps.length, samples: elapsedForwardJumps.slice(0, 20) });
|
||
const terminalElapsedGrowth = Array.isArray(sampleMetrics?.turnTimingTerminalElapsedGrowth) ? sampleMetrics.turnTimingTerminalElapsedGrowth : [];
|
||
if (terminalElapsedGrowth.length > 0) findings.push({ id: "turn-timing-terminal-elapsed-growth", severity: "amber", summary: "terminal Code Agent card total elapsed changed after terminal status; terminal timing alert is non-blocking", timingSourceOfTruth: "terminal-card-total-elapsed-seal", timingStatus: timingStatusFromRows(terminalElapsedGrowth, "business-turn-completed"), timingAlert: true, count: terminalElapsedGrowth.length, samples: terminalElapsedGrowth.slice(0, 20) });
|
||
const recentUpdateSawtoothJumps = Array.isArray(sampleMetrics?.turnTimingRecentUpdateSawtoothJumps)
|
||
? sampleMetrics.turnTimingRecentUpdateSawtoothJumps
|
||
: Array.isArray(sampleMetrics?.turnTimingNonMonotonic)
|
||
? sampleMetrics.turnTimingNonMonotonic.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump")
|
||
: [];
|
||
if (recentUpdateSawtoothJumps.length > 0) findings.push({ id: "turn-timing-recent-update-sawtooth-jump", severity: "amber", summary: "最近更新 value jumped faster than sample interval; expected sawtooth increase-or-reset", count: recentUpdateSawtoothJumps.length, samples: recentUpdateSawtoothJumps.slice(0, 20) });
|
||
const severeTimeoutRounds = Array.isArray(sampleMetrics?.rounds) ? sampleMetrics.rounds.filter((item) => Number(item.maxTotalElapsedSeconds) > alertThresholds.turnElapsedSevereTimeoutSeconds) : [];
|
||
const severeTimeoutSamples = Array.isArray(sampleMetrics?.timeline) ? sampleMetrics.timeline.filter((item) => Number(item.totalElapsedSeconds) > alertThresholds.turnElapsedSevereTimeoutSeconds) : [];
|
||
if (severeTimeoutRounds.length > 0 || severeTimeoutSamples.length > 0) findings.push({ id: "turn-elapsed-severe-timeout", severity: "amber", summary: "turn total elapsed exceeded the YAML-configured elapsed alert threshold; timing is a non-blocking alert unless the turn fails to complete or breaks multi-round continuity", timingSourceOfTruth: "dom-card-total-elapsed-yaml-threshold", timingStatus: timingStatusFromRows([...severeTimeoutRounds, ...severeTimeoutSamples], "observer-timeout"), timingAlert: true, thresholdSeconds: alertThresholds.turnElapsedSevereTimeoutSeconds, count: Math.max(severeTimeoutRounds.length, severeTimeoutSamples.length), rounds: severeTimeoutRounds.slice(0, 20), samples: severeTimeoutSamples.slice(0, 20) });
|
||
const loadingSummary = sampleMetrics?.loading?.summary || {};
|
||
const visibleLoadingSlowSeconds = alertThresholds.visibleLoadingSlowMs / 1000;
|
||
if (Number(loadingSummary.longestContinuousSeconds ?? 0) > visibleLoadingSlowSeconds) findings.push({ id: "page-loading-visible-over-budget", severity: "red", summary: "visible 加载中 stayed on screen longer than configured YAML budget; fix real loading latency instead of revealing incomplete content early", count: loadingSummary.overBudgetSegmentCount ?? loadingSummary.overFiveSecondSegmentCount ?? 1, longestContinuousSeconds: loadingSummary.longestContinuousSeconds, budgetSeconds: visibleLoadingSlowSeconds, segments: sampleMetrics.loading.segments.slice(0, 20), owners: sampleMetrics.loading.owners.slice(0, 20) });
|
||
if (Number(loadingSummary.maxSimultaneousCount ?? 0) > 1) findings.push({ id: "page-loading-concurrent", severity: "info", summary: "multiple 加载中 indicators were visible in the same sampled DOM point", count: loadingSummary.concurrentLoadingSampleCount ?? 0, maxSimultaneousCount: loadingSummary.maxSimultaneousCount, owners: sampleMetrics.loading.owners.slice(0, 20) });
|
||
const sessionRailTitleSummary = sampleMetrics?.sessionRailTitles?.summary || {};
|
||
if (Number(sessionRailTitleSummary.overThresholdSampleCount ?? sessionRailTitleSummary.majorityFallbackSampleCount ?? 0) > 0) findings.push({
|
||
id: "session-rail-title-fallback-root-cause",
|
||
severity: "red",
|
||
summary: "INV-02 root cause visible: session rail is rendering fallback Session ses_* titles, so list projection/read model or rail binding is missing stable title/preview data before DOM render",
|
||
rootCause: "session_title_fallback_from_facts",
|
||
rootCauseStatus: "confirmed-from-dom-session-rail",
|
||
rootCauseConfidence: "high",
|
||
nextAction: "Check OTel session_list_read fallbackTitleCount/fallbackTitleRatio and emptyPreviewCount for the same run; fix session list projection/read model title/preview before changing DOM fallback text.",
|
||
count: sessionRailTitleSummary.overThresholdSampleCount ?? sessionRailTitleSummary.majorityFallbackSampleCount,
|
||
thresholdRatio: alertThresholds.sessionRailFallbackRatio,
|
||
maxFallbackRatio: sessionRailTitleSummary.maxFallbackRatio,
|
||
maxFallbackTitleCount: sessionRailTitleSummary.maxFallbackTitleCount,
|
||
evidence: {
|
||
thresholdRatio: alertThresholds.sessionRailFallbackRatio,
|
||
maxFallbackRatio: sessionRailTitleSummary.maxFallbackRatio,
|
||
maxFallbackTitleCount: sessionRailTitleSummary.maxFallbackTitleCount,
|
||
overThresholdSampleCount: sessionRailTitleSummary.overThresholdSampleCount ?? null,
|
||
majorityFallbackSampleCount: sessionRailTitleSummary.majorityFallbackSampleCount ?? null,
|
||
valuesRedacted: true
|
||
},
|
||
samples: sampleMetrics.sessionRailTitles.samples.slice(0, 20),
|
||
examples: sampleMetrics.sessionRailTitles.examples.slice(0, 20),
|
||
valuesRedacted: true
|
||
});
|
||
const traceEventsPageReadIssues = detectTraceEventsPageReadIssues(network);
|
||
if (traceEventsPageReadIssues.http404.length > 0) findings.push({
|
||
id: "trace-events-page-read-404-root-cause",
|
||
severity: "red",
|
||
summary: "INV-07 root cause visible: /v1/workbench/traces/:traceId/events returned HTTP 404 for a trace event page read, so the failure is in the trace-events API paging/read-model contract before DOM rendering",
|
||
rootCause: "trace_events_paging_contract_mismatch",
|
||
rootCauseStatus: "confirmed-from-browser-network",
|
||
rootCauseConfidence: "high",
|
||
nextAction: "Use OTel trace_events_read for the same trace to compare sinceSeq/afterProjectedSeq, returnedEvents, range, totalEvents, hasMore and fullTraceLoaded; fix backend paging contract or add missing instrumentation before changing renderer behavior.",
|
||
count: traceEventsPageReadIssues.http404.length,
|
||
evidence: traceEventsPageReadIssues.summary,
|
||
samples: traceEventsPageReadIssues.http404.slice(0, 20),
|
||
valuesRedacted: true
|
||
});
|
||
if (traceEventsPageReadIssues.httpErrors.length > 0) findings.push({
|
||
id: "trace-events-page-read-http-error-root-cause",
|
||
severity: "red",
|
||
summary: "trace events page read returned HTTP error status; monitor can localize this to the trace-events API instead of a generic Workbench render failure",
|
||
rootCause: "trace-events-api-page-read-http-error",
|
||
rootCauseStatus: "confirmed-from-browser-network",
|
||
rootCauseConfidence: "high",
|
||
nextAction: "Drill down by traceId and afterProjectedSeq, then compare with OTel trace_events_read paging fields.",
|
||
count: traceEventsPageReadIssues.httpErrors.length,
|
||
evidence: traceEventsPageReadIssues.summary,
|
||
samples: traceEventsPageReadIssues.httpErrors.slice(0, 20),
|
||
valuesRedacted: true
|
||
});
|
||
if (traceEventsPageReadIssues.requestFailed.length > 0) findings.push({
|
||
id: "trace-events-page-read-requestfailed-root-cause",
|
||
severity: "amber",
|
||
summary: "trace events page read failed at browser/network level; monitor localized the failure path, but HTTP status is unavailable so OTel/API instrumentation is needed to confirm the backend root cause",
|
||
rootCause: "trace-events-api-page-read-network-failed",
|
||
rootCauseStatus: "network-signal-needs-otel-confirmation",
|
||
rootCauseConfidence: "medium",
|
||
nextAction: "Check whether this happened during observer refresh/navigation; if not, query OTel by /v1/workbench/traces/:traceId/events and add route span fields when status/afterProjectedSeq are missing.",
|
||
count: traceEventsPageReadIssues.requestFailed.length,
|
||
evidence: traceEventsPageReadIssues.summary,
|
||
samples: traceEventsPageReadIssues.requestFailed.slice(0, 20),
|
||
valuesRedacted: true
|
||
});
|
||
if ((runtimeAlerts?.summary?.httpErrorCount ?? 0) > 0) findings.push({ id: "runtime-http-errors", severity: "amber", summary: "natural page requests returned HTTP error status during observation", count: runtimeAlerts.summary.httpErrorCount, groups: runtimeAlerts.networkHttpErrorsByPath.slice(0, 12) });
|
||
if ((runtimeAlerts?.summary?.significantRequestFailedCount ?? runtimeAlerts?.summary?.requestFailedCount ?? 0) > 0) findings.push({ id: "runtime-requestfailed", severity: "amber", summary: "browser requestfailed events were captured during observation", count: runtimeAlerts.summary.significantRequestFailedCount ?? runtimeAlerts.summary.requestFailedCount, groups: (runtimeAlerts.networkSignificantRequestFailedByPath ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 12) });
|
||
if ((runtimeAlerts?.summary?.domDiagnosticSampleCount ?? 0) > 0) findings.push({ id: "runtime-dom-diagnostics", severity: "amber", summary: "diagnostic/error/warning-like text was visible in sampled DOM", count: runtimeAlerts.summary.domDiagnosticSampleCount, groupCount: runtimeAlerts.summary.domDiagnosticGroupCount ?? 0, groups: runtimeAlerts.domDiagnosticsByText.slice(0, 12), samples: runtimeAlerts.domDiagnostics.slice(0, 12) });
|
||
if ((runtimeAlerts?.summary?.executionErrorCount ?? 0) > 0) findings.push({ id: "runtime-execution-errors", severity: "red", summary: "Workbench rendered execution failure/error rows during observation", count: runtimeAlerts.summary.executionErrorCount, groups: runtimeAlerts.runtimeExecutionErrorsByCode.slice(0, 12) });
|
||
if ((runtimeAlerts?.summary?.significantConsoleAlertCount ?? runtimeAlerts?.summary?.consoleAlertCount ?? 0) > 0) findings.push({ id: "runtime-console-alerts", severity: "amber", summary: "browser console warning/error entries were captured during observation", count: runtimeAlerts.summary.significantConsoleAlertCount ?? runtimeAlerts.summary.consoleAlertCount, groups: (runtimeAlerts.significantConsoleAlertsByPath ?? runtimeAlerts.consoleAlertsByPath).slice(0, 12) });
|
||
const crossPageDiffs = mergeCrossPageDiffRows(
|
||
detectCrossPageProjectionDiffs(samples),
|
||
detectAdjacentCrossPageProjectionDiffs(samples)
|
||
);
|
||
const crossPageProjectionDiffs = crossPageDiffs.filter((item) => item.diffKind !== "trace-visibility");
|
||
const crossPageTraceVisibilityDiffs = crossPageDiffs.filter((item) => item.diffKind === "trace-visibility");
|
||
const crossPageProjectionBudgetMs = alertThresholds.crossPageProjectionDivergenceRedMs;
|
||
const sampleBySeq = new Map(samples.map((item) => [Number(item?.seq), item]).filter(([seq]) => Number.isFinite(seq)));
|
||
const appShellNotReadyRows = detectWorkbenchAppShellNotReady(samples);
|
||
const persistentAppShellNotReadyRows = appShellNotReadyRows.filter((item) => Number(item.observedSpanMs ?? 0) > crossPageProjectionBudgetMs);
|
||
const transientAppShellNotReadyRows = appShellNotReadyRows.filter((item) => Number(item.observedSpanMs ?? 0) <= crossPageProjectionBudgetMs);
|
||
if (persistentAppShellNotReadyRows.length > 0) findings.push({
|
||
id: "workbench-app-shell-not-ready",
|
||
severity: "red",
|
||
summary: "Workbench route document loaded but the app shell never mounted or hydrated within the configured budget; treat this as page/runtime startup evidence, not a Workbench projection divergence",
|
||
count: persistentAppShellNotReadyRows.length,
|
||
budgetMs: crossPageProjectionBudgetMs,
|
||
samples: persistentAppShellNotReadyRows.slice(0, 20),
|
||
rootCause: "workbench_page_app_shell_not_ready",
|
||
rootCauseStatus: "confirmed-from-dom-and-asset-provenance",
|
||
rootCauseConfidence: "high",
|
||
valuesRedacted: true
|
||
});
|
||
if (transientAppShellNotReadyRows.length > 0) findings.push({ id: "workbench-app-shell-transient-not-ready", severity: "info", summary: "Workbench route briefly had a document and assets but no mounted app shell; retained as startup context", count: transientAppShellNotReadyRows.length, budgetMs: crossPageProjectionBudgetMs, samples: transientAppShellNotReadyRows.slice(0, 20) });
|
||
const timedCrossPageProjectionDiffs = annotateCrossPageDiffTiming(crossPageProjectionDiffs);
|
||
const controlledNavigationHydrationProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq));
|
||
const appShellNotReadyProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => !controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq) && crossPageDiffHasWorkbenchAppShellNotReady(item, sampleBySeq));
|
||
const evaluatedCrossPageProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => !controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq) && !crossPageDiffHasWorkbenchAppShellNotReady(item, sampleBySeq));
|
||
const persistentCrossPageProjectionDiffs = evaluatedCrossPageProjectionDiffs.filter((item) => Number(item.observedSpanMs ?? 0) > crossPageProjectionBudgetMs);
|
||
const transientCrossPageProjectionDiffs = evaluatedCrossPageProjectionDiffs.filter((item) => Number(item.observedSpanMs ?? 0) <= crossPageProjectionBudgetMs);
|
||
if (persistentCrossPageProjectionDiffs.length > 0) findings.push({
|
||
id: "cross-page-projection-divergence",
|
||
severity: "red",
|
||
summary: "control and observer pages saw different projection state for the same sampled session beyond the configured budget",
|
||
count: persistentCrossPageProjectionDiffs.length,
|
||
budgetMs: crossPageProjectionBudgetMs,
|
||
samples: persistentCrossPageProjectionDiffs.slice(0, 20),
|
||
drilldown: buildCrossPageProjectionDrilldown(persistentCrossPageProjectionDiffs),
|
||
sourceOfTruth: "session list/detail/messages/turn APIs must read from the same durable projection snapshot across pages",
|
||
rootCause: "workbench_cross_page_projection_not_single_source",
|
||
rootCauseStatus: "confirmed-from-control-observer-dom-samples",
|
||
rootCauseConfidence: "high",
|
||
nextAction: "Use drilldown.traceIds with diagnose-code-agent and compare session_list_read/session_messages_read/turn_status_read rows before changing renderer fallback behavior.",
|
||
valuesRedacted: true
|
||
});
|
||
if (transientCrossPageProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-transient-divergence", severity: "info", summary: "control and observer pages briefly differed near a sampled transition; retained as transient evidence but not treated as persistent projection failure", count: transientCrossPageProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: transientCrossPageProjectionDiffs.slice(0, 20) });
|
||
if (controlledNavigationHydrationProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-controlled-navigation-hydration", severity: "info", summary: "control and observer pages differed while a non-blocking session-invariance navigation command still had an unhydrated blank page; retained as context but not treated as a red projection blocker", count: controlledNavigationHydrationProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: controlledNavigationHydrationProjectionDiffs.slice(0, 20) });
|
||
if (appShellNotReadyProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-app-shell-not-ready", severity: "info", summary: "cross-page projection differences were explained by a page whose Workbench app shell was not mounted; see workbench-app-shell-not-ready for the blocking root cause", count: appShellNotReadyProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: appShellNotReadyProjectionDiffs.slice(0, 20) });
|
||
if (crossPageTraceVisibilityDiffs.length > 0) findings.push({ id: "cross-page-trace-visibility-divergence", severity: "info", summary: "control and observer pages differed only in visible trace row count; this is local disclosure/hydration visibility, not session/message projection divergence", count: crossPageTraceVisibilityDiffs.length, samples: crossPageTraceVisibilityDiffs.slice(0, 20) });
|
||
const traceMessageDuplicates = detectTraceMessageDuplication(samples);
|
||
if (traceMessageDuplicates.length > 0) findings.push({ id: "trace-assistant-message-duplicates-final-response", severity: "amber", summary: "trace-frame rendered duplicate visible assistant final rows; the fixed Final Response renderer summary block is excluded", count: traceMessageDuplicates.length, finalResponseSummaryBlockCounted: false, traceFrameSource: "traceRows-only", samples: traceMessageDuplicates.slice(0, 20) });
|
||
const turnTraceMissing = detectTurnTraceIdMissing(samples);
|
||
if (turnTraceMissing.length > 0) findings.push({ id: "turn-trace-id-missing", severity: "red", summary: "Code Agent turn/card was visible without a trace id, so historical trace hydration cannot be reliable", count: turnTraceMissing.length, samples: turnTraceMissing.slice(0, 20) });
|
||
const pagePerformanceItems = Array.isArray(pagePerformance?.sameOriginApiByPath) ? pagePerformance.sameOriginApiByPath : [];
|
||
const slowApi = pagePerformanceItems.filter((item) => item.isLongLivedStream !== true && Number(item.overBudgetCount ?? item.overFiveSecondCount ?? 0) > 0);
|
||
if (slowApi.length > 0) findings.push({ id: "page-performance-slow-same-origin-api", severity: "red", summary: "same-origin API resource timing exceeded configured YAML usability budget", count: slowApi.length, budgetMs: alertThresholds.sameOriginApiSlowMs, groups: slowApi.slice(0, 20) });
|
||
const longLivedStreams = pagePerformanceItems.filter((item) => item.isLongLivedStream);
|
||
const slowStreamOpen = longLivedStreams.filter((item) => Number(item.streamOpenOverBudgetCount ?? item.streamOpenOverFiveSecondCount ?? 0) > 0);
|
||
if (slowStreamOpen.length > 0) findings.push({ id: "page-performance-slow-long-lived-stream-open", severity: "red", summary: "long-lived stream open latency exceeded configured YAML usability budget; stream lifetime is still reported separately", count: slowStreamOpen.length, budgetMs: alertThresholds.longLivedStreamOpenSlowMs, groups: slowStreamOpen.slice(0, 20) });
|
||
if (longLivedStreams.length > 0) findings.push({ id: "page-performance-long-lived-streams", severity: "info", summary: "same-origin long-lived streams are reported separately; lifetime is not treated as API load latency", count: longLivedStreams.length, groups: longLivedStreams.slice(0, 20) });
|
||
const requestRatePeaks = Array.isArray(requestRate?.peaks) ? requestRate.peaks : [];
|
||
const totalRequestRatePeaks = requestRatePeaks.filter((item) => item.scope === "total" && item.overThreshold === true);
|
||
const pageRequestRatePeaks = requestRatePeaks.filter((item) => item.scope === "page" && item.overThreshold === true);
|
||
const apiPathRequestRatePeaks = requestRatePeaks.filter((item) => item.scope === "apiPath" && item.overThreshold === true);
|
||
if (totalRequestRatePeaks.length > 0) findings.push({
|
||
id: "request-rate-total-peak",
|
||
severity: "red",
|
||
summary: "same-origin API request rate exceeded the YAML total peak threshold; this is request storm evidence from browser network events",
|
||
count: totalRequestRatePeaks.length,
|
||
thresholdPerMinute: alertThresholds.requestRateTotalRedPerMinute,
|
||
bucketMs: requestRate?.summary?.bucketMs ?? alertThresholds.requestRateBucketMs,
|
||
peaks: totalRequestRatePeaks.slice(0, 20),
|
||
valuesRedacted: true
|
||
});
|
||
if (pageRequestRatePeaks.length > 0) findings.push({
|
||
id: "request-rate-page-peak",
|
||
severity: "red",
|
||
summary: "a page-level same-origin API request curve exceeded the YAML peak threshold; inspect the page and top API paths before changing probe sampling",
|
||
count: pageRequestRatePeaks.length,
|
||
thresholdPerMinute: alertThresholds.requestRatePageRedPerMinute,
|
||
bucketMs: requestRate?.summary?.bucketMs ?? alertThresholds.requestRateBucketMs,
|
||
peaks: pageRequestRatePeaks.slice(0, 20),
|
||
valuesRedacted: true
|
||
});
|
||
if (apiPathRequestRatePeaks.length > 0) findings.push({
|
||
id: "request-rate-api-path-peak",
|
||
severity: "red",
|
||
summary: "an API-path request curve exceeded the YAML peak threshold; this pinpoints the route most likely involved in a request storm",
|
||
count: apiPathRequestRatePeaks.length,
|
||
thresholdPerMinute: alertThresholds.requestRateApiPathRedPerMinute,
|
||
bucketMs: requestRate?.summary?.bucketMs ?? alertThresholds.requestRateBucketMs,
|
||
peaks: apiPathRequestRatePeaks.slice(0, 20),
|
||
valuesRedacted: true
|
||
});
|
||
if ((pageProvenance?.summary?.segmentCount ?? 0) > 1) findings.push({ id: "page-provenance-segments", severity: "info", summary: "observer crossed page asset provenance segments; interpret runtime findings by segment", segmentCount: pageProvenance.summary.segmentCount, segments: pageProvenance.segments.slice(0, 20) });
|
||
const naturalApi = network.filter((item) => item.observerInitiated === false && item.type === "response" && /\/v1\/|\/auth\//u.test(String(item.url || "")));
|
||
const apiDomLagSummary = effectiveApiDomLag?.summary || {};
|
||
findings.push({ id: "natural-api-dom-lag-baseline", severity: "info", summary: "natural API responses and DOM samples are available for API-to-DOM lag correlation", naturalApiResponses: naturalApi.length, sampleCount: samples.length, apiDomLag: apiDomLagSummary });
|
||
findings.push({
|
||
id: "natural-api-dom-lag-candidates",
|
||
severity: Number(apiDomLagSummary.overBudgetCount ?? 0) > 0 ? "amber" : "info",
|
||
summary: "state-relevant natural API responses were correlated to the first subsequent DOM digest change; over-budget lag is a non-blocking investigation alert",
|
||
count: apiDomLagSummary.candidateCount ?? 0,
|
||
domChangedCount: apiDomLagSummary.domChangedCount ?? 0,
|
||
noDomChangeWithinWindowCount: apiDomLagSummary.noDomChangeWithinWindowCount ?? 0,
|
||
overBudgetCount: apiDomLagSummary.overBudgetCount ?? 0,
|
||
budgetMs: apiDomLagSummary.budgetMs ?? null,
|
||
p95DomChangeDeltaMs: apiDomLagSummary.p95DomChangeDeltaMs ?? null,
|
||
maxDomChangeDeltaMs: apiDomLagSummary.maxDomChangeDeltaMs ?? null,
|
||
groups: Array.isArray(effectiveApiDomLag?.groups) ? effectiveApiDomLag.groups.slice(0, 12) : [],
|
||
});
|
||
if (errors.length > 0) findings.push({ id: "browser-console-or-page-errors", severity: "amber", summary: "pageerror/runner errors were captured", count: errors.length, first: errors.slice(0, 5) });
|
||
if (samples.length === 0) findings.push({ id: "no-samples", severity: "red", summary: "observer produced no samples" });
|
||
return findings;
|
||
}
|
||
|
||
function buildCrossPageProjectionDrilldown(rows) {
|
||
const sourceRows = Array.isArray(rows) ? rows.filter(Boolean) : [];
|
||
const traceIds = uniqueSorted(sourceRows.flatMap((row) => collectIdsFromUnknown(row, "trace")).filter(Boolean)).slice(0, 12);
|
||
const sessionIds = uniqueSorted(sourceRows.flatMap((row) => collectIdsFromUnknown(row, "session")).filter(Boolean)).slice(0, 12);
|
||
const diffKinds = uniqueSorted(sourceRows.map((row) => row?.diffKind).filter(Boolean));
|
||
const groupsByKey = new Map();
|
||
for (const row of sourceRows) {
|
||
const rowTraceIds = collectIdsFromUnknown(row, "trace").slice(0, 8);
|
||
const rowSessionIds = collectIdsFromUnknown(row, "session").slice(0, 8);
|
||
const key = [
|
||
row?.diffKind || "projection",
|
||
rowSessionIds[0] || row?.sessionIdPrefix || "-",
|
||
rowTraceIds[0] || "-",
|
||
].join("|");
|
||
let group = groupsByKey.get(key);
|
||
if (!group) {
|
||
group = {
|
||
keyHash: sha256(key),
|
||
diffKind: row?.diffKind ?? null,
|
||
traceIds: new Set(),
|
||
sessionIds: new Set(),
|
||
count: 0,
|
||
firstSeq: row?.firstSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq ?? null,
|
||
lastSeq: row?.lastSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq ?? null,
|
||
firstAt: row?.firstAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null,
|
||
lastAt: row?.lastAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null,
|
||
maxObservedSpanMs: 0,
|
||
examples: [],
|
||
};
|
||
groupsByKey.set(key, group);
|
||
}
|
||
group.count += 1;
|
||
for (const traceId of rowTraceIds) group.traceIds.add(traceId);
|
||
for (const sessionId of rowSessionIds) group.sessionIds.add(sessionId);
|
||
group.firstSeq = minNumberOrValue(group.firstSeq, row?.firstSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq);
|
||
group.lastSeq = maxNumberOrValue(group.lastSeq, row?.lastSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq);
|
||
group.firstAt = minIso(group.firstAt, row?.firstAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null);
|
||
group.lastAt = maxIso(group.lastAt, row?.lastAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null);
|
||
group.maxObservedSpanMs = Math.max(group.maxObservedSpanMs, Number(row?.observedSpanMs ?? 0) || 0);
|
||
if (group.examples.length < 3) {
|
||
group.examples.push({
|
||
diffKind: row?.diffKind ?? null,
|
||
firstSeq: row?.firstSeq ?? null,
|
||
lastSeq: row?.lastSeq ?? null,
|
||
observedSpanMs: row?.observedSpanMs ?? null,
|
||
controlMessageCount: row?.controlMessageCount ?? null,
|
||
observerMessageCount: row?.observerMessageCount ?? null,
|
||
traceIds: rowTraceIds.slice(0, 6),
|
||
sessionIds: rowSessionIds.slice(0, 4),
|
||
valuesRedacted: true,
|
||
});
|
||
}
|
||
}
|
||
const groups = Array.from(groupsByKey.values()).map((group) => ({
|
||
keyHash: group.keyHash,
|
||
diffKind: group.diffKind,
|
||
traceIds: Array.from(group.traceIds).slice(0, 8),
|
||
sessionIds: Array.from(group.sessionIds).slice(0, 6),
|
||
count: group.count,
|
||
firstSeq: group.firstSeq,
|
||
lastSeq: group.lastSeq,
|
||
firstAt: group.firstAt,
|
||
lastAt: group.lastAt,
|
||
observedSpanMs: Math.max(group.maxObservedSpanMs, isoSpanMs(group.firstAt, group.lastAt)),
|
||
examples: group.examples,
|
||
valuesRedacted: true,
|
||
})).sort((left, right) => Number(right.observedSpanMs ?? 0) - Number(left.observedSpanMs ?? 0) || Number(right.count ?? 0) - Number(left.count ?? 0));
|
||
return {
|
||
summary: {
|
||
rowCount: sourceRows.length,
|
||
groupCount: groups.length,
|
||
diffKinds,
|
||
traceIds,
|
||
sessionIds,
|
||
maxObservedSpanMs: groups.reduce((value, item) => Math.max(value, Number(item.observedSpanMs ?? 0)), 0),
|
||
valuesRedacted: true,
|
||
},
|
||
traceIds,
|
||
sessionIds,
|
||
groups: groups.slice(0, 12),
|
||
otelDrilldown: buildWorkbenchTriadOtelDrilldown(traceIds.slice(0, 4)),
|
||
staticSourceHints: workbenchTriadStaticSourceHints(),
|
||
unitTestReproHints: [
|
||
"two independent Workbench pages must converge to identical session list/detail/messages/turn projection after the same durable events",
|
||
"late session-list or message refresh must not drop a trace that another page already read from durable projection",
|
||
"read-side APIs must expose enough OTel fields to compare projection version/cursor and trace ids across pages",
|
||
],
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function collectIdsFromUnknown(value, kind, output = new Set(), depth = 0) {
|
||
if (depth > 4 || value === null || value === undefined) return Array.from(output);
|
||
if (typeof value === "string") {
|
||
const pattern = kind === "session" ? /\bses_[A-Za-z0-9_-]+\b/gu : /\btrc_[A-Za-z0-9_-]+\b/gu;
|
||
for (const match of value.match(pattern) || []) output.add(match);
|
||
return Array.from(output);
|
||
}
|
||
if (Array.isArray(value)) {
|
||
for (const item of value.slice(0, 40)) collectIdsFromUnknown(item, kind, output, depth + 1);
|
||
return Array.from(output);
|
||
}
|
||
if (typeof value === "object") {
|
||
for (const item of Object.values(value).slice(0, 80)) collectIdsFromUnknown(item, kind, output, depth + 1);
|
||
}
|
||
return Array.from(output);
|
||
}
|
||
|
||
function buildFrontendFreezeFindings(errors, control) {
|
||
const findings = [];
|
||
const promptTimes = (control || [])
|
||
.filter((item) => item.type === "sendPrompt" && item.phase === "completed")
|
||
.map((item) => Date.parse(item.ts))
|
||
.filter(Number.isFinite)
|
||
.sort((a, b) => a - b);
|
||
const stopWindows = stopCommandWindows(control);
|
||
const events = (errors || [])
|
||
.map((item) => frontendFreezeErrorEvent(item, promptTimes))
|
||
.filter((item) => item && !errorInsideStopWindow(item, stopWindows));
|
||
const domEvents = events.filter((item) => item.kind === "dom-evaluate-timeout");
|
||
const controlDomBurst = firstBurst(
|
||
domEvents.filter((item) => item.pageRole === "control" || item.pageRole === null),
|
||
alertThresholds.domEvaluateTimeoutRedCount,
|
||
alertThresholds.domEvaluateTimeoutRedWindowMs,
|
||
);
|
||
if (controlDomBurst) findings.push(frontendFreezeBurstFinding({
|
||
id: "frontend-control-dom-evaluate-timeout-red",
|
||
summary: "control page DOM evaluation timed out repeatedly; treat the browser page as frozen and keep the sentinel red instead of refreshing or falling back",
|
||
burst: controlDomBurst,
|
||
thresholdCount: alertThresholds.domEvaluateTimeoutRedCount,
|
||
windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs,
|
||
pageRole: "control",
|
||
}));
|
||
const observerDomBurst = firstBurst(
|
||
domEvents.filter((item) => item.pageRole === "observer"),
|
||
alertThresholds.domEvaluateTimeoutRedCount,
|
||
alertThresholds.domEvaluateTimeoutRedWindowMs,
|
||
);
|
||
if (observerDomBurst) findings.push(frontendFreezeBurstFinding({
|
||
id: "frontend-observer-dom-evaluate-timeout-red",
|
||
summary: "observer page DOM evaluation timed out repeatedly; the observer page is frozen and later periodic refresh evidence must not clear this run",
|
||
burst: observerDomBurst,
|
||
thresholdCount: alertThresholds.domEvaluateTimeoutRedCount,
|
||
windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs,
|
||
pageRole: "observer",
|
||
}));
|
||
const screenshotBurst = firstBurst(
|
||
events.filter((item) => item.kind === "screenshot-timeout"),
|
||
alertThresholds.screenshotTimeoutRedCount,
|
||
alertThresholds.domEvaluateTimeoutRedWindowMs,
|
||
);
|
||
if (screenshotBurst) findings.push(frontendFreezeBurstFinding({
|
||
id: "frontend-screenshot-timeout-red",
|
||
summary: "browser screenshot capture timed out repeatedly; this is freeze evidence and the sentinel must stay red until investigated",
|
||
burst: screenshotBurst,
|
||
thresholdCount: alertThresholds.screenshotTimeoutRedCount,
|
||
windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs,
|
||
pageRole: null,
|
||
}));
|
||
const pageErrors = events.filter((item) => item.kind === "page-error");
|
||
const pageErrorBurst = firstBurst(pageErrors, alertThresholds.pageErrorRedCount, alertThresholds.domEvaluateTimeoutRedWindowMs);
|
||
if (pageErrorBurst) findings.push(frontendFreezeBurstFinding({
|
||
id: "frontend-page-error-red",
|
||
summary: "browser pageerror entries exceeded the YAML threshold; page runtime exceptions are blocking when repeated in the observation window",
|
||
burst: pageErrorBurst,
|
||
thresholdCount: alertThresholds.pageErrorRedCount,
|
||
windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs,
|
||
pageRole: null,
|
||
}));
|
||
return findings;
|
||
}
|
||
`;
|
||
}
|