// SPEC: PJ2026-01040111 long-running Workbench observation. // Responsibility: Source string for the offline web-probe observe analyzer. import { nodeWebObserveAnalyzerTimingSource } from "./hwlab-node-web-observe-analyzer-timing-source"; export function nodeWebObserveAnalyzerSource(): string { return String.raw`#!/usr/bin/env node import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import path from "node:path"; import { createInterface } from "node:readline"; const stateDir = path.resolve(process.env.UNIDESK_WEB_OBSERVE_STATE_DIR || process.argv[2] || ".state/web-observe/manual"); const archivePrefix = safeArchivePrefix(process.env.UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX || ""); const analyzeTailSamples = (() => { const raw = process.env.UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES; if (raw === undefined || raw === null || String(raw).trim() === "") return 360; const parsed = Number(raw); return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : 360; })(); const alertThresholds = parseAlertThresholds(process.env.UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON); const projectManagementConfig = parseProjectManagementConfig(process.env.UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON); const dataDir = archivePrefix ? path.join(stateDir, "archive") : stateDir; const dataFile = (name) => path.join(dataDir, archivePrefix ? archivePrefix + "-" + name : name); const analysisDir = path.join(stateDir, "analysis"); const reportJsonPath = path.join(analysisDir, "report.json"); const reportMdPath = path.join(analysisDir, "report.md"); const jsonlReadIssues = []; const sourceSamples = await readJsonl(dataFile("samples.jsonl"), { compact: compactSampleForAnalysis, tail: analyzeTailSamples }); const relatedJsonlTailLimit = analyzeTailSamples > 0 ? Math.max(1200, analyzeTailSamples * 50) : 0; const smallJsonlTailLimit = analyzeTailSamples > 0 ? Math.max(500, analyzeTailSamples * 8) : 0; const sourceSampleWindow = sampleTimeWindow(sourceSamples, 60_000); const sourceControlAll = await readJsonl(dataFile("control.jsonl"), { tail: relatedJsonlTailLimit }); const analysisFocus = analysisFocusFromControl(sourceControlAll); const sourceControl = filterRowsByTimeWindow(sourceControlAll, sourceSampleWindow); const samples = applyAnalysisFocus(sourceSamples, analysisFocus); const sampleWindow = sampleTimeWindow(samples, 60_000); const controlWindow = analysisControlWindow(sampleWindow, analysisFocus, 1000); const control = applyAnalysisFocus(filterRowsByTimeWindow(sourceControlAll, controlWindow), analysisFocus, 1000); const sourceNetworkAll = await readJsonl(dataFile("network.jsonl"), { tail: relatedJsonlTailLimit }); const network = applyAnalysisFocus(filterRowsByTimeWindow(sourceNetworkAll, sampleWindow), analysisFocus); const promptNetworkRows = applyAnalysisFocus(filterRowsByTimeWindow(sourceNetworkAll, controlWindow), analysisFocus); const consoleEvents = applyAnalysisFocus(filterRowsByTimeWindow(await readJsonl(dataFile("console.jsonl"), { tail: smallJsonlTailLimit }), sampleWindow), analysisFocus); const errors = applyAnalysisFocus(filterRowsByTimeWindow(await readJsonl(dataFile("errors.jsonl"), { tail: smallJsonlTailLimit }), sampleWindow), analysisFocus); const artifacts = applyAnalysisFocus(filterRowsByTimeWindow(await readJsonl(dataFile("artifacts.jsonl"), { tail: smallJsonlTailLimit }), sampleWindow), analysisFocus); const manifest = await readJson(path.join(stateDir, "manifest.json")); const heartbeat = await readJson(path.join(stateDir, "heartbeat.json")); const commandState = await readCommandState(stateDir); await mkdir(analysisDir, { recursive: true, mode: 0o700 }); const transitions = buildTransitions(samples); const sampleMetrics = buildSampleMetrics(samples, control); const pageProvenance = buildPageProvenanceReport(samples, control, manifest); const pagePerformance = buildPagePerformanceReport(samples, manifest); const promptNetwork = buildPromptNetworkReport(control, promptNetworkRows); const runtimeAlerts = buildRuntimeAlerts(samples, control, network, consoleEvents, errors); const apiDomLag = buildApiDomLagReport(samples, network); const projectManagement = buildProjectManagementReport(samples, control, network, pagePerformance, projectManagementConfig); const runnerErrors = errors.slice(-8).map((item) => { const details = item.error?.details && typeof item.error.details === "object" ? item.error.details : {}; const attempts = Array.isArray(item.error?.attempts) ? item.error.attempts : Array.isArray(details.attempts) ? details.attempts : []; const lastAttempt = attempts.length > 0 ? attempts[attempts.length - 1] : null; const readiness = lastAttempt?.readiness || lastAttempt?.readinessBeforeClick || details.readinessBeforeClick || details.readinessAfterWait || item.error?.navigationReadiness || null; const readinessSnapshot = readiness?.snapshot || readiness; return { ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, sampleSeq: item.sampleSeq ?? null, message: limitText(item.error?.message ?? item.message ?? "", 240), retry: item.error?.auth?.lastRetryLabel ?? null, retryExhausted: item.error?.auth?.retryExhausted === true, lastError: limitText(item.error?.auth?.lastError ?? "", 160), attemptCount: attempts.length, lastFailureKind: lastAttempt?.failureKind ?? null, lastReadinessReason: readiness?.reason ?? null, lastReadiness: readinessSnapshot ? { reason: readiness?.reason ?? readinessSnapshot.reason ?? null, path: readinessSnapshot.path ?? null, readyState: readinessSnapshot.readyState ?? null, workbenchShellVisible: readinessSnapshot.workbenchShellVisible === true, sessionCreatePresent: readinessSnapshot.sessionCreatePresent === true, sessionCreateVisible: readinessSnapshot.sessionCreateVisible === true, sessionRailPresent: readinessSnapshot.sessionRailPresent === true, sessionRailCollapsed: readinessSnapshot.sessionRailCollapsed ?? null, sessionCollapseTogglePresent: readinessSnapshot.sessionCollapseTogglePresent === true, sessionCollapseToggleVisible: readinessSnapshot.sessionCollapseToggleVisible === true, sessionCollapseToggleExpanded: readinessSnapshot.sessionCollapseToggleExpanded ?? null, commandInputPresent: readinessSnapshot.commandInputPresent === true, activeTabPresent: readinessSnapshot.activeTabPresent === true, warningPresent: readinessSnapshot.warningPresent === true, loginVisible: readinessSnapshot.loginVisible === true, bodyTextHash: readinessSnapshot.bodyTextHash ?? null, valuesRedacted: true } : null, navigationAttempts: attempts.slice(-5).map((attempt) => ({ attempt: attempt?.attempt ?? null, ok: attempt?.ok === true, failureKind: attempt?.failureKind ?? null, message: limitText(attempt?.message ?? "", 160), readinessReason: attempt?.readiness?.reason ?? null, valuesRedacted: true })), }; }); const commandFailures = summarizeCommandFailures(control); const toolFindings = buildToolFindings({ manifest, heartbeat, commandState }); const findings = [...toolFindings, ...buildProjectManagementFindings(projectManagement), ...buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures, manifest, apiDomLag)]; if (jsonlReadIssues.length > 0) findings.unshift({ id: "jsonl-read-issues", severity: "red", summary: "observer analyzer hit JSONL read/parse issues", count: jsonlReadIssues.length, issues: jsonlReadIssues.slice(0, 20) }); const recentWindow = buildRecentAnalysisWindow({ samples, control, network, consoleEvents, errors, manifest }); const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl })); const report = { ok: findings.filter((item) => item.severity === "red").length === 0, command: "web-probe-observe analyze", generatedAt: new Date().toISOString(), stateDir, jsonlScope: { mode: archivePrefix ? "archive" : "current", archivePrefix: archivePrefix || null, dataDir, analyzeTailSamples, sourceSampleCount: sourceSamples.length, effectiveSampleCount: samples.length, sourceControlCount: sourceControlAll.length, sampleWindow, focus: analysisFocus, valuesRedacted: true }, alertThresholds, manifest: compactManifest(manifest), heartbeat: compactHeartbeat(heartbeat), counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length }, commandTimeline, transitions, sampleMetrics, pageProvenance, pagePerformance, projectManagement, promptNetwork, runtimeAlerts, apiDomLag, runnerErrors, commandFailures, commandState, toolFindings, findings, windows: { recent: recentWindow }, readIssues: jsonlReadIssues, artifactSummary: await artifactSummary(artifacts), safety: { offlineOnly: true, browserDriven: false, apiFetch: false, valuesRedacted: true }, }; await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 }); await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 }); const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]); console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, jsonlScope: report.jsonlScope, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, archiveSummary: { sampleMetrics: sampleMetrics.summary, pagePerformance: pagePerformance.summary, projectManagement: projectManagement.summary, runtimeAlerts: runtimeAlerts.summary, apiDomLag: apiDomLag.summary, findingCount: findings.length, redFindingCount: findings.filter((item) => item.severity === "red").length, redFindings: prioritizeFindings(findings.filter((item) => item.severity === "red")).slice(0, 12).map((item) => ({ kind: item.id ?? item.kind ?? item.code, severity: item.severity, count: item.count ?? item.sampleCount ?? null, summary: String(item.summary ?? item.message ?? "").slice(0, 180) })), }, analysisWindow: recentWindow.summary, jsonlReadIssues: jsonlReadIssues.slice(0, 3).map((item) => ({ file: item.file, line: item.line ?? null, error: String(item.error ?? "").slice(0, 160) })), readIssues: jsonlReadIssues.slice(0, 3).map((item) => ({ file: item.file, line: item.line ?? null, error: String(item.error ?? "").slice(0, 160) })), sampleMetrics: { ...recentWindow.sampleMetrics.summary, rounds: recentWindow.sampleMetrics.rounds.slice(-8).map((item) => ({ promptIndex: item.promptIndex, promptTextHash: item.promptTextHash, sampleCount: item.sampleCount, firstSeq: item.firstSeq, lastSeq: item.lastSeq, lastTotalElapsedSeconds: item.lastTotalElapsedSeconds, lastRecentUpdateSeconds: item.lastRecentUpdateSeconds, loadingSamples: item.loadingSamples, maxLoadingCount: item.maxLoadingCount, loadingOwnerCount: item.loadingOwnerCount, diagnosticSamples: item.diagnosticSamples, terminalSamples: item.terminalSamples, finalTextSamples: item.finalTextSamples, turnTimingTotalElapsedZeroResetCount: item.turnTimingTotalElapsedZeroResetCount, turnTimingTotalElapsedForwardJumpCount: item.turnTimingTotalElapsedForwardJumpCount, turnTimingTotalElapsedForwardJumpMaxSeconds: item.turnTimingTotalElapsedForwardJumpMaxSeconds, turnTimingRecentUpdateJumpCount: item.turnTimingRecentUpdateJumpCount, turnTimingRecentUpdateMaxIncreaseSeconds: item.turnTimingRecentUpdateMaxIncreaseSeconds })), turnColumns: recentWindow.sampleMetrics.turnColumns.slice(-12).map((item) => ({ label: item.label, source: item.source, pageRole: item.pageRole ?? null, pageId: item.pageId ?? null, pageEpoch: item.pageEpoch ?? null, promptIndex: item.promptIndex, lastPromptIndex: item.lastPromptIndex, firstSeq: item.firstSeq, lastSeq: item.lastSeq, traceId: item.traceId, messageId: item.messageId })), loading: compactLoadingMetricsForOutput(recentWindow.sampleMetrics.loading), sessionRailTitles: compactSessionRailTitleMetricsForOutput(recentWindow.sampleMetrics.sessionRailTitles), }, pageProvenance: recentWindow.pageProvenance.summary, pagePerformance: recentWindow.pagePerformance.summary, projectManagement: compactProjectManagementForOutput(projectManagement), promptNetwork: recentWindow.promptNetwork.summary, runtimeAlerts: recentWindow.runtimeAlerts.summary, apiDomLag: compactApiDomLagForOutput(apiDomLag), runnerErrors, commandFailures: commandFailures.slice(-8), commandState, toolFindings: toolFindings.slice(0, 8).map((item) => ({ kind: item.id, severity: item.severity, count: item.count ?? null, summary: String(item.summary ?? "").slice(0, 180) })), httpErrorGroups: recentWindow.runtimeAlerts.networkHttpErrorsByPath.slice(0, 8).map((item) => ({ method: item.method ?? null, status: item.status ?? null, path: item.urlPath ?? null, count: item.count, firstAt: item.firstAt ?? null, lastAt: item.lastAt ?? null, promptIndexes: Array.isArray(item.promptIndexes) ? item.promptIndexes.slice(0, 6) : [], failureKinds: Array.isArray(item.failureKinds) ? item.failureKinds.slice(0, 4) : [], })), requestFailedGroups: recentWindow.runtimeAlerts.networkRequestFailedByPath.slice(0, 8).map((item) => ({ method: item.method ?? null, status: item.status ?? null, path: item.urlPath ?? null, count: item.count, firstAt: item.firstAt ?? null, lastAt: item.lastAt ?? null, promptIndexes: Array.isArray(item.promptIndexes) ? item.promptIndexes.slice(0, 6) : [], failureKinds: Array.isArray(item.failureKinds) ? item.failureKinds.slice(0, 4) : [], })), domDiagnosticGroups: recentWindow.runtimeAlerts.domDiagnosticsByFingerprint.slice(0, 5).map((item) => ({ diagnosticCode: item.diagnosticCode ?? null, count: item.count, firstAt: item.firstAt, lastAt: item.lastAt, promptIndexes: Array.isArray(item.promptIndexes) ? item.promptIndexes.slice(0, 6) : [], traceIds: Array.isArray(item.traceIds) ? item.traceIds.slice(0, 4) : [], text: String(item.preview ?? item.normalizedPreview ?? item.text ?? "").slice(0, 160), })), domDiagnosticSamples: recentWindow.runtimeAlerts.domDiagnostics.slice(-8).map((item) => ({ seq: item.seq ?? null, ts: item.ts ?? null, promptIndex: item.promptIndex ?? null, source: item.source ?? null, diagnosticCode: item.diagnosticCode ?? null, traceId: item.traceId ?? null, httpStatus: item.httpStatus ?? null, idleSeconds: item.idleSeconds ?? null, waitingFor: item.waitingFor ?? null, lastEventLabel: item.lastEventLabel ?? null, text: String(item.preview ?? item.text ?? "").slice(0, 180), })), consoleAlertGroups: recentWindow.runtimeAlerts.consoleAlertsByPath.slice(0, 8).map((item) => ({ type: item.type ?? null, status: item.status ?? null, path: item.urlPath ?? null, count: item.count, firstAt: item.firstAt ?? null, lastAt: item.lastAt ?? null, promptIndexes: Array.isArray(item.promptIndexes) ? item.promptIndexes.slice(0, 6) : [], traceIds: Array.isArray(item.traceIds) ? item.traceIds.slice(0, 4) : [], })), consoleAlertSamples: recentWindow.runtimeAlerts.consoleAlerts.slice(-8).map((item) => ({ ts: item.ts ?? null, promptIndex: item.promptIndex ?? null, type: item.type ?? null, status: item.status ?? null, path: item.urlPath ?? null, traceId: item.traceId ?? null, text: String(item.preview ?? item.text ?? "").slice(0, 180), })), turnTimingRecentUpdateJumps: recentWindow.sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 8).map((item) => ({ columnLabel: item.columnLabel ?? item.columnId ?? null, pageRole: item.pageRole ?? null, pageId: item.pageId ?? null, pageEpoch: item.pageEpoch ?? null, promptIndex: item.promptIndex ?? null, traceId: item.traceId ?? null, fromSeq: item.fromSeq ?? null, toSeq: item.toSeq ?? null, fromTs: item.fromTs ?? null, toTs: item.toTs ?? null, fromValue: item.fromValue ?? null, toValue: item.toValue ?? null, delta: item.delta ?? null, sampleDeltaSeconds: item.sampleDeltaSeconds ?? null, allowedIncreaseSeconds: item.allowedIncreaseSeconds ?? null, excessiveIncreaseSeconds: item.excessiveIncreaseSeconds ?? null, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, })), turnTimingElapsedZeroResets: recentWindow.sampleMetrics.turnTimingElapsedZeroResets.slice(0, 8).map((item) => ({ columnLabel: item.columnLabel ?? item.columnId ?? null, pageRole: item.pageRole ?? null, pageId: item.pageId ?? null, pageEpoch: item.pageEpoch ?? null, promptIndex: item.promptIndex ?? null, traceId: item.traceId ?? null, fromSeq: item.fromSeq ?? null, toSeq: item.toSeq ?? null, fromTs: item.fromTs ?? null, toTs: item.toTs ?? null, fromValue: item.fromValue ?? null, toValue: item.toValue ?? null, delta: item.delta ?? null, anomaly: item.anomaly ?? null, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, })), turnTimingTotalElapsedForwardJumps: recentWindow.sampleMetrics.turnTimingTotalElapsedForwardJumps.slice(0, 8).map((item) => ({ columnLabel: item.columnLabel ?? item.columnId ?? null, pageRole: item.pageRole ?? null, pageId: item.pageId ?? null, pageEpoch: item.pageEpoch ?? null, promptIndex: item.promptIndex ?? null, traceId: item.traceId ?? null, fromSeq: item.fromSeq ?? null, toSeq: item.toSeq ?? null, fromTs: item.fromTs ?? null, toTs: item.toTs ?? null, fromValue: item.fromValue ?? null, toValue: item.toValue ?? null, delta: item.delta ?? null, sampleDeltaSeconds: item.sampleDeltaSeconds ?? null, allowedIncreaseSeconds: item.allowedIncreaseSeconds ?? null, excessiveIncreaseSeconds: item.excessiveIncreaseSeconds ?? null, anomaly: item.anomaly ?? null, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, })), pagePerformanceSlowApi: recentWindow.pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream !== true && Number(item.overBudgetCount ?? item.overFiveSecondCount ?? 0) > 0).slice(0, 5).map((item) => ({ path: item.path, route: item.route, sampleCount: item.sampleCount, p95Ms: item.p95Ms, maxMs: item.maxMs, overBudgetCount: item.overBudgetCount, budgetMs: item.budgetMs, overFiveSecondCount: item.overFiveSecondCount, slowSamples: item.slowSamples })), archivePagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream !== true && Number(item.overBudgetCount ?? item.overFiveSecondCount ?? 0) > 0).slice(0, 8).map((item) => ({ path: item.path, route: item.route, sampleCount: item.sampleCount, p95Ms: item.p95Ms, maxMs: item.maxMs, overBudgetCount: item.overBudgetCount, budgetMs: item.budgetMs, overFiveSecondCount: item.overFiveSecondCount, slowSamples: item.slowSamples })), pagePerformancePartialApi: recentWindow.pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream !== true && Number(item.partialOverBudgetCount ?? item.partialOverFiveSecondCount ?? 0) > 0).slice(0, 5).map((item) => ({ path: item.path, route: item.route, sampleCount: item.sampleCount, completeTimingSampleCount: item.completeTimingSampleCount, partialTimingSampleCount: item.partialTimingSampleCount, partialOverBudgetCount: item.partialOverBudgetCount, budgetMs: item.partialBudgetMs, partialOverFiveSecondCount: item.partialOverFiveSecondCount, partialSamples: item.partialSamples })), pagePerformanceSseStreams: recentWindow.pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream === true).slice(0, 5).map((item) => ({ path: item.path, route: item.route, sampleCount: item.sampleCount, streamOpenSampleCount: item.streamOpenSampleCount, streamOpenP95Ms: item.streamOpenP95Ms, streamOpenMaxMs: item.streamOpenMaxMs, streamOpenOverBudgetCount: item.streamOpenOverBudgetCount, streamOpenBudgetMs: item.streamOpenBudgetMs, streamOpenOverFiveSecondCount: item.streamOpenOverFiveSecondCount, streamLifetimeOverFiveSecondCount: item.streamLifetimeOverFiveSecondCount, slowSamples: item.slowSamples })), findings: prioritizeFindings(recentWindow.findings).slice(0, 8).map((item) => ({ kind: item.id ?? item.kind ?? item.code, severity: item.severity, count: item.count ?? item.sampleCount ?? null, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, summary: String(item.summary ?? item.message ?? "").slice(0, 180) })), traceOrderAnomalies: (recentWindow.sampleMetrics?.traceOrder?.orderAnomalies || []).slice(0, 8).map((item) => ({ sampleSeq: item.sampleSeq ?? null, sampleIndex: item.sampleIndex ?? null, timestamp: item.timestamp ?? null, pageRole: item.pageRole ?? null, traceId: item.traceId ?? null, reasons: item.reasons ?? [], previousRowIndex: item.previousRowIndex ?? null, currentRowIndex: item.currentRowIndex ?? null, previousTotalSeconds: item.previousTotalSeconds ?? null, currentTotalSeconds: item.currentTotalSeconds ?? null, previousProjectedSeq: item.previousProjectedSeq ?? null, currentProjectedSeq: item.currentProjectedSeq ?? null, previousSourceSeq: item.previousSourceSeq ?? null, currentSourceSeq: item.currentSourceSeq ?? null, previousEventSeq: item.previousEventSeq ?? null, currentEventSeq: item.currentEventSeq ?? null, previousPreview: String(item.previousPreview || "").slice(0, 120), currentPreview: String(item.currentPreview || "").slice(0, 120), })), traceCompletionNotLast: (recentWindow.sampleMetrics?.traceOrder?.completionNotLast || []).slice(0, 8).map((item) => ({ sampleSeq: item.sampleSeq ?? null, sampleIndex: item.sampleIndex ?? null, timestamp: item.timestamp ?? null, pageRole: item.pageRole ?? null, traceId: item.traceId ?? null, completionRowIndex: item.completionRowIndex ?? null, laterRowIndex: item.laterRowIndex ?? null, completionTotalSeconds: item.completionTotalSeconds ?? null, laterTotalSeconds: item.laterTotalSeconds ?? null, completionProjectedSeq: item.completionProjectedSeq ?? null, laterProjectedSeq: item.laterProjectedSeq ?? null, completionPreview: String(item.completionPreview || "").slice(0, 120), laterPreview: String(item.laterPreview || "").slice(0, 120), })), roundCompletionElapsedMismatches: (recentWindow.sampleMetrics?.codeAgentCardTiming?.roundCompletion?.elapsedMismatches || []).slice(0, 8).map((item) => ({ seq: item.seq ?? null, ts: item.ts ?? null, pageRole: item.pageRole ?? null, promptIndex: item.promptIndex ?? null, traceId: item.traceId ?? null, completionElapsedSeconds: item.completionElapsedSeconds ?? null, cardTotalElapsedSeconds: item.cardTotalElapsedSeconds ?? null, deltaSeconds: item.deltaSeconds ?? null, toleranceSeconds: item.toleranceSeconds ?? null, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, })), codeAgentCardDurationUnderreported: (recentWindow.sampleMetrics?.codeAgentCardTiming?.durationUnderreported || []).slice(0, 8).map((item) => ({ sampleIndex: item.sampleIndex ?? null, timestamp: item.timestamp ?? null, pageRole: item.pageRole ?? null, traceId: item.traceId ?? null, cardTotalElapsedSeconds: item.cardTotalElapsedSeconds ?? null, expectedElapsedSeconds: item.expectedElapsedSeconds ?? null, deltaSeconds: item.deltaSeconds ?? null, evidenceKind: item.evidenceKind ?? null, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, evidencePreview: String(item.evidencePreview || "").slice(0, 120), })), codeAgentCardDurationMismatches: (recentWindow.sampleMetrics?.codeAgentCardTiming?.durationMismatches || []).slice(0, 8).map((item) => ({ sampleIndex: item.sampleIndex ?? null, timestamp: item.timestamp ?? null, pageRole: item.pageRole ?? null, traceId: item.traceId ?? null, direction: item.direction ?? null, cardTotalElapsedSeconds: item.cardTotalElapsedSeconds ?? null, expectedElapsedSeconds: item.expectedElapsedSeconds ?? null, signedDeltaSeconds: item.signedDeltaSeconds ?? null, deltaSeconds: item.deltaSeconds ?? null, evidenceKind: item.evidenceKind ?? null, exactEvidence: item.exactEvidence === true, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, evidencePreview: String(item.evidencePreview || "").slice(0, 120), })), valuesRedacted: true, })); async function readJson(file) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } } async function readCommandState(rootDir) { const buckets = {}; for (const bucket of ["pending", "processing", "done", "failed", "abandoned"]) { buckets[bucket] = await readCommandBucket(path.join(rootDir, "commands", bucket), bucket); } return { pendingCount: buckets.pending.count, processingCount: buckets.processing.count, doneCount: buckets.done.count, failedCount: buckets.failed.count, abandonedCount: buckets.abandoned.count, pending: buckets.pending.items, processing: buckets.processing.items, failed: buckets.failed.items.slice(0, 12), abandoned: buckets.abandoned.items.slice(0, 20), summary: { backlogCount: buckets.pending.count + buckets.processing.count, oldestPendingAgeSeconds: buckets.pending.oldestAgeSeconds, oldestProcessingAgeSeconds: buckets.processing.oldestAgeSeconds, valuesRedacted: true }, valuesRedacted: true }; } async function readCommandBucket(dir, bucket) { let names = []; try { names = (await readdir(dir)).filter((name) => name.endsWith(".json")).sort(); } catch (error) { if (!error || error.code !== "ENOENT") jsonlReadIssues.push({ file: path.basename(dir), kind: "command-dir-read-error", code: error && error.code ? String(error.code) : null, errorMessage: limitText(error && error.message ? error.message : String(error), 240) }); return { bucket, count: 0, oldestAgeSeconds: null, items: [] }; } const items = []; for (const name of names.slice(0, 200)) { const file = path.join(dir, name); const parsed = await readJson(file) || {}; let meta = null; try { meta = await stat(file); } catch {} const timestamp = parsed.createdAt || parsed.abandonedAt || parsed.completedAt || parsed.failedAt || (meta ? meta.mtime.toISOString() : null); const timestampMs = Date.parse(String(timestamp || "")); items.push({ bucket, id: parsed.id || parsed.commandId || name.replace(/[.]json$/u, ""), type: parsed.type || parsed.command?.type || null, createdAt: parsed.createdAt || null, completedAt: parsed.completedAt || null, failedAt: parsed.failedAt || null, abandonedAt: parsed.abandonedAt || null, reason: parsed.reason || parsed.error?.message || null, ageSeconds: Number.isFinite(timestampMs) ? Math.max(0, Math.round((Date.now() - timestampMs) / 1000)) : null, mtime: meta ? meta.mtime.toISOString() : null, valuesRedacted: true }); } const ages = items.map((item) => item.ageSeconds).filter((value) => Number.isFinite(value)); return { bucket, count: names.length, oldestAgeSeconds: ages.length > 0 ? Math.max(...ages) : null, items }; } function buildToolFindings({ manifest, heartbeat, commandState }) { const findings = []; const diagnostics = heartbeatDiagnostics(manifest, heartbeat); if (diagnostics.heartbeatStale) { findings.push({ id: "tool-runner-heartbeat-stale", severity: "red", summary: "web-probe observe runner heartbeat is stale; treat this as observer tooling failure, not Workbench behavior", count: 1, diagnostics, valuesRedacted: true }); } if ((commandState?.pendingCount ?? 0) > 0 || (commandState?.processingCount ?? 0) > 0) { findings.push({ id: "tool-pending-commands-unconsumed", severity: "red", summary: "web-probe observe has pending/processing control commands that were not consumed by the runner", count: (commandState?.pendingCount ?? 0) + (commandState?.processingCount ?? 0), pending: (commandState?.pending ?? []).slice(0, 12), processing: (commandState?.processing ?? []).slice(0, 12), valuesRedacted: true }); } if ((commandState?.abandonedCount ?? 0) > 0) { findings.push({ id: "tool-commands-abandoned", severity: "info", summary: "web-probe observe force-stop abandoned queued commands; do not interpret these as Workbench command failures", count: commandState.abandonedCount, commands: (commandState.abandoned ?? []).slice(0, 20), valuesRedacted: true }); } if (heartbeat?.forceStop || manifest?.forceStop) { findings.push({ id: "tool-runner-force-stopped", severity: "info", summary: "web-probe observe runner was stopped by the CLI outside the command queue", count: 1, forceStop: heartbeat?.forceStop || manifest?.forceStop, valuesRedacted: true }); } return findings; } function heartbeatDiagnostics(manifest, heartbeat) { const status = String(heartbeat?.status || manifest?.status || ""); const terminal = /^(completed|failed|force-stopped|stopped|abandoned)$/u.test(status); const sampleIntervalMs = Number(manifest?.sampling?.sampleIntervalMs) || 5000; const staleAfterMs = Math.max(60000, sampleIntervalMs * 3); const updatedAt = heartbeat?.updatedAt || heartbeat?.lastSampleAt || null; const updatedMs = Date.parse(String(updatedAt || "")); const ageSeconds = Number.isFinite(updatedMs) ? Math.max(0, Math.round((Date.now() - updatedMs) / 1000)) : null; const heartbeatStale = !terminal && (!Number.isFinite(updatedMs) || Date.now() - updatedMs > staleAfterMs); return { status: status || null, terminal, updatedAt, heartbeatAgeSeconds: ageSeconds, heartbeatStale, heartbeatStaleAfterSeconds: Math.round(staleAfterMs / 1000), sampleSeq: heartbeat?.sampleSeq ?? null, commandSeq: heartbeat?.commandSeq ?? null, activeCommandId: heartbeat?.activeCommandId ?? null, valuesRedacted: true }; } function safeArchivePrefix(value) { const text = String(value || "").trim(); if (!text) return ""; if (!/^[A-Za-z0-9_.-]+$/u.test(text) || text.includes("..")) throw new Error("unsafe archive prefix: " + text); return text; } function positiveNumber(value, fallback) { const parsed = Number(value); return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } function requiredPositiveThreshold(raw, key) { const parsed = Number(raw?.[key]); if (!Number.isFinite(parsed) || parsed <= 0) { throw new Error("UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON requires positive " + key + "; configure config/hwlab-node-lanes.yaml webProbe.alertThresholds"); } return parsed; } function parseAlertThresholds(value) { if (!value) { throw new Error("UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON is required; configure config/hwlab-node-lanes.yaml webProbe.alertThresholds for the selected node/lane"); } const raw = (() => { try { return JSON.parse(value); } catch (error) { throw new Error("UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON is invalid JSON: " + (error instanceof Error ? error.message : String(error))); } })(); const sessionRailFallbackRatio = requiredPositiveThreshold(raw, "sessionRailFallbackRatio"); if (sessionRailFallbackRatio > 1) { throw new Error("UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON sessionRailFallbackRatio must be <= 1"); } return { sameOriginApiSlowMs: requiredPositiveThreshold(raw, "sameOriginApiSlowMs"), partialApiSlowMs: requiredPositiveThreshold(raw, "partialApiSlowMs"), longLivedStreamOpenSlowMs: requiredPositiveThreshold(raw, "longLivedStreamOpenSlowMs"), visibleLoadingSlowMs: requiredPositiveThreshold(raw, "visibleLoadingSlowMs"), turnTimingSampleSlackSeconds: requiredPositiveThreshold(raw, "turnTimingSampleSlackSeconds"), turnElapsedSevereTimeoutSeconds: requiredPositiveThreshold(raw, "turnElapsedSevereTimeoutSeconds"), domEvaluateTimeoutRedCount: requiredPositiveThreshold(raw, "domEvaluateTimeoutRedCount"), domEvaluateTimeoutRedWindowMs: requiredPositiveThreshold(raw, "domEvaluateTimeoutRedWindowMs"), screenshotTimeoutRedCount: requiredPositiveThreshold(raw, "screenshotTimeoutRedCount"), pageErrorRedCount: requiredPositiveThreshold(raw, "pageErrorRedCount"), uncommandedStateChangeCommandWindowMs: requiredPositiveThreshold(raw, "uncommandedStateChangeCommandWindowMs"), scrollJumpCommandWindowMs: requiredPositiveThreshold(raw, "scrollJumpCommandWindowMs"), scrollJumpFromY: requiredPositiveThreshold(raw, "scrollJumpFromY"), scrollJumpToY: requiredPositiveThreshold(raw, "scrollJumpToY"), sessionRailFallbackRatio, crossPageProjectionDivergenceRedMs: positiveNumber(raw.crossPageProjectionDivergenceRedMs, requiredPositiveThreshold(raw, "visibleLoadingSlowMs")), source: "yaml-env", }; } function parseProjectManagementConfig(value) { if (!value || value === "null") { return { enabled: false, targetPaths: [], readinessSelectors: [], naturalApiPathPrefixes: [], commandAllowlist: [], launchRoute: "", slowApiBudgetMs: 0, source: "yaml-env", valuesRedacted: true }; } const raw = (() => { try { return JSON.parse(value); } catch (error) { throw new Error("UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON is invalid JSON: " + (error instanceof Error ? error.message : String(error))); } })(); if (raw?.enabled !== true && raw?.enabled !== false) throw new Error("UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON requires boolean enabled"); if (raw.enabled !== true) return { enabled: false, targetPaths: [], readinessSelectors: [], naturalApiPathPrefixes: [], commandAllowlist: [], launchRoute: "", slowApiBudgetMs: 0, source: "yaml-env", valuesRedacted: true }; const stringList = (key) => { const list = raw?.[key]; if (!Array.isArray(list) || list.some((item) => typeof item !== "string" || item.length === 0)) throw new Error("UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON requires string[] " + key + "; configure config/hwlab-node-lanes.yaml webProbe.projectManagement"); return list; }; const slowApiBudgetMs = Number(raw?.slowApiBudgetMs); if (!Number.isFinite(slowApiBudgetMs) || slowApiBudgetMs <= 0) throw new Error("UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON requires positive slowApiBudgetMs"); const launchRoute = String(raw.launchRoute || ""); if (!launchRoute.startsWith("/")) throw new Error("UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON launchRoute must be an absolute path"); return { enabled: true, targetPaths: stringList("targetPaths"), readinessSelectors: stringList("readinessSelectors"), naturalApiPathPrefixes: stringList("naturalApiPathPrefixes"), commandAllowlist: stringList("commandAllowlist"), launchRoute, slowApiBudgetMs, source: "yaml-env", valuesRedacted: true }; } async function readJsonl(file, options = {}) { const tailLimit = Number.isFinite(Number(options.tail)) && Number(options.tail) > 0 ? Math.floor(Number(options.tail)) : 0; if (tailLimit > 0) return readJsonlTail(file, tailLimit, options); const rows = []; try { const input = createReadStream(file, { encoding: "utf8" }); const lines = createInterface({ input, crlfDelay: Infinity }); let lineNo = 0; for await (const rawLine of lines) { lineNo += 1; const line = String(rawLine || "").trim(); if (!line) continue; try { const parsed = JSON.parse(line); rows.push(typeof options.compact === "function" ? options.compact(parsed) : parsed); } catch (error) { const item = { parseError: true, lineNo, rawHash: sha256(line), errorMessage: limitText(error && error.message ? error.message : String(error), 240) }; rows.push(item); if (jsonlReadIssues.length < 50) jsonlReadIssues.push({ file: path.basename(file), kind: "parse-error", lineNo, rawHash: item.rawHash, errorMessage: item.errorMessage }); } } return rows; } catch (error) { if (error && error.code === "ENOENT") return []; if (jsonlReadIssues.length < 50) jsonlReadIssues.push({ file: path.basename(file), kind: "read-error", code: error && error.code ? String(error.code) : null, errorMessage: limitText(error && error.message ? error.message : String(error), 240) }); return []; } } async function readJsonlTail(file, limit, options = {}) { try { const lines = await readTailLines(file, limit); const rows = []; let lineNo = 0; for (const rawLine of lines) { lineNo += 1; const line = String(rawLine || "").trim(); if (!line) continue; try { const parsed = JSON.parse(line); rows.push(typeof options.compact === "function" ? options.compact(parsed) : parsed); } catch (error) { const item = { parseError: true, lineNo, tail: true, rawHash: sha256(line), errorMessage: limitText(error && error.message ? error.message : String(error), 240) }; rows.push(item); if (jsonlReadIssues.length < 50) jsonlReadIssues.push({ file: path.basename(file), kind: "parse-error", lineNo, tail: true, rawHash: item.rawHash, errorMessage: item.errorMessage }); } } return rows; } catch (error) { if (error && error.code === "ENOENT") return []; if (jsonlReadIssues.length < 50) jsonlReadIssues.push({ file: path.basename(file), kind: "read-error", tail: true, code: error && error.code ? String(error.code) : null, errorMessage: limitText(error && error.message ? error.message : String(error), 240) }); return []; } } async function readTailLines(file, limit) { const info = await stat(file); if (!info.size || limit <= 0) return []; const chunkSize = 64 * 1024; const maxBytes = Math.max(32 * 1024 * 1024, Math.min(256 * 1024 * 1024, limit * 512 * 1024)); const chunks = []; let position = info.size; let readBytes = 0; let newlineCount = 0; while (position > 0 && newlineCount <= limit && readBytes < maxBytes) { const readSize = Math.min(chunkSize, position, maxBytes - readBytes); position -= readSize; const chunk = await readFileSlice(file, position, readSize); chunks.unshift(chunk); readBytes += chunk.length; for (let index = 0; index < chunk.length; index += 1) { if (chunk[index] === 10) newlineCount += 1; } } if (position > 0 && newlineCount <= limit && jsonlReadIssues.length < 50) { jsonlReadIssues.push({ file: path.basename(file), kind: "tail-scan-truncated", limit, readBytes, fileBytes: info.size }); } const text = Buffer.concat(chunks).toString("utf8"); let lines = text.split(/\r?\n/u); if (lines.length > 0 && lines[lines.length - 1] === "") lines.pop(); if (position > 0 && lines.length > 0) lines.shift(); return lines.slice(-limit); } async function readFileSlice(file, start, length) { const chunks = []; await new Promise((resolve, reject) => { const stream = createReadStream(file, { start, end: start + length - 1 }); stream.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))); stream.on("error", reject); stream.on("end", resolve); }); return Buffer.concat(chunks); } function sampleTimeWindow(samples, paddingMs) { const times = samples .map((item) => Date.parse(String(item && item.ts || ""))) .filter((value) => Number.isFinite(value)); if (times.length === 0) return { startMs: null, endMs: null, startAt: null, endAt: null, paddingMs }; const startMs = Math.max(0, Math.min(...times) - Math.max(0, paddingMs || 0)); const endMs = Math.max(...times) + Math.max(0, paddingMs || 0); return { startMs, endMs, startAt: new Date(startMs).toISOString(), endAt: new Date(endMs).toISOString(), paddingMs }; } function filterRowsByTimeWindow(rows, window) { if (!window || !Number.isFinite(window.startMs) || !Number.isFinite(window.endMs)) return rows; return rows.filter((item) => { const value = item && (item.ts || item.observedAt || item.startedAt || item.finishedAt || item.createdAt || item.updatedAt); const ms = Date.parse(String(value || "")); if (!Number.isFinite(ms)) return true; return ms >= window.startMs && ms <= window.endMs; }); } function analysisFocusFromControl(control) { const completedNewSessions = (Array.isArray(control) ? control : []) .filter((item) => item?.type === "newSession" && item?.phase === "completed" && Number.isFinite(Date.parse(String(item.ts || "")))) .sort((a, b) => Date.parse(String(a.ts || "")) - Date.parse(String(b.ts || ""))); const latest = completedNewSessions.at(-1) ?? null; if (!latest) return { mode: "all", reason: "no-new-session-command", startAt: null, startMs: null, commandId: null, valuesRedacted: true }; const startMs = Date.parse(String(latest.ts)); return { mode: "after-new-session", reason: "latest-completed-new-session", startAt: new Date(startMs).toISOString(), startMs, commandId: latest.commandId ?? null, valuesRedacted: true }; } function applyAnalysisFocus(rows, focus, graceMs = 0) { if (!focus || !Number.isFinite(focus.startMs)) return rows; const minMs = focus.startMs - Math.max(0, Number(graceMs) || 0); return (Array.isArray(rows) ? rows : []).filter((item) => { const value = item && (item.ts || item.observedAt || item.startedAt || item.finishedAt || item.createdAt || item.updatedAt); const ms = Date.parse(String(value || "")); return Number.isFinite(ms) ? ms >= minMs : true; }); } function analysisControlWindow(sampleWindow, focus, graceMs = 0) { if (!sampleWindow || !Number.isFinite(sampleWindow.endMs)) return sampleWindow; if (!focus || !Number.isFinite(focus.startMs)) return sampleWindow; const startMs = Math.max(0, Math.min(Number(sampleWindow.startMs ?? focus.startMs), focus.startMs - Math.max(0, Number(graceMs) || 0))); return { ...sampleWindow, startMs, startAt: new Date(startMs).toISOString() }; } function compactSampleForAnalysis(sample) { if (!sample || typeof sample !== "object") return sample; return { seq: sample.seq ?? null, ts: sample.ts ?? null, reason: sample.reason ?? null, sampleGroupSeq: sample.sampleGroupSeq ?? null, pageId: sample.pageId ?? null, pageRole: sample.pageRole ?? null, commandId: sample.commandId ?? null, observerInitiated: sample.observerInitiated ?? null, url: sample.url ?? null, path: sample.path ?? null, title: sample.title ?? null, routeSessionId: sample.routeSessionId ?? null, activeSessionId: sample.activeSessionId ?? null, messages: compactDomItems(sample.messages), traceRows: compactDomItems(sample.traceRows), loadings: compactLoadingItems(sample.loadings), sessionRail: compactSessionRail(sample.sessionRail), turns: compactDomItems(sample.turns), diagnostics: compactDomItems(sample.diagnostics), composer: compactComposer(sample.composer), projectManagement: compactProjectManagementSample(sample.projectManagement), pageProvenance: compactSamplePageProvenance(sample.pageProvenance), performance: compactPerformanceItems(sample.performance) }; } function compactProjectManagementSample(value) { if (!value || typeof value !== "object") return null; return { pageKind: value.pageKind ?? null, configuredPath: value.configuredPath === true, rootVisible: value.rootVisible === true, mdtodoVisible: value.mdtodoVisible === true, sourceCount: value.sourceCount ?? null, fileCount: value.fileCount ?? null, taskCount: value.taskCount ?? null, taskRefMissingCount: value.taskRefMissingCount ?? null, selectedSourceId: value.selectedSourceId ?? null, selectedFileRef: value.selectedFileRef ?? null, selectedTaskRef: value.selectedTaskRef ?? null, selectedTaskStatus: value.selectedTaskStatus ?? null, sourceSelectVisible: value.sourceSelectVisible === true, fileSelectVisible: value.fileSelectVisible === true, sourceConfigVisible: value.sourceConfigVisible === true, taskEditorVisible: value.taskEditorVisible === true, taskBodyVisible: value.taskBodyVisible === true, taskBody: value.taskBody ?? null, reportLinkCount: value.reportLinkCount ?? 0, reportPreviewVisible: value.reportPreviewVisible === true, reportPreview: value.reportPreview ?? null, reportFullscreenVisible: value.reportFullscreenVisible === true, newTaskDraftVisible: value.newTaskDraftVisible === true, taskStatusCounts: value.taskStatusCounts && typeof value.taskStatusCounts === "object" ? value.taskStatusCounts : {}, launchButtonVisible: value.launchButtonVisible === true, launchButtonEnabled: value.launchButtonEnabled === true, launchButtonText: value.launchButtonText ?? null, blockerCount: value.blockerCount ?? 0, blockers: Array.isArray(value.blockers) ? value.blockers.slice(0, 6) : [], paneGaps: Array.isArray(value.paneGaps) ? value.paneGaps.slice(0, 8).map((item) => ({ name: item?.name ?? null, visible: item?.visible === true, widthPx: item?.widthPx ?? null, heightPx: item?.heightPx ?? null, bottomGapPx: item?.bottomGapPx ?? null, bottomGapRatio: item?.bottomGapRatio ?? null, contentNodeCount: item?.contentNodeCount ?? null, valuesRedacted: true })) : [], workbenchLinkCount: value.workbenchLinkCount ?? 0, valuesRedacted: true }; } function compactSessionRail(value) { if (!value || typeof value !== "object") return null; const items = Array.isArray(value.items) ? value.items.slice(0, 80).map((item) => ({ index: item?.index ?? null, tag: item?.tag ?? null, testId: item?.testId ?? null, role: item?.role ?? null, active: item?.active === true, sessionIdPrefix: item?.sessionIdPrefix ?? (item?.sessionId ? String(item.sessionId).slice(0, 12) : null), fallbackTitle: item?.fallbackTitle === true, titlePreview: limitText(String(item?.titlePreview || item?.titleText || ""), 180), titleHash: item?.titleHash ?? sha256(String(item?.titlePreview || item?.titleText || "")), titleBytes: item?.titleBytes ?? null, })) : []; const fallbackItems = Array.isArray(value.fallbackItems) ? value.fallbackItems.slice(0, 20).map((item) => ({ index: item?.index ?? null, active: item?.active === true, sessionIdPrefix: item?.sessionIdPrefix ?? (item?.sessionId ? String(item.sessionId).slice(0, 12) : null), titlePreview: limitText(String(item?.titlePreview || item?.titleText || ""), 180), titleHash: item?.titleHash ?? sha256(String(item?.titlePreview || item?.titleText || "")), })) : items.filter((item) => item.fallbackTitle).slice(0, 20); const visibleCount = Number(value.visibleCount ?? items.length); const fallbackTitleCount = Number(value.fallbackTitleCount ?? fallbackItems.length); return { visibleCount: Number.isFinite(visibleCount) ? visibleCount : items.length, fallbackTitleCount: Number.isFinite(fallbackTitleCount) ? fallbackTitleCount : fallbackItems.length, fallbackTitleRatio: Number.isFinite(Number(value.fallbackTitleRatio)) ? Number(value.fallbackTitleRatio) : (items.length > 0 ? Number((fallbackItems.length / items.length).toFixed(4)) : 0), items, fallbackItems, valuesRedacted: true }; } function compactComposer(value) { if (!value || typeof value !== "object") return null; return { inputPresent: value.inputPresent === true, inputDisabled: value.inputDisabled === true, warningPresent: value.warningPresent === true, submitPresent: value.submitPresent === true, submitDisabled: value.submitDisabled === true, submitAction: value.submitAction ?? null, activeStatus: value.activeStatus ?? null, valuesRedacted: true }; } function compactLoadingItems(items) { if (!Array.isArray(items)) return []; return items.map((item) => { if (!item || typeof item !== "object") return item; const rawText = String(item.text ?? item.textPreview ?? ""); return { index: item.index ?? null, tag: item.tag ?? null, testId: item.testId ?? null, role: item.role ?? null, ownerKind: item.ownerKind ?? null, ownerKey: item.ownerKey ?? null, ownerLabel: item.ownerLabel ?? null, owner: item.owner && typeof item.owner === "object" ? { tag: item.owner.tag ?? null, testId: item.owner.testId ?? null, role: item.owner.role ?? null, id: item.owner.id ?? null, className: item.owner.className ?? null, status: item.owner.status ?? null, sessionId: item.owner.sessionId ?? null, messageId: item.owner.messageId ?? null, traceId: item.owner.traceId ?? null, ariaLabel: item.owner.ariaLabel ?? null, } : null, text: limitText(rawText, 400), textPreview: limitText(String(item.textPreview ?? rawText), 240), textHash: item.textHash ?? sha256(rawText), textBytes: item.textBytes ?? Buffer.byteLength(rawText), ownerTextHash: item.ownerTextHash ?? null, ownerTextPreview: item.ownerTextPreview ? limitText(item.ownerTextPreview, 240) : null, }; }); } function compactDomItems(items) { if (!Array.isArray(items)) return []; return items.map(compactDomItem); } function compactDomItem(item) { if (!item || typeof item !== "object") return item; const rawText = String(item.text ?? item.textPreview ?? ""); const preview = String(item.textPreview ?? limitText(rawText, 240)); return { index: item.index ?? null, tag: item.tag ?? null, testId: item.testId ?? null, role: item.role ?? null, dataRole: item.dataRole ?? null, status: item.status ?? null, sessionId: item.sessionId ?? null, messageId: item.messageId ?? null, traceId: item.traceId ?? null, turnId: item.turnId ?? null, projectedSeq: Number.isFinite(Number(item.projectedSeq)) ? Number(item.projectedSeq) : null, sourceSeq: Number.isFinite(Number(item.sourceSeq)) ? Number(item.sourceSeq) : null, eventSeq: Number.isFinite(Number(item.eventSeq)) ? Number(item.eventSeq) : null, eventTimestamp: item.eventTimestamp ?? null, eventTimeText: item.eventTimeText ?? null, eventKind: item.eventKind ?? null, durationText: item.durationText ?? null, activityText: item.activityText ?? null, className: item.className ?? null, diagnosticCode: item.diagnosticCode ?? null, source: item.source ?? null, sources: Array.isArray(item.sources) ? item.sources.slice(0, 8) : undefined, text: limitText(rawText, 4000), textPreview: limitText(preview, 600), textHash: item.textHash ?? sha256(rawText), textBytes: item.textBytes ?? Buffer.byteLength(rawText) }; } function compactPerformanceItems(items) { if (!Array.isArray(items)) return []; return items.map((item) => ({ name: item?.name ?? null, initiatorType: item?.initiatorType ?? null, startTime: item?.startTime ?? null, duration: item?.duration ?? null, workerStart: item?.workerStart ?? null, redirectStart: item?.redirectStart ?? null, redirectEnd: item?.redirectEnd ?? null, fetchStart: item?.fetchStart ?? null, domainLookupStart: item?.domainLookupStart ?? null, domainLookupEnd: item?.domainLookupEnd ?? null, connectStart: item?.connectStart ?? null, connectEnd: item?.connectEnd ?? null, secureConnectionStart: item?.secureConnectionStart ?? null, requestStart: item?.requestStart ?? null, responseStart: item?.responseStart ?? null, responseEnd: item?.responseEnd ?? null, transferSize: item?.transferSize ?? null, encodedBodySize: item?.encodedBodySize ?? null, decodedBodySize: item?.decodedBodySize ?? null, nextHopProtocol: item?.nextHopProtocol ?? null, responseStatus: Number.isFinite(Number(item?.responseStatus)) ? Number(item.responseStatus) : null })); } function compactLoadingMetricsForOutput(value) { if (!value || typeof value !== "object") return null; return { summary: value.summary ?? null, longestSegments: Array.isArray(value.segments) ? value.segments.slice(0, 8) : [], owners: Array.isArray(value.owners) ? value.owners.slice(0, 8) : [], timeline: Array.isArray(value.timeline) ? value.timeline.slice(-12) : [], valuesRedacted: true }; } function compactSessionRailTitleMetricsForOutput(value) { if (!value || typeof value !== "object") return null; return { summary: value.summary ?? null, samples: Array.isArray(value.samples) ? value.samples.slice(0, 12) : [], examples: Array.isArray(value.examples) ? value.examples.slice(0, 12) : [], timeline: Array.isArray(value.timeline) ? value.timeline.slice(-12) : [], valuesRedacted: true }; } function compactSamplePageProvenance(value) { if (!value || typeof value !== "object") return null; return { pageLoadSeq: value.pageLoadSeq ?? null, reason: value.reason ?? null, observedAt: value.observedAt ?? null, urlPath: value.urlPath ?? null, documentReadyState: value.documentReadyState ?? null, timeOrigin: value.timeOrigin ?? null, httpStatus: value.httpStatus ?? null, assetFingerprint: value.assetFingerprint ?? null, scriptCount: value.scriptCount ?? null, stylesheetCount: value.stylesheetCount ?? null, metaCount: value.metaCount ?? null, scripts: Array.isArray(value.scripts) ? value.scripts.slice(0, 20) : [], stylesheets: Array.isArray(value.stylesheets) ? value.stylesheets.slice(0, 20) : [], error: value.error ?? null, valuesRedacted: true }; } function summarizeCommandFailures(control) { return control.filter((item) => item?.phase === "failed").map((item) => { const detail = item?.detail && typeof item.detail === "object" ? item.detail : {}; const error = detail?.error && typeof detail.error === "object" ? detail.error : detail; return { ts: item.ts ?? null, commandId: item.commandId ?? null, type: item.type ?? item.input?.type ?? null, source: item.source ?? null, durationMs: detail.durationMs ?? null, beforePath: urlPath(detail.beforeUrl || item.beforeUrl), afterPath: urlPath(detail.afterUrl || item.afterUrl), name: error?.name ?? null, message: limitText(error?.message ?? detail?.message ?? "", 240), failureKind: error?.failureKind ?? detail?.failureKind ?? null, failureSampleOk: detail?.failureSample?.ok === true, sampleSeq: detail?.failureSample?.sampleSeq ?? null, valuesRedacted: true }; }); } function buildProjectManagementReport(samples, control, network, pagePerformance, config) { const enabled = config?.enabled === true; const targetPathSamples = (samples || []).filter((sample) => enabled && config.targetPaths.some((target) => String(sample?.path || "").startsWith(target))); const projectSamples = (samples || []).filter((sample) => sample?.projectManagement && typeof sample.projectManagement === "object"); const latest = projectSamples[projectSamples.length - 1] || null; const latestProject = latest?.projectManagement || null; const pageKindCounts = countBy(projectSamples.map((sample) => sample.projectManagement?.pageKind).filter(Boolean)); const latestTaskStatusCounts = latestProject?.taskStatusCounts && typeof latestProject.taskStatusCounts === "object" ? latestProject.taskStatusCounts : {}; const commandRows = projectManagementCommandRows(control, config); const launchCommands = commandRows.filter((item) => item.type === "launchWorkbenchFromTask" || item.type === "launchWorkbenchFromMdtodo"); const launchSuccess = launchCommands.filter((item) => item.phase === "completed" && Number(item.launchStatus ?? 0) >= 200 && Number(item.launchStatus ?? 0) < 300); const launchFailed = launchCommands.filter((item) => item.phase === "failed" || Number(item.launchStatus ?? 200) >= 400); const projectApiEvents = projectManagementNetworkRows(network, config); const projectApiResponses = projectApiEvents.filter((item) => item.type === "response"); const projectApiFailures = projectApiResponses.filter((item) => Number(item.status ?? 0) >= 400); const projectApiFailedRequests = projectApiEvents.filter((item) => item.type === "requestfailed"); const projectApiByPath = groupProjectApiEvents(projectApiEvents); const projectApiPerformance = projectManagementPerformanceRows(pagePerformance, config); const slowProjectApiPerformance = projectApiPerformance.filter((item) => Number(item.overBudgetCount ?? 0) > 0 || Number(item.p95Ms ?? 0) > Number(config?.slowApiBudgetMs ?? 0)); const selectedTaskSamples = projectSamples.filter((sample) => sample.projectManagement?.selectedTaskRef?.hash); const launchEnabledSamples = projectSamples.filter((sample) => sample.projectManagement?.launchButtonEnabled === true); const launchVisibleSamples = projectSamples.filter((sample) => sample.projectManagement?.launchButtonVisible === true); const mdtodoSamples = projectSamples.filter((sample) => sample.projectManagement?.pageKind === "project-management-mdtodo"); const selectedFileLabelBadSamples = projectSamples.filter((sample) => sample.projectManagement?.selectedFileLabel && sample.projectManagement?.selectedFileLabelLooksDirect === false); const suspiciousFileLabelSamples = projectSamples.filter((sample) => Number(sample.projectManagement?.fileOptionSuspiciousLabelCount ?? 0) > 0); const bodyVisibleSamples = selectedTaskSamples.filter((sample) => sample.projectManagement?.taskBodyVisible === true && Number(sample.projectManagement?.taskBody?.textBytes ?? 0) > 0); const reportLinkSamples = projectSamples.filter((sample) => Number(sample.projectManagement?.reportLinkCount ?? 0) > 0); const reportPreviewSamples = projectSamples.filter((sample) => sample.projectManagement?.reportPreviewVisible === true && Number(sample.projectManagement?.reportPreview?.textBytes ?? 0) > 0); const reportFullscreenSamples = projectSamples.filter((sample) => sample.projectManagement?.reportFullscreenVisible === true); const hwpodBlockerSamples = projectManagementHwpodBlockerRows(projectSamples); const projectionReportSamples = projectManagementProjectionReportRows(projectSamples); const hwpodApiFailures = projectManagementHwpodApiFailureRows(projectApiFailures); const paneGapRows = projectManagementPaneGapRows(projectSamples); const severePaneGapSamples = paneGapRows.actionable; const ignoredPaneGapSamples = paneGapRows.ignored; const previewCommands = commandRows.filter((item) => item.type === "openMdtodoReportPreview" || item.type === "toggleMdtodoReportFullscreen"); const launchNonEmpty = launchSuccess.filter((item) => item.chatObserved === true && (Number(item.workbenchMessageCount ?? 0) > 0 || Number(item.workbenchTraceRowCount ?? 0) > 0)); const launchEmpty = launchSuccess.filter((item) => item.chatObserved !== true || (Number(item.workbenchMessageCount ?? 0) === 0 && Number(item.workbenchTraceRowCount ?? 0) === 0)); const minMdtodoTaskCount = minNumber(mdtodoSamples.map((sample) => sample.projectManagement?.taskCount)); const maxMdtodoTaskCount = maxNumber(mdtodoSamples.map((sample) => sample.projectManagement?.taskCount)); return { enabled, config: config || null, summary: { enabled, targetPathSampleCount: targetPathSamples.length, projectSampleCount: projectSamples.length, mdtodoSampleCount: mdtodoSamples.length, pageKindCounts, latestPageKind: latestProject?.pageKind ?? null, latestPath: latest?.path ?? null, latestSeq: latest?.seq ?? null, latestTs: latest?.ts ?? null, latestSourceCount: latestProject?.sourceCount ?? null, latestFileCount: latestProject?.fileCount ?? null, latestTaskCount: latestProject?.taskCount ?? null, maxSourceCount: maxNumber(projectSamples.map((sample) => sample.projectManagement?.sourceCount)), maxFileCount: maxNumber(projectSamples.map((sample) => sample.projectManagement?.fileCount)), maxTaskCount: maxNumber(projectSamples.map((sample) => sample.projectManagement?.taskCount)), taskRefMissingMax: maxNumber(projectSamples.map((sample) => sample.projectManagement?.taskRefMissingCount)), latestSelectedTaskRefHash: latestProject?.selectedTaskRef?.hash ?? null, latestSelectedTaskRefPreview: latestProject?.selectedTaskRef?.preview ?? null, latestSelectedFileLabelPreview: latestProject?.selectedFileLabel?.textPreview ?? null, latestSelectedFileLabelLooksDirect: latestProject?.selectedFileLabelLooksDirect ?? null, selectedFileLabelBadSampleCount: selectedFileLabelBadSamples.length, fileOptionSuspiciousLabelSampleCount: suspiciousFileLabelSamples.length, maxFileOptionSuspiciousLabelCount: maxNumber(projectSamples.map((sample) => sample.projectManagement?.fileOptionSuspiciousLabelCount)), latestSelectedTaskStatus: latestProject?.selectedTaskStatus ?? null, latestTaskStatusCounts, selectedTaskBodyVisibleSamples: bodyVisibleSamples.length, reportLinkVisibleSamples: reportLinkSamples.length, maxReportLinkCount: maxNumber(projectSamples.map((sample) => sample.projectManagement?.reportLinkCount)), reportPreviewVisibleSamples: reportPreviewSamples.length, reportFullscreenVisibleSamples: reportFullscreenSamples.length, hwpodBlockerSampleCount: hwpodBlockerSamples.length, projectionReportSampleCount: projectionReportSamples.length, hwpodApiFailureCount: hwpodApiFailures.length, severePaneGapSampleCount: severePaneGapSamples.length, ignoredPaneGapSampleCount: ignoredPaneGapSamples.length, maxPaneBottomGapPx: maxNumber(severePaneGapSamples.map((item) => item.maxBottomGapPx)), maxPaneBottomGapRatio: maxNumber(severePaneGapSamples.map((item) => item.maxBottomGapRatio)), launchButtonVisibleSamples: launchVisibleSamples.length, launchButtonEnabledSamples: launchEnabledSamples.length, launchButtonDisabledSamples: Math.max(0, launchVisibleSamples.length - launchEnabledSamples.length), latestWorkbenchLinkCount: latestProject?.workbenchLinkCount ?? null, maxWorkbenchLinkCount: maxNumber(projectSamples.map((sample) => sample.projectManagement?.workbenchLinkCount)), maxBlockerCount: maxNumber(projectSamples.map((sample) => sample.projectManagement?.blockerCount)), selectedTaskSampleCount: selectedTaskSamples.length, projectCommandCount: commandRows.length, launchCommandCount: launchCommands.length, launchSuccessCount: launchSuccess.length, launchFailureCount: launchFailed.length, launchNonEmptyCount: launchNonEmpty.length, launchEmptyCount: launchEmpty.length, launchWithOtelTraceHeaderCount: launchSuccess.filter((item) => item.otelTraceId).length, reportPreviewCommandCount: previewCommands.length, mdtodoTaskCountMin: minMdtodoTaskCount, mdtodoTaskCountMax: maxMdtodoTaskCount, projectApiEventCount: projectApiEvents.length, projectApiResponseCount: projectApiResponses.length, projectApiFailureCount: projectApiFailures.length, projectApiRequestFailedCount: projectApiFailedRequests.length, projectApiSlowPathCount: slowProjectApiPerformance.length, slowApiBudgetMs: config?.slowApiBudgetMs ?? null, valuesRedacted: true }, latest: latestProject, samples: projectSamples.slice(-80).map((sample) => ({ seq: sample.seq ?? null, ts: sample.ts ?? null, pageRole: sample.pageRole ?? null, path: sample.path ?? null, pageKind: sample.projectManagement?.pageKind ?? null, sourceCount: sample.projectManagement?.sourceCount ?? null, fileCount: sample.projectManagement?.fileCount ?? null, taskCount: sample.projectManagement?.taskCount ?? null, taskRefMissingCount: sample.projectManagement?.taskRefMissingCount ?? null, selectedTaskRefHash: sample.projectManagement?.selectedTaskRef?.hash ?? null, selectedFileLabelPreview: sample.projectManagement?.selectedFileLabel?.textPreview ?? null, selectedFileLabelLooksDirect: sample.projectManagement?.selectedFileLabelLooksDirect ?? null, fileOptionSuspiciousLabelCount: sample.projectManagement?.fileOptionSuspiciousLabelCount ?? 0, selectedTaskStatus: sample.projectManagement?.selectedTaskStatus ?? null, taskBodyVisible: sample.projectManagement?.taskBodyVisible === true, taskBodyBytes: sample.projectManagement?.taskBody?.textBytes ?? 0, reportLinkCount: sample.projectManagement?.reportLinkCount ?? 0, reportPreviewVisible: sample.projectManagement?.reportPreviewVisible === true, reportPreviewBytes: sample.projectManagement?.reportPreview?.textBytes ?? 0, reportFullscreenVisible: sample.projectManagement?.reportFullscreenVisible === true, paneGaps: Array.isArray(sample.projectManagement?.paneGaps) ? sample.projectManagement.paneGaps.slice(0, 4) : [], launchButtonVisible: sample.projectManagement?.launchButtonVisible === true, launchButtonEnabled: sample.projectManagement?.launchButtonEnabled === true, blockerCount: sample.projectManagement?.blockerCount ?? 0, workbenchLinkCount: sample.projectManagement?.workbenchLinkCount ?? 0, valuesRedacted: true })), targetPathWithoutProjectSummary: targetPathSamples.filter((sample) => !sample.projectManagement).slice(0, 20).map(ref), commands: commandRows, launchCommands, projectApiByPath, projectApiFailures: projectApiFailures.slice(0, 40), projectApiRequestFailed: projectApiFailedRequests.slice(0, 40), hwpodBlockerSamples: hwpodBlockerSamples.slice(0, 40), projectionReportSamples: projectionReportSamples.slice(0, 40), hwpodApiFailures: hwpodApiFailures.slice(0, 40), severePaneGapSamples: severePaneGapSamples.slice(0, 40), ignoredPaneGapSamples: ignoredPaneGapSamples.slice(0, 40), projectApiPerformance, slowProjectApiPerformance, valuesRedacted: true }; } function compactProjectManagementForOutput(report) { if (!report || typeof report !== "object") return null; const compactCommand = (item) => ({ ts: item?.ts ?? null, phase: item?.phase ?? null, type: item?.type ?? null, commandId: item?.commandId ?? null, afterPath: item?.afterPath ?? null, launchStatus: item?.launchStatus ?? null, sessionId: item?.sessionId ?? null, workbenchUrl: item?.workbenchUrl ?? null, otelTraceId: item?.otelTraceId ?? null, chatObserved: item?.chatObserved ?? null, chatStatus: item?.chatStatus ?? null, chatTraceId: item?.chatTraceId ?? null, workbenchMessageCount: item?.workbenchMessageCount ?? null, workbenchTraceRowCount: item?.workbenchTraceRowCount ?? null, contractVersion: item?.contractVersion ?? null, selectedTaskRefHash: item?.selectedTaskRefHash ?? null, errorMessageHash: item?.errorMessageHash ?? null, message: item?.message ? limitText(item.message, 180) : null, valuesRedacted: true }); const compactApiGroup = (item) => ({ method: item?.method ?? null, path: item?.path ?? item?.urlPath ?? null, status: item?.status ?? null, type: item?.type ?? null, count: item?.count ?? item?.sampleCount ?? null, firstAt: item?.firstAt ?? null, lastAt: item?.lastAt ?? null, failureKinds: Array.isArray(item?.failureKinds) ? item.failureKinds.slice(0, 4) : [], valuesRedacted: true }); const compactSlowSample = (item) => ({ ts: item?.ts ?? null, seq: item?.seq ?? null, path: item?.path ?? item?.rawPath ?? null, durationMs: item?.durationMs ?? null, requestToResponseStartMs: item?.requestToResponseStartMs ?? item?.streamOpenMs ?? null, responseTransferMs: item?.responseTransferMs ?? null, timingStatus: item?.timingStatus ?? null, initiatorType: item?.initiatorType ?? null, nextHopProtocol: item?.nextHopProtocol ?? null, serverTimingNames: Array.isArray(item?.serverTimingNames) ? item.serverTimingNames.slice(0, 4) : [], otelTraceId: item?.otelTraceId ?? null, valuesRedacted: true }); const compactPerformance = (item) => ({ path: item?.path ?? item?.route ?? null, sampleCount: item?.sampleCount ?? null, p95Ms: item?.p95Ms ?? item?.p95 ?? null, maxMs: item?.maxMs ?? item?.max ?? null, budgetMs: item?.projectSlowBudgetMs ?? item?.budgetMs ?? report.summary?.slowApiBudgetMs ?? null, overBudgetCount: item?.overBudgetCount ?? item?.overFiveSecondCount ?? null, slowSamples: Array.isArray(item?.slowSamples) ? item.slowSamples.slice(0, 3).map(compactSlowSample) : [], valuesRedacted: true }); const compactSample = (item) => ({ seq: item?.seq ?? null, ts: item?.ts ?? null, pageRole: item?.pageRole ?? null, path: item?.path ?? null, selectedTaskRefHash: item?.selectedTaskRefHash ?? null, selectedFileLabelPreview: item?.selectedFileLabelPreview ?? null, pageKind: item?.pageKind ?? null, reason: item?.reason ?? null, severePaneCount: item?.severePaneCount ?? null, maxBottomGapPx: item?.maxBottomGapPx ?? null, maxBottomGapRatio: item?.maxBottomGapRatio ?? null, paneGaps: Array.isArray(item?.paneGaps) ? item.paneGaps.slice(0, 4) : undefined, valuesRedacted: true }); return { summary: report.summary ?? null, samples: Array.isArray(report.samples) ? report.samples.slice(-8) : [], commands: Array.isArray(report.commands) ? report.commands.slice(-8).map(compactCommand) : [], launchCommands: Array.isArray(report.launchCommands) ? report.launchCommands.slice(-8).map(compactCommand) : [], projectApiByPath: Array.isArray(report.projectApiByPath) ? report.projectApiByPath.slice(0, 8).map(compactApiGroup) : [], hwpodBlockerSamples: Array.isArray(report.hwpodBlockerSamples) ? report.hwpodBlockerSamples.slice(0, 8).map(compactSample) : [], projectionReportSamples: Array.isArray(report.projectionReportSamples) ? report.projectionReportSamples.slice(0, 8).map(compactSample) : [], hwpodApiFailures: Array.isArray(report.hwpodApiFailures) ? report.hwpodApiFailures.slice(0, 8).map(compactApiGroup) : [], severePaneGapSamples: Array.isArray(report.severePaneGapSamples) ? report.severePaneGapSamples.slice(0, 8).map(compactSample) : [], ignoredPaneGapSamples: Array.isArray(report.ignoredPaneGapSamples) ? report.ignoredPaneGapSamples.slice(0, 8).map(compactSample) : [], projectApiPerformance: Array.isArray(report.projectApiPerformance) ? report.projectApiPerformance.slice(0, 8).map(compactPerformance) : [], slowProjectApiPerformance: Array.isArray(report.slowProjectApiPerformance) ? report.slowProjectApiPerformance.slice(0, 8).map(compactPerformance) : [], valuesRedacted: true }; } function projectManagementCommandRows(control, config) { const allowed = new Set(config?.commandAllowlist || []); const mdtodoCommandTypes = new Set(["gotoProjectMdtodo", "openMdtodoSourceConfig", "configureMdtodoHwpodSource", "probeMdtodoSource", "reindexMdtodoSource", "expandMdtodoTask", "openMdtodoReportPreview", "toggleMdtodoReportFullscreen", "editMdtodoTaskInline", "editMdtodoTaskTitle", "editMdtodoTaskBody", "toggleMdtodoTaskStatus", "addMdtodoRootTask", "addMdtodoSubTask", "continueMdtodoTask", "deleteMdtodoTask", "launchWorkbenchFromMdtodo"]); return (control || []) .filter((item) => allowed.has(item?.type) || mdtodoCommandTypes.has(item?.type) || String(item?.type || "").startsWith("selectMdtodo") || item?.type === "selectProjectSource" || item?.type === "launchWorkbenchFromTask") .filter((item) => item.phase === "completed" || item.phase === "failed") .map((item) => { const detail = item.detail && typeof item.detail === "object" ? item.detail : {}; const error = detail.error && typeof detail.error === "object" ? detail.error : {}; return { ts: item.ts ?? null, phase: item.phase ?? null, type: item.type ?? null, commandId: item.commandId ?? null, afterPath: urlPath(item.afterUrl), launchStatus: detail.launchStatus ?? error.details?.launchStatus ?? null, sessionId: detail.sessionId ?? error.details?.sessionId ?? null, workbenchUrl: detail.workbenchUrl ?? error.details?.workbenchUrl ?? null, otelTraceId: detail.otelTraceId ?? error.details?.otelTraceId ?? null, chatObserved: detail.chatObserved ?? error.details?.chatObserved ?? null, chatStatus: detail.chatStatus ?? error.details?.chatStatus ?? null, chatSessionId: detail.chatSessionId ?? error.details?.chatSessionId ?? null, chatTraceId: detail.chatTraceId ?? error.details?.chatTraceId ?? null, chatOtelTraceId: detail.chatOtelTraceId ?? error.details?.chatOtelTraceId ?? null, workbenchMessageCount: detail.workbenchSnapshot?.messageCount ?? error.details?.workbenchSnapshot?.messageCount ?? null, workbenchTraceRowCount: detail.workbenchSnapshot?.traceRowCount ?? error.details?.workbenchSnapshot?.traceRowCount ?? null, workbenchComposerReady: detail.workbenchSnapshot?.composerReady ?? error.details?.workbenchSnapshot?.composerReady ?? null, contractVersion: detail.contractVersion ?? error.details?.contractVersion ?? null, selectedTaskRefHash: detail.selectedTask?.hash ?? detail.projectBeforeClick?.selectedTaskRef?.hash ?? null, errorName: error.name ?? null, errorMessageHash: error.message ? sha256(error.message) : null, message: error.message ? limitText(error.message, 180) : null, valuesRedacted: true }; }); } function projectManagementNetworkRows(network, config) { const prefixes = config?.naturalApiPathPrefixes || []; return (network || []) .filter((item) => item?.observerInitiated !== true) .map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, method: String(item.method || "GET").toUpperCase(), status: Number.isFinite(Number(item.status)) ? Number(item.status) : null, path: urlPath(item.url), failureKind: item.failure ? limitText(item.failure, 120) : null, valuesRedacted: true })) .filter((item) => prefixes.some((prefix) => String(item.path || "").startsWith(prefix))); } function groupProjectApiEvents(events) { const groups = new Map(); for (const item of events || []) { const key = [item.method, item.path, item.status ?? "-", item.type].join(" "); const existing = groups.get(key) || { method: item.method, path: item.path, status: item.status, type: item.type, count: 0, firstAt: item.ts, lastAt: item.ts, failureKinds: [], valuesRedacted: true }; existing.count += 1; existing.lastAt = item.ts; if (item.failureKind && !existing.failureKinds.includes(item.failureKind)) existing.failureKinds.push(item.failureKind); groups.set(key, existing); } return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(a.path).localeCompare(String(b.path))); } function projectManagementPerformanceRows(pagePerformance, config) { const prefixes = config?.naturalApiPathPrefixes || []; const rows = Array.isArray(pagePerformance?.sameOriginApiByPath) ? pagePerformance.sameOriginApiByPath : []; return rows .filter((item) => prefixes.some((prefix) => String(item?.path || "").startsWith(prefix))) .map((item) => ({ ...item, projectSlowBudgetMs: config?.slowApiBudgetMs ?? null })); } function projectManagementDigestText(value) { if (!value || typeof value !== "object") return ""; return String(value.textPreview ?? value.preview ?? value.text ?? "").trim(); } function projectManagementSampleRef(sample) { return { seq: sample?.seq ?? null, ts: sample?.ts ?? null, pageRole: sample?.pageRole ?? null, path: sample?.path ?? null, pageKind: sample?.projectManagement?.pageKind ?? null, selectedTaskRefHash: sample?.projectManagement?.selectedTaskRef?.hash ?? null, selectedFileLabelPreview: sample?.projectManagement?.selectedFileLabel?.textPreview ?? null, valuesRedacted: true }; } function projectManagementHwpodBlockerRows(projectSamples) { const pattern = /(?:no outbound WebSocket hwpod-node|HWLAB_HWPOD_NODE_OPS_URL|hwpod-node-ops contract)/iu; const rows = []; for (const sample of projectSamples || []) { const blockers = Array.isArray(sample?.projectManagement?.blockers) ? sample.projectManagement.blockers : []; const matched = blockers .filter((item) => pattern.test(projectManagementDigestText(item))) .map((item) => ({ index: item?.index ?? null, testId: item?.testId ?? null, role: item?.role ?? null, textHash: item?.textHash ?? null, textPreview: item?.textPreview ?? null, valuesRedacted: true })); if (matched.length > 0) rows.push({ ...projectManagementSampleRef(sample), blockers: matched.slice(0, 4), valuesRedacted: true }); } return rows; } function projectManagementProjectionReportRows(projectSamples) { const pattern = /(?:报告索引待刷新|projection-only|任务投影确认存在报告链接)/iu; return (projectSamples || []) .filter((sample) => pattern.test(projectManagementDigestText(sample?.projectManagement?.reportPreview))) .map((sample) => ({ ...projectManagementSampleRef(sample), reportPreviewHash: sample?.projectManagement?.reportPreview?.textHash ?? null, reportPreviewPreview: sample?.projectManagement?.reportPreview?.textPreview ?? null, reportPreviewBytes: sample?.projectManagement?.reportPreview?.textBytes ?? null, valuesRedacted: true })); } function projectManagementHwpodApiFailureRows(projectApiFailures) { const pattern = /^\/v1\/project-management\/mdtodo\/(?:task-detail|report-preview)\b/u; return (projectApiFailures || []) .filter((item) => pattern.test(String(item?.path || "")) && Number(item?.status ?? 0) >= 500) .map((item) => ({ ts: item?.ts ?? null, method: item?.method ?? null, path: item?.path ?? null, status: item?.status ?? null, type: item?.type ?? null, failureKind: item?.failureKind ?? null, valuesRedacted: true })); } function projectManagementPaneGapRows(projectSamples) { const actionable = []; const ignored = []; for (const sample of projectSamples || []) { const paneGaps = Array.isArray(sample?.projectManagement?.paneGaps) ? sample.projectManagement.paneGaps : []; const severeGaps = paneGaps .filter((item) => item?.visible === true) .filter((item) => { const bottomGapPx = Number(item?.bottomGapPx ?? 0); const bottomGapRatio = Number(item?.bottomGapRatio ?? 0); const heightPx = Number(item?.heightPx ?? 0); return heightPx >= 120 && bottomGapPx >= 180 && bottomGapRatio >= 0.28; }) .map((item) => ({ name: item?.name ?? null, widthPx: item?.widthPx ?? null, heightPx: item?.heightPx ?? null, bottomGapPx: item?.bottomGapPx ?? null, bottomGapRatio: item?.bottomGapRatio ?? null, contentNodeCount: item?.contentNodeCount ?? null, valuesRedacted: true })); if (severeGaps.length === 0) continue; const maxGapPx = maxNumber(severeGaps.map((item) => item.bottomGapPx)); const maxGapRatio = maxNumber(severeGaps.map((item) => item.bottomGapRatio)); const multiPane = severeGaps.length >= 2; const singleExtreme = maxGapPx >= 240 && maxGapRatio >= 0.45; if (!multiPane && !singleExtreme) continue; const selectedTaskRefHash = sample?.projectManagement?.selectedTaskRef?.hash ?? null; const isMdtodo = sample?.projectManagement?.pageKind === "project-management-mdtodo"; const isInitialEmptyDetail = isMdtodo && !selectedTaskRefHash; const row = { ...projectManagementSampleRef(sample), severePaneCount: severeGaps.length, maxBottomGapPx: maxGapPx, maxBottomGapRatio: maxGapRatio, paneGaps: severeGaps.slice(0, 4), valuesRedacted: true }; if (isInitialEmptyDetail) { ignored.push({ ...row, reason: "mdtodo-initial-empty-detail-no-selected-task", valuesRedacted: true }); continue; } actionable.push(row); } return { actionable, ignored, valuesRedacted: true }; } function buildProjectManagementFindings(report) { if (!report?.enabled) return []; const findings = []; const summary = report.summary || {}; if (Number(summary.targetPathSampleCount ?? 0) > 0 && Number(summary.projectSampleCount ?? 0) === 0) { findings.push({ id: "project-management-route-not-ready", severity: "red", summary: "project management target path was sampled but no project-management DOM summary was detected", count: summary.targetPathSampleCount, samples: report.targetPathWithoutProjectSummary, valuesRedacted: true }); } if (Number(summary.taskRefMissingMax ?? 0) > 0) { findings.push({ id: "mdtodo-taskref-missing", severity: "red", summary: "mdtodo task rows were visible without stable data-task-ref; Workbench launch must bind by opaque public task id", count: summary.taskRefMissingMax, samples: report.samples.filter((item) => Number(item.taskRefMissingCount ?? 0) > 0).slice(0, 20), valuesRedacted: true }); } if (Number(summary.mdtodoSampleCount ?? 0) > 0 && Number(summary.latestTaskCount ?? 0) > 0 && Number(summary.launchButtonEnabledSamples ?? 0) === 0) { findings.push({ id: "workbench-launch-button-unavailable", severity: "red", summary: "mdtodo tasks were sampled but the Workbench launch button was never enabled", count: summary.mdtodoSampleCount, latest: report.latest, valuesRedacted: true }); } if (Number(summary.selectedFileLabelBadSampleCount ?? 0) > 0 || summary.latestSelectedFileLabelLooksDirect === false) { findings.push({ id: "mdtodo-file-label-not-filename", severity: "red", summary: "MDTODO file dropdown selected label is not a direct markdown filename", count: summary.selectedFileLabelBadSampleCount, latestSelectedFileLabelPreview: summary.latestSelectedFileLabelPreview, samples: report.samples.filter((item) => item.selectedFileLabelLooksDirect === false).slice(0, 12), valuesRedacted: true }); } if (Number(summary.maxFileOptionSuspiciousLabelCount ?? 0) > 0) { findings.push({ id: "mdtodo-nondirect-files-visible", severity: "red", summary: "MDTODO file dropdown includes non-direct or report-like markdown labels; docs/MDTODO discovery must be direct files only", count: summary.maxFileOptionSuspiciousLabelCount, samples: report.samples.filter((item) => Number(item.fileOptionSuspiciousLabelCount ?? 0) > 0).slice(0, 12), valuesRedacted: true }); } if (Number(summary.selectedTaskSampleCount ?? 0) > 0 && Number(summary.selectedTaskBodyVisibleSamples ?? 0) === 0) { findings.push({ id: "mdtodo-task-body-not-visible", severity: "red", summary: "selected MDTODO task was sampled but no rendered task body was visible", count: summary.selectedTaskSampleCount, samples: report.samples.filter((item) => item.selectedTaskRefHash).slice(-12), valuesRedacted: true }); } if (Number(summary.hwpodBlockerSampleCount ?? 0) > 0) { findings.push({ id: "mdtodo-hwpod-node-disconnected", severity: "red", summary: "MDTODO surfaced the hwpod-node disconnected / HWLAB_HWPOD_NODE_OPS_URL fallback blocker", count: summary.hwpodBlockerSampleCount, samples: report.hwpodBlockerSamples.slice(0, 12), valuesRedacted: true }); } if (Number(summary.projectionReportSampleCount ?? 0) > 0) { findings.push({ id: "mdtodo-report-projection-only", severity: "red", summary: "MDTODO report preview is projection-only instead of opening the full markdown report from the HWPOD source", count: summary.projectionReportSampleCount, samples: report.projectionReportSamples.slice(0, 12), valuesRedacted: true }); } if (Number(summary.maxReportLinkCount ?? 0) > 0 && Number(summary.reportPreviewVisibleSamples ?? 0) === 0) { const severity = Number(summary.reportPreviewCommandCount ?? 0) > 0 ? "red" : "amber"; findings.push({ id: "mdtodo-report-preview-missing", severity, summary: "MDTODO report links were visible but no markdown report preview was sampled", count: summary.maxReportLinkCount, previewCommandCount: summary.reportPreviewCommandCount, samples: report.samples.filter((item) => Number(item.reportLinkCount ?? 0) > 0).slice(-12), valuesRedacted: true }); } if (Number(summary.reportPreviewCommandCount ?? 0) > 0 && Number(summary.reportFullscreenVisibleSamples ?? 0) === 0 && report.commands.some((item) => item.type === "toggleMdtodoReportFullscreen" && item.phase === "completed")) { findings.push({ id: "mdtodo-report-fullscreen-missing", severity: "red", summary: "toggleMdtodoReportFullscreen command completed but fullscreen report dialog was never sampled", count: summary.reportPreviewCommandCount, commands: report.commands.filter((item) => item.type === "toggleMdtodoReportFullscreen").slice(-8), valuesRedacted: true }); } if (Number(summary.launchEmptyCount ?? 0) > 0) { findings.push({ id: "mdtodo-workbench-launch-empty", severity: "red", summary: "MDTODO Workbench launch created a session without observing agent chat or visible message/trace content", count: summary.launchEmptyCount, commands: report.launchCommands.filter((item) => item.chatObserved !== true || (Number(item.workbenchMessageCount ?? 0) === 0 && Number(item.workbenchTraceRowCount ?? 0) === 0)).slice(0, 12), valuesRedacted: true }); } if (Number(summary.mdtodoTaskCountMin ?? 0) > 0 && Number(summary.mdtodoTaskCountMax ?? 0) > 0 && (Number(summary.mdtodoTaskCountMax) - Number(summary.mdtodoTaskCountMin) >= 10 || Number(summary.mdtodoTaskCountMax) / Math.max(1, Number(summary.mdtodoTaskCountMin)) >= 2)) { findings.push({ id: "mdtodo-task-count-diverged", severity: "amber", summary: "MDTODO task count varied sharply during observation; compare control commands and observer samples for projection divergence", minTaskCount: summary.mdtodoTaskCountMin, maxTaskCount: summary.mdtodoTaskCountMax, samples: report.samples.slice(-20), valuesRedacted: true }); } if (Number(summary.severePaneGapSampleCount ?? 0) > 0) { findings.push({ id: "mdtodo-pane-bottom-gap", severity: "red", summary: "MDTODO task tree, main detail, or report sidebar left large unused bottom gaps in actionable selected-task samples", count: summary.severePaneGapSampleCount, ignoredInitialEmptyDetailCount: summary.ignoredPaneGapSampleCount, maxBottomGapPx: summary.maxPaneBottomGapPx, maxBottomGapRatio: summary.maxPaneBottomGapRatio, samples: report.severePaneGapSamples.slice(0, 12), valuesRedacted: true }); } if (Number(summary.hwpodApiFailureCount ?? 0) > 0) { findings.push({ id: "project-management-hwpod-api-failed", severity: "red", summary: "HWPOD-backed MDTODO task detail or report preview API returned a server error during natural page use", count: summary.hwpodApiFailureCount, failures: report.hwpodApiFailures.slice(0, 12), valuesRedacted: true }); } if (Number(summary.projectApiFailureCount ?? 0) > 0 || Number(summary.projectApiRequestFailedCount ?? 0) > 0) { findings.push({ id: "project-management-api-failed", severity: "amber", summary: "natural project-management or Workbench launch API requests failed during observation", count: Number(summary.projectApiFailureCount ?? 0) + Number(summary.projectApiRequestFailedCount ?? 0), groups: report.projectApiByPath.slice(0, 12), valuesRedacted: true }); } if (Number(summary.projectApiSlowPathCount ?? 0) > 0) { findings.push({ id: "project-management-api-slow", severity: "red", summary: "project-management API resource timing exceeded YAML projectManagement.slowApiBudgetMs", count: summary.projectApiSlowPathCount, budgetMs: summary.slowApiBudgetMs, groups: report.slowProjectApiPerformance.slice(0, 12), valuesRedacted: true }); } if (Number(summary.launchFailureCount ?? 0) > 0) { findings.push({ id: "mdtodo-workbench-launch-failed", severity: "red", summary: "MDTODO Workbench launch command failed or returned an HTTP error", count: summary.launchFailureCount, commands: report.launchCommands.filter((item) => item.phase === "failed" || Number(item.launchStatus ?? 200) >= 400).slice(0, 12), valuesRedacted: true }); } if (Number(summary.launchSuccessCount ?? 0) > 0 && Number(summary.launchWithOtelTraceHeaderCount ?? 0) === 0) { findings.push({ id: "mdtodo-workbench-launch-otel-trace-missing", severity: "amber", summary: "Workbench launch succeeded but no x-hwlab-otel-trace-id header was captured for Tempo drill-down", count: summary.launchSuccessCount, commands: report.launchCommands.slice(0, 12), valuesRedacted: true }); } return findings; } function countBy(values) { const out = {}; for (const value of values || []) out[value] = (out[value] || 0) + 1; return out; } function maxNumber(values) { const numeric = (values || []).map((value) => Number(value)).filter(Number.isFinite); return numeric.length > 0 ? Math.max(...numeric) : 0; } function minNumber(values) { const numeric = (values || []).map((value) => Number(value)).filter(Number.isFinite); return numeric.length > 0 ? Math.min(...numeric) : 0; } function buildSessionInvariantFindings(control, manifest = {}) { const findings = []; for (const row of control || []) { if (row?.type !== "assertSessionInvariant" || row?.phase !== "completed") continue; const detail = objectValue(row.detail); const messageOrder = objectValue(detail.messageOrder); if (messageOrder.userClustered !== true) continue; const afterRound = numberOrNull(detail.afterRound ?? row.input?.afterRound); const consecutiveUserMessageCount = numberOrNull(messageOrder.consecutiveUserMessageCount) ?? 0; const sentinelRange = stringOrNull(messageOrder.sentinelRange) ?? stringOrNull(detail.expectedSentinelRange); const traceIds = arrayStrings(messageOrder.traceIds).slice(0, 12); const findingId = stringOrNull(detail.findingId) ?? "workbench-message-order-user-clustered-after-navigation"; const severity = stringOrNull(detail.severity) ?? "amber"; const rootCause = "session_message_role_clustered"; const rootCauseStatus = "confirmed-from-controlled-refresh-dom;check-otel-session_messages_read-role-sequence"; const rootCauseConfidence = "medium"; const nextAction = "Use OTel session_messages_read/session detail for the same canarySessionId and traceIds. Compare roleSequencePrefix and adjacentSameRoleCount; if the read model is already clustered, fix Workbench projection/read-model timeline ordering before changing renderer code."; findings.push({ id: findingId, severity, summary: "message-order root cause visible: controlled refresh/switch-back afterRound=" + (afterRound ?? "-") + " left consecutive user message cards without interleaved assistant/code-agent terminal cards" + (sentinelRange ? " (" + sentinelRange + ")" : ""), rootCause, rootCauseStatus, rootCauseConfidence, nextAction, count: Math.max(1, consecutiveUserMessageCount), blocking: detail.blocking === true ? true : false, afterRound, canarySessionId: stringOrNull(detail.canarySessionId), routeSessionId: stringOrNull(detail.routeSessionId), activeSessionId: stringOrNull(detail.activeSessionId), consecutiveUserMessageCount, sentinelRange, sampleSeq: numberOrNull(detail.sampleSeq), traceIds, pageRole: stringOrNull(detail.pageRole) ?? "control", pageId: stringOrNull(detail.pageId), observerId: stringOrNull(manifest.jobId), stateDir: stringOrNull(manifest.stateDir), commandId: stringOrNull(row.commandId), commandTs: stringOrNull(row.ts), evidence: { afterRound, consecutiveUserMessageCount, sentinelRange, sampleSeq: numberOrNull(detail.sampleSeq), traceIds, canarySessionId: stringOrNull(detail.canarySessionId), routeSessionId: stringOrNull(detail.routeSessionId), activeSessionId: stringOrNull(detail.activeSessionId), valuesRedacted: true, }, messageOrder: { sequence: Array.isArray(messageOrder.sequence) ? messageOrder.sequence.slice(-20) : [], clusters: Array.isArray(messageOrder.clusters) ? messageOrder.clusters.slice(0, 8) : [], valuesRedacted: true, }, valuesRedacted: true, }); } return findings; } function buildControlledNavigationRootCauseFindings(control, manifest = {}) { const commands = []; for (const row of control || []) { if (row?.phase !== "completed") continue; if (row?.type !== "refreshCurrentSession" && row?.type !== "switchAwayAndBack") continue; const detail = objectValue(row.detail); const navigation = objectValue(detail.navigation); const readiness = objectValue(navigation.readiness); const snapshot = objectValue(readiness.snapshot); const pageProvenance = objectValue(navigation.pageProvenance); const blankShell = snapshot.workbenchShellVisible === false && snapshot.sessionRailPresent === false && snapshot.commandInputPresent === false && snapshot.bodyTextHash === "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; const shellOrComposerMissing = snapshot.workbenchShellVisible === false || snapshot.sessionRailPresent === false || snapshot.commandInputPresent === false; const degraded = navigation.degraded === true || readiness.ok === false || detail.routeOk === false || blankShell || (detail.activeOk === false && shellOrComposerMissing) || (detail.composerReady === false && snapshot.commandInputPresent === false); if (!degraded) continue; const rootCause = stringOrNull(navigation.degradedReason) ?? (readiness.ok === false ? stringOrNull(readiness.reason) : null) ?? (detail.routeOk === false ? "route-session-not-hydrated" : null) ?? (blankShell ? "workbench-blank-shell-after-navigation" : null) ?? (detail.activeOk === false && shellOrComposerMissing ? "active-session-not-hydrated" : null) ?? (detail.composerReady === false && snapshot.commandInputPresent === false ? "composer-not-ready" : null) ?? "controlled-navigation-degraded"; commands.push({ commandId: stringOrNull(row.commandId), type: stringOrNull(row.type), commandTs: stringOrNull(row.ts), afterRound: numberOrNull(detail.afterRound ?? row.input?.afterRound), rootCause, blocking: true, canarySessionId: stringOrNull(detail.canarySessionId), routeSessionId: stringOrNull(detail.routeSessionId), activeSessionId: stringOrNull(detail.activeSessionId), routeOk: detail.routeOk === true, activeOk: detail.activeOk === true, composerReady: detail.composerReady === true, navigation: { httpStatus: numberOrNull(navigation.httpStatus), degraded: navigation.degraded === true, degradedReason: stringOrNull(navigation.degradedReason), beforePath: urlPath(navigation.beforeUrl), afterPath: urlPath(navigation.afterUrl), valuesRedacted: true, }, readiness: { ok: readiness.ok === true, reason: stringOrNull(readiness.reason), durationMs: numberOrNull(readiness.durationMs), path: stringOrNull(snapshot.path), readyState: stringOrNull(snapshot.readyState), workbenchShellVisible: snapshot.workbenchShellVisible === true, sessionCreatePresent: snapshot.sessionCreatePresent === true, sessionRailPresent: snapshot.sessionRailPresent === true, commandInputPresent: snapshot.commandInputPresent === true, activeTabPresent: snapshot.activeTabPresent === true, loginVisible: snapshot.loginVisible === true, blankShell, bodyTextHash: stringOrNull(snapshot.bodyTextHash), valuesRedacted: true, }, pageProvenance: { pageLoadSeq: numberOrNull(pageProvenance.pageLoadSeq), reason: stringOrNull(pageProvenance.reason), observedAt: stringOrNull(pageProvenance.observedAt), urlPath: stringOrNull(pageProvenance.urlPath), documentReadyState: stringOrNull(pageProvenance.documentReadyState), timeOrigin: numberOrNull(pageProvenance.timeOrigin), httpStatus: numberOrNull(pageProvenance.httpStatus), assetFingerprint: stringOrNull(pageProvenance.assetFingerprint), scriptCount: numberOrNull(pageProvenance.scriptCount), stylesheetCount: numberOrNull(pageProvenance.stylesheetCount), scripts: arrayStrings(pageProvenance.scripts).slice(0, 8), stylesheets: arrayStrings(pageProvenance.stylesheets).slice(0, 8), valuesRedacted: true, }, observer: { ok: detail.observer?.ok === true, pageRole: stringOrNull(detail.observer?.pageRole), pageId: stringOrNull(detail.observer?.pageId), changed: detail.observer?.changed === true, valuesRedacted: true, }, observerId: stringOrNull(manifest.jobId), stateDir: stringOrNull(manifest.stateDir), valuesRedacted: true, }); } if (commands.length === 0) return []; return [{ id: "workbench-controlled-navigation-degraded-root-cause", severity: "red", summary: "controlled Workbench refresh/switch completed degraded; route may be correct but app shell, active session, or composer was not ready, so later Code Agent turns cannot continue", count: commands.length, blocking: true, rootCauses: Array.from(new Set(commands.map((item) => item.rootCause))).slice(0, 12), commands: commands.slice(0, 20), next: "Investigate the first degraded command, then correlate browser requestfailed/static asset failures and Workbench hydration state before changing Code Agent/provider logic.", valuesRedacted: true, }]; } function sessionInvariantNavigationWindows(control) { const started = new Map(); const windows = []; for (const row of control || []) { if (row?.type !== "switchAwayAndBack" && row?.type !== "refreshCurrentSession") continue; const commandId = stringOrNull(row.commandId) ?? String(row.seq ?? ""); if (row.phase === "started") { started.set(commandId, row); continue; } if (row.phase !== "completed") continue; const detail = objectValue(row.detail); const canarySessionId = stringOrNull(detail.canarySessionId); const alternateSessionId = stringOrNull(detail.alternateSessionId); const startRow = started.get(commandId); const startMs = timestampMs(startRow?.ts ?? row.ts); const endMs = timestampMs(row.ts); if (!canarySessionId || !Number.isFinite(startMs) || !Number.isFinite(endMs)) continue; windows.push({ commandId, afterRound: numberOrNull(detail.afterRound ?? row.input?.afterRound), startMs: Math.max(0, startMs - 1000), endMs: endMs + 5000, startAt: new Date(startMs).toISOString(), endAt: new Date(endMs).toISOString(), canarySessionId, alternateSessionId, routeOk: detail.routeOk === true, activeOk: detail.activeOk === true, valuesRedacted: true, }); } return windows; } function sessionInvariantCanarySessionIds(control) { const ids = new Set(); for (const row of control || []) { const detail = objectValue(row?.detail); if (row?.type === "newSession" && row?.phase === "completed") { const sessionId = stringOrNull(detail.sessionId) ?? stringOrNull(detail.result?.sessionId) ?? stringOrNull(detail.createSession?.createdSessionId); if (sessionId) ids.add(sessionId); } const canarySessionId = stringOrNull(detail.canarySessionId); if (canarySessionId) ids.add(canarySessionId); } return ids; } function sessionChangeSamplesOutsideControlledNavigation(samples, key, windows) { const canaryIds = new Set((windows || []).map((item) => item.canarySessionId).filter(Boolean)); if (canaryIds.size === 0) return samples.filter((item) => item?.[key]); return (samples || []).filter((sample) => { const value = stringOrNull(sample?.[key]); if (!value) return false; if (canaryIds.has(value)) return false; return !sampleInControlledNavigationWindow(sample, windows); }); } function sampleInControlledNavigationWindow(sample, windows) { const ms = timestampMs(sample?.ts); if (!Number.isFinite(ms)) return false; return (windows || []).some((window) => ms >= window.startMs && ms <= window.endMs); } function sampleRefInControlledNavigationSessionWindow(sample, windows) { const ms = timestampMs(sample?.ts); if (!Number.isFinite(ms)) return false; if (!sampleRefMatchesControlledNavigationSession(sample, windows)) return false; return (windows || []).some((window) => ms >= window.startMs && ms <= window.endMs); } function sampleRefMatchesControlledNavigationSession(sample, windows) { const routeSessionId = stringOrNull(sample?.routeSessionId); const activeSessionId = stringOrNull(sample?.activeSessionId); return (windows || []).some((window) => { const expected = [window.canarySessionId, window.alternateSessionId].filter(Boolean); return expected.some((sessionId) => sessionId === routeSessionId || sessionId === activeSessionId); }); } function isBlankHydrationProjectionSample(sample) { if (!sample) return false; const messageCount = Array.isArray(sample.messages) ? sample.messages.length : Number(sample.messageCount ?? 0); const traceRowCount = Array.isArray(sample.traceRows) ? sample.traceRows.length : Number(sample.traceRowCount ?? 0); return !stringOrNull(sample.activeSessionId) && Number(messageCount) === 0 && Number(traceRowCount) === 0; } function controlledNavigationHydrationCrossPageDiff(row, windows, sampleBySeq) { if (row?.diffKind !== "projection") return false; if (!sampleRefMatchesControlledNavigationSession(row.control, windows) || !sampleRefMatchesControlledNavigationSession(row.observer, windows)) return false; const control = sampleBySeq.get(Number(row?.control?.seq)); const observer = sampleBySeq.get(Number(row?.observer?.seq)); return isBlankHydrationProjectionSample(control) || isBlankHydrationProjectionSample(observer); } function crossPageDiffHasWorkbenchAppShellNotReady(row, sampleBySeq) { const control = sampleBySeq.get(Number(row?.control?.seq)); const observer = sampleBySeq.get(Number(row?.observer?.seq)); return workbenchSampleAppShellNotReady(control) || workbenchSampleAppShellNotReady(observer); } function workbenchSampleAppShellNotReady(sample) { if (!sample || !isWorkbenchPathSample(sample)) return false; const routeSessionId = stringOrNull(sample.routeSessionId) || workbenchSessionIdFromPath(samplePathname(sample)); if (!routeSessionId) return false; const messageCount = Array.isArray(sample.messages) ? sample.messages.length : Number(sample.messageCount ?? 0); const traceRowCount = Array.isArray(sample.traceRows) ? sample.traceRows.length : Number(sample.traceRowCount ?? 0); const turnCount = Array.isArray(sample.turns) ? sample.turns.length : Number(sample.turnCount ?? 0); const loadingCount = Array.isArray(sample.loadings) ? sample.loadings.length : 0; const diagnosticCount = Array.isArray(sample.diagnostics) ? sample.diagnostics.length : 0; const railVisibleCount = Number(sample?.sessionRail?.visibleCount ?? 0); const composer = objectValue(sample.composer); const composerPresent = composer.inputPresent === true || composer.submitPresent === true; const provenance = objectValue(sample.pageProvenance); const hasWorkbenchAssets = Number(provenance.scriptCount ?? 0) > 0 || Number(provenance.stylesheetCount ?? 0) > 0; return !stringOrNull(sample.activeSessionId) && Number(messageCount) === 0 && Number(traceRowCount) === 0 && Number(turnCount) === 0 && Number(loadingCount) === 0 && Number(diagnosticCount) === 0 && Number(railVisibleCount) === 0 && !composerPresent && hasWorkbenchAssets; } function detectWorkbenchAppShellNotReady(samples) { const rows = (Array.isArray(samples) ? samples : []) .filter(workbenchSampleAppShellNotReady) .map((sample) => workbenchAppShellNotReadyRef(sample)); return annotateWorkbenchAppShellNotReadyTiming(rows); } function annotateWorkbenchAppShellNotReadyTiming(rows) { const groups = new Map(); const splitGapMs = Math.max(1000, Number(alertThresholds.crossPageProjectionDivergenceRedMs || alertThresholds.visibleLoadingSlowMs || 10_000)); for (const row of rows.slice().sort((a, b) => timestampMs(a.ts) - timestampMs(b.ts))) { const ms = timestampMs(row.ts); const key = [row.pageRole || "control", row.pageId || "default", row.routeSessionId || row.url || ""].join(":"); const group = groups.get(key) || []; let segment = group.at(-1); if (!segment || !Number.isFinite(ms) || !Number.isFinite(segment.lastMs) || ms - segment.lastMs > splitGapMs) { segment = { rows: [], firstMs: Number.isFinite(ms) ? ms : null, lastMs: Number.isFinite(ms) ? ms : null }; group.push(segment); } segment.rows.push(row); if (Number.isFinite(ms)) { if (segment.firstMs === null || ms < segment.firstMs) segment.firstMs = ms; if (segment.lastMs === null || ms > segment.lastMs) segment.lastMs = ms; } groups.set(key, group); } const result = []; for (const group of groups.values()) { for (let segmentIndex = 0; segmentIndex < group.length; segmentIndex += 1) { const segment = group[segmentIndex]; const observedSpanMs = segment.firstMs === null || segment.lastMs === null ? null : segment.lastMs - segment.firstMs; for (const row of segment.rows) { result.push({ ...row, segmentIndex, observedFirstAt: segment.firstMs === null ? null : new Date(segment.firstMs).toISOString(), observedLastAt: segment.lastMs === null ? null : new Date(segment.lastMs).toISOString(), observedSpanMs, }); } } } return result; } function workbenchAppShellNotReadyRef(sample) { const provenance = objectValue(sample?.pageProvenance); const performance = workbenchAssetPerformanceSummary(sample); return { ...ref(sample), title: stringOrNull(sample?.title), messageCount: Array.isArray(sample?.messages) ? sample.messages.length : 0, turnCount: Array.isArray(sample?.turns) ? sample.turns.length : 0, traceRowCount: Array.isArray(sample?.traceRows) ? sample.traceRows.length : 0, sessionRailVisibleCount: Number(sample?.sessionRail?.visibleCount ?? 0), composerInputPresent: sample?.composer?.inputPresent === true, composerSubmitPresent: sample?.composer?.submitPresent === true, pageProvenance: { documentReadyState: stringOrNull(provenance.documentReadyState), pageLoadSeq: numberOrNull(provenance.pageLoadSeq), timeOrigin: numberOrNull(provenance.timeOrigin), assetFingerprint: stringOrNull(provenance.assetFingerprint), scriptCount: numberOrNull(provenance.scriptCount), stylesheetCount: numberOrNull(provenance.stylesheetCount), scripts: arrayStrings(provenance.scripts).slice(0, 8), stylesheets: arrayStrings(provenance.stylesheets).slice(0, 8), valuesRedacted: true }, assetPerformance: performance, valuesRedacted: true }; } function workbenchAssetPerformanceSummary(sample) { const assets = (Array.isArray(sample?.performance) ? sample.performance : []) .filter((entry) => /^(script|link|css)$/iu.test(String(entry?.initiatorType || "")) || /\.(?:js|css)$/iu.test(String(entry?.name || ""))) .map((entry) => ({ path: urlPath(entry?.name), initiatorType: entry?.initiatorType ?? null, duration: numberOrNull(entry?.duration), responseStatus: numberOrNull(entry?.responseStatus), transferSize: numberOrNull(entry?.transferSize), encodedBodySize: numberOrNull(entry?.encodedBodySize), decodedBodySize: numberOrNull(entry?.decodedBodySize), nextHopProtocol: entry?.nextHopProtocol ?? null, valuesRedacted: true })); const responseStatusCounts = {}; for (const item of assets) { const key = item.responseStatus === null ? "unknown" : String(item.responseStatus); responseStatusCounts[key] = (responseStatusCounts[key] || 0) + 1; } return { assetCount: assets.length, zeroStatusCount: assets.filter((item) => item.responseStatus === 0).length, missingStatusCount: assets.filter((item) => item.responseStatus === null).length, responseStatusCounts, assets: assets.slice(0, 12), valuesRedacted: true }; } function objectValue(value) { return value && typeof value === "object" && !Array.isArray(value) ? value : {}; } function stringOrNull(value) { return typeof value === "string" && value.length > 0 ? value : null; } function numberOrNull(value) { const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } function arrayStrings(value) { return Array.isArray(value) ? value.map((item) => String(item || "")).filter(Boolean) : []; } function timestampMs(value) { const parsed = Date.parse(String(value || "")); return Number.isFinite(parsed) ? parsed : NaN; } function buildApiDomLagReport(samples, network) { const windowMs = 30_000; const budgetMs = Number.isFinite(Number(alertThresholds.sameOriginApiSlowMs)) ? Number(alertThresholds.sameOriginApiSlowMs) : 10_000; const sampleRows = (Array.isArray(samples) ? samples : []) .map((sample) => { const tsMs = timestampMs(sample?.ts); return { sample, tsMs, pageKey: samplePageKey(sample), digest: digestSample(sample), sessionIds: new Set([sample?.routeSessionId, sample?.activeSessionId].filter(Boolean).map(String)), traceIds: sampleTraceIds(sample) }; }) .filter((item) => Number.isFinite(item.tsMs)) .sort((a, b) => a.tsMs - b.tsMs); const samplesByPage = new Map(); for (const row of sampleRows) { const rows = samplesByPage.get(row.pageKey) || []; rows.push(row); samplesByPage.set(row.pageKey, rows); } const naturalApiResponses = (Array.isArray(network) ? network : []) .filter((item) => item?.observerInitiated !== true && item?.type === "response" && isApiLikePath(urlPath(item?.url))); const telemetryExcluded = []; const nonStateRelevant = []; const stateRelevantResponses = []; for (const item of naturalApiResponses) { const event = compactApiDomLagResponseEvent(item); if (!Number.isFinite(event.tsMs)) { nonStateRelevant.push(event); continue; } if (isApiDomLagTelemetryPath(event.path)) telemetryExcluded.push(event); else if (!isApiDomLagStateRelevantPath(event.path)) nonStateRelevant.push(event); else stateRelevantResponses.push(event); } const candidates = []; for (const event of stateRelevantResponses) { const pageSamples = samplesByPage.get(event.pageKey) || []; const before = lastSampleAtOrBefore(pageSamples, event.tsMs, event); const firstAfter = firstSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event); const baselineDigest = before?.digest ?? null; const change = firstSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event, (row) => !baselineDigest || row.digest !== baselineDigest); candidates.push({ ...event, windowMs, budgetMs, firstSampleDeltaMs: firstAfter ? Math.max(0, Math.round(firstAfter.tsMs - event.tsMs)) : null, domChangeDeltaMs: change ? Math.max(0, Math.round(change.tsMs - event.tsMs)) : null, overBudget: change ? (change.tsMs - event.tsMs) > budgetMs : false, domChanged: Boolean(change), noDomChangeWithinWindow: !change, beforeSample: compactApiDomLagSample(before), firstAfterSample: compactApiDomLagSample(firstAfter), changeSample: compactApiDomLagSample(change), confidence: apiDomLagConfidence(event.path), valuesRedacted: true }); } const changedDeltas = candidates.map((item) => nullableNumber(item.domChangeDeltaMs)).filter(Number.isFinite).sort((a, b) => a - b); const groups = groupApiDomLagCandidates(candidates); const overBudget = candidates.filter((item) => item.overBudget === true); return { summary: { windowMs, budgetMs, naturalApiResponseCount: naturalApiResponses.length, telemetryExcludedCount: telemetryExcluded.length, nonStateRelevantResponseCount: nonStateRelevant.length, stateRelevantResponseCount: stateRelevantResponses.length, candidateCount: candidates.length, domChangedCount: changedDeltas.length, noDomChangeWithinWindowCount: candidates.filter((item) => item.noDomChangeWithinWindow === true).length, lowConfidenceStreamOpenCount: candidates.filter((item) => item.confidence === "low-stream-open-only").length, overBudgetCount: overBudget.length, p50DomChangeDeltaMs: percentile(changedDeltas, 50), p95DomChangeDeltaMs: percentile(changedDeltas, 95), maxDomChangeDeltaMs: changedDeltas.length > 0 ? Math.max(...changedDeltas) : null, groupCount: groups.length, valuesRedacted: true }, groups, worstCandidates: candidates .filter((item) => Number.isFinite(nullableNumber(item.domChangeDeltaMs))) .sort((a, b) => nullableNumber(b.domChangeDeltaMs) - nullableNumber(a.domChangeDeltaMs)) .slice(0, 20), recentCandidates: candidates.slice(-40), telemetryExcluded: telemetryExcluded.slice(0, 20), nonStateRelevant: nonStateRelevant.slice(0, 20), valuesRedacted: true }; } function compactApiDomLagResponseEvent(item) { const parsed = parseApiDomLagUrl(item?.url); const tsMs = timestampMs(item?.ts); return { ts: item?.ts ?? null, tsMs, pageRole: item?.pageRole ?? null, pageId: item?.pageId ?? null, pageKey: String(item?.pageRole || "control") + ":" + String(item?.pageId || "default"), commandId: item?.commandId ?? null, method: String(item?.method || "GET").toUpperCase(), status: Number.isFinite(Number(item?.status)) ? Number(item.status) : null, path: parsed.path, rawPath: parsed.rawPath, queryKeys: parsed.queryKeys, sessionId: parsed.sessionId, traceId: parsed.traceId, urlHash: item?.url ? sha256(item.url) : null, routeKind: apiDomLagRouteKind(parsed.path), valuesRedacted: true }; } function parseApiDomLagUrl(value) { try { const parsed = new URL(String(value || "http://invalid.local/")); const rawPath = parsed.pathname || "-"; const queryKeys = Array.from(parsed.searchParams.keys()).sort().slice(0, 12); const sessionId = parsed.searchParams.get("sessionId") || parsed.searchParams.get("includeSessionId") || firstIdInText(parsed.pathname + " " + parsed.search, /\bses_[A-Za-z0-9_-]+\b/u); const traceId = parsed.searchParams.get("traceId") || firstIdInText(parsed.pathname + " " + parsed.search, /\btrc_[A-Za-z0-9_-]+\b/u); return { rawPath, path: normalizeApiPath(rawPath), queryKeys, sessionId, traceId }; } catch { const rawPath = urlPath(value); return { rawPath, path: normalizeApiPath(rawPath), queryKeys: [], sessionId: firstIdInText(String(value || ""), /\bses_[A-Za-z0-9_-]+\b/u), traceId: firstIdInText(String(value || ""), /\btrc_[A-Za-z0-9_-]+\b/u) }; } } function firstIdInText(text, pattern) { const match = String(text || "").match(pattern); return match ? match[0] : null; } function nullableNumber(value) { if (value === null || value === undefined || value === "") return NaN; const numeric = Number(value); return Number.isFinite(numeric) ? numeric : NaN; } function isApiDomLagTelemetryPath(path) { const value = String(path || ""); return value === "/v1/web-performance" || value === "/v1/health" || value === "/health"; } function isApiDomLagStateRelevantPath(path) { const value = String(path || ""); return value.startsWith("/auth/") || value.startsWith("/v1/workbench/") || value === "/v1/agent/chat" || value === "/v1/agent/chat/steer"; } function apiDomLagRouteKind(path) { const value = String(path || ""); if (value === "/v1/workbench/events") return "workbench-events-stream"; if (value.startsWith("/v1/workbench/sessions")) return "workbench-sessions"; if (value.startsWith("/v1/workbench/traces")) return "workbench-traces"; if (value.startsWith("/v1/workbench/turns")) return "workbench-turns"; if (value === "/v1/agent/chat" || value === "/v1/agent/chat/steer") return "agent-chat-submit"; if (value.startsWith("/auth/")) return "auth"; return "state-api"; } function apiDomLagConfidence(path) { return String(path || "") === "/v1/workbench/events" ? "low-stream-open-only" : "medium-response-to-dom"; } function sampleTraceIds(sample) { const ids = new Set(); for (const group of [sample?.messages, sample?.traceRows, sample?.turns, sample?.diagnostics]) { if (!Array.isArray(group)) continue; for (const item of group) if (item?.traceId) ids.add(String(item.traceId)); } return ids; } function lastSampleAtOrBefore(rows, tsMs, event) { let result = null; for (const row of rows) { if (row.tsMs > tsMs) break; if (apiDomLagSampleMatchesEvent(row, event)) result = row; } return result; } function firstSampleAfter(rows, startMs, endMs, event, predicate = null) { for (const row of rows) { if (row.tsMs < startMs) continue; if (row.tsMs > endMs) break; if (!apiDomLagSampleMatchesEvent(row, event)) continue; if (typeof predicate === "function" && !predicate(row)) continue; return row; } return null; } function apiDomLagSampleMatchesEvent(row, event) { if (!row || !event) return false; if (event.sessionId && !row.sessionIds.has(String(event.sessionId))) return false; if (event.traceId && row.traceIds.size > 0 && !row.traceIds.has(String(event.traceId))) return false; return true; } function compactApiDomLagSample(row) { if (!row) return null; const sample = row.sample || {}; return { seq: sample.seq ?? null, ts: sample.ts ?? null, pageRole: sample.pageRole ?? null, pageId: sample.pageId ?? null, routeSessionId: sample.routeSessionId ?? null, activeSessionId: sample.activeSessionId ?? null, messageCount: Array.isArray(sample.messages) ? sample.messages.length : null, traceRowCount: Array.isArray(sample.traceRows) ? sample.traceRows.length : null, diagnosticCount: Array.isArray(sample.diagnostics) ? sample.diagnostics.length : null, valuesRedacted: true }; } function groupApiDomLagCandidates(candidates) { const groups = new Map(); for (const item of candidates || []) { const key = [item.method || "-", item.path || "-", item.status ?? "-", item.confidence || "-"].join(" "); const group = groups.get(key) || { method: item.method ?? null, path: item.path ?? "-", routeKind: item.routeKind ?? null, status: item.status ?? null, confidence: item.confidence ?? null, count: 0, domChangedCount: 0, noDomChangeWithinWindowCount: 0, overBudgetCount: 0, firstAt: item.ts ?? null, lastAt: item.ts ?? null, deltas: [], examples: [] }; group.count += 1; group.firstAt = minIso(group.firstAt, item.ts ?? null); group.lastAt = maxIso(group.lastAt, item.ts ?? null); if (item.domChanged === true && Number.isFinite(Number(item.domChangeDeltaMs))) { group.domChangedCount += 1; group.deltas.push(Number(item.domChangeDeltaMs)); } if (item.noDomChangeWithinWindow === true) group.noDomChangeWithinWindowCount += 1; if (item.overBudget === true) group.overBudgetCount += 1; if (group.examples.length < 6) { group.examples.push({ ts: item.ts ?? null, sessionId: item.sessionId ?? null, traceId: item.traceId ?? null, domChangeDeltaMs: item.domChangeDeltaMs ?? null, firstSampleDeltaMs: item.firstSampleDeltaMs ?? null, changeSeq: item.changeSample?.seq ?? null, beforeSeq: item.beforeSample?.seq ?? null, valuesRedacted: true }); } groups.set(key, group); } return Array.from(groups.values()) .map((item) => { const deltas = item.deltas.slice().sort((a, b) => a - b); return { method: item.method, path: item.path, routeKind: item.routeKind, status: item.status, confidence: item.confidence, count: item.count, domChangedCount: item.domChangedCount, noDomChangeWithinWindowCount: item.noDomChangeWithinWindowCount, overBudgetCount: item.overBudgetCount, p50DomChangeDeltaMs: percentile(deltas, 50), p95DomChangeDeltaMs: percentile(deltas, 95), maxDomChangeDeltaMs: deltas.length > 0 ? Math.max(...deltas) : null, firstAt: item.firstAt, lastAt: item.lastAt, examples: item.examples, valuesRedacted: true }; }) .sort((a, b) => Number(b.maxDomChangeDeltaMs ?? -1) - Number(a.maxDomChangeDeltaMs ?? -1) || b.count - a.count || String(a.path).localeCompare(String(b.path))); } function detectTraceEventsPageReadIssues(network) { const events = (Array.isArray(network) ? network : []) .filter((item) => item?.observerInitiated !== true && (item?.type === "response" || item?.type === "requestfailed")) .map(compactTraceEventsPageReadEvent) .filter((item) => item !== null); const http404 = events.filter((item) => item.type === "response" && Number(item.status) === 404); const httpErrors = events.filter((item) => item.type === "response" && Number(item.status) >= 400 && Number(item.status) !== 404); const requestFailed = events.filter((item) => item.type === "requestfailed"); return { events, http404, httpErrors, requestFailed, summary: traceEventsPageReadIssueSummary(events), valuesRedacted: true }; } function compactTraceEventsPageReadEvent(item) { const parsed = parseTraceEventsPageReadUrl(item?.url); if (!parsed.match) return null; const failureText = item?.failureKind ?? item?.failure ?? item?.errorText ?? null; return { ts: item?.ts ?? null, pageRole: item?.pageRole ?? null, pageId: item?.pageId ?? null, commandId: item?.commandId ?? null, method: String(item?.method || "GET").toUpperCase(), type: item?.type ?? null, status: Number.isFinite(Number(item?.status)) ? Number(item.status) : null, path: "/v1/workbench/traces/:traceId/events", rawPath: parsed.rawPath, traceId: parsed.traceId, afterProjectedSeq: parsed.afterProjectedSeq, sinceSeq: parsed.sinceSeq, limit: parsed.limit, tail: parsed.tail, queryKeys: parsed.queryKeys, failureKind: failureText ? limitText(String(failureText), 120) : null, urlHash: item?.url ? sha256(item.url) : null, valuesRedacted: true }; } function parseTraceEventsPageReadUrl(value) { const fallback = { match: false, rawPath: urlPath(value), traceId: firstIdInText(String(value || ""), /\btrc_[A-Za-z0-9_-]+\b/u), afterProjectedSeq: null, sinceSeq: null, limit: null, tail: null, queryKeys: [], }; try { const parsed = new URL(String(value || ""), "http://invalid.local/"); const rawPath = parsed.pathname || "-"; const match = rawPath.match(/^\/v1\/workbench\/traces\/([^/]+)\/events$/u); return { match: Boolean(match), rawPath, traceId: match ? decodeURIComponent(match[1]) : fallback.traceId, afterProjectedSeq: numericSearchParam(parsed.searchParams, "afterProjectedSeq"), sinceSeq: numericSearchParam(parsed.searchParams, "sinceSeq") ?? numericSearchParam(parsed.searchParams, "afterSeq"), limit: numericSearchParam(parsed.searchParams, "limit"), tail: numericSearchParam(parsed.searchParams, "tail"), queryKeys: Array.from(parsed.searchParams.keys()).sort().slice(0, 12), }; } catch { return fallback; } } function numericSearchParam(searchParams, key) { const raw = searchParams?.get?.(key); if (raw === null || raw === undefined || raw === "") return null; const parsed = Number(raw); return Number.isFinite(parsed) ? parsed : null; } function traceEventsPageReadIssueSummary(events) { const items = Array.isArray(events) ? events : []; const statuses = uniqueSorted(items.map((item) => item.status).filter((item) => item !== null && item !== undefined).map(String)); const traceIds = uniqueSorted(items.map((item) => item.traceId).filter(Boolean).map(String)).slice(0, 8); const afterProjectedSeqs = uniqueSorted(items.map((item) => item.afterProjectedSeq).filter((item) => item !== null && item !== undefined).map(String)).slice(0, 8); const sinceSeqs = uniqueSorted(items.map((item) => item.sinceSeq).filter((item) => item !== null && item !== undefined).map(String)).slice(0, 8); const failureKinds = uniqueSorted(items.map((item) => item.failureKind).filter(Boolean).map(String)).slice(0, 6); return { eventCount: items.length, responseErrorCount: items.filter((item) => item.type === "response" && Number(item.status) >= 400).length, http404Count: items.filter((item) => item.type === "response" && Number(item.status) === 404).length, requestFailedCount: items.filter((item) => item.type === "requestfailed").length, statuses, traceIds, afterProjectedSeqs, sinceSeqs, failureKinds, firstAt: items.reduce((value, item) => minIso(value, item.ts ?? null), null), lastAt: items.reduce((value, item) => maxIso(value, item.ts ?? null), null), rootCauseVisibility: "browser network rows identify trace-events page-read path; OTel trace_events_read should confirm backend paging fields", valuesRedacted: true }; } function uniqueSorted(values) { return Array.from(new Set((values || []).filter((item) => item !== null && item !== undefined).map(String))).sort(); } function compactApiDomLagForOutput(report) { if (!report || typeof report !== "object") return null; return { summary: report.summary ?? null, groups: Array.isArray(report.groups) ? report.groups.slice(0, 8) : [], worstCandidates: Array.isArray(report.worstCandidates) ? report.worstCandidates.slice(0, 8) : [], recentCandidates: Array.isArray(report.recentCandidates) ? report.recentCandidates.slice(-8) : [], valuesRedacted: true }; } function detectWorkbenchInPlaceProjectionLag(samples, network, control = []) { const canarySessionIds = sessionInvariantCanarySessionIds(control); const terminalTraceMissing = detectWorkbenchTerminalTraceMissing(samples, canarySessionIds); const terminalApiDomLag = detectWorkbenchTerminalApiDomLag(samples, network, canarySessionIds); return { terminalTraceMissing, terminalApiDomLag, valuesRedacted: true }; } function detectWorkbenchTerminalTraceMissing(samples, canarySessionIds = new Set()) { const budgetMs = Number.isFinite(Number(alertThresholds.sameOriginApiSlowMs)) ? Number(alertThresholds.sameOriginApiSlowMs) : 10_000; const rows = []; const open = new Map(); const sortedSamples = (Array.isArray(samples) ? samples : []) .filter(isWorkbenchPathSample) .filter((sample) => workbenchSampleMatchesCanarySessions(sample, canarySessionIds)) .slice() .sort((a, b) => Date.parse(a?.ts || "") - Date.parse(b?.ts || "")); const closeSegment = (key, lastSample = null) => { const segment = open.get(key); if (!segment) return; open.delete(key); const firstMs = Date.parse(segment.first.ts || ""); const lastMs = Date.parse((lastSample ?? segment.last).ts || ""); const observedMs = Number.isFinite(firstMs) && Number.isFinite(lastMs) && lastMs >= firstMs ? Math.round(lastMs - firstMs) : 0; if (observedMs <= budgetMs) return; rows.push({ ...ref(segment.first), lastSeq: segment.last.seq ?? null, lastAt: segment.last.ts ?? null, traceId: segment.traceId, messageCount: Array.isArray(segment.last.messages) ? segment.last.messages.length : 0, turnCount: Array.isArray(segment.last.turns) ? segment.last.turns.length : 0, traceRowCount: 0, sampleCount: segment.sampleCount, observedMs, budgetMs, finalMessageVisible: workbenchFinalMessageVisible(segment.last, segment.traceId), terminalTurnVisible: workbenchTerminalTurnVisible(segment.last, segment.traceId), detail: "terminal turn/message stayed visible for this trace while the same in-place Workbench page had zero trace rows beyond the configured budget", valuesRedacted: true }); }; for (const sample of sortedSamples) { if (!isWorkbenchPathSample(sample)) continue; const tsMs = Date.parse(sample?.ts || ""); if (!Number.isFinite(tsMs)) continue; const terminalTraceIds = workbenchTerminalTraceIdsFromDom(sample); const presentKeys = new Set(); for (const traceId of terminalTraceIds) { const key = [samplePageKey(sample), sample?.routeSessionId ?? "", sample?.activeSessionId ?? "", traceId].join("|"); presentKeys.add(key); const traceRows = workbenchTraceRowsForTrace(sample, traceId); if (traceRows.length > 0 || workbenchSampleHasTerminalProjection(sample, { traceIds: [traceId] })) { closeSegment(key, sample); continue; } const existing = open.get(key); if (existing) { existing.last = sample; existing.sampleCount += 1; } else { open.set(key, { first: sample, last: sample, traceId, sampleCount: 1 }); } } for (const [key, segment] of Array.from(open.entries())) { if (samplePageKey(sample) === samplePageKey(segment.first) && !presentKeys.has(key)) closeSegment(key, sample); } } for (const key of Array.from(open.keys())) closeSegment(key); return rows; } function detectWorkbenchTerminalApiDomLag(samples, network, canarySessionIds = new Set()) { const windowMs = 30_000; const budgetMs = Number.isFinite(Number(alertThresholds.sameOriginApiSlowMs)) ? Number(alertThresholds.sameOriginApiSlowMs) : 10_000; const sampleRows = (Array.isArray(samples) ? samples : []) .filter(isWorkbenchPathSample) .filter((sample) => workbenchSampleMatchesCanarySessions(sample, canarySessionIds)) .map((sample) => ({ sample, tsMs: Date.parse(sample?.ts || ""), pageKey: samplePageKey(sample) })) .filter((item) => Number.isFinite(item.tsMs)) .sort((a, b) => a.tsMs - b.tsMs); const rowsByPage = new Map(); for (const row of sampleRows) { const rows = rowsByPage.get(row.pageKey) || []; rows.push(row); rowsByPage.set(row.pageKey, rows); } const terminalEvents = (Array.isArray(network) ? network : []) .map(compactWorkbenchTerminalApiEvent) .filter((item) => workbenchTerminalEventMatchesCanarySessions(item, canarySessionIds)) .filter((item) => item !== null); const overBudget = []; for (const event of terminalEvents) { const pageSamples = rowsByPage.get(event.pageKey) || []; if (pageSamples.length === 0 || event.tsMs < pageSamples[0].tsMs) continue; const alreadyVisible = lastWorkbenchSampleAtOrBefore(pageSamples, event.tsMs, event, (row) => workbenchSampleHasTerminalProjection(row.sample, event)); if (alreadyVisible) continue; const firstAfter = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event); const firstMiss = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + budgetMs, event, (row) => !workbenchSampleHasTerminalProjection(row.sample, event)); const resolved = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event, (row) => workbenchSampleHasTerminalProjection(row.sample, event)); const deltaMs = resolved ? Math.max(0, Math.round(resolved.tsMs - event.tsMs)) : null; const unresolved = !resolved; const exceedsBudget = unresolved || (Number.isFinite(deltaMs) && deltaMs > budgetMs); if (!firstMiss) continue; if (!exceedsBudget) continue; overBudget.push({ ts: event.ts, pageRole: event.pageRole, pageId: event.pageId, routeKind: event.routeKind, method: event.method, path: event.path, traceIds: event.traceIds.slice(0, 6), sessionIds: event.sessionIds.slice(0, 4), terminalEvidenceCount: event.terminalEvidenceCount, traceEventLikeCount: event.traceEventLikeCount, finalTextFieldCount: event.finalTextFieldCount, budgetMs, windowMs, resolvedDeltaMs: deltaMs, unresolvedWithinWindow: unresolved, firstAfterSample: compactWorkbenchProjectionSample(firstAfter?.sample, event), firstMissSample: compactWorkbenchProjectionSample(firstMiss?.sample, event), resolvedSample: compactWorkbenchProjectionSample(resolved?.sample, event), valuesRedacted: true }); } return { summary: { terminalEventCount: terminalEvents.length, overBudgetCount: overBudget.length, budgetMs, windowMs, valuesRedacted: true }, overBudget, terminalEvents: terminalEvents.slice(-20), valuesRedacted: true }; } function workbenchSampleMatchesCanarySessions(sample, canarySessionIds) { if (!(canarySessionIds instanceof Set) || canarySessionIds.size === 0) return true; const sessionIds = [sample?.routeSessionId, sample?.activeSessionId].filter(Boolean).map(String); return sessionIds.some((id) => canarySessionIds.has(id)); } function workbenchTerminalEventMatchesCanarySessions(event, canarySessionIds) { if (!event) return false; if (!(canarySessionIds instanceof Set) || canarySessionIds.size === 0) return true; const eventSessionIds = Array.isArray(event.sessionIds) ? event.sessionIds.map(String) : []; return eventSessionIds.some((id) => canarySessionIds.has(id)); } function compactWorkbenchTerminalApiEvent(item) { if (!item || item.type !== "response" || item.observerInitiated === true) return null; const summary = objectValue(item.bodySummary); if (!summary || Number(summary.terminalEvidenceCount ?? 0) <= 0) return null; const parsed = parseApiDomLagUrl(item.url); if (!String(parsed.path || "").startsWith("/v1/workbench/") && parsed.path !== "/v1/agent/chat" && parsed.path !== "/v1/agent/chat/steer") return null; const routeKind = summary.pathKind ?? apiDomLagRouteKind(parsed.path); if (!isReliableWorkbenchTerminalApiEvent(summary, routeKind)) return null; const terminalTraceIds = uniqueSorted(Array.isArray(summary.terminalTraceIds) ? summary.terminalTraceIds : []).slice(0, 12); if (terminalTraceIds.length === 0) return null; const tsMs = Date.parse(item.ts || ""); if (!Number.isFinite(tsMs)) return null; return { ts: item.ts, tsMs, pageRole: item.pageRole ?? null, pageId: item.pageId ?? null, pageKey: String(item.pageRole || "control") + ":" + String(item.pageId || "default"), method: String(item.method || "GET").toUpperCase(), path: parsed.path, routeKind, traceIds: terminalTraceIds, observedTraceIds: uniqueSorted([...(Array.isArray(summary.traceIds) ? summary.traceIds : []), parsed.traceId].filter(Boolean)).slice(0, 12), sessionIds: uniqueSorted([...(Array.isArray(summary.sessionIds) ? summary.sessionIds : []), parsed.sessionId].filter(Boolean)).slice(0, 12), terminalEvidenceCount: Number(summary.terminalEvidenceCount ?? 0), terminalStatusCount: Number(summary.terminalStatusCount ?? 0), terminalTextCount: Number(summary.terminalTextCount ?? 0), traceEventLikeCount: Number(summary.traceEventLikeCount ?? 0), finalTextFieldCount: Number(summary.finalTextFieldCount ?? 0), valuesRedacted: true }; } function isReliableWorkbenchTerminalApiEvent(summary, routeKind) { if (!summary || routeKind !== "workbench-turn") return false; if (Number(summary.terminalEvidenceCount ?? 0) <= 0) return false; return Number(summary.runningStatusCount ?? 0) <= 0; } function lastWorkbenchSampleAtOrBefore(rows, tsMs, event, predicate = null) { let result = null; for (const row of rows || []) { if (row.tsMs > tsMs) break; if (!workbenchSampleMatchesTerminalEvent(row.sample, event)) continue; if (typeof predicate === "function" && !predicate(row)) continue; result = row; } return result; } function firstWorkbenchSampleAfter(rows, startMs, endMs, event, predicate = null) { for (const row of rows || []) { if (row.tsMs < startMs) continue; if (row.tsMs > endMs) break; if (!workbenchSampleMatchesTerminalEvent(row.sample, event)) continue; if (typeof predicate === "function" && !predicate(row)) continue; return row; } return null; } function workbenchSampleMatchesTerminalEvent(sample, event) { if (!sample || !event) return false; if (event.sessionIds.length > 0) { const sessionIds = new Set([sample.routeSessionId, sample.activeSessionId].filter(Boolean).map(String)); if (!event.sessionIds.some((id) => sessionIds.has(id))) return false; } if (event.traceIds.length > 0) { const traces = sampleTraceIds(sample); if (traces.size === 0) return false; if (!event.traceIds.some((id) => traces.has(id))) return false; } return true; } function workbenchSampleHasTerminalProjection(sample, event) { const traceIds = event.traceIds.length > 0 ? event.traceIds : Array.from(sampleTraceIds(sample)); if (traceIds.length === 0) return false; return traceIds.some((traceId) => workbenchFinalMessageVisible(sample, traceId) || workbenchTerminalTurnVisible(sample, traceId)); } function compactWorkbenchProjectionSample(sample, event = null) { if (!sample) return null; const eventTraceIds = event?.traceIds && event.traceIds.length > 0 ? event.traceIds : Array.from(sampleTraceIds(sample)); const visibleTraceIds = eventTraceIds.slice(0, 6); return { ...ref(sample), messageCount: Array.isArray(sample.messages) ? sample.messages.length : 0, turnCount: Array.isArray(sample.turns) ? sample.turns.length : 0, traceRowCount: Array.isArray(sample.traceRows) ? sample.traceRows.length : 0, traceIds: visibleTraceIds, finalMessageVisible: visibleTraceIds.some((traceId) => workbenchFinalMessageVisible(sample, traceId)), terminalTurnVisible: visibleTraceIds.some((traceId) => workbenchTerminalTurnVisible(sample, traceId)), valuesRedacted: true }; } function isWorkbenchPathSample(sample) { return /\/workbench(?:\/|$)/u.test(String(sample?.path || sample?.url || "")); } function workbenchTerminalTraceIdsFromDom(sample) { const ids = new Set(); for (const groupName of ["turns", "messages"]) { const group = Array.isArray(sample?.[groupName]) ? sample[groupName] : []; for (const item of group) { const traceId = stringOrNull(item?.traceId); if (!traceId) continue; if (workbenchDomItemIsTerminal(item)) ids.add(traceId); } } return Array.from(ids).sort(); } function workbenchTraceRowsForTrace(sample, traceId) { const rows = Array.isArray(sample?.traceRows) ? sample.traceRows : []; return rows.filter((item) => !traceId || item?.traceId === traceId); } function workbenchFinalMessageVisible(sample, traceId) { const messages = Array.isArray(sample?.messages) ? sample.messages : []; return messages.some((item) => (!traceId || item?.traceId === traceId) && workbenchDomItemLooksFinal(item)); } function workbenchTerminalTurnVisible(sample, traceId) { const turns = Array.isArray(sample?.turns) ? sample.turns : []; return turns.some((item) => (!traceId || item?.traceId === traceId) && workbenchDomItemIsTerminal(item)); } function workbenchDomItemIsTerminal(item) { const status = String(item?.status || "").toLowerCase(); if (/^(completed|complete|succeeded|success|failed|failure|error|canceled|cancelled|done)$/u.test(status)) return true; return isTerminalTraceText([item?.status, item?.textPreview, item?.text, item?.durationText, item?.activityText].filter(Boolean).join(" ")); } function workbenchDomItemLooksFinal(item) { const text = [item?.status, item?.textPreview, item?.text].filter(Boolean).join(" "); return workbenchDomItemIsTerminal(item) && isFinalResultText(text); } function promptCommandHasAuthoritativeSubmitSideEffect(control, promptRound) { const commandId = stringOrNull(promptRound?.promptCommandId); if (!commandId) return false; const row = (control || []).find((item) => item?.type === "sendPrompt" && item?.phase === "completed" && item?.commandId === commandId); const detail = objectValue(row?.detail); const chatSubmit = objectValue(detail.chatSubmit); const sideEffect = objectValue(chatSubmit.sideEffect); return chatSubmit.sideEffectObserved === true || sideEffect.submitted === true || Number(sideEffect.messageCountDelta ?? 0) > 0; } function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures = [], manifest = {}, apiDomLag = null) { const findings = []; const effectiveApiDomLag = apiDomLag || buildApiDomLagReport(samples, network); if (commandFailures.length > 0) findings.push({ id: "observer-command-failed", severity: "red", summary: "observer control commands failed; analyze must surface command failure instead of hiding it in command artifacts", count: commandFailures.length, commands: commandFailures.slice(0, 20) }); findings.push(...buildFrontendFreezeFindings(errors, control)); findings.push(...buildControlledNavigationRootCauseFindings(control, manifest)); findings.push(...buildSessionInvariantFindings(control, manifest)); const commandTimes = control .filter((item) => item.phase === "completed" || item.phase === "started" || item.type === "observer-periodic-refresh") .map((item) => Date.parse(item.ts)) .filter(Number.isFinite); const controlledNavigationWindows = sessionInvariantNavigationWindows(control); const routeSessions = new Set(samples.map((item) => item.routeSessionId).filter(Boolean)); const activeSessions = new Set(samples.map((item) => item.activeSessionId).filter(Boolean)); const routeSessionUnexpected = sessionChangeSamplesOutsideControlledNavigation(samples, "routeSessionId", controlledNavigationWindows); const activeSessionUnexpected = sessionChangeSamplesOutsideControlledNavigation(samples, "activeSessionId", controlledNavigationWindows); if (routeSessions.size > 1 && routeSessionUnexpected.length > 0) findings.push({ id: "session-route-changed", severity: "amber", summary: "route session changed outside controlled session-invariance navigation windows", routeSessionCount: routeSessions.size, samples: sampleRefs(routeSessionUnexpected, (item) => item.routeSessionId) }); if (activeSessions.size > 1 && activeSessionUnexpected.length > 0) findings.push({ id: "active-session-changed", severity: "amber", summary: "active session changed outside controlled session-invariance navigation windows", activeSessionCount: activeSessions.size, samples: sampleRefs(activeSessionUnexpected, (item) => item.activeSessionId) }); const mismatches = samples.filter((item) => item.routeSessionId && item.activeSessionId && item.routeSessionId !== item.activeSessionId && !sampleInControlledNavigationWindow(item, controlledNavigationWindows)); if (mismatches.length > 0) findings.push({ id: "route-active-session-mismatch", severity: "red", summary: "routeSessionId and activeSessionId diverged", count: mismatches.length, samples: mismatches.slice(0, 10).map(ref) }); const uncommandedChanges = []; const commandedPromptSeqs = new Set((sampleMetrics?.timeline ?? []).filter((item) => Number(item?.promptIndex) > 0).map((item) => item.seq).filter((seq) => seq !== null && seq !== undefined)); const previousDigestByPage = new Map(); for (const sample of samples) { const pageKey = samplePageKey(sample); const previous = previousDigestByPage.get(pageKey); const next = digestSample(sample); if (previous && previous.digest !== next && !commandedPromptSeqs.has(sample?.seq) && !nearCommand(sample, commandTimes, alertThresholds.uncommandedStateChangeCommandWindowMs)) { uncommandedChanges.push({ ...ref(sample), previousSeq: previous.sample?.seq ?? null, previousTs: previous.sample?.timestamp ?? previous.sample?.ts ?? null, fromDigest: previous.digest, toDigest: next, fromMessageCount: previous.sample?.messageCount ?? previous.sample?.messages?.length ?? null, toMessageCount: sample?.messageCount ?? sample?.messages?.length ?? null, fromTraceRowCount: Array.isArray(previous.sample?.traceRows) ? previous.sample.traceRows.length : previous.sample?.traceRowCount ?? null, toTraceRowCount: Array.isArray(sample?.traceRows) ? sample.traceRows.length : sample?.traceRowCount ?? null, fromPath: previous.sample?.path ?? previous.sample?.urlPath ?? null, toPath: sample?.path ?? sample?.urlPath ?? null, detail: "visible digest changed without nearby command", }); } previousDigestByPage.set(pageKey, { digest: next, sample }); } if (uncommandedChanges.length > 0) findings.push({ id: "uncommanded-visible-state-change", severity: "amber", summary: "visible message/trace digest changed without a nearby command", count: uncommandedChanges.length, samples: uncommandedChanges.slice(0, 20) }); const finalFlicker = detectFinalFlicker(samples); if (finalFlicker.length > 0) findings.push({ id: "final-response-flicker", severity: "red", summary: "message text digest disappeared or switched to diagnostic-like text after non-empty final text", count: finalFlicker.length, samples: finalFlicker.slice(0, 20) }); const terminalZeroElapsed = detectTerminalZeroElapsed(samples); if (terminalZeroElapsed.length > 0) findings.push({ id: "turn-terminal-zero-elapsed", severity: "amber", summary: "terminal Code Agent card displayed 耗时 0 秒; terminal duration issue is a non-blocking timing alert", count: terminalZeroElapsed.length, samples: terminalZeroElapsed.slice(0, 20) }); const cardTiming = sampleMetrics?.codeAgentCardTiming || {}; const cardTimingSummary = cardTiming.summary || {}; if (Number(cardTimingSummary.missingElapsedCount ?? 0) > 0) findings.push({ id: "code-agent-card-elapsed-missing", severity: "amber", summary: "visible Code Agent card did not display total elapsed time; elapsed visibility is a non-blocking timing alert", count: cardTimingSummary.missingElapsedCount, samples: (cardTiming.missingElapsed || []).slice(0, 20) }); if (Number(cardTimingSummary.missingRecentUpdateCount ?? 0) > 0) findings.push({ id: "code-agent-card-running-recent-update-missing", severity: "amber", summary: "non-terminal Code Agent card did not display 最近更新; recent-update visibility is a non-blocking timing alert", count: cardTimingSummary.missingRecentUpdateCount, samples: (cardTiming.missingRecentUpdate || []).slice(0, 20) }); const roundCompletion = cardTiming.roundCompletion || {}; if (Number(cardTimingSummary.durationUnderreportedCount ?? 0) > 0) { findings.push({ id: "code-agent-card-duration-underreported", severity: "amber", summary: "completed Code Agent card total elapsed is shorter than trace/final-response duration evidence; timing mismatch is a non-blocking alert", timingSourceOfTruth: "trace-completion-total-or-final-response-duration", timingStatus: timingStatusFromRows(cardTiming.durationUnderreported, "business-turn-completed"), timingAlert: true, count: cardTimingSummary.durationUnderreportedCount, samples: (cardTiming.durationUnderreported || []).slice(0, 20), }); } if (Number(cardTimingSummary.durationMismatchCount ?? 0) > 0) { findings.push({ id: "code-agent-card-duration-mismatch", severity: "amber", summary: "completed Code Agent card total elapsed does not match sealed completion/final-response timing evidence; timing mismatch is a non-blocking alert", timingSourceOfTruth: "trace-completion-total-or-final-response-duration", timingStatus: timingStatusFromRows(cardTiming.durationMismatches, "business-turn-completed"), timingAlert: true, count: cardTimingSummary.durationMismatchCount, samples: (cardTiming.durationMismatches || []).slice(0, 20), }); } const traceOrder = sampleMetrics?.traceOrder || {}; const traceOrderSummary = traceOrder.summary || {}; if (Number(traceOrderSummary.orderAnomalyCount ?? 0) > 0) { findings.push({ id: "trace-row-order-nonmonotonic", severity: "red", summary: "visible trace rows are not monotonic by total time, clock time, or projected sequence in DOM order", count: traceOrderSummary.orderAnomalyCount, samples: (traceOrder.orderAnomalies || []).slice(0, 20), }); } if (Number(traceOrderSummary.completionNotLastCount ?? 0) > 0) { findings.push({ id: "trace-completion-row-not-last", severity: "red", summary: "visible trace shows a completion row before later trace rows for the same trace", count: traceOrderSummary.completionNotLastCount, samples: (traceOrder.completionNotLast || []).slice(0, 20), }); } if (Number(cardTimingSummary.roundCompletionElapsedMismatchCount ?? 0) > 0) findings.push({ id: "round-completion-elapsed-mismatch", severity: "amber", summary: "Trace row 轮次完成(总耗时 ...) does not match the visible Code Agent card total elapsed time within YAML timing slack; timing mismatch is a non-blocking alert", timingSourceOfTruth: "trace-round-completion-total", timingStatus: timingStatusFromRows(roundCompletion.elapsedMismatches, "business-turn-completed"), timingAlert: true, count: cardTimingSummary.roundCompletionElapsedMismatchCount, toleranceSeconds: cardTimingSummary.elapsedMismatchToleranceSeconds, samples: (roundCompletion.elapsedMismatches || []).slice(0, 20) }); if (Number(cardTimingSummary.roundCompletionFinalResponseMissingCount ?? 0) > 0) findings.push({ id: "round-completion-final-response-missing", severity: "red", summary: "Trace row showed 轮次完成, but no final response was visible in the Code Agent card afterward", count: cardTimingSummary.roundCompletionFinalResponseMissingCount, samples: (roundCompletion.finalResponseMissing || []).slice(0, 20) }); if (Number(cardTimingSummary.roundCompletionPostTimingChangeCount ?? 0) > 0) findings.push({ id: "round-completion-post-timing-change", severity: "amber", summary: "After 轮次完成, card total elapsed or 最近更新 continued changing; terminal timing alert is non-blocking", count: cardTimingSummary.roundCompletionPostTimingChangeCount, samples: (roundCompletion.postCompletionTimingChanges || []).slice(0, 20) }); if (Number(cardTimingSummary.roundCompletionPostRecentUpdateVisibleCount ?? 0) > 0) findings.push({ id: "round-completion-recent-update-still-visible", severity: "info", summary: "最近更新 was still visible after 轮次完成; inspect whether terminal cards should hide activity age or keep it sealed", count: cardTimingSummary.roundCompletionPostRecentUpdateVisibleCount, samples: (roundCompletion.postCompletionRecentUpdateVisible || []).slice(0, 20) }); const scrollJumps = []; for (let i = 1; i < samples.length; i += 1) { const prevY = Number(samples[i - 1]?.scroll?.y ?? 0); const nextY = Number(samples[i]?.scroll?.y ?? 0); if (prevY > alertThresholds.scrollJumpFromY && nextY < alertThresholds.scrollJumpToY && !nearCommand(samples[i], commandTimes, alertThresholds.scrollJumpCommandWindowMs)) scrollJumps.push({ from: ref(samples[i - 1]), to: ref(samples[i]) }); } if (scrollJumps.length > 0) findings.push({ id: "scroll-jump-top", severity: "amber", summary: "scroll position jumped near top without nearby command", count: scrollJumps.length, samples: scrollJumps.slice(0, 10) }); const traceTerminal = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.some((row) => isTerminalTraceText((row.status || "") + " " + (row.textPreview || "")))); const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0); if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) }); const workbenchInPlaceProjectionLag = detectWorkbenchInPlaceProjectionLag(samples, network, control); if (workbenchInPlaceProjectionLag.terminalTraceMissing.length > 0) findings.push({ id: "workbench-terminal-trace-not-hydrated-in-place", severity: "red", summary: "Workbench rendered a terminal turn/message in-place while the same trace still had no visible run-record trace rows", count: workbenchInPlaceProjectionLag.terminalTraceMissing.length, samples: workbenchInPlaceProjectionLag.terminalTraceMissing.slice(0, 20), rootCause: "workbench_trace_projection_not_hydrated_in_place", rootCauseStatus: "confirmed-from-dom-samples", rootCauseConfidence: "high", valuesRedacted: true }); if (workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.length > 0) findings.push({ id: "workbench-terminal-api-dom-not-refreshed-in-place", severity: "red", summary: "Workbench REST returned terminal/final trace evidence but the same in-place page did not render terminal message plus trace rows within the configured budget", count: workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.length, budgetMs: workbenchInPlaceProjectionLag.terminalApiDomLag.summary.budgetMs, windowMs: workbenchInPlaceProjectionLag.terminalApiDomLag.summary.windowMs, samples: workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.slice(0, 20), rootCause: "workbench_rest_terminal_projection_dom_lag", rootCauseStatus: "confirmed-from-network-body-summary-and-dom-samples", rootCauseConfidence: "high", valuesRedacted: true }); const promptFailures = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.chatPostOk === false && !promptCommandHasAuthoritativeSubmitSideEffect(control, item)) : []; if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat or /v1/agent/chat/steer POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) }); const promptSteerRounds = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.steerUsed === true) : []; if (promptSteerRounds.length > 0) findings.push({ id: "prompt-routed-to-steer", severity: "amber", summary: "sendPrompt was submitted through /v1/agent/chat/steer; verify the previous turn was truly in-flight and not an unsealed terminal failure", count: promptSteerRounds.length, rounds: promptSteerRounds.slice(0, 10) }); const elapsedZeroResets = Array.isArray(sampleMetrics?.turnTimingElapsedZeroResets) ? sampleMetrics.turnTimingElapsedZeroResets : []; if (elapsedZeroResets.length > 0) findings.push({ id: "turn-timing-total-elapsed-zero-reset", severity: "amber", summary: "Code Agent total elapsed jumped from a non-zero value back to 0 seconds; timing reset is a non-blocking alert", timingSourceOfTruth: "dom-card-total-elapsed-sequence", timingStatus: timingStatusFromRows(elapsedZeroResets), timingAlert: true, count: elapsedZeroResets.length, samples: elapsedZeroResets.slice(0, 20) }); const elapsedDecreases = Array.isArray(sampleMetrics?.turnTimingNonMonotonic) ? sampleMetrics.turnTimingNonMonotonic.filter((item) => item.metric === "totalElapsedSeconds" && item.anomaly !== "zero-reset") : []; if (elapsedDecreases.length > 0) findings.push({ id: "turn-timing-total-elapsed-decrease", severity: "amber", summary: "Code Agent total elapsed decreased between adjacent samples; timing decrease is a non-blocking alert", timingSourceOfTruth: "dom-card-total-elapsed-sequence", timingStatus: timingStatusFromRows(elapsedDecreases), timingAlert: true, count: elapsedDecreases.length, samples: elapsedDecreases.slice(0, 20) }); const elapsedForwardJumps = Array.isArray(sampleMetrics?.turnTimingTotalElapsedForwardJumps) ? sampleMetrics.turnTimingTotalElapsedForwardJumps : []; if (elapsedForwardJumps.length > 0) findings.push({ id: "turn-timing-total-elapsed-forward-jump", severity: "amber", summary: "Code Agent total elapsed jumped forward faster than browser sample interval; timing jump is a non-blocking alert", timingSourceOfTruth: "dom-card-total-elapsed-sequence", timingStatus: timingStatusFromRows(elapsedForwardJumps), timingAlert: true, count: elapsedForwardJumps.length, samples: elapsedForwardJumps.slice(0, 20) }); const terminalElapsedGrowth = Array.isArray(sampleMetrics?.turnTimingTerminalElapsedGrowth) ? sampleMetrics.turnTimingTerminalElapsedGrowth : []; if (terminalElapsedGrowth.length > 0) findings.push({ id: "turn-timing-terminal-elapsed-growth", severity: "amber", summary: "terminal Code Agent card total elapsed changed after terminal status; terminal timing alert is non-blocking", timingSourceOfTruth: "terminal-card-total-elapsed-seal", timingStatus: timingStatusFromRows(terminalElapsedGrowth, "business-turn-completed"), timingAlert: true, count: terminalElapsedGrowth.length, samples: terminalElapsedGrowth.slice(0, 20) }); const recentUpdateSawtoothJumps = Array.isArray(sampleMetrics?.turnTimingRecentUpdateSawtoothJumps) ? sampleMetrics.turnTimingRecentUpdateSawtoothJumps : Array.isArray(sampleMetrics?.turnTimingNonMonotonic) ? sampleMetrics.turnTimingNonMonotonic.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump") : []; if (recentUpdateSawtoothJumps.length > 0) findings.push({ id: "turn-timing-recent-update-sawtooth-jump", severity: "amber", summary: "最近更新 value jumped faster than sample interval; expected sawtooth increase-or-reset", count: recentUpdateSawtoothJumps.length, samples: recentUpdateSawtoothJumps.slice(0, 20) }); const severeTimeoutRounds = Array.isArray(sampleMetrics?.rounds) ? sampleMetrics.rounds.filter((item) => Number(item.maxTotalElapsedSeconds) > alertThresholds.turnElapsedSevereTimeoutSeconds) : []; const severeTimeoutSamples = Array.isArray(sampleMetrics?.timeline) ? sampleMetrics.timeline.filter((item) => Number(item.totalElapsedSeconds) > alertThresholds.turnElapsedSevereTimeoutSeconds) : []; if (severeTimeoutRounds.length > 0 || severeTimeoutSamples.length > 0) findings.push({ id: "turn-elapsed-severe-timeout", severity: "amber", summary: "turn total elapsed exceeded the YAML-configured elapsed alert threshold; timing is a non-blocking alert unless the turn fails to complete or breaks multi-round continuity", timingSourceOfTruth: "dom-card-total-elapsed-yaml-threshold", timingStatus: timingStatusFromRows([...severeTimeoutRounds, ...severeTimeoutSamples], "observer-timeout"), timingAlert: true, thresholdSeconds: alertThresholds.turnElapsedSevereTimeoutSeconds, count: Math.max(severeTimeoutRounds.length, severeTimeoutSamples.length), rounds: severeTimeoutRounds.slice(0, 20), samples: severeTimeoutSamples.slice(0, 20) }); const loadingSummary = sampleMetrics?.loading?.summary || {}; const visibleLoadingSlowSeconds = alertThresholds.visibleLoadingSlowMs / 1000; if (Number(loadingSummary.longestContinuousSeconds ?? 0) > visibleLoadingSlowSeconds) findings.push({ id: "page-loading-visible-over-budget", severity: "red", summary: "visible 加载中 stayed on screen longer than configured YAML budget; fix real loading latency instead of revealing incomplete content early", count: loadingSummary.overBudgetSegmentCount ?? loadingSummary.overFiveSecondSegmentCount ?? 1, longestContinuousSeconds: loadingSummary.longestContinuousSeconds, budgetSeconds: visibleLoadingSlowSeconds, segments: sampleMetrics.loading.segments.slice(0, 20), owners: sampleMetrics.loading.owners.slice(0, 20) }); if (Number(loadingSummary.maxSimultaneousCount ?? 0) > 1) findings.push({ id: "page-loading-concurrent", severity: "info", summary: "multiple 加载中 indicators were visible in the same sampled DOM point", count: loadingSummary.concurrentLoadingSampleCount ?? 0, maxSimultaneousCount: loadingSummary.maxSimultaneousCount, owners: sampleMetrics.loading.owners.slice(0, 20) }); const sessionRailTitleSummary = sampleMetrics?.sessionRailTitles?.summary || {}; if (Number(sessionRailTitleSummary.overThresholdSampleCount ?? sessionRailTitleSummary.majorityFallbackSampleCount ?? 0) > 0) findings.push({ id: "session-rail-title-fallback-root-cause", severity: "red", summary: "INV-02 root cause visible: session rail is rendering fallback Session ses_* titles, so list projection/read model or rail binding is missing stable title/preview data before DOM render", rootCause: "session_title_fallback_from_facts", rootCauseStatus: "confirmed-from-dom-session-rail", rootCauseConfidence: "high", nextAction: "Check OTel session_list_read fallbackTitleCount/fallbackTitleRatio and emptyPreviewCount for the same run; fix session list projection/read model title/preview before changing DOM fallback text.", count: sessionRailTitleSummary.overThresholdSampleCount ?? sessionRailTitleSummary.majorityFallbackSampleCount, thresholdRatio: alertThresholds.sessionRailFallbackRatio, maxFallbackRatio: sessionRailTitleSummary.maxFallbackRatio, maxFallbackTitleCount: sessionRailTitleSummary.maxFallbackTitleCount, evidence: { thresholdRatio: alertThresholds.sessionRailFallbackRatio, maxFallbackRatio: sessionRailTitleSummary.maxFallbackRatio, maxFallbackTitleCount: sessionRailTitleSummary.maxFallbackTitleCount, overThresholdSampleCount: sessionRailTitleSummary.overThresholdSampleCount ?? null, majorityFallbackSampleCount: sessionRailTitleSummary.majorityFallbackSampleCount ?? null, valuesRedacted: true }, samples: sampleMetrics.sessionRailTitles.samples.slice(0, 20), examples: sampleMetrics.sessionRailTitles.examples.slice(0, 20), valuesRedacted: true }); const traceEventsPageReadIssues = detectTraceEventsPageReadIssues(network); if (traceEventsPageReadIssues.http404.length > 0) findings.push({ id: "trace-events-page-read-404-root-cause", severity: "red", summary: "INV-07 root cause visible: /v1/workbench/traces/:traceId/events returned HTTP 404 for a trace event page read, so the failure is in the trace-events API paging/read-model contract before DOM rendering", rootCause: "trace_events_paging_contract_mismatch", rootCauseStatus: "confirmed-from-browser-network", rootCauseConfidence: "high", nextAction: "Use OTel trace_events_read for the same trace to compare sinceSeq/afterProjectedSeq, returnedEvents, range, totalEvents, hasMore and fullTraceLoaded; fix backend paging contract or add missing instrumentation before changing renderer behavior.", count: traceEventsPageReadIssues.http404.length, evidence: traceEventsPageReadIssues.summary, samples: traceEventsPageReadIssues.http404.slice(0, 20), valuesRedacted: true }); if (traceEventsPageReadIssues.httpErrors.length > 0) findings.push({ id: "trace-events-page-read-http-error-root-cause", severity: "red", summary: "trace events page read returned HTTP error status; monitor can localize this to the trace-events API instead of a generic Workbench render failure", rootCause: "trace-events-api-page-read-http-error", rootCauseStatus: "confirmed-from-browser-network", rootCauseConfidence: "high", nextAction: "Drill down by traceId and afterProjectedSeq, then compare with OTel trace_events_read paging fields.", count: traceEventsPageReadIssues.httpErrors.length, evidence: traceEventsPageReadIssues.summary, samples: traceEventsPageReadIssues.httpErrors.slice(0, 20), valuesRedacted: true }); if (traceEventsPageReadIssues.requestFailed.length > 0) findings.push({ id: "trace-events-page-read-requestfailed-root-cause", severity: "amber", summary: "trace events page read failed at browser/network level; monitor localized the failure path, but HTTP status is unavailable so OTel/API instrumentation is needed to confirm the backend root cause", rootCause: "trace-events-api-page-read-network-failed", rootCauseStatus: "network-signal-needs-otel-confirmation", rootCauseConfidence: "medium", nextAction: "Check whether this happened during observer refresh/navigation; if not, query OTel by /v1/workbench/traces/:traceId/events and add route span fields when status/afterProjectedSeq are missing.", count: traceEventsPageReadIssues.requestFailed.length, evidence: traceEventsPageReadIssues.summary, samples: traceEventsPageReadIssues.requestFailed.slice(0, 20), valuesRedacted: true }); if ((runtimeAlerts?.summary?.httpErrorCount ?? 0) > 0) findings.push({ id: "runtime-http-errors", severity: "amber", summary: "natural page requests returned HTTP error status during observation", count: runtimeAlerts.summary.httpErrorCount, groups: runtimeAlerts.networkHttpErrorsByPath.slice(0, 12) }); if ((runtimeAlerts?.summary?.significantRequestFailedCount ?? runtimeAlerts?.summary?.requestFailedCount ?? 0) > 0) findings.push({ id: "runtime-requestfailed", severity: "amber", summary: "browser requestfailed events were captured during observation", count: runtimeAlerts.summary.significantRequestFailedCount ?? runtimeAlerts.summary.requestFailedCount, groups: (runtimeAlerts.networkSignificantRequestFailedByPath ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 12) }); if ((runtimeAlerts?.summary?.domDiagnosticSampleCount ?? 0) > 0) findings.push({ id: "runtime-dom-diagnostics", severity: "amber", summary: "diagnostic/error/warning-like text was visible in sampled DOM", count: runtimeAlerts.summary.domDiagnosticSampleCount, groupCount: runtimeAlerts.summary.domDiagnosticGroupCount ?? 0, groups: runtimeAlerts.domDiagnosticsByText.slice(0, 12), samples: runtimeAlerts.domDiagnostics.slice(0, 12) }); if ((runtimeAlerts?.summary?.executionErrorCount ?? 0) > 0) findings.push({ id: "runtime-execution-errors", severity: "red", summary: "Workbench rendered execution failure/error rows during observation", count: runtimeAlerts.summary.executionErrorCount, groups: runtimeAlerts.runtimeExecutionErrorsByCode.slice(0, 12) }); if ((runtimeAlerts?.summary?.significantConsoleAlertCount ?? runtimeAlerts?.summary?.consoleAlertCount ?? 0) > 0) findings.push({ id: "runtime-console-alerts", severity: "amber", summary: "browser console warning/error entries were captured during observation", count: runtimeAlerts.summary.significantConsoleAlertCount ?? runtimeAlerts.summary.consoleAlertCount, groups: (runtimeAlerts.significantConsoleAlertsByPath ?? runtimeAlerts.consoleAlertsByPath).slice(0, 12) }); const crossPageDiffs = mergeCrossPageDiffRows( detectCrossPageProjectionDiffs(samples), detectAdjacentCrossPageProjectionDiffs(samples) ); const crossPageProjectionDiffs = crossPageDiffs.filter((item) => item.diffKind !== "trace-visibility"); const crossPageTraceVisibilityDiffs = crossPageDiffs.filter((item) => item.diffKind === "trace-visibility"); const crossPageProjectionBudgetMs = alertThresholds.crossPageProjectionDivergenceRedMs; const sampleBySeq = new Map(samples.map((item) => [Number(item?.seq), item]).filter(([seq]) => Number.isFinite(seq))); const appShellNotReadyRows = detectWorkbenchAppShellNotReady(samples); const persistentAppShellNotReadyRows = appShellNotReadyRows.filter((item) => Number(item.observedSpanMs ?? 0) > crossPageProjectionBudgetMs); const transientAppShellNotReadyRows = appShellNotReadyRows.filter((item) => Number(item.observedSpanMs ?? 0) <= crossPageProjectionBudgetMs); if (persistentAppShellNotReadyRows.length > 0) findings.push({ id: "workbench-app-shell-not-ready", severity: "red", summary: "Workbench route document loaded but the app shell never mounted or hydrated within the configured budget; treat this as page/runtime startup evidence, not a Workbench projection divergence", count: persistentAppShellNotReadyRows.length, budgetMs: crossPageProjectionBudgetMs, samples: persistentAppShellNotReadyRows.slice(0, 20), rootCause: "workbench_page_app_shell_not_ready", rootCauseStatus: "confirmed-from-dom-and-asset-provenance", rootCauseConfidence: "high", valuesRedacted: true }); if (transientAppShellNotReadyRows.length > 0) findings.push({ id: "workbench-app-shell-transient-not-ready", severity: "info", summary: "Workbench route briefly had a document and assets but no mounted app shell; retained as startup context", count: transientAppShellNotReadyRows.length, budgetMs: crossPageProjectionBudgetMs, samples: transientAppShellNotReadyRows.slice(0, 20) }); const timedCrossPageProjectionDiffs = annotateCrossPageDiffTiming(crossPageProjectionDiffs); const controlledNavigationHydrationProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq)); const appShellNotReadyProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => !controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq) && crossPageDiffHasWorkbenchAppShellNotReady(item, sampleBySeq)); const evaluatedCrossPageProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => !controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq) && !crossPageDiffHasWorkbenchAppShellNotReady(item, sampleBySeq)); const persistentCrossPageProjectionDiffs = evaluatedCrossPageProjectionDiffs.filter((item) => Number(item.observedSpanMs ?? 0) > crossPageProjectionBudgetMs); const transientCrossPageProjectionDiffs = evaluatedCrossPageProjectionDiffs.filter((item) => Number(item.observedSpanMs ?? 0) <= crossPageProjectionBudgetMs); if (persistentCrossPageProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-divergence", severity: "red", summary: "control and observer pages saw different projection state for the same sampled session beyond the configured budget", count: persistentCrossPageProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: persistentCrossPageProjectionDiffs.slice(0, 20) }); if (transientCrossPageProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-transient-divergence", severity: "info", summary: "control and observer pages briefly differed near a sampled transition; retained as transient evidence but not treated as persistent projection failure", count: transientCrossPageProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: transientCrossPageProjectionDiffs.slice(0, 20) }); if (controlledNavigationHydrationProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-controlled-navigation-hydration", severity: "info", summary: "control and observer pages differed while a non-blocking session-invariance navigation command still had an unhydrated blank page; retained as context but not treated as a red projection blocker", count: controlledNavigationHydrationProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: controlledNavigationHydrationProjectionDiffs.slice(0, 20) }); if (appShellNotReadyProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-app-shell-not-ready", severity: "info", summary: "cross-page projection differences were explained by a page whose Workbench app shell was not mounted; see workbench-app-shell-not-ready for the blocking root cause", count: appShellNotReadyProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: appShellNotReadyProjectionDiffs.slice(0, 20) }); if (crossPageTraceVisibilityDiffs.length > 0) findings.push({ id: "cross-page-trace-visibility-divergence", severity: "info", summary: "control and observer pages differed only in visible trace row count; this is local disclosure/hydration visibility, not session/message projection divergence", count: crossPageTraceVisibilityDiffs.length, samples: crossPageTraceVisibilityDiffs.slice(0, 20) }); const traceMessageDuplicates = detectTraceMessageDuplication(samples); if (traceMessageDuplicates.length > 0) findings.push({ id: "trace-assistant-message-duplicates-final-response", severity: "amber", summary: "trace-frame rendered duplicate visible assistant final rows; the fixed Final Response renderer summary block is excluded", count: traceMessageDuplicates.length, finalResponseSummaryBlockCounted: false, traceFrameSource: "traceRows-only", samples: traceMessageDuplicates.slice(0, 20) }); const turnTraceMissing = detectTurnTraceIdMissing(samples); if (turnTraceMissing.length > 0) findings.push({ id: "turn-trace-id-missing", severity: "red", summary: "Code Agent turn/card was visible without a trace id, so historical trace hydration cannot be reliable", count: turnTraceMissing.length, samples: turnTraceMissing.slice(0, 20) }); const pagePerformanceItems = Array.isArray(pagePerformance?.sameOriginApiByPath) ? pagePerformance.sameOriginApiByPath : []; const slowApi = pagePerformanceItems.filter((item) => item.isLongLivedStream !== true && Number(item.overBudgetCount ?? item.overFiveSecondCount ?? 0) > 0); if (slowApi.length > 0) findings.push({ id: "page-performance-slow-same-origin-api", severity: "red", summary: "same-origin API resource timing exceeded configured YAML usability budget", count: slowApi.length, budgetMs: alertThresholds.sameOriginApiSlowMs, groups: slowApi.slice(0, 20) }); const longLivedStreams = pagePerformanceItems.filter((item) => item.isLongLivedStream); const slowStreamOpen = longLivedStreams.filter((item) => Number(item.streamOpenOverBudgetCount ?? item.streamOpenOverFiveSecondCount ?? 0) > 0); if (slowStreamOpen.length > 0) findings.push({ id: "page-performance-slow-long-lived-stream-open", severity: "red", summary: "long-lived stream open latency exceeded configured YAML usability budget; stream lifetime is still reported separately", count: slowStreamOpen.length, budgetMs: alertThresholds.longLivedStreamOpenSlowMs, groups: slowStreamOpen.slice(0, 20) }); if (longLivedStreams.length > 0) findings.push({ id: "page-performance-long-lived-streams", severity: "info", summary: "same-origin long-lived streams are reported separately; lifetime is not treated as API load latency", count: longLivedStreams.length, groups: longLivedStreams.slice(0, 20) }); if ((pageProvenance?.summary?.segmentCount ?? 0) > 1) findings.push({ id: "page-provenance-segments", severity: "info", summary: "observer crossed page asset provenance segments; interpret runtime findings by segment", segmentCount: pageProvenance.summary.segmentCount, segments: pageProvenance.segments.slice(0, 20) }); const naturalApi = network.filter((item) => item.observerInitiated === false && item.type === "response" && /\/v1\/|\/auth\//u.test(String(item.url || ""))); const apiDomLagSummary = effectiveApiDomLag?.summary || {}; findings.push({ id: "natural-api-dom-lag-baseline", severity: "info", summary: "natural API responses and DOM samples are available for API-to-DOM lag correlation", naturalApiResponses: naturalApi.length, sampleCount: samples.length, apiDomLag: apiDomLagSummary }); findings.push({ id: "natural-api-dom-lag-candidates", severity: Number(apiDomLagSummary.overBudgetCount ?? 0) > 0 ? "amber" : "info", summary: "state-relevant natural API responses were correlated to the first subsequent DOM digest change; over-budget lag is a non-blocking investigation alert", count: apiDomLagSummary.candidateCount ?? 0, domChangedCount: apiDomLagSummary.domChangedCount ?? 0, noDomChangeWithinWindowCount: apiDomLagSummary.noDomChangeWithinWindowCount ?? 0, overBudgetCount: apiDomLagSummary.overBudgetCount ?? 0, budgetMs: apiDomLagSummary.budgetMs ?? null, p95DomChangeDeltaMs: apiDomLagSummary.p95DomChangeDeltaMs ?? null, maxDomChangeDeltaMs: apiDomLagSummary.maxDomChangeDeltaMs ?? null, groups: Array.isArray(effectiveApiDomLag?.groups) ? effectiveApiDomLag.groups.slice(0, 12) : [], }); if (errors.length > 0) findings.push({ id: "browser-console-or-page-errors", severity: "amber", summary: "pageerror/runner errors were captured", count: errors.length, first: errors.slice(0, 5) }); if (samples.length === 0) findings.push({ id: "no-samples", severity: "red", summary: "observer produced no samples" }); return findings; } function buildFrontendFreezeFindings(errors, control) { const findings = []; const promptTimes = (control || []) .filter((item) => item.type === "sendPrompt" && item.phase === "completed") .map((item) => Date.parse(item.ts)) .filter(Number.isFinite) .sort((a, b) => a - b); const stopWindows = stopCommandWindows(control); const events = (errors || []) .map((item) => frontendFreezeErrorEvent(item, promptTimes)) .filter((item) => item && !errorInsideStopWindow(item, stopWindows)); const domEvents = events.filter((item) => item.kind === "dom-evaluate-timeout"); const controlDomBurst = firstBurst( domEvents.filter((item) => item.pageRole === "control" || item.pageRole === null), alertThresholds.domEvaluateTimeoutRedCount, alertThresholds.domEvaluateTimeoutRedWindowMs, ); if (controlDomBurst) findings.push(frontendFreezeBurstFinding({ id: "frontend-control-dom-evaluate-timeout-red", summary: "control page DOM evaluation timed out repeatedly; treat the browser page as frozen and keep the sentinel red instead of refreshing or falling back", burst: controlDomBurst, thresholdCount: alertThresholds.domEvaluateTimeoutRedCount, windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs, pageRole: "control", })); const observerDomBurst = firstBurst( domEvents.filter((item) => item.pageRole === "observer"), alertThresholds.domEvaluateTimeoutRedCount, alertThresholds.domEvaluateTimeoutRedWindowMs, ); if (observerDomBurst) findings.push(frontendFreezeBurstFinding({ id: "frontend-observer-dom-evaluate-timeout-red", summary: "observer page DOM evaluation timed out repeatedly; the observer page is frozen and later periodic refresh evidence must not clear this run", burst: observerDomBurst, thresholdCount: alertThresholds.domEvaluateTimeoutRedCount, windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs, pageRole: "observer", })); const screenshotBurst = firstBurst( events.filter((item) => item.kind === "screenshot-timeout"), alertThresholds.screenshotTimeoutRedCount, alertThresholds.domEvaluateTimeoutRedWindowMs, ); if (screenshotBurst) findings.push(frontendFreezeBurstFinding({ id: "frontend-screenshot-timeout-red", summary: "browser screenshot capture timed out repeatedly; this is freeze evidence and the sentinel must stay red until investigated", burst: screenshotBurst, thresholdCount: alertThresholds.screenshotTimeoutRedCount, windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs, pageRole: null, })); const pageErrors = events.filter((item) => item.kind === "page-error"); const pageErrorBurst = firstBurst(pageErrors, alertThresholds.pageErrorRedCount, alertThresholds.domEvaluateTimeoutRedWindowMs); if (pageErrorBurst) findings.push(frontendFreezeBurstFinding({ id: "frontend-page-error-red", summary: "browser pageerror entries exceeded the YAML threshold; page runtime exceptions are blocking when repeated in the observation window", burst: pageErrorBurst, thresholdCount: alertThresholds.pageErrorRedCount, windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs, pageRole: null, })); return findings; } function frontendFreezeErrorEvent(item, promptTimes) { const details = objectValue(item?.error?.details); const message = String(item?.error?.message ?? item?.message ?? item?.error ?? ""); const type = String(item?.type || ""); const tsMs = Date.parse(String(item?.ts || "")); if (!Number.isFinite(tsMs)) return null; const kind = classifyFrontendFreezeError(type, message); if (!kind) return null; return { ts: item.ts ?? null, tsMs, promptIndex: promptIndexForTs(promptTimes, item.ts), kind, type: item.type ?? null, pageRole: stringOrNull(item?.pageRole) ?? stringOrNull(details.pageRole) ?? pageRoleFromErrorType(type), pageId: stringOrNull(item?.pageId) ?? stringOrNull(details.pageId), routeSessionId: stringOrNull(item?.routeSessionId) ?? stringOrNull(details.routeSessionId), activeSessionId: stringOrNull(item?.activeSessionId) ?? stringOrNull(details.activeSessionId), commandId: stringOrNull(item?.commandId) ?? stringOrNull(details.commandId), sampleSeq: numberOrNull(item?.sampleSeq ?? details.sampleSeq), timeoutMs: timeoutMsFromMessage(message), messageHash: message ? sha256(message) : null, preview: limitText(message, 240), valuesRedacted: true, }; } function pageRoleFromErrorType(type) { const value = String(type || ""); if (/^control-/iu.test(value)) return "control"; if (/^observer-/iu.test(value)) return "observer"; return null; } function classifyFrontendFreezeError(type, message) { const value = String(message || ""); if (/sampleOnePage\s+DOM\s+evaluate\s+exceeded/iu.test(value) && /(?:control|observer)-sample-error/iu.test(type)) return "dom-evaluate-timeout"; if (/screenshot|captureScreenshot|page\.screenshot/iu.test(type + " " + value) && /timeout|timed\s*out|exceeded/iu.test(value)) return "screenshot-timeout"; if (/pageerror|uncaught|unhandledrejection/iu.test(type) || /^(?:Error|TypeError|ReferenceError|RangeError|SyntaxError):/u.test(value)) return "page-error"; return null; } function firstBurst(events, thresholdCount, windowMs) { const count = Math.max(1, Math.floor(Number(thresholdCount || 0))); const budgetMs = Math.max(1, Number(windowMs || 0)); const sorted = (events || []).filter((item) => Number.isFinite(item?.tsMs)).sort((a, b) => a.tsMs - b.tsMs); if (sorted.length < count) return null; for (let start = 0; start <= sorted.length - count; start += 1) { const end = start + count - 1; if (sorted[end].tsMs - sorted[start].tsMs <= budgetMs) return sorted.slice(start, end + 1); } return null; } function frontendFreezeBurstFinding({ id, summary, burst, thresholdCount, windowMs, pageRole }) { const first = burst[0]; const last = burst[burst.length - 1]; const pageIds = uniqueStrings(burst.map((item) => item.pageId)); const routeSessionIds = uniqueStrings(burst.map((item) => item.routeSessionId)); const activeSessionIds = uniqueStrings(burst.map((item) => item.activeSessionId)); return { id, severity: "red", summary, count: burst.length, thresholdCount, windowMs, firstAt: first?.ts ?? null, lastAt: last?.ts ?? null, pageRole, pageIds, routeSessionIds, activeSessionIds, timeoutMsMax: maxPresentNumber(burst.map((item) => item.timeoutMs)), rootCause: "frontend_page_freeze_or_runtime_exception", rootCauseStatus: "confirmed-from-browser-observer-errors", rootCauseConfidence: "high", fallbackAllowed: false, observerRefreshMayNotClear: true, nextAction: "Keep this run red; do not auto-refresh, fallback, or mark healthy until OTel/browser evidence explains why the page stopped responding.", events: burst.map((item) => ({ ts: item.ts, promptIndex: item.promptIndex, type: item.type, pageRole: item.pageRole, pageId: item.pageId, routeSessionId: item.routeSessionId, activeSessionId: item.activeSessionId, commandId: item.commandId, sampleSeq: item.sampleSeq, timeoutMs: item.timeoutMs, messageHash: item.messageHash, preview: item.preview, valuesRedacted: true, })), valuesRedacted: true, }; } function stopCommandWindows(control) { return (control || []) .filter((item) => /^(?:stop|forceStop|cancel|close)$/iu.test(String(item?.type || item?.command || ""))) .map((item) => { const tsMs = Date.parse(String(item?.ts || "")); return Number.isFinite(tsMs) ? { fromMs: tsMs - 1000, toMs: tsMs + 10000 } : null; }) .filter(Boolean); } function errorInsideStopWindow(event, windows) { return (windows || []).some((window) => event.tsMs >= window.fromMs && event.tsMs <= window.toMs); } function timeoutMsFromMessage(value) { const match = String(value || "").match(/\b(?:exceeded|timeout|timed\s*out\s*after)\s+(\d{2,})\s*ms\b/iu) || String(value || "").match(/\b(\d{2,})\s*ms\b/iu); return match ? Number(match[1]) : null; } function uniqueStrings(values) { return Array.from(new Set((values || []).filter((item) => typeof item === "string" && item.length > 0))).slice(0, 12); } function maxPresentNumber(values) { const numbers = (values || []).filter((item) => item !== null && item !== undefined && Number.isFinite(Number(item))).map((item) => Number(item)); return numbers.length > 0 ? Math.max(...numbers) : null; } function buildRecentAnalysisWindow({ samples, control, network, consoleEvents, errors, manifest }) { const latestSampleMs = latestTimestampMs(samples); const windowMs = 5 * 60 * 1000; const fromMs = Number.isFinite(latestSampleMs) ? latestSampleMs - windowMs : Number.NEGATIVE_INFINITY; const toMs = Number.isFinite(latestSampleMs) ? latestSampleMs : Number.POSITIVE_INFINITY; const inWindow = (item) => { const tsMs = Date.parse(item?.ts); return Number.isFinite(tsMs) && tsMs >= fromMs && tsMs <= toMs; }; const windowSamples = samples.filter(inWindow); const windowControl = control.filter(inWindow); const windowNetwork = network.filter(inWindow); const windowConsole = consoleEvents.filter(inWindow); const windowErrors = errors.filter(inWindow); const sampleMetrics = buildSampleMetrics(windowSamples, control); const pageProvenance = buildPageProvenanceReport(windowSamples, windowControl, manifest); const pagePerformance = buildPagePerformanceReport(windowSamples, manifest); const promptNetwork = buildPromptNetworkReport(windowControl, windowNetwork); const runtimeAlerts = buildRuntimeAlerts(windowSamples, control, windowNetwork, windowConsole, windowErrors); const apiDomLag = buildApiDomLagReport(windowSamples, windowNetwork); const findings = buildFindings(windowSamples, control, windowNetwork, windowErrors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, [], {}, apiDomLag); return { summary: { name: "recent-5m", windowMs, fromAt: Number.isFinite(fromMs) ? new Date(fromMs).toISOString() : null, toAt: Number.isFinite(toMs) ? new Date(toMs).toISOString() : null, samples: windowSamples.length, control: windowControl.length, network: windowNetwork.length, console: windowConsole.length, errors: windowErrors.length, valuesRedacted: true }, sampleMetrics, pageProvenance, pagePerformance, promptNetwork, runtimeAlerts, apiDomLag, findings, valuesRedacted: true }; } function latestTimestampMs(items) { let latest = Number.NEGATIVE_INFINITY; for (const item of items || []) { const tsMs = Date.parse(item?.ts); if (Number.isFinite(tsMs) && tsMs > latest) latest = tsMs; } return latest; } function buildPageProvenanceReport(samples, control, manifest) { const groups = new Map(); for (const sample of samples) { const provenance = sample?.pageProvenance; if (!provenance) continue; const key = provenance.assetFingerprint || "unknown"; const group = groups.get(key) || { assetFingerprint: provenance.assetFingerprint || null, pageLoadSeqs: [], sampleCount: 0, firstSeq: sample.seq ?? null, lastSeq: sample.seq ?? null, firstAt: sample.ts ?? null, lastAt: sample.ts ?? null, urlPaths: [], scriptCount: provenance.scriptCount ?? null, stylesheetCount: provenance.stylesheetCount ?? null, metaCount: provenance.metaCount ?? null, scripts: Array.isArray(provenance.scripts) ? provenance.scripts.slice(0, 12) : [], stylesheets: Array.isArray(provenance.stylesheets) ? provenance.stylesheets.slice(0, 12) : [], valuesRedacted: true }; group.sampleCount += 1; group.lastSeq = sample.seq ?? null; group.lastAt = sample.ts ?? null; if (provenance.pageLoadSeq !== null && provenance.pageLoadSeq !== undefined && !group.pageLoadSeqs.includes(provenance.pageLoadSeq)) group.pageLoadSeqs.push(provenance.pageLoadSeq); if (provenance.urlPath && !group.urlPaths.includes(provenance.urlPath)) group.urlPaths.push(provenance.urlPath); groups.set(key, group); } const segments = Array.from(groups.values()).sort((a, b) => Number(a.firstSeq ?? 0) - Number(b.firstSeq ?? 0)); const controlSegments = control .filter((item) => item.type === "page-provenance" || item?.pageProvenance) .map((item) => ({ ts: item.ts ?? null, reason: item.reason ?? item.detail?.reason ?? null, httpStatus: item.httpStatus ?? item.detail?.httpStatus ?? null, pageProvenance: item.pageProvenance ?? item.detail?.pageProvenance ?? null, })) .slice(0, 80); return { summary: { segmentCount: segments.length, sampleCount: segments.reduce((sum, item) => sum + item.sampleCount, 0), manifestFingerprint: manifest?.pageProvenance?.assetFingerprint ?? null, controlSegmentCount: controlSegments.length }, segments, controlSegments, valuesRedacted: true }; } function buildPagePerformanceReport(samples, manifest) { const base = manifest?.baseUrl || "http://invalid.local"; const seen = new Set(); const groups = new Map(); const sampleTimes = samples.map((sample) => Date.parse(sample?.ts || "")).filter(Number.isFinite); const windowStartMs = sampleTimes.length > 0 ? Math.min(...sampleTimes) : null; const windowEndMs = sampleTimes.length > 0 ? Math.max(...sampleTimes) : null; for (const sample of samples) { const entries = Array.isArray(sample?.performance) ? sample.performance : []; for (const entry of entries) { const durationMs = Number(entry?.duration); if (!Number.isFinite(durationMs) || durationMs < 0) continue; const entryCompletedMs = performanceEntryCompletedEpochMs(sample, entry); if (windowStartMs !== null && entryCompletedMs !== null && entryCompletedMs < windowStartMs) continue; if (windowEndMs !== null && entryCompletedMs !== null && entryCompletedMs > windowEndMs + 1000) continue; const entryTs = entryCompletedMs === null ? (sample.ts ?? null) : new Date(entryCompletedMs).toISOString(); const parsed = parsePerformanceUrl(entry?.name, base); if (!parsed.sameOrigin || !isApiLikePath(parsed.path)) continue; const normalizedPath = normalizeApiPath(parsed.path); const routeKind = classifyApiPerformanceRoute(normalizedPath, entry); const isLongLivedStream = routeKind === "same-origin-api-stream"; const streamOpenMs = streamOpenLatencyMs(entry); const timingStatus = resourceTimingPhaseStatus(entry); const dedupeKey = [parsed.path, entry.initiatorType || "", sample?.pageProvenance?.pageLoadSeq ?? "", sample?.pageProvenance?.timeOrigin ?? "", entry.startTime ?? "", Math.round(durationMs)].join("|"); if (seen.has(dedupeKey)) continue; seen.add(dedupeKey); const group = groups.get(normalizedPath) || { routeKind, path: normalizedPath, isLongLivedStream, budgetMetric: isLongLivedStream ? "streamOpenMs" : "durationMs", rawPathSamples: [], sampleCount: 0, completeTimingSampleCount: 0, partialTimingSampleCount: 0, durationsMs: [], streamOpenDurationsMs: [], overFiveSecondCount: 0, overBudgetCount: 0, partialOverFiveSecondCount: 0, partialOverBudgetCount: 0, streamLifetimeOverFiveSecondCount: 0, streamOpenOverFiveSecondCount: 0, streamOpenOverBudgetCount: 0, firstAt: entryTs, lastAt: entryTs, firstSeq: sample.seq ?? null, lastSeq: sample.seq ?? null, initiatorTypes: [], pageAssetFingerprints: [], slowSamples: [], partialSamples: [], valuesRedacted: true }; group.sampleCount += 1; const partialOrdinaryTiming = !isLongLivedStream && timingStatus.status !== "complete"; let overBudget = false; if (partialOrdinaryTiming) { group.partialTimingSampleCount += 1; if (durationMs > 5000) group.partialOverFiveSecondCount += 1; if (durationMs > alertThresholds.partialApiSlowMs) { group.partialOverBudgetCount += 1; if (group.partialSamples.length < 80) group.partialSamples.push(compactPagePerformanceSlowSample({ sample, entry, entryTs, normalizedPath, rawPath: parsed.path, durationMs, streamOpenMs })); } } else { group.completeTimingSampleCount += 1; group.durationsMs.push(durationMs); if (isLongLivedStream) { if (durationMs > 5000) group.streamLifetimeOverFiveSecondCount += 1; if (streamOpenMs !== null) { group.streamOpenDurationsMs.push(streamOpenMs); if (streamOpenMs > 5000) { group.streamOpenOverFiveSecondCount += 1; group.overFiveSecondCount += 1; } if (streamOpenMs > alertThresholds.longLivedStreamOpenSlowMs) { group.streamOpenOverBudgetCount += 1; group.overBudgetCount += 1; overBudget = true; } } } else { if (durationMs > 5000) group.overFiveSecondCount += 1; if (durationMs > alertThresholds.sameOriginApiSlowMs) { group.overBudgetCount += 1; overBudget = true; } } } if (overBudget && group.slowSamples.length < 80) group.slowSamples.push(compactPagePerformanceSlowSample({ sample, entry, entryTs, normalizedPath, rawPath: parsed.path, durationMs, streamOpenMs })); group.lastAt = entryTs; group.lastSeq = sample.seq ?? null; if (parsed.path && !group.rawPathSamples.includes(parsed.path)) group.rawPathSamples.push(parsed.path); if (entry.initiatorType && !group.initiatorTypes.includes(entry.initiatorType)) group.initiatorTypes.push(entry.initiatorType); const assetFingerprint = sample?.pageProvenance?.assetFingerprint; if (assetFingerprint && !group.pageAssetFingerprints.includes(assetFingerprint)) group.pageAssetFingerprints.push(assetFingerprint); groups.set(normalizedPath, group); } } const sameOriginApiByPath = Array.from(groups.values()).map((group) => { const durations = group.durationsMs.slice().sort((a, b) => a - b); const streamOpenDurations = group.streamOpenDurationsMs.slice().sort((a, b) => a - b); return { routeKind: group.routeKind, path: group.path, isLongLivedStream: group.isLongLivedStream === true, budgetMetric: group.budgetMetric, sampleCount: group.sampleCount, budgetMs: group.isLongLivedStream === true ? alertThresholds.longLivedStreamOpenSlowMs : alertThresholds.sameOriginApiSlowMs, partialBudgetMs: alertThresholds.partialApiSlowMs, streamOpenBudgetMs: alertThresholds.longLivedStreamOpenSlowMs, completeTimingSampleCount: group.completeTimingSampleCount, partialTimingSampleCount: group.partialTimingSampleCount, p50Ms: percentile(durations, 50), p75Ms: percentile(durations, 75), p95Ms: percentile(durations, 95), maxMs: durations.length > 0 ? durations[durations.length - 1] : null, streamOpenSampleCount: streamOpenDurations.length, streamOpenP50Ms: percentile(streamOpenDurations, 50), streamOpenP75Ms: percentile(streamOpenDurations, 75), streamOpenP95Ms: percentile(streamOpenDurations, 95), streamOpenMaxMs: streamOpenDurations.length > 0 ? streamOpenDurations[streamOpenDurations.length - 1] : null, streamOpenOverFiveSecondCount: group.streamOpenOverFiveSecondCount, streamOpenOverBudgetCount: group.streamOpenOverBudgetCount, streamLifetimeOverFiveSecondCount: group.streamLifetimeOverFiveSecondCount, overFiveSecondCount: group.overFiveSecondCount, overBudgetCount: group.overBudgetCount, partialOverFiveSecondCount: group.partialOverFiveSecondCount, partialOverBudgetCount: group.partialOverBudgetCount, overFiveSecondRatio: group.sampleCount > 0 ? Number((group.overFiveSecondCount / group.sampleCount).toFixed(3)) : 0, overBudgetRatio: group.sampleCount > 0 ? Number((group.overBudgetCount / group.sampleCount).toFixed(3)) : 0, firstAt: group.firstAt, lastAt: group.lastAt, firstSeq: group.firstSeq, lastSeq: group.lastSeq, initiatorTypes: group.initiatorTypes, rawPathSamples: group.rawPathSamples.slice(0, 8), pageAssetFingerprints: group.pageAssetFingerprints.slice(0, 8), slowSamples: group.slowSamples .slice() .sort((a, b) => Number(b.durationMs ?? 0) - Number(a.durationMs ?? 0)) .slice(0, 12), partialSamples: group.partialSamples .slice() .sort((a, b) => Number(b.durationMs ?? 0) - Number(a.durationMs ?? 0)) .slice(0, 12), valuesRedacted: true }; }).sort((a, b) => (Number(b.overBudgetCount ?? b.overFiveSecondCount ?? 0) - Number(a.overBudgetCount ?? a.overFiveSecondCount ?? 0)) || (Number(b.p95Ms ?? 0) - Number(a.p95Ms ?? 0)) || a.path.localeCompare(b.path)); const longLivedStreams = sameOriginApiByPath.filter((item) => item.isLongLivedStream); const ordinaryApi = sameOriginApiByPath.filter((item) => item.isLongLivedStream !== true); const slow = ordinaryApi.filter((item) => Number(item.overBudgetCount ?? item.overFiveSecondCount ?? 0) > 0); const slowFiveSecond = ordinaryApi.filter((item) => Number(item.overFiveSecondCount ?? 0) > 0); const partialSlow = ordinaryApi.filter((item) => Number(item.partialOverBudgetCount ?? item.partialOverFiveSecondCount ?? 0) > 0); const partialFiveSecond = ordinaryApi.filter((item) => Number(item.partialOverFiveSecondCount ?? 0) > 0); const slowStreamOpen = longLivedStreams.filter((item) => Number(item.streamOpenOverBudgetCount ?? item.streamOpenOverFiveSecondCount ?? 0) > 0); const slowStreamOpenFiveSecond = longLivedStreams.filter((item) => Number(item.streamOpenOverFiveSecondCount ?? 0) > 0); const budgetP95Values = sameOriginApiByPath .map((item) => Number(item.isLongLivedStream ? (item.streamOpenP95Ms ?? 0) : (item.p95Ms ?? 0))) .filter((value) => Number.isFinite(value)); return { summary: { budgetMs: alertThresholds.sameOriginApiSlowMs, alertThresholds, sameOriginApiPathCount: sameOriginApiByPath.length, sameOriginApiSampleCount: sameOriginApiByPath.reduce((sum, item) => sum + item.sampleCount, 0), longLivedStreamPathCount: longLivedStreams.length, longLivedStreamSampleCount: longLivedStreams.reduce((sum, item) => sum + item.sampleCount, 0), longLivedStreamOpenOverFiveSecondPathCount: slowStreamOpenFiveSecond.length, longLivedStreamOpenOverFiveSecondSampleCount: slowStreamOpenFiveSecond.reduce((sum, item) => sum + Number(item.streamOpenOverFiveSecondCount ?? 0), 0), longLivedStreamOpenOverBudgetPathCount: slowStreamOpen.length, longLivedStreamOpenOverBudgetSampleCount: slowStreamOpen.reduce((sum, item) => sum + Number(item.streamOpenOverBudgetCount ?? item.streamOpenOverFiveSecondCount ?? 0), 0), longLivedStreamLifetimeOverFiveSecondSampleCount: longLivedStreams.reduce((sum, item) => sum + Number(item.streamLifetimeOverFiveSecondCount ?? 0), 0), slowPathCount: slow.length, slowSampleCount: slow.reduce((sum, item) => sum + Number(item.overBudgetCount ?? item.overFiveSecondCount ?? 0), 0), overFiveSecondPathCount: slowFiveSecond.length, overFiveSecondSampleCount: slowFiveSecond.reduce((sum, item) => sum + Number(item.overFiveSecondCount ?? 0), 0), partialTimingSampleCount: ordinaryApi.reduce((sum, item) => sum + Number(item.partialTimingSampleCount ?? 0), 0), partialOverFiveSecondPathCount: partialFiveSecond.length, partialOverFiveSecondSampleCount: partialFiveSecond.reduce((sum, item) => sum + Number(item.partialOverFiveSecondCount ?? 0), 0), partialOverBudgetPathCount: partialSlow.length, partialOverBudgetSampleCount: partialSlow.reduce((sum, item) => sum + Number(item.partialOverBudgetCount ?? item.partialOverFiveSecondCount ?? 0), 0), worstP95Ms: budgetP95Values.length > 0 ? Math.max(...budgetP95Values) : null, valuesRedacted: true }, sameOriginApiByPath, valuesRedacted: true }; } function performanceEntryCompletedEpochMs(sample, entry) { const origin = Number(sample?.pageProvenance?.timeOrigin); const responseEnd = Number(entry?.responseEnd); const startTime = Number(entry?.startTime); const offset = Number.isFinite(responseEnd) && responseEnd > 0 ? responseEnd : startTime; if (Number.isFinite(origin) && origin > 0 && Number.isFinite(offset) && offset >= 0) return Math.round(origin + offset); const sampleTs = Date.parse(sample?.ts || ""); return Number.isFinite(sampleTs) ? sampleTs : null; } function compactPagePerformanceSlowSample({ sample, entry, entryTs, normalizedPath, rawPath, durationMs, streamOpenMs }) { const timingStatus = resourceTimingPhaseStatus(entry); const serverTiming = compactServerTiming(entry?.serverTiming); return { ts: entryTs ?? sample?.ts ?? null, sampleTs: sample?.ts ?? null, seq: sample?.seq ?? null, path: normalizedPath ?? null, rawPath: rawPath ?? null, initiatorType: entry?.initiatorType ?? null, durationMs: roundFinite(durationMs), startTimeMs: roundFinite(entry?.startTime), fetchStartMs: roundFinite(entry?.fetchStart), requestStartMs: roundFinite(entry?.requestStart), responseStartMs: roundFinite(entry?.responseStart), responseEndMs: roundFinite(entry?.responseEnd), streamOpenMs: roundFinite(streamOpenMs), dnsMs: phaseDeltaMs(entry, "domainLookupEnd", "domainLookupStart"), tcpMs: phaseDeltaMs(entry, "connectEnd", "connectStart"), tlsStartMs: roundFinite(entry?.secureConnectionStart), requestToResponseStartMs: phaseDeltaMs(entry, "responseStart", "requestStart"), responseTransferMs: phaseDeltaMs(entry, "responseEnd", "responseStart"), timingStatus: timingStatus.status, invalidTimingPhases: timingStatus.invalidPhases, partialTimingPhases: timingStatus.partialPhases, transferSize: Number.isFinite(Number(entry?.transferSize)) ? Number(entry.transferSize) : null, encodedBodySize: Number.isFinite(Number(entry?.encodedBodySize)) ? Number(entry.encodedBodySize) : null, decodedBodySize: Number.isFinite(Number(entry?.decodedBodySize)) ? Number(entry.decodedBodySize) : null, nextHopProtocol: entry?.nextHopProtocol ?? null, serverTiming, serverTimingNames: serverTiming.map((item) => item.name).filter(Boolean).slice(0, 8), otelTraceId: extractOtelTraceIdFromServerTiming(serverTiming), valuesRedacted: true }; } function phaseDeltaMs(entry, endKey, startKey) { const end = Number(entry?.[endKey]); const start = Number(entry?.[startKey]); if (!Number.isFinite(end) || !Number.isFinite(start) || end <= 0 || start <= 0 || end < start) return null; return Math.round(end - start); } function resourceTimingPhaseStatus(entry) { const pairs = [ ["requestToResponseStart", "requestStart", "responseStart"], ["responseTransfer", "responseStart", "responseEnd"], ]; const invalidPhases = []; const partialPhases = []; for (const [label, startKey, endKey] of pairs) { const start = Number(entry?.[startKey]); const end = Number(entry?.[endKey]); if (!Number.isFinite(start) || !Number.isFinite(end) || start <= 0 || end <= 0) { partialPhases.push(label); } else if (end < start) { invalidPhases.push(label); } } return { status: invalidPhases.length > 0 ? "invalid" : (partialPhases.length > 0 ? "partial" : "complete"), invalidPhases, partialPhases, }; } function compactServerTiming(value) { const items = Array.isArray(value) ? value : []; return items.slice(0, 8).map((item) => ({ name: truncate(String(item?.name || ""), 80), duration: Number.isFinite(Number(item?.duration)) ? Math.round(Number(item.duration)) : null, description: truncate(String(item?.description || ""), 120), })).filter((item) => item.name || item.description || item.duration !== null); } function extractOtelTraceIdFromServerTiming(items) { const text = (Array.isArray(items) ? items : []).map((item) => [item.name, item.description].filter(Boolean).join(" ")).join(" "); const match = text.match(/\b[0-9a-f]{32}\b/iu); return match ? match[0].toLowerCase() : null; } function roundFinite(value) { const numeric = Number(value); return Number.isFinite(numeric) ? Math.round(numeric) : null; } function classifyApiPerformanceRoute(normalizedPath, entry = {}) { if (normalizedPath === "/v1/workbench/events") return "same-origin-api-stream"; if (String(entry?.initiatorType ?? "").toLowerCase() === "eventsource") return "same-origin-api-stream"; return "same-origin-api"; } function streamOpenLatencyMs(entry = {}) { const responseStart = Number(entry?.responseStart); const startTime = Number(entry?.startTime); if (!Number.isFinite(responseStart) || responseStart <= 0) return null; if (!Number.isFinite(startTime) || startTime < 0) return Math.max(0, responseStart); if (responseStart < startTime) return null; return Math.max(0, responseStart - startTime); } function parsePerformanceUrl(value, base) { try { const url = new URL(String(value || ""), base); const origin = new URL(String(base || "http://invalid.local")).origin; return { sameOrigin: url.origin === origin, path: url.pathname }; } catch { return { sameOrigin: false, path: "-" }; } } function isApiLikePath(path) { return /^\/(?:v1(?:\/|$)|auth(?:\/|$)|health(?:\/|$))/u.test(String(path || "")); } function normalizeApiPath(path) { return String(path || "-") .replace(/\/v1\/workbench\/sessions\/ses_[^/]+/gu, "/v1/workbench/sessions/:id") .replace(/\/v1\/workbench\/turns\/trc_[^/]+/gu, "/v1/workbench/turns/:traceId") .replace(/\/v1\/workbench\/traces\/trc_[^/]+/gu, "/v1/workbench/traces/:traceId") .replace(/\/v1\/workbench\/sessions\/[0-9a-f-]{12,}/giu, "/v1/workbench/sessions/:id") .replace(/\/v1\/[^/]+\/[0-9a-f-]{16,}(?=\/|$)/giu, (match) => match.replace(/\/[0-9a-f-]{16,}$/iu, "/:id")); } function percentile(sortedValues, percentileValue) { if (!Array.isArray(sortedValues) || sortedValues.length === 0) return null; if (sortedValues.length === 1) return Math.round(sortedValues[0]); const rank = (percentileValue / 100) * (sortedValues.length - 1); const lower = Math.floor(rank); const upper = Math.ceil(rank); if (lower === upper) return Math.round(sortedValues[lower]); const weight = rank - lower; return Math.round(sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight); } function buildPromptNetworkReport(control, network) { const promptsById = new Map(); for (const item of control) { if (item?.type !== "sendPrompt" || !item.commandId) continue; const existing = promptsById.get(item.commandId) || { commandId: item.commandId, promptIndex: promptsById.size + 1, promptTextHash: item.input?.textHash ?? null, promptTextBytes: item.input?.textBytes ?? null, startedAt: null, completedAt: null, failedAt: null, phase: null }; if (!existing.promptTextHash && item.input?.textHash) existing.promptTextHash = item.input.textHash; if (!existing.promptTextBytes && item.input?.textBytes) existing.promptTextBytes = item.input.textBytes; if (item.phase === "started") existing.startedAt = item.ts ?? existing.startedAt; if (item.phase === "completed") existing.completedAt = item.ts ?? existing.completedAt; if (item.phase === "failed") existing.failedAt = item.ts ?? existing.failedAt; existing.phase = item.phase ?? existing.phase; promptsById.set(item.commandId, existing); } const prompts = Array.from(promptsById.values()).sort((a, b) => Date.parse(a.startedAt || a.completedAt || a.failedAt || "") - Date.parse(b.startedAt || b.completedAt || b.failedAt || "")); prompts.forEach((item, index) => { item.promptIndex = index + 1; }); const chatEvents = network .filter((item) => String(item?.method || "").toUpperCase() === "POST" && promptSubmitModeForUrl(item?.url) !== null) .map((item) => { const failureText = item.failureKind ?? item.failure ?? item.errorText ?? null; const urlPathValue = urlPath(item.url); return { ts: item.ts ?? null, tsMs: Date.parse(item.ts), type: item.type ?? null, status: Number.isFinite(Number(item.status)) ? Number(item.status) : null, commandId: item.commandId ?? null, urlPath: urlPathValue, submitMode: promptSubmitModeForUrl(item.url), failureKind: failureText ? String(failureText) : null, errorTextHash: failureText ? sha256(failureText) : null }; }) .filter((item) => Number.isFinite(item.tsMs)) .sort((a, b) => a.tsMs - b.tsMs); const rounds = prompts.map((prompt) => { const startMs = Date.parse(prompt.startedAt || prompt.completedAt || prompt.failedAt || ""); const endAnchorMs = Date.parse(prompt.completedAt || prompt.failedAt || prompt.startedAt || ""); const fromMs = Number.isFinite(startMs) ? startMs - 3000 : Number.NEGATIVE_INFINITY; const toMs = Number.isFinite(endAnchorMs) ? endAnchorMs + 30000 : Number.POSITIVE_INFINITY; const events = chatEvents.filter((event) => { if (event.commandId && prompt.commandId && event.commandId === prompt.commandId) return true; return event.tsMs >= fromMs && event.tsMs <= toMs; }); const responses = events.filter((event) => event.type === "response"); const failures = events.filter((event) => event.type === "requestfailed"); const responseStatuses = responses.map((event) => event.status).filter((status) => status !== null); const submitModes = Array.from(new Set(events.map((event) => event.submitMode).filter(Boolean))).sort(); const chatPostOk = responseStatuses.some((status) => status >= 200 && status < 300); const failureKind = chatPostOk ? null : failures.length > 0 ? "requestfailed" : responseStatuses.length === 0 ? "missing-response" : "http-status"; return { promptIndex: prompt.promptIndex, promptCommandId: prompt.commandId, promptTextHash: prompt.promptTextHash, promptTextBytes: prompt.promptTextBytes, startedAt: prompt.startedAt, completedAt: prompt.completedAt, failedAt: prompt.failedAt, chatPostOk, failureKind, requestCount: events.filter((event) => event.type === "request").length, responseCount: responses.length, requestFailedCount: failures.length, responseStatuses, submitModes, steerUsed: submitModes.includes("steer"), firstChatEventAt: events[0]?.ts ?? null, lastChatEventAt: events[events.length - 1]?.ts ?? null, events: events.slice(0, 12).map((event) => ({ ts: event.ts, type: event.type, status: event.status, urlPath: event.urlPath, submitMode: event.submitMode, failureKind: event.failureKind, errorTextHash: event.errorTextHash })) }; }); return { summary: { promptCount: rounds.length, chatPostOk: rounds.filter((item) => item.chatPostOk === true).length, chatPostFailed: rounds.filter((item) => item.chatPostOk === false).length, chatPostMissing: rounds.filter((item) => item.failureKind === "missing-response").length }, rounds }; } function promptSubmitModeForUrl(value) { const pathValue = urlPath(value); if (pathValue === "/v1/agent/chat") return "chat"; if (pathValue === "/v1/agent/chat/steer") return "steer"; return null; } function parseDomDiagnosticSummary(text) { const value = String(text || ""); const traceMatch = value.match(/\b(?:trace_id=)?(trc_[A-Za-z0-9_-]+|[a-f0-9]{16,64})\b/iu); const httpStatusMatch = value.match(/\bHTTP\s+([1-5][0-9]{2})\b/iu); const idleMatch = value.match(/\bidle\s+(\d+)s\b/iu); const waitingForMatch = value.match(/\bwaitingFor=([^\s;;,,)]+)/iu); const lastEventLabelMatch = value.match(/\blastEventLabel=([^\s;;,,)]+)/iu); const diagnosticCode = httpStatusMatch ? "http-" + httpStatusMatch[1] : /turn\s*超过|无新活动/iu.test(value) ? "turn-idle-no-activity" : /Failed to fetch/iu.test(value) ? "failed-to-fetch" : "diagnostic"; return { diagnosticCode, traceId: traceMatch?.[1] || null, httpStatus: httpStatusMatch ? Number(httpStatusMatch[1]) : null, idleSeconds: idleMatch ? Number(idleMatch[1]) : null, waitingFor: waitingForMatch?.[1] || null, lastEventLabel: lastEventLabelMatch?.[1] || null }; } function isDomDiagnosticSampleText(text) { const value = String(text || "").replace(/\s+/g, " ").trim(); if (!value) return false; const strongDiagnostic = [ /\bHTTP\s+[45][0-9]{2}\b(?:[\s\S]{0,120}\btrace_id=|\b)/iu, /\btrace_id=(?:trc_[A-Za-z0-9_-]+|[a-f0-9]{16,64})\b/iu, /workbench\s+turn\s*超过\s*\d+ms\s*无新活动/iu, /\bturn\s*超过\b[\s\S]{0,120}\b无新活动\b/iu, /\bprojection-resume:sync-failed\b/iu, /\bAgentRun\s+GET\b[\s\S]*\/result\b[\s\S]*timed out after\s+\d+ms\b/iu, /\bFailed to fetch\b/iu, /\bFailed to load resource\b[\s\S]{0,180}\bstatus of\s+[45][0-9]{2}\b/iu, /\bserver responded with a status of\s+[45][0-9]{2}\b/iu, /Code Agent\b[\s\S]{0,120}(?:无法连接上游|请求已结束)/iu ].some((pattern) => pattern.test(value)); if (!strongDiagnostic) return false; const looksLikeToolStdout = /\b(?:stdout|stderr):/iu.test(value) && /(?:\becho\s+["']?===|\bnode\s+|\.tspy\b|tspy\/|===\s*[A-Za-z0-9_.-]+\s*===)/iu.test(value); if (!looksLikeToolStdout) return true; return /\b(?:trace_id=|HTTP\s+[45][0-9]{2}|workbench\s+turn\s*超过|projection-resume:sync-failed|Failed to fetch|Failed to load resource|server responded with a status of\s+[45][0-9]{2}|Code Agent\b[\s\S]{0,120}(?:无法连接上游|请求已结束))\b/iu.test(value); } function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) { const promptTimes = control .filter((item) => item.type === "sendPrompt" && item.phase === "completed") .map((item) => Date.parse(item.ts)) .filter(Number.isFinite) .sort((a, b) => a - b); const observerRefreshTimes = control .filter((item) => item.type === "observer-periodic-refresh") .map((item) => Date.parse(item.ts)) .filter(Number.isFinite) .sort((a, b) => a - b); const naturalNetwork = network.filter((item) => item?.observerInitiated !== true); const httpErrors = naturalNetwork .filter((item) => item?.type === "response" && Number(item.status) >= 400) .map((item) => networkAlertEvent(item, promptTimes)); const requestFailed = naturalNetwork .filter((item) => item?.type === "requestfailed") .map((item) => networkAlertEvent(item, promptTimes)); const significantRequestFailed = requestFailed.filter( (item) => !isBenignLongLivedStreamClosureAlert(item) && !isObserverRefreshClosureAlert(item, observerRefreshTimes), ); const domDiagnostics = []; const executionErrors = []; const baselineExecutionErrors = []; const canarySessionIds = sessionInvariantCanarySessionIds(control); const firstPromptMs = promptTimes.length > 0 ? promptTimes[0] : Infinity; const firstSeenExecutionErrorMs = new Map(); for (const sample of samples) { const tsMs = Date.parse(sample?.ts); const promptIndex = Number.isFinite(tsMs) ? latestPromptIndex(promptTimes, tsMs) : 0; if (Array.isArray(sample?.diagnostics)) { for (const diagnostic of sample.diagnostics.slice(0, 12)) { const text = diagnostic?.textPreview || diagnostic?.text || ""; if (!String(text).trim()) continue; const parsedDiagnostic = parseDomDiagnosticSummary(text); domDiagnostics.push({ seq: sample.seq ?? null, ts: sample.ts ?? null, promptIndex, source: "diagnostic-node", className: diagnostic.className ?? null, diagnosticCode: diagnostic.diagnosticCode ?? parsedDiagnostic.diagnosticCode, traceId: diagnostic.traceId ?? parsedDiagnostic.traceId, httpStatus: diagnostic.httpStatus ?? parsedDiagnostic.httpStatus, idleSeconds: diagnostic.idleSeconds ?? parsedDiagnostic.idleSeconds, waitingFor: diagnostic.waitingFor ?? parsedDiagnostic.waitingFor, lastEventLabel: diagnostic.lastEventLabel ?? parsedDiagnostic.lastEventLabel, compact: diagnostic.compact ?? null, expanded: diagnostic.expanded ?? null, routeSessionId: sample.routeSessionId ?? null, activeSessionId: sample.activeSessionId ?? null, textHash: diagnostic.textHash || sha256(text), preview: limitText(text, 260) }); } } const texts = sampleTexts(sample).filter(isDomDiagnosticSampleText); for (const text of texts.slice(0, 4)) { const parsedDiagnostic = parseDomDiagnosticSummary(text); domDiagnostics.push({ seq: sample.seq ?? null, ts: sample.ts ?? null, promptIndex, source: "sample-text", diagnosticCode: parsedDiagnostic.diagnosticCode, traceId: parsedDiagnostic.traceId, httpStatus: parsedDiagnostic.httpStatus, idleSeconds: parsedDiagnostic.idleSeconds, waitingFor: parsedDiagnostic.waitingFor, lastEventLabel: parsedDiagnostic.lastEventLabel, routeSessionId: sample.routeSessionId ?? null, activeSessionId: sample.activeSessionId ?? null, textHash: sha256(text), preview: limitText(text, 220) }); } const seenExecutionErrors = new Set(); for (const candidate of sampleExecutionErrorCandidates(sample)) { const parsed = parseExecutionErrorText(candidate.text); if (!parsed) continue; const textHash = sha256(candidate.text); const dedupeKey = [candidate.source, candidate.traceId || "-", parsed.backend || "-", parsed.code || "-", parsed.status || "-", textHash].join("|"); if (seenExecutionErrors.has(dedupeKey)) continue; seenExecutionErrors.add(dedupeKey); const firstSeenMs = firstSeenExecutionErrorMs.has(dedupeKey) ? firstSeenExecutionErrorMs.get(dedupeKey) : tsMs; if (!firstSeenExecutionErrorMs.has(dedupeKey) && Number.isFinite(tsMs)) firstSeenExecutionErrorMs.set(dedupeKey, tsMs); const baseline = Number.isFinite(firstSeenMs) && firstSeenMs < firstPromptMs; const event = { seq: sample.seq ?? null, ts: sample.ts ?? null, promptIndex, baseline, firstSeenAt: Number.isFinite(firstSeenMs) ? new Date(firstSeenMs).toISOString() : null, source: candidate.source, backend: parsed.backend, status: parsed.status, code: parsed.code, rawCode: parsed.rawCode, totalSeconds: parsed.totalSeconds, traceId: candidate.traceId || parsed.traceId || null, messageId: candidate.messageId || null, routeSessionId: sample.routeSessionId ?? null, activeSessionId: sample.activeSessionId ?? null, textHash, preview: limitText(candidate.text, 260) }; const eventSessions = [event.routeSessionId, event.activeSessionId].filter(Boolean); const nonCanarySession = canarySessionIds.size > 0 && !eventSessions.some((sessionId) => canarySessionIds.has(sessionId)); if (baseline || nonCanarySession) baselineExecutionErrors.push({ ...event, baseline: true, baselineReason: nonCanarySession ? "non-canary-session" : "pre-first-prompt" }); else executionErrors.push(event); domDiagnostics.push({ seq: sample.seq ?? null, ts: sample.ts ?? null, promptIndex, source: "execution-row", diagnosticCode: parsed.rawCode || parsed.code || "execution-error", traceId: candidate.traceId || parsed.traceId || null, routeSessionId: sample.routeSessionId ?? null, activeSessionId: sample.activeSessionId ?? null, textHash, preview: limitText(candidate.text, 220) }); } } const consoleAlerts = consoleEvents .filter((item) => /error|warning|warn|assert/iu.test(String(item?.type || "")) || isDiagnosticText(item?.text)) .map((item) => consoleAlertEvent(item, promptTimes)); const significantConsoleAlerts = consoleAlerts.filter((item) => !isBenignLongLivedStreamClosureAlert(item) && !isObserverRefreshClosureAlert(item, observerRefreshTimes)); const pageErrors = errors.map((item) => ({ ts: item.ts ?? null, promptIndex: promptIndexForTs(promptTimes, item.ts), type: item.type ?? null, pageRole: item.pageRole ?? item.error?.details?.pageRole ?? null, pageId: item.pageId ?? item.error?.details?.pageId ?? null, routeSessionId: item.routeSessionId ?? item.error?.details?.routeSessionId ?? null, activeSessionId: item.activeSessionId ?? item.error?.details?.activeSessionId ?? null, commandId: item.commandId ?? item.error?.details?.commandId ?? null, sampleSeq: item.sampleSeq ?? item.error?.details?.sampleSeq ?? null, timeoutMs: timeoutMsFromMessage(item.error?.message || item.message || item.error || ""), errorName: item.error?.name ?? item.name ?? null, messageHash: item.error?.message ? sha256(item.error.message) : item.message ? sha256(item.message) : null, preview: limitText(item.error?.message || item.message || item.error || "", 220) })); return { summary: { httpErrorCount: httpErrors.length, requestFailedCount: requestFailed.length, significantRequestFailedCount: significantRequestFailed.length, benignLongLivedStreamClosureCount: requestFailed.length - significantRequestFailed.length, domDiagnosticSampleCount: domDiagnostics.length, domDiagnosticGroupCount: groupDomDiagnostics(domDiagnostics).length, executionErrorCount: executionErrors.length, baselineExecutionErrorCount: baselineExecutionErrors.length, consoleAlertCount: consoleAlerts.length, significantConsoleAlertCount: significantConsoleAlerts.length, pageErrorCount: pageErrors.length, networkErrorGroupCount: groupNetworkAlerts(httpErrors).length, requestFailedGroupCount: groupNetworkAlerts(requestFailed).length, significantRequestFailedGroupCount: groupNetworkAlerts(significantRequestFailed).length, executionErrorGroupCount: groupExecutionErrors(executionErrors).length, baselineExecutionErrorGroupCount: groupExecutionErrors(baselineExecutionErrors).length, consoleAlertGroupCount: groupConsoleAlerts(consoleAlerts).length, significantConsoleAlertGroupCount: groupConsoleAlerts(significantConsoleAlerts).length }, networkHttpErrorsByPath: groupNetworkAlerts(httpErrors), networkRequestFailedByPath: groupNetworkAlerts(requestFailed), networkSignificantRequestFailedByPath: groupNetworkAlerts(significantRequestFailed), domDiagnostics: domDiagnostics.slice(-80), domDiagnosticsByText: groupDomDiagnostics(domDiagnostics), domDiagnosticsByFingerprint: groupDomDiagnostics(domDiagnostics).slice(0, 80), runtimeExecutionErrors: executionErrors.slice(0, 120), runtimeExecutionErrorsByCode: groupExecutionErrors(executionErrors), baselineRuntimeExecutionErrors: baselineExecutionErrors.slice(0, 80), baselineRuntimeExecutionErrorsByCode: groupExecutionErrors(baselineExecutionErrors), consoleAlerts: consoleAlerts.slice(0, 80), consoleAlertsByPath: groupConsoleAlerts(consoleAlerts), significantConsoleAlerts: significantConsoleAlerts.slice(0, 80), significantConsoleAlertsByPath: groupConsoleAlerts(significantConsoleAlerts), pageErrors: pageErrors.slice(0, 40) }; } function groupDomDiagnostics(events) { const groups = new Map(); for (const item of events || []) { const preview = String(item?.preview || "").trim(); if (!isReportableDomDiagnostic(item, preview)) continue; const normalizedPreview = normalizeDiagnosticPreview(preview); const key = [ item?.diagnosticCode || "", normalizedPreview ].join("|"); const existing = groups.get(key) || { source: item?.source || null, sources: new Set(), diagnosticCode: item?.diagnosticCode || null, textHash: item?.textHash || null, normalizedPreview, preview, count: 0, firstAt: item?.ts || null, lastAt: item?.ts || null, promptIndexes: new Set(), traceIds: new Set(), sampleSeqs: [] }; if (item?.source) existing.sources.add(String(item.source)); existing.count += 1; existing.firstAt = minIso(existing.firstAt, item?.ts || null); existing.lastAt = maxIso(existing.lastAt, item?.ts || null); if (Number.isFinite(Number(item?.promptIndex))) existing.promptIndexes.add(Number(item.promptIndex)); for (const traceId of extractDiagnosticTraceIds(item, preview)) existing.traceIds.add(traceId); if (existing.sampleSeqs.length < 12 && item?.seq !== undefined && item?.seq !== null) existing.sampleSeqs.push(item.seq); groups.set(key, existing); } return Array.from(groups.values()) .map((item) => ({ source: item.source, sources: Array.from(item.sources).sort(), diagnosticCode: item.diagnosticCode, textHash: item.textHash, normalizedPreview: item.normalizedPreview, preview: item.preview, count: item.count, firstAt: item.firstAt, lastAt: item.lastAt, promptIndexes: Array.from(item.promptIndexes).sort((a, b) => a - b), traceIds: Array.from(item.traceIds).sort(), sampleSeqs: item.sampleSeqs })) .sort((a, b) => (b.count - a.count) || String(a.firstAt || "").localeCompare(String(b.firstAt || ""))); } function isReportableDomDiagnostic(item, preview) { if (item?.source === "diagnostic-node" || item?.source === "execution-row") return true; return /trace_id=|HTTP\s+\d{3}\b|Failed to load resource|ERR_[A-Z_]+|provider-unavailable|AgentRun error|超过\s*\d+\s*ms\s*无新活动|代理暂时无法连接上游|Trace 更新超时|加载失败/iu.test(String(preview || "")); } function normalizeDiagnosticPreview(text) { return String(text || "") .replace(/trace_id=[A-Za-z0-9_-]+/gu, "trace_id=:traceId") .replace(/\btrc_[A-Za-z0-9_-]+\b/gu, "trc_:traceId") .replace(/\bses_[A-Za-z0-9_-]+\b/gu, "ses_:sessionId") .replace(/\brun_[A-Za-z0-9_-]+\b/gu, "run_:runId") .replace(/\bcmd_[A-Za-z0-9_-]+\b/gu, "cmd_:commandId") .replace(/[!!]+$/gu, "") .replace(/\s+/gu, " ") .trim(); } function extractDiagnosticTraceIds(item, preview) { const ids = new Set(); if (item?.traceId) ids.add(String(item.traceId)); const text = String(preview || ""); for (const match of text.matchAll(/\btrc_[A-Za-z0-9_-]+\b/gu)) ids.add(match[0]); for (const match of text.matchAll(/trace_id=([A-Za-z0-9_-]+)/gu)) ids.add(match[1]); return ids; } function minIso(a, b) { if (!a) return b || null; if (!b) return a || null; return Date.parse(a) <= Date.parse(b) ? a : b; } function maxIso(a, b) { if (!a) return b || null; if (!b) return a || null; return Date.parse(a) >= Date.parse(b) ? a : b; } function sampleExecutionErrorCandidates(sample) { const candidates = []; const add = (source, items) => { if (!Array.isArray(items)) return; for (const item of items) { const text = String(item?.textPreview || item?.text || item?.preview || "").trim(); if (!text) continue; if (!parseExecutionErrorText(text)) continue; candidates.push({ source, text, traceId: item?.traceId ?? null, messageId: item?.messageId ?? null, status: item?.status ?? null }); } }; add("diagnostic-node", sample?.diagnostics); add("message", sample?.messages); add("trace-row", sample?.traceRows); add("turn", sample?.turns); const specific = candidates.filter((candidate) => { const parsed = parseExecutionErrorText(candidate.text); return parsed && parsed.code !== "error"; }); return specific.length > 0 ? specific : candidates; } function parseExecutionErrorText(text) { const value = String(text || ""); const agentRunCodeMatch = value.match(/\bagentrun:error:([A-Za-z0-9_.:-]+)/u); const agentRunText = /\bAgentRun\s+error\b|\bagentrun:error:/iu.test(value); const providerUnavailable = /\bprovider[-_\s]*unavailable\b/iu.test(value); if (!agentRunCodeMatch && !agentRunText && !providerUnavailable) return null; const statusMatch = value.match(/\b(fail(?:ed)?|error|blocked|cancel(?:ed)?)\b/iu); const traceMatch = value.match(/\btrc_[A-Za-z0-9_-]+\b/u); const totalMatch = value.match(/\btotal\s*=\s*([0-9]{1,2}:[0-9]{2}(?::[0-9]{2})?)\b/iu) || value.match(/总耗时\s*[::]?\s*([0-9]{1,2}:[0-9]{2}(?::[0-9]{2})?)/iu); const agentRunCode = cleanExecutionCode(agentRunCodeMatch?.[1] || ""); const rawCode = agentRunCode ? "agentrun:error:" + agentRunCode : providerUnavailable ? "provider-unavailable" : "agentrun:error"; return { backend: agentRunText || agentRunCodeMatch ? "agentrun" : "unknown", status: normalizeExecutionStatus(statusMatch?.[1] || "error"), code: agentRunCode || (providerUnavailable ? "provider-unavailable" : "error"), rawCode, totalSeconds: totalMatch ? parseClockDurationSeconds(totalMatch[1]) : null, traceId: traceMatch?.[0] || null }; } function cleanExecutionCode(code) { const value = String(code || "").replace(/(?:AgentRun|Error|Failed).*$/u, "").replace(/[^A-Za-z0-9_.:-].*$/u, ""); return value || null; } function normalizeExecutionStatus(status) { const value = String(status || "").toLowerCase(); if (value === "failed") return "fail"; if (value === "cancelled" || value === "canceled") return "cancel"; return value || "error"; } function parseClockDurationSeconds(value) { const parts = String(value || "").split(":").map((part) => Number(part)); if (parts.length === 2 && parts.every(Number.isFinite)) return parts[0] * 60 + parts[1]; if (parts.length === 3 && parts.every(Number.isFinite)) return parts[0] * 3600 + parts[1] * 60 + parts[2]; return null; } function groupExecutionErrors(events) { const groups = new Map(); for (const event of events) { const key = [event.backend || "-", event.status || "-", event.code || "-"].join(" "); const group = groups.get(key) || { backend: event.backend ?? null, status: event.status ?? null, code: event.code ?? null, rawCode: event.rawCode ?? null, count: 0, firstAt: event.ts, lastAt: event.ts, promptIndexes: [], traceIds: [], sources: [] }; group.count += 1; group.lastAt = event.ts; if (event.promptIndex && !group.promptIndexes.includes(event.promptIndex)) group.promptIndexes.push(event.promptIndex); if (event.traceId && !group.traceIds.includes(event.traceId)) group.traceIds.push(event.traceId); if (event.source && !group.sources.includes(event.source)) group.sources.push(event.source); groups.set(key, group); } return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(a.code).localeCompare(String(b.code))); } function consoleAlertEvent(item, promptTimes) { const text = String(item?.text || ""); const statusMatch = text.match(/\bstatus\s+of\s+([1-5][0-9]{2})\b/iu) || text.match(/\bHTTP\s+([1-5][0-9]{2})\b/iu); const location = compactLocation(item.location); const traceMatch = (location?.urlPath || text).match(/\btrc_[A-Za-z0-9_-]+\b/u); return { ts: item.ts ?? null, promptIndex: promptIndexForTs(promptTimes, item.ts), type: item.type ?? null, status: statusMatch ? Number(statusMatch[1]) : null, urlPath: location?.urlPath || "-", traceId: traceMatch?.[0] || null, textHash: item.text ? sha256(item.text) : null, preview: limitText(text, 220), location }; } function groupConsoleAlerts(events) { const groups = new Map(); for (const event of events) { const key = [event.type || "-", event.status ?? "-", event.urlPath || "-"].join(" "); const group = groups.get(key) || { type: event.type ?? null, status: event.status ?? null, urlPath: event.urlPath || "-", count: 0, firstAt: event.ts, lastAt: event.ts, promptIndexes: [], traceIds: [] }; group.count += 1; group.lastAt = event.ts; if (event.promptIndex && !group.promptIndexes.includes(event.promptIndex)) group.promptIndexes.push(event.promptIndex); if (event.traceId && !group.traceIds.includes(event.traceId)) group.traceIds.push(event.traceId); groups.set(key, group); } return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(a.urlPath).localeCompare(String(b.urlPath))); } function isBenignLongLivedStreamClosureAlert(event) { if (event?.urlPath !== "/v1/workbench/events") return false; if (event.status !== null && event.status !== undefined && Number(event.status) > 0) return false; const text = String(event.failureKind || event.errorPreview || event.preview || ""); return /ERR_NETWORK_CHANGED|ERR_ABORTED|net::ERR_NETWORK_CHANGED|net::ERR_ABORTED|aborted|network changed/iu.test(text); } function isObserverRefreshClosureAlert(event, observerRefreshTimes) { const urlPath = String(event?.urlPath || ""); if (!["/v1/workbench/events", "/v1/web-performance"].includes(urlPath) && !/^\/v1\/workbench\/traces\/[^/]+\/events$/u.test(urlPath)) return false; if (event.status !== null && event.status !== undefined && Number(event.status) > 0) return false; const text = String(event.failureKind || event.errorPreview || event.preview || ""); if (!/ERR_NETWORK_CHANGED|ERR_ABORTED|ERR_INCOMPLETE_CHUNKED_ENCODING|ERR_INVALID_CHUNKED_ENCODING|net::ERR_NETWORK_CHANGED|net::ERR_ABORTED|aborted|network changed|incomplete chunked|invalid chunked/iu.test(text)) return false; const ts = Date.parse(String(event.ts || "")); return Number.isFinite(ts) && observerRefreshTimes.some((refreshTs) => Math.abs(ts - refreshTs) <= 8000); } function networkAlertEvent(item, promptTimes) { const failureText = item.failureKind ?? item.failure ?? item.errorText ?? null; return { ts: item.ts ?? null, promptIndex: promptIndexForTs(promptTimes, item.ts), method: String(item.method || "GET").toUpperCase(), status: Number.isFinite(Number(item.status)) ? Number(item.status) : null, type: item.type ?? null, urlPath: urlPath(item.url), urlHash: item.url ? sha256(item.url) : null, failureKind: failureText ? String(failureText) : null, errorTextHash: failureText ? sha256(failureText) : null, errorPreview: failureText ? limitText(failureText, 160) : null }; } function groupNetworkAlerts(events) { const groups = new Map(); for (const event of events) { const key = [event.method, event.urlPath, event.status ?? "-", event.type].join(" "); const group = groups.get(key) || { method: event.method, urlPath: event.urlPath, status: event.status, type: event.type, count: 0, firstAt: event.ts, lastAt: event.ts, promptIndexes: [], failureKinds: [], errorTextHashes: [] }; group.count += 1; group.lastAt = event.ts; if (event.promptIndex && !group.promptIndexes.includes(event.promptIndex)) group.promptIndexes.push(event.promptIndex); if (event.failureKind && !group.failureKinds.includes(event.failureKind)) group.failureKinds.push(event.failureKind); if (event.errorTextHash && !group.errorTextHashes.includes(event.errorTextHash)) group.errorTextHashes.push(event.errorTextHash); groups.set(key, group); } return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(a.urlPath).localeCompare(String(b.urlPath))); } function isDiagnosticText(text) { const value = String(text || ""); return /Failed to (?:fetch|load resource)|request failed|net::ERR_[A-Z0-9_:-]+|server responded with a status of [45][0-9]{2}|HTTP\s+[45][0-9]{2}\b|trace_id=|workbench turn\s*超过|turn\s*超过|无新活动|idle\s+\d+s|waitingFor=|lastEventLabel=|无法连接上游|代理暂时无法连接上游|provider-unavailable|agentrun:error|AgentRun error|projection-resume|sync-failed|durable projection store|realtime-gap|Trace 更新超时|加载失败|请求失败|请求已失败/iu.test(value); } function prioritizeFindings(findings) { const items = Array.isArray(findings) ? findings : []; const severityRank = (severity) => { const value = String(severity || "").toLowerCase(); if (value === "red") return 0; if (value === "amber" || value === "warning") return 1; if (value === "info") return 3; return 2; }; const kindRank = (item) => { const id = String(item?.id ?? item?.kind ?? item?.code ?? ""); if (id.startsWith("project-management-") || id.startsWith("mdtodo-") || id === "workbench-launch-button-unavailable") return 0; if (id === "page-performance-slow-same-origin-api") return 0; if (id === "session-rail-title-fallback-majority") return 0.5; if (id.startsWith("workbench-terminal-")) return 0.6; if (id.startsWith("code-agent-card-")) return 0.8; if (id.startsWith("round-completion-")) return 0.9; if (id.startsWith("turn-timing-total-elapsed")) return 1; if (id.startsWith("turn-timing-terminal-elapsed")) return 1.1; if (id.startsWith("turn-timing-recent-update")) return 2; if (id.includes("runtime-execution") || id.includes("prompt-chat-submit-failed")) return 3; return 10; }; return items.slice().sort((left, right) => { const kindDelta = kindRank(left) - kindRank(right); if (kindDelta !== 0) return kindDelta; return severityRank(left?.severity ?? left?.level) - severityRank(right?.severity ?? right?.level); }); } function isTerminalTraceText(text) { return /轮次完成|轮次失败|轮次取消|已记录|已完成第\d+轮|final response|sealed final response|turn completed|turn failed|turn canceled|terminal result|\bcompleted\b|\bfailed\b|\bcanceled\b|\bcancelled\b|\bterminal\b|\bdone\b/iu.test(String(text || "")); } function isFinalResultText(text) { return /已完成第\d+轮|已按第\d+轮完成|final response|sealed final response|最终结果|已完成[::]|smoke\s*测试结果|benchmark|PVC\/workspace|修改文件|Results:/iu.test(String(text || "")); } function buildSampleMetrics(samples, control) { const promptCommands = buildSendPromptCommandTimeline(control); const promptTimes = promptCommands.map((item) => item.tsMs); const timeline = samples.map((sample) => { const texts = sampleTexts(sample); const timingTexts = sampleTurnTimingTexts(sample); const tsMs = Date.parse(sample.ts); const promptIndex = Number.isFinite(tsMs) ? latestPromptIndex(promptTimes, tsMs) : 0; const totalElapsedValues = timingTexts.flatMap(parseTotalElapsedSeconds).filter(Number.isFinite); const recentUpdateValues = timingTexts.flatMap(parseRecentUpdateSeconds).filter(Number.isFinite); const diagnosticTexts = texts.filter(isDiagnosticText).slice(0, 5); const terminalTexts = texts.filter(isTerminalTraceText).slice(0, 5); const finalResultTexts = texts.filter(isFinalResultText).slice(0, 5); const loadings = Array.isArray(sample.loadings) ? sample.loadings : []; const loadingOwners = uniqueLoadingOwners(loadings); return { seq: sample.seq ?? null, ts: sample.ts ?? null, routeSessionId: sample.routeSessionId ?? null, activeSessionId: sample.activeSessionId ?? null, promptIndex, messageCount: Array.isArray(sample.messages) ? sample.messages.length : 0, traceRowCount: Array.isArray(sample.traceRows) ? sample.traceRows.length : 0, loadingCount: loadings.length, loadingOwnerCount: loadingOwners.length, loadingOwners: loadingOwners.map((item) => ({ ownerKey: item.ownerKey, ownerKind: item.ownerKind, ownerLabel: item.ownerLabel, count: item.count })).slice(0, 12), sessionRailVisibleCount: Number(sample?.sessionRail?.visibleCount ?? 0), sessionRailFallbackTitleCount: Number(sample?.sessionRail?.fallbackTitleCount ?? 0), sessionRailFallbackTitleRatio: Number(sample?.sessionRail?.fallbackTitleRatio ?? 0), totalElapsedSeconds: totalElapsedValues.length > 0 ? Math.max(...totalElapsedValues) : null, recentUpdateSeconds: recentUpdateValues.length > 0 ? Math.max(...recentUpdateValues) : null, terminalSeen: terminalTexts.length > 0, finalResultTextSeen: finalResultTexts.length > 0, diagnosticSeen: diagnosticTexts.length > 0, diagnosticTextHashes: diagnosticTexts.map(sha256).slice(0, 5), textDigest: digestSample(sample) }; }); const turnTiming = buildTurnTimingTable(samples, timeline); const traceOrder = buildTraceOrderMetrics(samples, timeline); const codeAgentCardTiming = buildCodeAgentCardTimingMetrics(samples, timeline, turnTiming); const codeAgentCardDurationUnderreported = buildCodeAgentCardDurationUnderreportedMetrics(samples, timeline); const codeAgentCardDurationMismatches = buildCodeAgentCardDurationMismatchMetrics(samples, timeline); if (codeAgentCardTiming && codeAgentCardTiming.summary) { codeAgentCardTiming.summary.durationUnderreportedCount = codeAgentCardDurationUnderreported.length; codeAgentCardTiming.summary.durationMismatchCount = codeAgentCardDurationMismatches.length; codeAgentCardTiming.durationUnderreported = codeAgentCardDurationUnderreported; codeAgentCardTiming.durationMismatches = codeAgentCardDurationMismatches; } const turnCells = turnTiming.rows.flatMap((row) => Object.values(row.cells || {})); const turnTimingNonMonotonic = Array.isArray(turnTiming.nonMonotonic) ? turnTiming.nonMonotonic : []; const turnTimingElapsedZeroResets = Array.isArray(turnTiming.elapsedZeroResets) ? turnTiming.elapsedZeroResets : []; const turnTimingTotalElapsedForwardJumps = Array.isArray(turnTiming.totalElapsedForwardJumps) ? turnTiming.totalElapsedForwardJumps : []; const turnTimingRecentUpdateSawtoothJumps = turnTimingNonMonotonic.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump"); const turnTimingTerminalElapsedGrowth = Array.isArray(turnTiming.terminalElapsedGrowth) ? turnTiming.terminalElapsedGrowth : []; const turnTimingRecentUpdateResets = Array.isArray(turnTiming.recentUpdateResets) ? turnTiming.recentUpdateResets : []; const turnTimingRecentUpdateSteps = Array.isArray(turnTiming.recentUpdateSteps) ? turnTiming.recentUpdateSteps : []; const turnTimingTerminalElapsedGrowthDeltas = turnTimingTerminalElapsedGrowth .map((item) => Number(item.delta)) .filter((value) => Number.isFinite(value) && value > 0); const turnTimingRecentUpdateLargestSteps = turnTimingRecentUpdateSteps .filter((item) => Number.isFinite(Number(item.delta))) .slice() .sort((a, b) => Number(b.delta) - Number(a.delta)) .slice(0, 200); const turnTimingRecentUpdatePositiveSteps = turnTimingRecentUpdateSteps .map((item) => Number(item.delta)) .filter((value) => Number.isFinite(value) && value >= 0); const turnTimingRecentUpdateExcessSteps = turnTimingRecentUpdateSteps .map((item) => Number(item.excessiveIncreaseSeconds)) .filter((value) => Number.isFinite(value) && value > 0); const withTotal = timeline.filter((item) => item.totalElapsedSeconds !== null).length; const withRecent = timeline.filter((item) => item.recentUpdateSeconds !== null).length; const diagnostics = timeline.filter((item) => item.diagnosticSeen).length; const loading = buildLoadingMetrics(samples, timeline); const sessionRailTitles = buildSessionRailTitleMetrics(samples, timeline); const reportTurnTimingRows = boundedTurnTimingRowsForReport(turnTiming.rows); const reportTimeline = boundedRowsForReport(timeline); const rounds = buildRoundMetricSummaries(timeline, promptCommands, { columns: turnTiming.columns, rows: turnTiming.rows, nonMonotonic: turnTimingNonMonotonic, elapsedZeroResets: turnTimingElapsedZeroResets, totalElapsedForwardJumps: turnTimingTotalElapsedForwardJumps, terminalElapsedGrowth: turnTimingTerminalElapsedGrowth, recentUpdateResets: turnTimingRecentUpdateResets, recentUpdateSteps: turnTimingRecentUpdateSteps }); const recentUpdateJumpCount = turnTimingRecentUpdateSawtoothJumps.length; return { summary: { sampleCount: timeline.length, withTotalElapsed: withTotal, withRecentUpdate: withRecent, diagnostics, loadingSampleCount: loading.summary.loadingSampleCount, loadingMaxCount: loading.summary.maxSimultaneousCount, loadingMaxOwnerCount: loading.summary.maxSimultaneousOwnerCount, loadingOwnerCount: loading.summary.ownerCount, loadingConcurrentSampleCount: loading.summary.concurrentLoadingSampleCount, loadingLongestContinuousSeconds: loading.summary.longestContinuousSeconds, loadingCurrentContinuousSeconds: loading.summary.currentContinuousSeconds, loadingOverFiveSecondSegmentCount: loading.summary.overFiveSecondSegmentCount, loadingOverBudgetSegmentCount: loading.summary.overBudgetSegmentCount, sessionRailSampleCount: sessionRailTitles.summary.sampleCount, sessionRailVisibleSampleCount: sessionRailTitles.summary.visibleSampleCount, sessionRailFallbackMajoritySampleCount: sessionRailTitles.summary.majorityFallbackSampleCount, sessionRailFallbackMaxRatio: sessionRailTitles.summary.maxFallbackRatio, sessionRailFallbackMaxVisibleCount: sessionRailTitles.summary.maxVisibleCount, sessionRailFallbackMaxCount: sessionRailTitles.summary.maxFallbackTitleCount, promptSegments: Math.max(0, promptTimes.length), rounds: rounds.length, turnColumns: turnTiming.columns.length, turnTimingRows: turnTiming.rows.length, turnCellsWithTotalElapsed: turnCells.filter((item) => item.totalElapsedSeconds !== null).length, turnCellsWithRecentUpdate: turnCells.filter((item) => item.recentUpdateSeconds !== null).length, turnTimingNonMonotonicCount: turnTimingNonMonotonic.length, turnTimingTotalElapsedDecreaseCount: turnTimingNonMonotonic.filter((item) => item.metric === "totalElapsedSeconds").length, turnTimingTotalElapsedZeroResetCount: turnTimingElapsedZeroResets.length, turnTimingTotalElapsedForwardJumpCount: turnTimingTotalElapsedForwardJumps.length, turnTimingTotalElapsedForwardJumpMaxSeconds: maxPositiveDelta(turnTimingTotalElapsedForwardJumps), turnTimingTerminalElapsedGrowthCount: turnTimingTerminalElapsedGrowth.length, turnTimingTerminalElapsedGrowthMaxSeconds: turnTimingTerminalElapsedGrowthDeltas.length > 0 ? Math.max(...turnTimingTerminalElapsedGrowthDeltas) : 0, turnTimingRecentUpdateJumpCount: recentUpdateJumpCount, turnTimingRecentUpdateSawtoothJumpCount: recentUpdateJumpCount, turnTimingRecentUpdateStepCount: turnTimingRecentUpdateSteps.length, turnTimingRecentUpdateMaxIncreaseSeconds: turnTimingRecentUpdatePositiveSteps.length > 0 ? Math.max(...turnTimingRecentUpdatePositiveSteps) : null, turnTimingRecentUpdateMaxExcessSeconds: turnTimingRecentUpdateExcessSteps.length > 0 ? Math.max(...turnTimingRecentUpdateExcessSteps) : 0, turnTimingRecentUpdateResetCount: turnTimingRecentUpdateResets.length, turnTimingRecentUpdateDecreaseCount: turnTimingRecentUpdateResets.length, codeAgentCardSampleCount: codeAgentCardTiming.summary.cardSampleCount, codeAgentCardMissingElapsedCount: codeAgentCardTiming.summary.missingElapsedCount, codeAgentCardMissingRecentUpdateCount: codeAgentCardTiming.summary.missingRecentUpdateCount, roundCompletionEventCount: codeAgentCardTiming.summary.roundCompletionEventCount, roundCompletionElapsedMismatchCount: codeAgentCardTiming.summary.roundCompletionElapsedMismatchCount, roundCompletionFinalResponseMissingCount: codeAgentCardTiming.summary.roundCompletionFinalResponseMissingCount, roundCompletionPostTimingChangeCount: codeAgentCardTiming.summary.roundCompletionPostTimingChangeCount, codeAgentCardDurationUnderreportedCount: codeAgentCardTiming.summary.durationUnderreportedCount, codeAgentCardDurationMismatchCount: codeAgentCardTiming.summary.durationMismatchCount, traceRowCount: traceOrder.summary.traceRowCount, traceRowOrderAnomalyCount: traceOrder.summary.orderAnomalyCount, traceRowCompletionNotLastCount: traceOrder.summary.completionNotLastCount, roundsWithTurnTimingNonMonotonic: rounds.filter((item) => item.turnTimingNonMonotonicCount > 0).length, roundsWithTurnTimingTotalElapsedForwardJumps: rounds.filter((item) => item.turnTimingTotalElapsedForwardJumpCount > 0).length, roundsWithTerminalElapsedGrowth: rounds.filter((item) => item.turnTimingTerminalElapsedGrowthCount > 0).length, roundsWithRecentUpdateJumps: rounds.filter((item) => item.turnTimingRecentUpdateJumpCount > 0).length }, loading, sessionRailTitles, codeAgentCardTiming, traceOrder, rounds, turnColumns: turnTiming.columns, turnTimingTable: reportTurnTimingRows.rows, turnTimingTableDisclosure: reportTurnTimingRows.disclosure, turnTimingNonMonotonic, turnTimingElapsedZeroResets, turnTimingTotalElapsedForwardJumps, turnTimingTerminalElapsedGrowth, turnTimingRecentUpdateSawtoothJumps, turnTimingRecentUpdateSteps, turnTimingRecentUpdateLargestSteps, turnTimingRecentUpdateResets, timeline: reportTimeline.rows, timelineDisclosure: reportTimeline.disclosure }; } function boundedRowsForReport(rows) { const sourceRows = Array.isArray(rows) ? rows : []; const maxRows = 1200; const headRows = 120; if (sourceRows.length <= maxRows) return { rows: sourceRows, disclosure: { truncated: false, totalRows: sourceRows.length, includedRows: sourceRows.length, omittedRows: 0, headRows: sourceRows.length, tailRows: 0 } }; const tailRows = Math.max(0, maxRows - headRows); return { rows: [...sourceRows.slice(0, headRows), ...sourceRows.slice(-tailRows)], disclosure: { truncated: true, totalRows: sourceRows.length, includedRows: maxRows, omittedRows: Math.max(0, sourceRows.length - maxRows), headRows, tailRows, policy: "report-bounded-head-tail; summary metrics are computed before truncation", valuesRedacted: true } }; } function buildSendPromptCommandTimeline(control) { const byCommand = new Map(); let ordinal = 0; for (const item of Array.isArray(control) ? control : []) { if (item?.type !== "sendPrompt") continue; const commandId = item.commandId ? String(item.commandId) : "sendPrompt-" + String(ordinal++); const existing = byCommand.get(commandId) || { commandId, startedAt: null, startedTsMs: null, completedAt: null, completedTsMs: null, failedAt: null, failedTsMs: null, textHash: null, textBytes: null, }; if (!existing.textHash && item.input?.textHash) existing.textHash = item.input.textHash; if (!existing.textBytes && item.input?.textBytes) existing.textBytes = item.input.textBytes; const tsMs = Date.parse(item.ts); if (item.phase === "started" && Number.isFinite(tsMs)) { existing.startedAt = item.ts ?? existing.startedAt; existing.startedTsMs = tsMs; } else if (item.phase === "completed" && Number.isFinite(tsMs)) { existing.completedAt = item.ts ?? existing.completedAt; existing.completedTsMs = tsMs; } else if (item.phase === "failed" && Number.isFinite(tsMs)) { existing.failedAt = item.ts ?? existing.failedAt; existing.failedTsMs = tsMs; } byCommand.set(commandId, existing); } return Array.from(byCommand.values()) .map((item) => { const tsMs = Number.isFinite(item.startedTsMs) ? item.startedTsMs : Number.isFinite(item.completedTsMs) ? item.completedTsMs : item.failedTsMs; const ts = Number.isFinite(item.startedTsMs) ? item.startedAt : Number.isFinite(item.completedTsMs) ? item.completedAt : item.failedAt; return { ...item, ts, tsMs }; }) .filter((item) => Number.isFinite(item.tsMs)) .sort((a, b) => a.tsMs - b.tsMs) .map((item, index) => ({ ...item, promptIndex: index + 1 })); } function buildSessionRailTitleMetrics(samples, timeline) { const rows = []; const examplesByHash = new Map(); for (let index = 0; index < (Array.isArray(samples) ? samples : []).length; index += 1) { const sample = samples[index]; const rail = sample?.sessionRail && typeof sample.sessionRail === "object" ? sample.sessionRail : null; if (!rail) continue; const visibleCount = Number(rail.visibleCount ?? 0); const fallbackTitleCount = Number(rail.fallbackTitleCount ?? 0); const safeVisibleCount = Number.isFinite(visibleCount) && visibleCount > 0 ? visibleCount : 0; const safeFallbackTitleCount = Number.isFinite(fallbackTitleCount) && fallbackTitleCount > 0 ? fallbackTitleCount : 0; const fallbackTitleRatio = safeVisibleCount > 0 ? Number((safeFallbackTitleCount / safeVisibleCount).toFixed(4)) : 0; const fallbackItems = Array.isArray(rail.fallbackItems) ? rail.fallbackItems : []; for (const item of fallbackItems) { const hash = String(item?.titleHash || item?.titlePreview || item?.sessionIdPrefix || "").trim(); if (!hash || examplesByHash.has(hash)) continue; examplesByHash.set(hash, { titleHash: item?.titleHash ?? null, titlePreview: limitText(String(item?.titlePreview || ""), 160), sessionIdPrefix: item?.sessionIdPrefix ?? null, active: item?.active === true, firstSeq: sample?.seq ?? null, firstAt: sample?.ts ?? null, pageRole: sample?.pageRole ?? null, }); } rows.push({ ...ref(sample), promptIndex: timeline[index]?.promptIndex ?? 0, visibleCount: safeVisibleCount, fallbackTitleCount: safeFallbackTitleCount, fallbackTitleRatio, majorityFallback: safeVisibleCount > 0 && safeFallbackTitleCount > safeVisibleCount / 2, overThreshold: safeVisibleCount > 0 && fallbackTitleRatio > alertThresholds.sessionRailFallbackRatio, examples: fallbackItems.slice(0, 5).map((item) => ({ titleHash: item?.titleHash ?? null, titlePreview: limitText(String(item?.titlePreview || ""), 160), sessionIdPrefix: item?.sessionIdPrefix ?? null, active: item?.active === true, })), }); } const visibleRows = rows.filter((item) => item.visibleCount > 0); const majorityRows = rows.filter((item) => item.majorityFallback); const overThresholdRows = rows.filter((item) => item.overThreshold); const fallbackRows = rows.filter((item) => item.fallbackTitleCount > 0); const maxFallbackRatio = rows.length > 0 ? Math.max(...rows.map((item) => Number(item.fallbackTitleRatio) || 0)) : 0; const maxVisibleCount = rows.length > 0 ? Math.max(...rows.map((item) => Number(item.visibleCount) || 0)) : 0; const maxFallbackTitleCount = rows.length > 0 ? Math.max(...rows.map((item) => Number(item.fallbackTitleCount) || 0)) : 0; return { summary: { sampleCount: rows.length, visibleSampleCount: visibleRows.length, fallbackSampleCount: fallbackRows.length, majorityFallbackSampleCount: majorityRows.length, overThresholdSampleCount: overThresholdRows.length, thresholdRatio: alertThresholds.sessionRailFallbackRatio, maxFallbackRatio, maxVisibleCount, maxFallbackTitleCount, }, samples: majorityRows.slice(0, 80), examples: Array.from(examplesByHash.values()).slice(0, 80), timeline: rows.slice(-200), valuesRedacted: true }; } function uniqueLoadingOwners(loadings) { const groups = new Map(); for (let index = 0; index < (Array.isArray(loadings) ? loadings : []).length; index += 1) { const item = loadings[index]; const ownerKey = loadingOwnerKey(item, index); const ownerIdentity = loadingOwnerIdentity(item); const existing = groups.get(ownerKey) || { ownerKey, ownerKind: item?.ownerKind ?? "unknown", ownerLabel: loadingOwnerLabel(item, ownerKey), ...ownerIdentity, count: 0, textHashes: [] }; existing.count += 1; if (item?.textHash && !existing.textHashes.includes(item.textHash)) existing.textHashes.push(item.textHash); for (const key of ["ownerSessionId", "ownerMessageId", "ownerTraceId"]) { if (!existing[key] && ownerIdentity[key]) existing[key] = ownerIdentity[key]; } groups.set(ownerKey, existing); } return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(a.ownerLabel).localeCompare(String(b.ownerLabel))); } function loadingOwnerIdentity(item) { const owner = item?.owner && typeof item.owner === "object" ? item.owner : {}; return { ownerSessionId: owner.sessionId ?? null, ownerMessageId: owner.messageId ?? null, ownerTraceId: owner.traceId ?? null, }; } function loadingOwnerKey(item, index = 0) { const key = String(item?.ownerKey || "").trim(); if (key) return key.slice(0, 240); const owner = item?.owner && typeof item.owner === "object" ? item.owner : {}; return [ item?.ownerKind || "unknown", owner.testId || item?.testId || owner.id || owner.role || owner.className || item?.role || item?.tag || "node", owner.sessionId || owner.messageId || owner.traceId || item?.textHash || String(index) ].filter(Boolean).join(":").slice(0, 240); } function loadingOwnerLabel(item, fallback) { return limitText(String(item?.ownerLabel || item?.owner?.ariaLabel || item?.owner?.testId || item?.owner?.className || fallback || "unknown"), 160); } function buildLoadingMetrics(samples, timeline) { const events = samples.map((sample, index) => { const tsMs = Date.parse(sample?.ts); const loadings = Array.isArray(sample?.loadings) ? sample.loadings : []; const owners = uniqueLoadingOwners(loadings); return { seq: sample?.seq ?? null, ts: sample?.ts ?? null, tsMs, promptIndex: timeline[index]?.promptIndex ?? 0, routeSessionId: sample?.routeSessionId ?? null, activeSessionId: sample?.activeSessionId ?? null, loadingCount: loadings.length, ownerCount: owners.length, owners, ownerKeys: owners.map((item) => item.ownerKey), ownerLabels: owners.map((item) => item.ownerLabel).slice(0, 8) }; }).filter((item) => Number.isFinite(item.tsMs)); const continuityThresholdMs = loadingContinuityThresholdMs(events); const segments = buildLoadingSegments(events, continuityThresholdMs, (event) => event.loadingCount, (event) => event.owners) .sort((a, b) => Number(b.durationSeconds ?? 0) - Number(a.durationSeconds ?? 0) || Number(b.maxCount ?? 0) - Number(a.maxCount ?? 0)); const ownerMap = new Map(); for (const event of events) { for (const owner of event.owners) { const existing = ownerMap.get(owner.ownerKey) || { ownerKey: owner.ownerKey, ownerKind: owner.ownerKind, ownerLabel: owner.ownerLabel, ownerSessionId: owner.ownerSessionId ?? null, ownerMessageId: owner.ownerMessageId ?? null, ownerTraceId: owner.ownerTraceId ?? null, sampleCount: 0, occurrenceCount: 0, maxSimultaneousCount: 0, firstAt: event.ts, lastAt: event.ts, firstSeq: event.seq, lastSeq: event.seq, promptIndexes: new Set(), events: [] }; existing.sampleCount += 1; existing.occurrenceCount += owner.count; existing.maxSimultaneousCount = Math.max(existing.maxSimultaneousCount, owner.count); existing.lastAt = event.ts; existing.lastSeq = event.seq; for (const key of ["ownerSessionId", "ownerMessageId", "ownerTraceId"]) { if (!existing[key] && owner[key]) existing[key] = owner[key]; } if (Number.isFinite(Number(event.promptIndex))) existing.promptIndexes.add(Number(event.promptIndex)); existing.events.push({ ...event, loadingCount: owner.count, owners: [owner] }); ownerMap.set(owner.ownerKey, existing); } } const owners = Array.from(ownerMap.values()).map((owner) => { const ownerSegments = buildLoadingSegments(owner.events, continuityThresholdMs, (event) => event.loadingCount, (event) => event.owners); const longest = ownerSegments.reduce((max, item) => Math.max(max, Number(item.durationSeconds ?? 0)), 0); return { ownerKey: owner.ownerKey, ownerKind: owner.ownerKind, ownerLabel: owner.ownerLabel, ownerSessionId: owner.ownerSessionId ?? null, ownerMessageId: owner.ownerMessageId ?? null, ownerTraceId: owner.ownerTraceId ?? null, sampleCount: owner.sampleCount, occurrenceCount: owner.occurrenceCount, maxSimultaneousCount: owner.maxSimultaneousCount, longestContinuousSeconds: longest, firstAt: owner.firstAt, lastAt: owner.lastAt, firstSeq: owner.firstSeq, lastSeq: owner.lastSeq, promptIndexes: Array.from(owner.promptIndexes).sort((a, b) => a - b), segments: ownerSegments.sort((a, b) => Number(b.durationSeconds ?? 0) - Number(a.durationSeconds ?? 0)).slice(0, 8), valuesRedacted: true }; }).sort((a, b) => Number(b.longestContinuousSeconds ?? 0) - Number(a.longestContinuousSeconds ?? 0) || Number(b.occurrenceCount ?? 0) - Number(a.occurrenceCount ?? 0)); const latest = events[events.length - 1] || null; const currentSegment = latest && latest.loadingCount > 0 ? segments.find((segment) => segment.ongoing === true && segment.lastSeq === latest.seq) || null : null; const timelineRows = events .filter((event, index) => event.loadingCount > 0 || (index > 0 && events[index - 1]?.loadingCount > 0)) .slice(0, 500) .map((event) => ({ seq: event.seq, ts: event.ts, promptIndex: event.promptIndex, loadingCount: event.loadingCount, ownerCount: event.ownerCount, owners: event.owners.map((owner) => ({ ownerKind: owner.ownerKind, ownerLabel: owner.ownerLabel, ownerSessionId: owner.ownerSessionId ?? null, ownerMessageId: owner.ownerMessageId ?? null, ownerTraceId: owner.ownerTraceId ?? null, count: owner.count })).slice(0, 8) })); return { summary: { sampleCount: events.length, loadingSampleCount: events.filter((event) => event.loadingCount > 0).length, maxSimultaneousCount: events.reduce((max, event) => Math.max(max, event.loadingCount), 0), maxSimultaneousOwnerCount: events.reduce((max, event) => Math.max(max, event.ownerCount), 0), concurrentLoadingSampleCount: events.filter((event) => event.loadingCount > 1).length, ownerCount: owners.length, segmentCount: segments.length, overFiveSecondSegmentCount: segments.filter((segment) => Number(segment.durationSeconds ?? 0) > 5).length, overBudgetSegmentCount: segments.filter((segment) => Number(segment.durationSeconds ?? 0) > alertThresholds.visibleLoadingSlowMs / 1000).length, budgetSeconds: alertThresholds.visibleLoadingSlowMs / 1000, longestContinuousSeconds: segments.length > 0 ? Number(segments[0].durationSeconds ?? 0) : 0, currentContinuousSeconds: currentSegment ? Number(currentSegment.durationSeconds ?? 0) : 0, continuityThresholdMs, latestLoadingCount: latest?.loadingCount ?? 0, latestOwnerCount: latest?.ownerCount ?? 0, valuesRedacted: true }, segments: segments.slice(0, 80), owners: owners.slice(0, 80), timeline: timelineRows, valuesRedacted: true }; } function loadingContinuityThresholdMs(events) { const deltas = []; for (let index = 1; index < events.length; index += 1) { const delta = events[index].tsMs - events[index - 1].tsMs; if (Number.isFinite(delta) && delta > 0) deltas.push(delta); } if (deltas.length === 0) return 5000; const sorted = deltas.slice().sort((a, b) => a - b); const median = sorted[Math.floor(sorted.length / 2)]; return Math.min(15000, Math.max(1500, Math.round(median * 2.5))); } function buildLoadingSegments(events, continuityThresholdMs, countForEvent, ownersForEvent) { const segments = []; let segment = null; let previousTsMs = null; for (const event of events) { const count = Number(countForEvent(event) ?? 0); const gapOk = previousTsMs === null || !Number.isFinite(event.tsMs) || event.tsMs - previousTsMs <= continuityThresholdMs; if (count > 0) { if (!segment || !gapOk) { if (segment) segments.push(finalizeLoadingSegment(segment, null)); segment = { firstAt: event.ts, lastAt: event.ts, firstSeq: event.seq, lastSeq: event.seq, promptIndexes: new Set(), ownerKeys: new Set(), ownerLabels: new Map(), sampleCount: 0, maxCount: 0, ongoing: true }; } segment.lastAt = event.ts; segment.lastSeq = event.seq; segment.sampleCount += 1; segment.maxCount = Math.max(segment.maxCount, count); if (Number.isFinite(Number(event.promptIndex))) segment.promptIndexes.add(Number(event.promptIndex)); for (const owner of ownersForEvent(event) || []) { if (!owner?.ownerKey) continue; segment.ownerKeys.add(owner.ownerKey); if (!segment.ownerLabels.has(owner.ownerKey)) segment.ownerLabels.set(owner.ownerKey, { ownerKey: owner.ownerKey, ownerKind: owner.ownerKind, ownerLabel: owner.ownerLabel, count: 0 }); const label = segment.ownerLabels.get(owner.ownerKey); label.count += owner.count ?? 1; } } else if (segment) { segment.ongoing = false; segment.endedAt = event.ts; segment.endSeq = event.seq; segments.push(finalizeLoadingSegment(segment, event)); segment = null; } previousTsMs = event.tsMs; } if (segment) segments.push(finalizeLoadingSegment(segment, null)); return segments; } function finalizeLoadingSegment(segment, absentEvent) { const startMs = Date.parse(segment.firstAt || ""); const lastMs = Date.parse(segment.lastAt || ""); const absentMs = Date.parse(absentEvent?.ts || ""); const durationSeconds = Number.isFinite(startMs) && Number.isFinite(lastMs) && lastMs >= startMs ? Number(((lastMs - startMs) / 1000).toFixed(3)) : 0; const upperBoundSeconds = Number.isFinite(startMs) && Number.isFinite(absentMs) && absentMs >= startMs ? Number(((absentMs - startMs) / 1000).toFixed(3)) : durationSeconds; const endedGapSeconds = Number.isFinite(lastMs) && Number.isFinite(absentMs) && absentMs >= lastMs ? Number(((absentMs - lastMs) / 1000).toFixed(3)) : null; return { firstAt: segment.firstAt, lastAt: segment.lastAt, endedAt: absentEvent?.ts ?? null, firstSeq: segment.firstSeq, lastSeq: segment.lastSeq, endSeq: absentEvent?.seq ?? null, durationSeconds, upperBoundSeconds, endedGapSeconds, sampleCount: segment.sampleCount, maxCount: segment.maxCount, ownerCount: segment.ownerKeys.size, owners: Array.from(segment.ownerLabels.values()).sort((a, b) => b.count - a.count || String(a.ownerLabel).localeCompare(String(b.ownerLabel))).slice(0, 12), promptIndexes: Array.from(segment.promptIndexes).sort((a, b) => a - b), ongoing: absentEvent ? false : segment.ongoing === true, valuesRedacted: true }; } function boundedTurnTimingRowsForReport(rows) { const sourceRows = Array.isArray(rows) ? rows : []; const maxRows = 1200; const headRows = 120; if (sourceRows.length <= maxRows) return { rows: sourceRows, disclosure: { truncated: false, totalRows: sourceRows.length, includedRows: sourceRows.length, omittedRows: 0, headRows: sourceRows.length, tailRows: 0 } }; const tailRows = Math.max(0, maxRows - headRows); return { rows: [...sourceRows.slice(0, headRows), ...sourceRows.slice(-tailRows)], disclosure: { truncated: true, totalRows: sourceRows.length, includedRows: maxRows, omittedRows: Math.max(0, sourceRows.length - maxRows), headRows, tailRows, policy: "report-bounded-head-tail; full anomaly counters are computed before truncation", valuesRedacted: true } }; } ` + nodeWebObserveAnalyzerTimingSource(); }