diff --git a/scripts/src/hwlab-node-web-observe-analyzer-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-source.ts index 757eb4ee..178b19e1 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-source.ts @@ -1132,6 +1132,7 @@ function compactWorkbenchTurnStateTriadForOutput(value) { if (!value || typeof value !== "object") return null; return { summary: value.summary ?? null, + drilldown: value.drilldown ?? null, invalidFullTriads: Array.isArray(value.invalidFullTriads) ? value.invalidFullTriads.slice(0, 8) : [], cardFinalResponseMismatches: Array.isArray(value.cardFinalResponseMismatches) ? value.cardFinalResponseMismatches.slice(0, 8) : [], collectorMissingRows: Array.isArray(value.collectorMissingRows) ? value.collectorMissingRows.slice(0, 8) : [], @@ -2858,6 +2859,8 @@ function buildWorkbenchTurnStateTriadMetrics(samples, timeline = []) { const collectorMissingRows = rows.filter((row) => Array.isArray(row.collectorMissing) && row.collectorMissing.length > 0); const invalidRowKeys = new Set([...invalidFullTriads, ...cardFinalResponseMismatches].map((row) => row.rowKey)); const collectorMissingFields = uniqueSorted(collectorMissingRows.flatMap((row) => row.collectorMissing || [])); + const offendingRows = Array.from(new Map([...invalidFullTriads, ...cardFinalResponseMismatches].map((row) => [row.rowKey, row])).values()); + const drilldown = buildWorkbenchTurnStateTriadDrilldown(offendingRows); return { summary: { sampleCount: sourceSamples.length, @@ -2866,10 +2869,14 @@ function buildWorkbenchTurnStateTriadMetrics(samples, timeline = []) { invalidRowCount: invalidRowKeys.size, invalidFullTriadCount: invalidFullTriads.length, cardFinalResponseMismatchCount: cardFinalResponseMismatches.length, + invalidGroupCount: drilldown.summary.groupCount, + invalidTraceIdCount: drilldown.summary.traceIds.length, + invalidMaxObservedSpanMs: drilldown.summary.maxObservedSpanMs, collectorMissingRowCount: collectorMissingRows.length, collectorMissingFields, valuesRedacted: true }, + drilldown, invalidFullTriads: invalidFullTriads.slice(0, 120), cardFinalResponseMismatches: cardFinalResponseMismatches.slice(0, 120), collectorMissingRows: collectorMissingRows.slice(0, 120), @@ -2878,6 +2885,210 @@ function buildWorkbenchTurnStateTriadMetrics(samples, timeline = []) { }; } +function buildWorkbenchTurnStateTriadDrilldown(rows) { + const uniqueRows = Array.from(new Map((Array.isArray(rows) ? rows : []) + .filter(Boolean) + .map((row) => [row?.rowKey || [row?.seq, row?.traceId, row?.messageId, row?.pageRole].join("|"), row])).values()); + const groupsByKey = new Map(); + let firstAt = null; + let lastAt = null; + for (const row of uniqueRows) { + const traceId = firstString(row?.traceId); + const mismatchKind = workbenchTriadMismatchKind(row); + const tuple = workbenchTriadTuple(row); + const key = [ + traceId || row?.messageId || row?.sessionIdPrefix || "-", + row?.sessionIdPrefix || "-", + row?.pageRole || "-", + mismatchKind, + tuple, + ].join("|"); + let group = groupsByKey.get(key); + if (!group) { + group = { + keyHash: sha256(key), + mismatchKind, + statusTuple: tuple, + traceId: traceId || null, + messageId: row?.messageId ?? null, + sessionIdPrefix: row?.sessionIdPrefix ?? null, + pageRole: row?.pageRole ?? null, + pageId: row?.pageId ?? null, + count: 0, + firstSeq: row?.seq ?? null, + lastSeq: row?.seq ?? null, + firstAt: row?.ts ?? null, + lastAt: row?.ts ?? null, + promptIndexes: new Set(), + commandIds: new Set(), + rawRailStatuses: new Set(), + rawCardStatuses: new Set(), + finalResponseTextBytes: new Set(), + examples: [], + valuesRedacted: true, + }; + groupsByKey.set(key, group); + } + group.count += 1; + group.firstSeq = minNumberOrValue(group.firstSeq, row?.seq); + group.lastSeq = maxNumberOrValue(group.lastSeq, row?.seq); + group.firstAt = minIso(group.firstAt, row?.ts ?? null); + group.lastAt = maxIso(group.lastAt, row?.ts ?? null); + firstAt = minIso(firstAt, row?.ts ?? null); + lastAt = maxIso(lastAt, row?.ts ?? null); + if (row?.promptIndex !== null && row?.promptIndex !== undefined) group.promptIndexes.add(String(row.promptIndex)); + if (row?.nearestCommandId) group.commandIds.add(String(row.nearestCommandId)); + if (row?.railStatusRaw) group.rawRailStatuses.add(String(row.railStatusRaw)); + if (row?.cardStatusRaw) group.rawCardStatuses.add(String(row.cardStatusRaw)); + if (row?.finalResponseTextBytes !== null && row?.finalResponseTextBytes !== undefined) group.finalResponseTextBytes.add(String(row.finalResponseTextBytes)); + if (group.examples.length < 3) { + group.examples.push({ + seq: row?.seq ?? null, + ts: row?.ts ?? null, + traceId, + railStatus: row?.railStatus ?? null, + railStatusRaw: row?.railStatusRaw ?? null, + cardStatus: row?.cardStatus ?? null, + cardStatusRaw: row?.cardStatusRaw ?? null, + finalResponsePresent: row?.finalResponsePresent ?? null, + finalResponseTextBytes: row?.finalResponseTextBytes ?? null, + nearestCommandId: row?.nearestCommandId ?? null, + valuesRedacted: true, + }); + } + } + const groups = Array.from(groupsByKey.values()).map((group) => { + const observedSpanMs = isoSpanMs(group.firstAt, group.lastAt); + return { + keyHash: group.keyHash, + mismatchKind: group.mismatchKind, + statusTuple: group.statusTuple, + traceId: group.traceId, + messageId: group.messageId, + sessionIdPrefix: group.sessionIdPrefix, + pageRole: group.pageRole, + pageId: group.pageId, + count: group.count, + firstSeq: group.firstSeq, + lastSeq: group.lastSeq, + firstAt: group.firstAt, + lastAt: group.lastAt, + observedSpanMs, + promptIndexes: Array.from(group.promptIndexes).slice(0, 8), + commandIds: Array.from(group.commandIds).slice(0, 8), + rawRailStatuses: Array.from(group.rawRailStatuses).slice(0, 8), + rawCardStatuses: Array.from(group.rawCardStatuses).slice(0, 8), + finalResponseTextBytes: Array.from(group.finalResponseTextBytes).slice(0, 8), + examples: group.examples, + valuesRedacted: true, + }; + }).sort((left, right) => Number(right.observedSpanMs ?? 0) - Number(left.observedSpanMs ?? 0) || Number(right.count ?? 0) - Number(left.count ?? 0)); + const traceIds = uniqueSorted(uniqueRows.map((row) => row?.traceId).filter(Boolean)).slice(0, 12); + const sessionPrefixes = uniqueSorted(uniqueRows.map((row) => row?.sessionIdPrefix).filter(Boolean)).slice(0, 12); + const mismatchKinds = uniqueSorted(uniqueRows.map(workbenchTriadMismatchKind).filter(Boolean)); + return { + summary: { + invalidUniqueRows: uniqueRows.length, + groupCount: groups.length, + traceIds, + sessionPrefixes, + mismatchKinds, + firstAt, + lastAt, + maxObservedSpanMs: groups.reduce((value, item) => Math.max(value, Number(item.observedSpanMs ?? 0)), 0), + valuesRedacted: true, + }, + groups: groups.slice(0, 12), + otelDrilldown: buildWorkbenchTriadOtelDrilldown(traceIds.slice(0, 4)), + staticSourceHints: workbenchTriadStaticSourceHints(), + unitTestReproHints: workbenchTriadUnitTestReproHints(), + valuesRedacted: true, + }; +} + +function buildWorkbenchTriadOtelDrilldown(traceIds) { + const target = otelTargetForAnalyzer(); + const ids = (Array.isArray(traceIds) ? traceIds : []).filter(Boolean).slice(0, 4); + return { + target, + commands: ids.flatMap((traceId) => [ + "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target " + target + " --business-trace-id " + traceId + " --full", + "bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id --grep session_ --limit 60 --full", + "bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id --grep turn_status_read --limit 40 --full", + "bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id --grep projection --limit 120 --full", + ]).slice(0, 16), + valuesRedacted: true, + }; +} + +function otelTargetForAnalyzer() { + const explicit = firstString(process.env.UNIDESK_WEB_OBSERVE_OTEL_TARGET, process.env.UNIDESK_OBSERVABILITY_TARGET); + if (explicit) return explicit; + const parts = String(stateDir || "").split(/[\\\/]+/u); + const index = parts.lastIndexOf("web-observe"); + if (index >= 0 && parts[index + 1]) return String(parts[index + 1]).toUpperCase(); + return "JD01"; +} + +function workbenchTriadMismatchKind(row) { + if (!row || typeof row !== "object") return "unknown"; + if (row.railCardMismatch === true) return "rail-card-status-mismatch"; + if (row.cardFinalResponseMismatch === true) return "completed-card-final-response-absent"; + if (row.legacyCardFinalResponseMismatch === true) return "completed-card-final-response-uncollected-or-absent"; + if (row.tupleAllowed === false) return "tuple-not-allowed"; + return "unknown"; +} + +function workbenchTriadTuple(row) { + return [ + "rail=" + (row?.railStatus ?? "-"), + "card=" + (row?.cardStatus ?? "-"), + "final=" + String(row?.finalResponsePresent ?? "unknown"), + ].join(","); +} + +function minNumberOrValue(left, right) { + const l = Number(left); + const r = Number(right); + if (!Number.isFinite(l)) return right ?? left ?? null; + if (!Number.isFinite(r)) return left ?? right ?? null; + return Math.min(l, r); +} + +function maxNumberOrValue(left, right) { + const l = Number(left); + const r = Number(right); + if (!Number.isFinite(l)) return right ?? left ?? null; + if (!Number.isFinite(r)) return left ?? right ?? null; + return Math.max(l, r); +} + +function isoSpanMs(firstAt, lastAt) { + const start = Date.parse(firstAt || ""); + const end = Date.parse(lastAt || ""); + if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return 0; + return end - start; +} + +function workbenchTriadStaticSourceHints() { + return [ + { repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-session.ts", reason: "session rail/list status and active session projection source" }, + { repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-server-state.ts", reason: "server snapshot merge path that can overwrite sealed turn state" }, + { repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.ts", reason: "turn card and Final Response projection/runtime merge path" }, + { repo: "pikasTech/HWLAB", path: "internal/cloud/server-workbench-http*.go", reason: "session list/detail/messages/turn REST read-model contract" }, + { repo: "pikasTech/HWLAB", path: "internal/cloud/workbench-projection-*.go", reason: "durable projection writer and terminal seal contract" }, + ]; +} + +function workbenchTriadUnitTestReproHints() { + return [ + "backend projector: terminal event must produce a single sealed turn tuple consumed by session list, session detail, messages and turn-status APIs", + "backend read model: completed rail status must not coexist with running turn card or missing Final Response for the same trace", + "frontend server-state merge: stale running/empty snapshots must not overwrite a sealed completed+Final Response turn", + "frontend projection runtime: cross-page hydration must converge from the same durable projection without DOM-only repair", + ]; +} + function workbenchTurnStateTriadRow(sample, turn, rail, promptIndex) { const collectorMissing = []; const railRawStatus = firstString(rail?.status, rail?.dataStatus); @@ -3168,7 +3379,10 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN { railStatus: "running", cardStatus: "running", finalResponsePresent: false } ], samples: turnStateTriadRows.slice(0, 20), + drilldown: turnStateTriad.drilldown ?? buildWorkbenchTurnStateTriadDrilldown(turnStateTriadRows), collectorMissingSamples: Array.isArray(turnStateTriad.collectorMissingRows) ? turnStateTriad.collectorMissingRows.slice(0, 10) : [], + sourceOfTruth: "durable Workbench projection/read model; do not repair via DOM fallback or GET-side state mutation", + nextAction: "Use drilldown.otelDrilldown.commands for the listed traceIds, then inspect staticSourceHints and add unit tests from unitTestReproHints before changing UI rendering.", rootCause: "workbench_projection_state_triad_not_sealed", rootCauseStatus: "confirmed-from-dom-samples", rootCauseConfidence: "high", @@ -3301,7 +3515,21 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN const evaluatedCrossPageProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => !controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq) && !crossPageDiffHasWorkbenchAppShellNotReady(item, sampleBySeq)); const persistentCrossPageProjectionDiffs = evaluatedCrossPageProjectionDiffs.filter((item) => Number(item.observedSpanMs ?? 0) > crossPageProjectionBudgetMs); const transientCrossPageProjectionDiffs = evaluatedCrossPageProjectionDiffs.filter((item) => Number(item.observedSpanMs ?? 0) <= crossPageProjectionBudgetMs); - if (persistentCrossPageProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-divergence", severity: "red", summary: "control and observer pages saw different projection state for the same sampled session beyond the configured budget", count: persistentCrossPageProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: persistentCrossPageProjectionDiffs.slice(0, 20) }); + if (persistentCrossPageProjectionDiffs.length > 0) findings.push({ + id: "cross-page-projection-divergence", + severity: "red", + summary: "control and observer pages saw different projection state for the same sampled session beyond the configured budget", + count: persistentCrossPageProjectionDiffs.length, + budgetMs: crossPageProjectionBudgetMs, + samples: persistentCrossPageProjectionDiffs.slice(0, 20), + drilldown: buildCrossPageProjectionDrilldown(persistentCrossPageProjectionDiffs), + sourceOfTruth: "session list/detail/messages/turn APIs must read from the same durable projection snapshot across pages", + rootCause: "workbench_cross_page_projection_not_single_source", + rootCauseStatus: "confirmed-from-control-observer-dom-samples", + rootCauseConfidence: "high", + nextAction: "Use drilldown.traceIds with diagnose-code-agent and compare session_list_read/session_messages_read/turn_status_read rows before changing renderer fallback behavior.", + valuesRedacted: true + }); if (transientCrossPageProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-transient-divergence", severity: "info", summary: "control and observer pages briefly differed near a sampled transition; retained as transient evidence but not treated as persistent projection failure", count: transientCrossPageProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: transientCrossPageProjectionDiffs.slice(0, 20) }); if (controlledNavigationHydrationProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-controlled-navigation-hydration", severity: "info", summary: "control and observer pages differed while a non-blocking session-invariance navigation command still had an unhydrated blank page; retained as context but not treated as a red projection blocker", count: controlledNavigationHydrationProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: controlledNavigationHydrationProjectionDiffs.slice(0, 20) }); if (appShellNotReadyProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-app-shell-not-ready", severity: "info", summary: "cross-page projection differences were explained by a page whose Workbench app shell was not mounted; see workbench-app-shell-not-ready for the blocking root cause", count: appShellNotReadyProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: appShellNotReadyProjectionDiffs.slice(0, 20) }); @@ -3339,6 +3567,114 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN return findings; } +function buildCrossPageProjectionDrilldown(rows) { + const sourceRows = Array.isArray(rows) ? rows.filter(Boolean) : []; + const traceIds = uniqueSorted(sourceRows.flatMap((row) => collectIdsFromUnknown(row, "trace")).filter(Boolean)).slice(0, 12); + const sessionIds = uniqueSorted(sourceRows.flatMap((row) => collectIdsFromUnknown(row, "session")).filter(Boolean)).slice(0, 12); + const diffKinds = uniqueSorted(sourceRows.map((row) => row?.diffKind).filter(Boolean)); + const groupsByKey = new Map(); + for (const row of sourceRows) { + const rowTraceIds = collectIdsFromUnknown(row, "trace").slice(0, 8); + const rowSessionIds = collectIdsFromUnknown(row, "session").slice(0, 8); + const key = [ + row?.diffKind || "projection", + rowSessionIds[0] || row?.sessionIdPrefix || "-", + rowTraceIds[0] || "-", + ].join("|"); + let group = groupsByKey.get(key); + if (!group) { + group = { + keyHash: sha256(key), + diffKind: row?.diffKind ?? null, + traceIds: new Set(), + sessionIds: new Set(), + count: 0, + firstSeq: row?.firstSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq ?? null, + lastSeq: row?.lastSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq ?? null, + firstAt: row?.firstAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null, + lastAt: row?.lastAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null, + maxObservedSpanMs: 0, + examples: [], + }; + groupsByKey.set(key, group); + } + group.count += 1; + for (const traceId of rowTraceIds) group.traceIds.add(traceId); + for (const sessionId of rowSessionIds) group.sessionIds.add(sessionId); + group.firstSeq = minNumberOrValue(group.firstSeq, row?.firstSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq); + group.lastSeq = maxNumberOrValue(group.lastSeq, row?.lastSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq); + group.firstAt = minIso(group.firstAt, row?.firstAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null); + group.lastAt = maxIso(group.lastAt, row?.lastAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null); + group.maxObservedSpanMs = Math.max(group.maxObservedSpanMs, Number(row?.observedSpanMs ?? 0) || 0); + if (group.examples.length < 3) { + group.examples.push({ + diffKind: row?.diffKind ?? null, + firstSeq: row?.firstSeq ?? null, + lastSeq: row?.lastSeq ?? null, + observedSpanMs: row?.observedSpanMs ?? null, + controlMessageCount: row?.controlMessageCount ?? null, + observerMessageCount: row?.observerMessageCount ?? null, + traceIds: rowTraceIds.slice(0, 6), + sessionIds: rowSessionIds.slice(0, 4), + valuesRedacted: true, + }); + } + } + const groups = Array.from(groupsByKey.values()).map((group) => ({ + keyHash: group.keyHash, + diffKind: group.diffKind, + traceIds: Array.from(group.traceIds).slice(0, 8), + sessionIds: Array.from(group.sessionIds).slice(0, 6), + count: group.count, + firstSeq: group.firstSeq, + lastSeq: group.lastSeq, + firstAt: group.firstAt, + lastAt: group.lastAt, + observedSpanMs: Math.max(group.maxObservedSpanMs, isoSpanMs(group.firstAt, group.lastAt)), + examples: group.examples, + valuesRedacted: true, + })).sort((left, right) => Number(right.observedSpanMs ?? 0) - Number(left.observedSpanMs ?? 0) || Number(right.count ?? 0) - Number(left.count ?? 0)); + return { + summary: { + rowCount: sourceRows.length, + groupCount: groups.length, + diffKinds, + traceIds, + sessionIds, + maxObservedSpanMs: groups.reduce((value, item) => Math.max(value, Number(item.observedSpanMs ?? 0)), 0), + valuesRedacted: true, + }, + traceIds, + sessionIds, + groups: groups.slice(0, 12), + otelDrilldown: buildWorkbenchTriadOtelDrilldown(traceIds.slice(0, 4)), + staticSourceHints: workbenchTriadStaticSourceHints(), + unitTestReproHints: [ + "two independent Workbench pages must converge to identical session list/detail/messages/turn projection after the same durable events", + "late session-list or message refresh must not drop a trace that another page already read from durable projection", + "read-side APIs must expose enough OTel fields to compare projection version/cursor and trace ids across pages", + ], + valuesRedacted: true, + }; +} + +function collectIdsFromUnknown(value, kind, output = new Set(), depth = 0) { + if (depth > 4 || value === null || value === undefined) return Array.from(output); + if (typeof value === "string") { + const pattern = kind === "session" ? /\bses_[A-Za-z0-9_-]+\b/gu : /\btrc_[A-Za-z0-9_-]+\b/gu; + for (const match of value.match(pattern) || []) output.add(match); + return Array.from(output); + } + if (Array.isArray(value)) { + for (const item of value.slice(0, 40)) collectIdsFromUnknown(item, kind, output, depth + 1); + return Array.from(output); + } + if (typeof value === "object") { + for (const item of Object.values(value).slice(0, 80)) collectIdsFromUnknown(item, kind, output, depth + 1); + } + return Array.from(output); +} + function buildFrontendFreezeFindings(errors, control) { const findings = []; const promptTimes = (control || []) diff --git a/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts index 6df9340a..04e3cfda 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts @@ -2219,6 +2219,9 @@ function renderMarkdown(report) { const loadingSummary = loading.summary || {}; const sessionRailTitles = report.sampleMetrics?.sessionRailTitles || {}; const sessionRailTitleSummary = sessionRailTitles.summary || {}; + const workbenchTurnStateTriad = report.sampleMetrics?.workbenchTurnStateTriad || {}; + const workbenchTurnStateTriadSummary = workbenchTurnStateTriad.summary || {}; + const workbenchTurnStateTriadDrilldown = workbenchTurnStateTriad.drilldown || {}; const codeAgentCardTiming = report.sampleMetrics?.codeAgentCardTiming || {}; const codeAgentCardTimingSummary = codeAgentCardTiming.summary || {}; const roundCompletion = codeAgentCardTiming.roundCompletion || {}; @@ -2312,6 +2315,18 @@ function renderMarkdown(report) { const sessionRailTitleExampleLines = Array.isArray(sessionRailTitles.examples) && sessionRailTitles.examples.length > 0 ? sessionRailTitles.examples.slice(0, 80).map((item) => "- firstSeq=" + (item.firstSeq ?? "-") + " role=" + (item.pageRole || "-") + " active=" + String(item.active === true) + " sessionPrefix=" + (item.sessionIdPrefix || "-") + " titleHash=" + (item.titleHash || "-") + " preview=" + escapeMarkdownCell(item.titlePreview || "")).join("\n") : "- 无 fallback 标题示例。"; + const workbenchTurnStateTriadGroupLines = Array.isArray(workbenchTurnStateTriadDrilldown.groups) && workbenchTurnStateTriadDrilldown.groups.length > 0 + ? workbenchTurnStateTriadDrilldown.groups.slice(0, 80).map((item) => "- mismatch=" + (item.mismatchKind || "-") + " tuple=" + (item.statusTuple || "-") + " traceId=" + (item.traceId || "-") + " session=" + (item.sessionIdPrefix || "-") + " role=" + (item.pageRole || "-") + " count=" + (item.count ?? 0) + " spanMs=" + (item.observedSpanMs ?? 0) + " seq=" + (item.firstSeq ?? "-") + ".." + (item.lastSeq ?? "-") + " commands=" + ((Array.isArray(item.commandIds) ? item.commandIds : []).slice(0, 4).join(",") || "-") + " railRaw=" + ((Array.isArray(item.rawRailStatuses) ? item.rawRailStatuses : []).slice(0, 4).join(",") || "-") + " cardRaw=" + ((Array.isArray(item.rawCardStatuses) ? item.rawCardStatuses : []).slice(0, 4).join(",") || "-") + " finalBytes=" + ((Array.isArray(item.finalResponseTextBytes) ? item.finalResponseTextBytes : []).slice(0, 4).join(",") || "-")).join("\n") + : "- 无三态不一致聚合分组。"; + const workbenchTurnStateTriadOtelLines = Array.isArray(workbenchTurnStateTriadDrilldown.otelDrilldown?.commands) && workbenchTurnStateTriadDrilldown.otelDrilldown.commands.length > 0 + ? workbenchTurnStateTriadDrilldown.otelDrilldown.commands.slice(0, 16).map((item) => "- command=" + String(item).replaceAll(String.fromCharCode(96), "'")).join("\n") + : "- 无可用 traceId 生成 OTel drill-down 命令。"; + const workbenchTurnStateTriadStaticLines = Array.isArray(workbenchTurnStateTriadDrilldown.staticSourceHints) && workbenchTurnStateTriadDrilldown.staticSourceHints.length > 0 + ? workbenchTurnStateTriadDrilldown.staticSourceHints.map((item) => "- " + (item.repo || "-") + " " + (item.path || "-") + ": " + escapeMarkdownCell(item.reason || "")).join("\n") + : "- 无静态源码候选。"; + const workbenchTurnStateTriadUnitLines = Array.isArray(workbenchTurnStateTriadDrilldown.unitTestReproHints) && workbenchTurnStateTriadDrilldown.unitTestReproHints.length > 0 + ? workbenchTurnStateTriadDrilldown.unitTestReproHints.map((item) => "- " + escapeMarkdownCell(item)).join("\n") + : "- 无单元测试复现提示。"; const provenanceLines = Array.isArray(report.pageProvenance?.segments) && report.pageProvenance.segments.length > 0 ? report.pageProvenance.segments.slice(0, 40).map((item) => "- fingerprint=" + (item.assetFingerprint || "-") + " samples=" + item.sampleCount + " seq=" + (item.firstSeq ?? "-") + ".." + (item.lastSeq ?? "-") + " ts=" + (item.firstAt || "-") + ".." + (item.lastAt || "-") + " scripts=" + (item.scriptCount ?? "-") + " styles=" + (item.stylesheetCount ?? "-") + " urlPaths=" + (Array.isArray(item.urlPaths) ? item.urlPaths.slice(0, 4).join(",") : "-")).join("\n") : "- 无页面 provenance segment。"; @@ -2471,6 +2486,19 @@ function renderMarkdown(report) { + "- policy: 可见 session 列表中 'Session ses_...' fallback 标题超过一半必须报警;修复应让上游 session list projection 直接携带名称,不能靠点击详情后下游修补。\n\n" + "#### Session rail fallback samples\n\n" + sessionRailTitleSampleLines + "\n\n" + "#### Session rail fallback examples\n\n" + sessionRailTitleExampleLines + "\n\n" + + "### Workbench turn-state triad\n\n" + + "- rowCount: " + (workbenchTurnStateTriadSummary.rowCount ?? 0) + "\n" + + "- invalidRowCount: " + (workbenchTurnStateTriadSummary.invalidRowCount ?? 0) + "\n" + + "- invalidFullTriadCount: " + (workbenchTurnStateTriadSummary.invalidFullTriadCount ?? 0) + "\n" + + "- cardFinalResponseMismatchCount: " + (workbenchTurnStateTriadSummary.cardFinalResponseMismatchCount ?? 0) + "\n" + + "- invalidGroupCount: " + (workbenchTurnStateTriadSummary.invalidGroupCount ?? workbenchTurnStateTriadDrilldown.summary?.groupCount ?? 0) + "\n" + + "- invalidTraceIdCount: " + (workbenchTurnStateTriadSummary.invalidTraceIdCount ?? workbenchTurnStateTriadDrilldown.summary?.traceIds?.length ?? 0) + "\n" + + "- invalidMaxObservedSpanMs: " + (workbenchTurnStateTriadSummary.invalidMaxObservedSpanMs ?? workbenchTurnStateTriadDrilldown.summary?.maxObservedSpanMs ?? 0) + "\n" + + "- allowedTuples: completed/completed/final-present 或 running/running/final-absent;其他组合都应按 projection seal/read-model 合同处理,禁止 UI fallback repair。\n\n" + + "#### Triad mismatch groups\n\n" + workbenchTurnStateTriadGroupLines + "\n\n" + + "#### Triad OTel drill-down\n\n" + workbenchTurnStateTriadOtelLines + "\n\n" + + "#### Triad static source hints\n\n" + workbenchTurnStateTriadStaticLines + "\n\n" + + "#### Triad unit-test repro hints\n\n" + workbenchTurnStateTriadUnitLines + "\n\n" + "### Page provenance\n\n" + "- segmentCount: " + (report.pageProvenance?.summary?.segmentCount ?? 0) + "\n" + "- controlSegmentCount: " + (report.pageProvenance?.summary?.controlSegmentCount ?? 0) + "\n\n" diff --git a/scripts/src/platform-infra-observability/diagnose-code-agent-script.ts b/scripts/src/platform-infra-observability/diagnose-code-agent-script.ts index 1635d044..59e19da3 100644 --- a/scripts/src/platform-infra-observability/diagnose-code-agent-script.ts +++ b/scripts/src/platform-infra-observability/diagnose-code-agent-script.ts @@ -1621,6 +1621,8 @@ def agentrun_summary(spans, authority=None): def hwlab_read_model_summary(spans): trace_read_spans = [] turn_read_spans = [] + session_list_spans = [] + session_message_spans = [] projection_spans = [] source_event_count = None requested_since_seq = None @@ -1648,6 +1650,10 @@ def hwlab_read_model_summary(spans): status_value = attrs.get("turnStatus") if attrs.get("turnStatus") is not None else attrs.get("status") turn_statuses.append(status_value) turn_status_counts[str(status_value)] += 1 + if name == "session_list_read": + session_list_spans.append(public_span(item)) + if name == "session_messages_read": + session_message_spans.append(public_span(item)) if name == "projection_sync": projection_spans.append(public_span(item)) after_seq = to_int(attrs.get("afterSeq")) @@ -1683,6 +1689,10 @@ def hwlab_read_model_summary(spans): "turnStatusCounts": [{"status": status, "count": count} for status, count in turn_status_counts.most_common()], "traceEventReadSpans": trace_read_spans, "turnStatusReadSpans": turn_read_spans, + "sessionListReadCount": len(session_list_spans), + "sessionMessageReadCount": len(session_message_spans), + "sessionListReadSpans": session_list_spans, + "sessionMessageReadSpans": session_message_spans, "projectionSpans": projection_spans, } @@ -2144,6 +2154,8 @@ payload = { "traceStatus": read_model.get("traceStatus"), "turnStatus": read_model.get("turnStatus"), "turnStatusCounts": read_model.get("turnStatusCounts"), + "sessionListReadCount": read_model.get("sessionListReadCount"), + "sessionMessageReadCount": read_model.get("sessionMessageReadCount"), }, "http": { "statusCounts": http_summary.get("statusCounts", [])[:20], @@ -2159,6 +2171,7 @@ payload = { "diagnoseFull": DIAGNOSE_FULL_COMMAND, "traceSummary": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --limit 80" % resolved_trace_id, "traceReads": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep trace_events_read --limit 20 --full" % resolved_trace_id, + "sessionReads": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep session_ --limit 60 --full" % resolved_trace_id, "turnStatus": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep turn_status_read --limit 20 --full" % resolved_trace_id, "projection": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep projection_sync --limit 120 --full" % resolved_trace_id, "runnerTerminal": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep runner_terminal --limit 20 --full" % resolved_trace_id, @@ -2170,11 +2183,13 @@ if FULL: payload["full"] = { "traceEventReadSpans": read_model.get("traceEventReadSpans", []), "turnStatusReadSpans": read_model.get("turnStatusReadSpans", []), + "sessionListReadSpans": read_model.get("sessionListReadSpans", []), + "sessionMessageReadSpans": read_model.get("sessionMessageReadSpans", []), "projectionSpans": read_model.get("projectionSpans", [])[:LIMIT], "terminalSpans": agentrun.get("terminalSpans", [])[:LIMIT], "errorSpans": [public_span(item) for item in error_spans[:LIMIT]], "httpProblemSpans": [public_span(item) for item in http_summary.get("problemSpans", [])[:LIMIT]], - "selectedSpans": [public_span(item) for item in spans if item.get("name") in ("trace_events_read", "turn_status_read", "projection_sync", "command_result")][:LIMIT], + "selectedSpans": [public_span(item) for item in spans if item.get("name") in ("trace_events_read", "turn_status_read", "session_list_read", "session_messages_read", "projection_sync", "command_result")][:LIMIT], } if RAW: payload["raw"] = raw_trace_shape(trace_parsed) diff --git a/scripts/src/platform-infra-observability/render.ts b/scripts/src/platform-infra-observability/render.ts index 6a822771..395498b8 100644 --- a/scripts/src/platform-infra-observability/render.ts +++ b/scripts/src/platform-infra-observability/render.ts @@ -749,9 +749,12 @@ export function renderDiagnoseCodeAgentTable(input: { textValue(next?.diagnoseFull), textValue(next?.traceSummary), textValue(next?.traceReads), + textValue(next?.sessionReads), + textValue(next?.turnStatus), + textValue(next?.projection), ].filter((item) => item !== "-"); if (nextCommands.length > 0) { - for (const command of nextCommands.slice(0, 3)) lines.push(` ${command}`); + for (const command of nextCommands.slice(0, 6)) lines.push(` ${command}`); } else { lines.push(` ${buildDiagnoseCommand(input.target, input.options, true)}`); if (traceId.length > 0 && traceId !== "-") lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${input.target.id} --trace-id ${traceId}`);