From f3075d2999f35c6e3f0373821b8882a9ff434f95 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Wed, 24 Jun 2026 03:45:28 +0800 Subject: [PATCH] fix(web-probe): stabilize round timing attribution (#770) Co-authored-by: Codex --- scripts/src/hwlab-node-impl.ts | 7 ++- .../hwlab-node-web-observe-runner-source.ts | 59 +++++++++++++++---- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index e4661f78..6c04f61f 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -8907,7 +8907,12 @@ function renderWebObserveAnalyzeTable(value: Record): string { const sessionRailTitleSummary = record(sessionRailTitles?.summary); const sessionRailTitleSamples = Array.isArray(sessionRailTitles?.samples) ? sessionRailTitles.samples.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const sessionRailTitleExamples = Array.isArray(sessionRailTitles?.examples) ? sessionRailTitles.examples.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; - const rounds = webObserveArray(sampleMetrics?.roundItems ?? sampleMetrics?.rounds).map(record).filter((item): item is Record => item !== null).slice(-8); + const roundSource = Array.isArray(sampleMetrics?.roundItems) && sampleMetrics.roundItems.length > 0 + ? sampleMetrics.roundItems + : Array.isArray(sampleMetrics?.rounds) + ? sampleMetrics.rounds + : sampleMetrics?.roundItems ?? sampleMetrics?.rounds; + const rounds = webObserveArray(roundSource).map(record).filter((item): item is Record => item !== null).slice(-8); const turnColumns = webObserveArray(sampleMetrics?.turnColumns).map(record).filter((item): item is Record => item !== null).slice(-12); const roundCount = Array.isArray(sampleMetrics?.rounds) ? sampleMetrics.rounds.length : sampleMetrics?.rounds; const turnColumnCount = Array.isArray(sampleMetrics?.turnColumns) ? sampleMetrics.turnColumns.length : sampleMetrics?.turnColumns; diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 98db13db..e4c22aee 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -4372,12 +4372,21 @@ function finalizeLoadingSegment(segment, absentEvent) { function buildTurnTimingTable(samples, timeline) { const columns = []; const registry = new Map(); + const promptAssignmentByKey = new Map(); const rows = []; for (let index = 0; index < samples.length; index += 1) { const sample = samples[index]; const timelineItem = timeline[index] || {}; const cells = {}; - for (const metric of turnMetricItems(sample, timelineItem)) { + const rawMetrics = turnMetricItems(sample, timelineItem); + const domIndexes = rawMetrics.map((item) => Number(item.domIndex)).filter(Number.isFinite); + const maxDomIndex = domIndexes.length > 0 ? Math.max(...domIndexes) : null; + for (const rawMetric of rawMetrics) { + const samplePromptIndex = Number(timelineItem.promptIndex ?? 0); + const evidencePromptIndex = inferTurnMetricPromptIndex(rawMetric, samplePromptIndex, maxDomIndex); + if (evidencePromptIndex !== null && !promptAssignmentByKey.has(rawMetric.key)) promptAssignmentByKey.set(rawMetric.key, evidencePromptIndex); + const assignedPromptIndex = promptAssignmentByKey.get(rawMetric.key) ?? null; + const metric = { ...rawMetric, promptIndex: assignedPromptIndex, samplePromptIndex }; let column = registry.get(metric.key); if (!column) { column = { @@ -4402,6 +4411,7 @@ function buildTurnTimingTable(samples, timeline) { } else { column.lastSeq = sample.seq ?? null; column.lastTs = sample.ts ?? null; + if (!column.promptIndex && metric.promptIndex) column.promptIndex = metric.promptIndex; column.lastPromptIndex = metric.promptIndex ?? column.lastPromptIndex ?? null; if (!column.traceId && metric.traceId) column.traceId = metric.traceId; if (!column.messageId && metric.messageId) column.messageId = metric.messageId; @@ -4417,7 +4427,8 @@ function buildTurnTimingTable(samples, timeline) { sampleGroupSeq: sample.sampleGroupSeq ?? null, traceId: metric.traceId ?? null, messageId: metric.messageId ?? null, - textHash: metric.textHash ?? null + textHash: metric.textHash ?? null, + samplePromptIndex: metric.samplePromptIndex ?? null }; } rows.push({ @@ -4445,6 +4456,28 @@ function buildTurnTimingTable(samples, timeline) { }; } +function inferTurnMetricPromptIndex(metric, samplePromptIndex, maxDomIndex) { + const promptIndex = Number(samplePromptIndex); + if (!Number.isFinite(promptIndex) || promptIndex <= 0) return null; + if (metric?.source === "aggregate") return promptIndex; + if (isActiveTurnStatus(metric?.status)) return promptIndex; + const domIndex = Number(metric?.domIndex); + if (!isTerminalTurnStatus(metric?.status) && Number.isFinite(domIndex) && Number.isFinite(maxDomIndex) && domIndex === maxDomIndex && metric?.recentUpdateSeconds !== null && metric?.recentUpdateSeconds !== undefined) return promptIndex; + return null; +} + +function isActiveTurnStatus(status) { + const value = String(status || "").toLowerCase(); + if (!value) return false; + if (isTerminalTurnStatus(value)) return false; + return /running|pending|admitted|queued|dispatch|execut|progress|thinking|working|active|streaming|created|started/u.test(value); +} + +function isTerminalTurnStatus(status) { + const value = String(status || "").toLowerCase(); + return /completed|succeeded|success|failed|error|canceled|cancelled|blocked|done|terminal/u.test(value); +} + function detectTurnTimingNonMonotonic(columns, rows) { const anomalies = []; const elapsedZeroResets = []; @@ -4480,7 +4513,8 @@ function detectTurnTimingNonMonotonic(columns, rows) { delta: current - previous.value, traceId: cell.traceId ?? column.traceId ?? null, messageId: cell.messageId ?? column.messageId ?? null, - promptIndex: cell.promptIndex ?? row.promptIndex ?? null, + promptIndex: cell.promptIndex ?? null, + samplePromptIndex: row.promptIndex ?? null, source: cell.source ?? column.source ?? null, pageRole: cell.pageRole ?? column.pageRole ?? null, pageId: cell.pageId ?? column.pageId ?? null, @@ -4515,7 +4549,8 @@ function detectTurnTimingNonMonotonic(columns, rows) { excessiveIncreaseSeconds: Number((delta - allowedIncreaseSeconds).toFixed(3)), traceId: cell.traceId ?? column.traceId ?? null, messageId: cell.messageId ?? column.messageId ?? null, - promptIndex: cell.promptIndex ?? row.promptIndex ?? null, + promptIndex: cell.promptIndex ?? null, + samplePromptIndex: row.promptIndex ?? null, source: cell.source ?? column.source ?? null, pageRole: cell.pageRole ?? column.pageRole ?? null, pageId: cell.pageId ?? column.pageId ?? null, @@ -4542,7 +4577,8 @@ function detectTurnTimingNonMonotonic(columns, rows) { delta: current - previousTerminalTotal.value, traceId: cell.traceId ?? column.traceId ?? null, messageId: cell.messageId ?? column.messageId ?? null, - promptIndex: cell.promptIndex ?? row.promptIndex ?? null, + promptIndex: cell.promptIndex ?? null, + samplePromptIndex: row.promptIndex ?? null, source: cell.source ?? column.source ?? null, pageRole: cell.pageRole ?? column.pageRole ?? null, pageId: cell.pageId ?? column.pageId ?? null, @@ -4577,7 +4613,8 @@ function detectTurnTimingNonMonotonic(columns, rows) { excessiveIncreaseSeconds: excessiveIncrease, traceId: cell.traceId ?? column.traceId ?? null, messageId: cell.messageId ?? column.messageId ?? null, - promptIndex: cell.promptIndex ?? row.promptIndex ?? null, + promptIndex: cell.promptIndex ?? null, + samplePromptIndex: row.promptIndex ?? null, source: cell.source ?? column.source ?? null, pageRole: cell.pageRole ?? column.pageRole ?? null, pageId: cell.pageId ?? column.pageId ?? null, @@ -4599,7 +4636,8 @@ function detectTurnTimingNonMonotonic(columns, rows) { sampleDeltaSeconds: elapsedSeconds, traceId: cell.traceId ?? column.traceId ?? null, messageId: cell.messageId ?? column.messageId ?? null, - promptIndex: cell.promptIndex ?? row.promptIndex ?? null, + promptIndex: cell.promptIndex ?? null, + samplePromptIndex: row.promptIndex ?? null, source: cell.source ?? column.source ?? null, pageRole: cell.pageRole ?? column.pageRole ?? null, pageId: cell.pageId ?? column.pageId ?? null, @@ -4625,7 +4663,8 @@ function detectTurnTimingNonMonotonic(columns, rows) { excessiveIncreaseSeconds: excessiveIncrease, traceId: cell.traceId ?? column.traceId ?? null, messageId: cell.messageId ?? column.messageId ?? null, - promptIndex: cell.promptIndex ?? row.promptIndex ?? null, + promptIndex: cell.promptIndex ?? null, + samplePromptIndex: row.promptIndex ?? null, source: cell.source ?? column.source ?? null, pageRole: cell.pageRole ?? column.pageRole ?? null, pageId: cell.pageId ?? column.pageId ?? null, @@ -6051,7 +6090,7 @@ function detectCrossPageProjectionDiffs(samples) { const controlMessageDigest = digestMessageTexts(control); const observerMessageDigest = digestMessageTexts(observer); const messageTextDigestDiff = controlMessageDigest !== observerMessageDigest; - const projectionDivergent = sameSession && (Math.abs(controlMessages - observerMessages) > 0 || controlZero !== observerZero || messageTextDigestDiff); + const projectionDivergent = sameSession && (Math.abs(controlMessages - observerMessages) > 0 || controlZero !== observerZero); const traceVisibilityDivergent = sameSession && !projectionDivergent && (missingInObserver.length > 0 || Math.abs(controlTraceRows - observerTraceRows) > 0); if (!projectionDivergent && !traceVisibilityDivergent) continue; rows.push({ @@ -6184,7 +6223,7 @@ function detectAdjacentCrossPageProjectionDiffs(samples) { const controlMessageDigest = digestMessageTexts(control); const observerMessageDigest = digestMessageTexts(observer); const messageTextDigestDiff = controlMessageDigest !== observerMessageDigest; - const projectionDivergent = controlMessages !== observerMessages || controlZero !== observerZero || messageTextDigestDiff; + const projectionDivergent = controlMessages !== observerMessages || controlZero !== observerZero; const traceVisibilityDivergent = !projectionDivergent && (missingInObserver.length > 0 || controlTraceRows !== observerTraceRows); if (!projectionDivergent && !traceVisibilityDivergent) continue; rows.push({