From 3b6190959b954b5049920dcab1de63569ae9a1bd Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Wed, 24 Jun 2026 00:36:37 +0800 Subject: [PATCH] fix(web-probe): attribute final responses by trace role (#763) Co-authored-by: Codex --- .../hwlab-node-web-observe-runner-source.ts | 96 ++++++++++++++++++- 1 file changed, 92 insertions(+), 4 deletions(-) diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 94e87561..ab6ec8b5 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -5293,11 +5293,12 @@ function buildRoundCompletionMetrics(samples, timeline, turnTiming) { const timelineItem = timeline[index] || {}; for (const event of roundCompletionEventsForSample(sample, timelineItem)) events.push(event); } + const completionEvents = dedupeRoundCompletionEvents(events); const elapsedMismatches = []; const finalResponseMissing = []; const postCompletionTimingChanges = []; const postCompletionRecentUpdateVisible = []; - for (const event of events) { + for (const event of completionEvents) { const sampleIndex = samples.findIndex((sample) => sample?.seq === event.seq && sample?.pageRole === event.pageRole && sample?.pageId === event.pageId); const sameSample = sampleIndex >= 0 ? samples[sampleIndex] : null; const sameTimelineItem = sampleIndex >= 0 ? timeline[sampleIndex] || {} : {}; @@ -5318,10 +5319,11 @@ function buildRoundCompletionMetrics(samples, timeline, turnTiming) { }); } } - if (!hasFinalResponseAfterCompletion(samples, timeline, event)) { + if (!(sameSample && sampleHasTerminalAgentResultCard(sameSample, event)) && !(sameSample && sampleHasVisibleFinalResponse(sameSample, event)) && !hasFinalResponseAfterCompletion(samples, timeline, event)) { finalResponseMissing.push({ ...eventRef(event), completionElapsedSeconds: event.elapsedSeconds, + finalResponseProbe: sameSample ? terminalAgentResultProbe(sameSample, event) : { ok: false, reason: "sample-not-found" }, valuesRedacted: true }); } @@ -5330,7 +5332,7 @@ function buildRoundCompletionMetrics(samples, timeline, turnTiming) { postCompletionRecentUpdateVisible.push(...postTiming.recentUpdateVisible); } return { - events: dedupeRoundCompletionRows(events).slice(0, 200), + events: completionEvents.slice(0, 200), elapsedMismatches: dedupeRoundCompletionRows(elapsedMismatches).slice(0, 200), finalResponseMissing: dedupeRoundCompletionRows(finalResponseMissing).slice(0, 200), postCompletionTimingChanges: dedupeRoundCompletionRows(postCompletionTimingChanges).slice(0, 200), @@ -5414,17 +5416,65 @@ function hasFinalResponseAfterCompletion(samples, timeline, event) { if (sampleSession && eventSession && sampleSession !== eventSession) continue; const promptIndex = timeline[index]?.promptIndex ?? 0; if (!event.traceId && !event.messageId && event.promptIndex && promptIndex !== event.promptIndex) continue; + if (sampleHasTerminalAgentResultCard(sample, event)) return true; if (sampleHasVisibleFinalResponse(sample, event)) return true; } return false; } +function sampleHasTerminalAgentResultCard(sample, event = {}) { + return terminalAgentResultProbe(sample, event).ok === true; +} + +function terminalAgentResultProbe(sample, event = {}) { + const decisions = []; + for (const item of codeAgentCardsForSample(sample)) { + const text = normalizedText(codeAgentCardText(item)); + const traceMatched = Boolean(event.traceId && item?.traceId && item.traceId === event.traceId); + const decision = { + source: item?.source || null, + role: item?.role ?? null, + dataRole: item?.dataRole ?? null, + status: item?.status ?? null, + traceId: item?.traceId ?? null, + messageId: item?.messageId ?? null, + textBytes: Buffer.byteLength(text), + valuesRedacted: true + }; + if (event.traceId && item?.traceId && item.traceId !== event.traceId) { + decisions.push({ ...decision, decision: "skip-trace" }); + continue; + } + if (!traceMatched && event.messageId && item?.messageId && item.messageId !== event.messageId) { + decisions.push({ ...decision, decision: "skip-message" }); + continue; + } + if (!isAssistantFinalResponseCandidate(item, item?.source || "turn")) { + decisions.push({ ...decision, decision: "skip-role" }); + continue; + } + if (!isCodeAgentCardTerminal(item)) { + decisions.push({ ...decision, decision: "skip-non-terminal" }); + continue; + } + if (text.length < 24) { + decisions.push({ ...decision, decision: "skip-short" }); + continue; + } + decisions.push({ ...decision, decision: "accept-terminal-agent-card" }); + return { ok: true, decisions: decisions.slice(-8), valuesRedacted: true }; + } + return { ok: false, decisions: decisions.slice(-8), valuesRedacted: true }; +} + function sampleHasVisibleFinalResponse(sample, event = {}) { for (const [groupName, group] of [["messages", sample?.messages], ["turns", sample?.turns]]) { if (!Array.isArray(group)) continue; for (const item of group) { + const traceMatched = Boolean(event.traceId && item?.traceId && item.traceId === event.traceId); if (event.traceId && item?.traceId && item.traceId !== event.traceId) continue; - if (event.messageId && item?.messageId && item.messageId !== event.messageId) continue; + if (!traceMatched && event.messageId && item?.messageId && item.messageId !== event.messageId) continue; + if (!isAssistantFinalResponseCandidate(item, groupName)) continue; const text = normalizedText([item?.text, item?.textPreview].filter(Boolean).join(" ")); if (text.length < 24) continue; if (groupName === "messages" && isTerminalTurnStatus(item?.status)) return true; @@ -5436,6 +5486,23 @@ function sampleHasVisibleFinalResponse(sample, event = {}) { return false; } +function normalizedDomRole(item) { + return String(item?.dataRole ?? item?.role ?? item?.ariaRole ?? "") + .trim() + .toLowerCase() + .replace(/[_\s]+/gu, "-"); +} + +function isAssistantFinalResponseCandidate(item, groupName) { + const role = normalizedDomRole(item); + if (/agent|assistant|code-agent|bot/iu.test(role)) return true; + if (/user|human|client|prompt/iu.test(role)) return false; + if (groupName === "turns") return isCodeAgentCardLike(item); + const text = codeAgentCardText(item); + if (isTerminalTurnStatus(item?.status) && /Code Agent|运行记录|assistant|agent/iu.test(text)) return true; + return false; +} + function detectPostCompletionTimingChanges(turnTiming, event) { const timingChanges = []; const recentUpdateVisible = []; @@ -5520,6 +5587,26 @@ function detectPostCompletionTimingChanges(turnTiming, event) { return { timingChanges, recentUpdateVisible }; } +function dedupeRoundCompletionEvents(rows) { + const result = []; + const seen = new Set(); + for (const row of Array.isArray(rows) ? rows : []) { + const key = [ + row?.pageRole ?? "", + row?.pageId ?? "", + row?.promptIndex ?? "", + row?.traceId ?? "", + row?.messageId ?? "", + row?.textHash ?? "", + row?.elapsedSeconds ?? "" + ].join("|"); + if (seen.has(key)) continue; + seen.add(key); + result.push(row); + } + return result; +} + function eventRef(event) { return { seq: event?.seq ?? null, @@ -5531,6 +5618,7 @@ function eventRef(event) { activeSessionId: event?.activeSessionId ?? null, promptIndex: event?.promptIndex ?? null, traceId: event?.traceId ?? null, + messageId: event?.messageId ?? null, }; }