feat(web-probe): detect trace order and card timing drift (#737)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
@@ -2966,7 +2966,7 @@ function withNodeRuntimeTriggerRendered(result: Record<string, unknown>, scoped:
|
||||
const warning = record(result.warning ?? pipelineWait.warning);
|
||||
const postFlush = record(result.postFlush);
|
||||
const gitMirror = record(result.gitMirror);
|
||||
const gitMirrorSummary = record(gitMirror.statusSummary ?? gitMirror.afterSummary ?? gitMirror.beforeSummary ?? gitMirror.summary);
|
||||
const gitMirrorSummary = record(postFlush.afterSummary ?? postFlush.beforeSummary ?? gitMirror.statusSummary ?? gitMirror.afterSummary ?? gitMirror.beforeSummary ?? gitMirror.summary);
|
||||
const refresh = record(result.refresh);
|
||||
const create = record(result.create);
|
||||
const next = record(result.next);
|
||||
|
||||
@@ -1600,11 +1600,14 @@ async function sampleOnePage(targetPage, { reason, groupSeq, pageRole, targetPag
|
||||
tag: element.tagName.toLowerCase(),
|
||||
testId: element.getAttribute("data-testid"),
|
||||
role: element.getAttribute("role"),
|
||||
dataRole: element.getAttribute("data-role"),
|
||||
status: element.getAttribute("data-status") || element.getAttribute("aria-busy") || null,
|
||||
sessionId: element.getAttribute("data-session-id") || null,
|
||||
messageId: element.getAttribute("data-message-id") || element.getAttribute("id") || null,
|
||||
traceId: element.getAttribute("data-trace-id") || null,
|
||||
turnId: element.getAttribute("data-turn-id") || null,
|
||||
durationText: trim(element.querySelector(".message-duration-meta")?.textContent || "", 120),
|
||||
activityText: trim(element.querySelector(".message-activity-meta")?.textContent || "", 120),
|
||||
text: stableMessageText(element),
|
||||
rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) },
|
||||
};
|
||||
@@ -2449,6 +2452,7 @@ function compactDomItem(item) {
|
||||
tag: item.tag ?? null,
|
||||
testId: item.testId ?? null,
|
||||
role: item.role ?? null,
|
||||
dataRole: item.dataRole ?? null,
|
||||
status: item.status ?? null,
|
||||
sessionId: item.sessionId ?? null,
|
||||
messageId: item.messageId ?? null,
|
||||
@@ -2580,6 +2584,44 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN
|
||||
if (finalFlicker.length > 0) findings.push({ id: "final-response-flicker", severity: "red", summary: "message text digest disappeared or switched to diagnostic-like text after non-empty final text", count: finalFlicker.length, samples: finalFlicker.slice(0, 20) });
|
||||
const terminalZeroElapsed = detectTerminalZeroElapsed(samples);
|
||||
if (terminalZeroElapsed.length > 0) findings.push({ id: "turn-terminal-zero-elapsed", severity: "red", summary: "terminal Code Agent card displayed 耗时 0 秒; terminal duration must come from durable timing projection, not a missing/zero display fallback", count: terminalZeroElapsed.length, samples: terminalZeroElapsed.slice(0, 20) });
|
||||
const cardTiming = sampleMetrics?.codeAgentCardTiming || {};
|
||||
const cardTimingSummary = cardTiming.summary || {};
|
||||
if (Number(cardTimingSummary.missingElapsedCount ?? 0) > 0) findings.push({ id: "code-agent-card-elapsed-missing", severity: "red", summary: "visible Code Agent card did not display total elapsed time; elapsed must be visible for running and terminal cards", count: cardTimingSummary.missingElapsedCount, samples: (cardTiming.missingElapsed || []).slice(0, 20) });
|
||||
if (Number(cardTimingSummary.missingRecentUpdateCount ?? 0) > 0) findings.push({ id: "code-agent-card-running-recent-update-missing", severity: "red", summary: "non-terminal Code Agent card did not display 最近更新", count: cardTimingSummary.missingRecentUpdateCount, samples: (cardTiming.missingRecentUpdate || []).slice(0, 20) });
|
||||
const roundCompletion = cardTiming.roundCompletion || {};
|
||||
if (Number(cardTimingSummary.durationUnderreportedCount ?? 0) > 0) {
|
||||
findings.push({
|
||||
id: "code-agent-card-duration-underreported",
|
||||
severity: "red",
|
||||
summary: "completed Code Agent card total elapsed is shorter than trace/final-response duration evidence",
|
||||
count: cardTimingSummary.durationUnderreportedCount,
|
||||
samples: (cardTiming.durationUnderreported || []).slice(0, 20),
|
||||
});
|
||||
}
|
||||
const traceOrder = sampleMetrics?.traceOrder || {};
|
||||
const traceOrderSummary = traceOrder.summary || {};
|
||||
if (Number(traceOrderSummary.orderAnomalyCount ?? 0) > 0) {
|
||||
findings.push({
|
||||
id: "trace-row-order-nonmonotonic",
|
||||
severity: "red",
|
||||
summary: "visible trace rows are not monotonic by total time, clock time, or projected sequence in DOM order",
|
||||
count: traceOrderSummary.orderAnomalyCount,
|
||||
samples: (traceOrder.orderAnomalies || []).slice(0, 20),
|
||||
});
|
||||
}
|
||||
if (Number(traceOrderSummary.completionNotLastCount ?? 0) > 0) {
|
||||
findings.push({
|
||||
id: "trace-completion-row-not-last",
|
||||
severity: "red",
|
||||
summary: "visible trace shows a completion row before later trace rows for the same trace",
|
||||
count: traceOrderSummary.completionNotLastCount,
|
||||
samples: (traceOrder.completionNotLast || []).slice(0, 20),
|
||||
});
|
||||
}
|
||||
if (Number(cardTimingSummary.roundCompletionElapsedMismatchCount ?? 0) > 0) findings.push({ id: "round-completion-elapsed-mismatch", severity: "red", summary: "Trace row 轮次完成（总耗时 ...） does not match the visible Code Agent card total elapsed time within YAML timing slack", count: cardTimingSummary.roundCompletionElapsedMismatchCount, toleranceSeconds: cardTimingSummary.elapsedMismatchToleranceSeconds, samples: (roundCompletion.elapsedMismatches || []).slice(0, 20) });
|
||||
if (Number(cardTimingSummary.roundCompletionFinalResponseMissingCount ?? 0) > 0) findings.push({ id: "round-completion-final-response-missing", severity: "red", summary: "Trace row showed 轮次完成, but no final response was visible in the Code Agent card afterward", count: cardTimingSummary.roundCompletionFinalResponseMissingCount, samples: (roundCompletion.finalResponseMissing || []).slice(0, 20) });
|
||||
if (Number(cardTimingSummary.roundCompletionPostTimingChangeCount ?? 0) > 0) findings.push({ id: "round-completion-post-timing-change", severity: "red", summary: "After 轮次完成, card total elapsed or 最近更新 continued changing; terminal timing should be sealed", count: cardTimingSummary.roundCompletionPostTimingChangeCount, samples: (roundCompletion.postCompletionTimingChanges || []).slice(0, 20) });
|
||||
if (Number(cardTimingSummary.roundCompletionPostRecentUpdateVisibleCount ?? 0) > 0) findings.push({ id: "round-completion-recent-update-still-visible", severity: "info", summary: "最近更新 was still visible after 轮次完成; inspect whether terminal cards should hide activity age or keep it sealed", count: cardTimingSummary.roundCompletionPostRecentUpdateVisibleCount, samples: (roundCompletion.postCompletionRecentUpdateVisible || []).slice(0, 20) });
|
||||
const scrollJumps = [];
|
||||
for (let i = 1; i < samples.length; i += 1) {
|
||||
const prevY = Number(samples[i - 1]?.scroll?.y ?? 0);
|
||||
@@ -3673,6 +3715,8 @@ function prioritizeFindings(findings) {
|
||||
const id = String(item?.id ?? item?.kind ?? item?.code ?? "");
|
||||
if (id === "page-performance-slow-same-origin-api") return 0;
|
||||
if (id === "session-rail-title-fallback-majority") return 0.5;
|
||||
if (id.startsWith("code-agent-card-")) return 0.8;
|
||||
if (id.startsWith("round-completion-")) return 0.9;
|
||||
if (id.startsWith("turn-timing-total-elapsed")) return 1;
|
||||
if (id.startsWith("turn-timing-recent-update")) return 2;
|
||||
if (id.includes("runtime-execution") || id.includes("prompt-chat-submit-failed")) return 3;
|
||||
@@ -3736,6 +3780,13 @@ function buildSampleMetrics(samples, control) {
|
||||
};
|
||||
});
|
||||
const turnTiming = buildTurnTimingTable(samples, timeline);
|
||||
const traceOrder = buildTraceOrderMetrics(samples, timeline);
|
||||
const codeAgentCardTiming = buildCodeAgentCardTimingMetrics(samples, timeline, turnTiming);
|
||||
const codeAgentCardDurationUnderreported = buildCodeAgentCardDurationUnderreportedMetrics(samples, timeline);
|
||||
if (codeAgentCardTiming && codeAgentCardTiming.summary) {
|
||||
codeAgentCardTiming.summary.durationUnderreportedCount = codeAgentCardDurationUnderreported.length;
|
||||
codeAgentCardTiming.durationUnderreported = codeAgentCardDurationUnderreported;
|
||||
}
|
||||
const turnCells = turnTiming.rows.flatMap((row) => Object.values(row.cells || {}));
|
||||
const turnTimingNonMonotonic = Array.isArray(turnTiming.nonMonotonic) ? turnTiming.nonMonotonic : [];
|
||||
const turnTimingElapsedZeroResets = Array.isArray(turnTiming.elapsedZeroResets) ? turnTiming.elapsedZeroResets : [];
|
||||
@@ -3813,6 +3864,17 @@ function buildSampleMetrics(samples, control) {
|
||||
turnTimingRecentUpdateMaxExcessSeconds: turnTimingRecentUpdateExcessSteps.length > 0 ? Math.max(...turnTimingRecentUpdateExcessSteps) : 0,
|
||||
turnTimingRecentUpdateResetCount: turnTimingRecentUpdateResets.length,
|
||||
turnTimingRecentUpdateDecreaseCount: turnTimingRecentUpdateResets.length,
|
||||
codeAgentCardSampleCount: codeAgentCardTiming.summary.cardSampleCount,
|
||||
codeAgentCardMissingElapsedCount: codeAgentCardTiming.summary.missingElapsedCount,
|
||||
codeAgentCardMissingRecentUpdateCount: codeAgentCardTiming.summary.missingRecentUpdateCount,
|
||||
roundCompletionEventCount: codeAgentCardTiming.summary.roundCompletionEventCount,
|
||||
roundCompletionElapsedMismatchCount: codeAgentCardTiming.summary.roundCompletionElapsedMismatchCount,
|
||||
roundCompletionFinalResponseMissingCount: codeAgentCardTiming.summary.roundCompletionFinalResponseMissingCount,
|
||||
roundCompletionPostTimingChangeCount: codeAgentCardTiming.summary.roundCompletionPostTimingChangeCount,
|
||||
codeAgentCardDurationUnderreportedCount: codeAgentCardTiming.summary.durationUnderreportedCount,
|
||||
traceRowCount: traceOrder.summary.traceRowCount,
|
||||
traceRowOrderAnomalyCount: traceOrder.summary.orderAnomalyCount,
|
||||
traceRowCompletionNotLastCount: traceOrder.summary.completionNotLastCount,
|
||||
roundsWithTurnTimingNonMonotonic: rounds.filter((item) => item.turnTimingNonMonotonicCount > 0).length,
|
||||
roundsWithTurnTimingTotalElapsedForwardJumps: rounds.filter((item) => item.turnTimingTotalElapsedForwardJumpCount > 0).length,
|
||||
roundsWithTerminalElapsedGrowth: rounds.filter((item) => item.turnTimingTerminalElapsedGrowthCount > 0).length,
|
||||
@@ -3820,6 +3882,8 @@ function buildSampleMetrics(samples, control) {
|
||||
},
|
||||
loading,
|
||||
sessionRailTitles,
|
||||
codeAgentCardTiming,
|
||||
traceOrder,
|
||||
rounds,
|
||||
turnColumns: turnTiming.columns,
|
||||
turnTimingTable: turnTiming.rows,
|
||||
@@ -4421,6 +4485,758 @@ function isTerminalTurnStatus(value) {
|
||||
return ["completed", "failed", "blocked", "timeout", "canceled", "cancelled", "stale", "thread-resume-failed"].includes(status);
|
||||
}
|
||||
|
||||
|
||||
|
||||
function buildTraceOrderMetrics(samples, timeline) {
|
||||
const rows = [];
|
||||
const orderAnomalies = [];
|
||||
const completionNotLast = [];
|
||||
const groups = new Map();
|
||||
for (let sampleIndex = 0; sampleIndex < samples.length; sampleIndex += 1) {
|
||||
const sample = samples[sampleIndex] || {};
|
||||
const sampleRows = traceTimingRowsForSample(sample, timeline[sampleIndex] || {});
|
||||
for (const row of sampleRows) {
|
||||
const normalized = {
|
||||
...row,
|
||||
sampleIndex,
|
||||
timestamp: sample.timestamp || sample.collectedAt || sample.time || null,
|
||||
pageRole: row.pageRole || sample.pageRole || sample.role || sample.contextRole || null,
|
||||
pageId: row.pageId || sample.pageId || sample.contextId || null,
|
||||
sessionId: row.sessionId || sample.sessionId || sample.workbenchSessionId || null,
|
||||
};
|
||||
rows.push(normalized);
|
||||
const key = traceRowGroupKey(normalized);
|
||||
if (!groups.has(key)) groups.set(key, []);
|
||||
groups.get(key).push(normalized);
|
||||
}
|
||||
}
|
||||
const slackSeconds = Math.max(1, Number(alertThresholds?.turnTimingSampleSlackSeconds || 0));
|
||||
for (const groupRows of groups.values()) {
|
||||
const sorted = groupRows.slice().sort((a, b) => {
|
||||
if (a.sampleIndex !== b.sampleIndex) return a.sampleIndex - b.sampleIndex;
|
||||
return (a.rowIndex ?? 0) - (b.rowIndex ?? 0);
|
||||
});
|
||||
let previous = null;
|
||||
for (const row of sorted) {
|
||||
if (previous) {
|
||||
const reasons = [];
|
||||
if (Number.isFinite(previous.totalSeconds) && Number.isFinite(row.totalSeconds) && row.totalSeconds + slackSeconds < previous.totalSeconds) {
|
||||
reasons.push('total-decreased');
|
||||
}
|
||||
if (Number.isFinite(previous.projectedSeq) && Number.isFinite(row.projectedSeq) && row.projectedSeq < previous.projectedSeq) {
|
||||
reasons.push('projected-seq-decreased');
|
||||
}
|
||||
if (Number.isFinite(previous.clockSeconds) && Number.isFinite(row.clockSeconds)) {
|
||||
const diff = previous.clockSeconds - row.clockSeconds;
|
||||
if (diff > slackSeconds && diff < 43200) reasons.push('clock-decreased');
|
||||
}
|
||||
if (reasons.length) {
|
||||
orderAnomalies.push({
|
||||
sampleIndex: row.sampleIndex,
|
||||
timestamp: row.timestamp,
|
||||
pageRole: row.pageRole,
|
||||
pageId: row.pageId,
|
||||
sessionId: row.sessionId,
|
||||
traceId: row.traceId || previous.traceId || null,
|
||||
previousRowIndex: previous.rowIndex,
|
||||
currentRowIndex: row.rowIndex,
|
||||
reasons,
|
||||
previousTotalSeconds: previous.totalSeconds,
|
||||
currentTotalSeconds: row.totalSeconds,
|
||||
previousProjectedSeq: previous.projectedSeq,
|
||||
currentProjectedSeq: row.projectedSeq,
|
||||
previousClockSeconds: previous.clockSeconds,
|
||||
currentClockSeconds: row.clockSeconds,
|
||||
previousPreview: previous.preview,
|
||||
currentPreview: row.preview,
|
||||
});
|
||||
}
|
||||
}
|
||||
previous = row;
|
||||
}
|
||||
for (let index = 0; index < sorted.length; index += 1) {
|
||||
const row = sorted[index];
|
||||
if (!row.isCompletion) continue;
|
||||
const later = sorted.slice(index + 1).find((candidate) => {
|
||||
if (candidate.isCompletion && candidate.preview === row.preview) return false;
|
||||
return Number.isFinite(candidate.totalSeconds) || Number.isFinite(candidate.clockSeconds) || Number.isFinite(candidate.projectedSeq);
|
||||
});
|
||||
if (later) {
|
||||
completionNotLast.push({
|
||||
sampleIndex: row.sampleIndex,
|
||||
timestamp: row.timestamp,
|
||||
pageRole: row.pageRole,
|
||||
pageId: row.pageId,
|
||||
sessionId: row.sessionId,
|
||||
traceId: row.traceId || later.traceId || null,
|
||||
completionRowIndex: row.rowIndex,
|
||||
laterRowIndex: later.rowIndex,
|
||||
completionTotalSeconds: row.totalSeconds,
|
||||
laterTotalSeconds: later.totalSeconds,
|
||||
completionPreview: row.preview,
|
||||
laterPreview: later.preview,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
summary: {
|
||||
sampleCount: samples.length,
|
||||
traceRowCount: rows.length,
|
||||
orderAnomalyCount: orderAnomalies.length,
|
||||
completionNotLastCount: completionNotLast.length,
|
||||
},
|
||||
rows: rows.slice(0, 200),
|
||||
orderAnomalies: orderAnomalies.slice(0, 100),
|
||||
completionNotLast: completionNotLast.slice(0, 100),
|
||||
};
|
||||
}
|
||||
|
||||
function buildCodeAgentCardDurationUnderreportedMetrics(samples, timeline) {
|
||||
const findings = [];
|
||||
const slackSeconds = Math.max(5, Number(alertThresholds?.turnTimingSampleSlackSeconds || 0));
|
||||
for (let sampleIndex = 0; sampleIndex < samples.length; sampleIndex += 1) {
|
||||
const sample = samples[sampleIndex] || {};
|
||||
const cards = codeAgentCardsForSample(sample);
|
||||
if (!cards.length) continue;
|
||||
const traceRows = traceTimingRowsForSample(sample, timeline[sampleIndex] || {});
|
||||
const terminalCards = cards.filter((card) => isCodeAgentCardTerminal(card));
|
||||
const sampleText = sampleVisibleText(sample, timeline[sampleIndex] || {});
|
||||
for (const card of terminalCards) {
|
||||
const cardText = codeAgentCardText(card);
|
||||
const parsedCardSeconds = parseTotalElapsedSeconds(cardText).filter(Number.isFinite);
|
||||
const cardSeconds = Number.isFinite(Number(card.totalElapsedSeconds)) ? Number(card.totalElapsedSeconds) : parsedCardSeconds.length > 0 ? Math.max(...parsedCardSeconds) : NaN;
|
||||
if (!Number.isFinite(cardSeconds)) continue;
|
||||
const traceMatched = traceRows.filter((row) => traceRowMatchesCard(row, card, terminalCards.length));
|
||||
const traceEvidence = maxTraceDurationEvidence(traceMatched.length ? traceMatched : traceRows);
|
||||
const textEvidence = maxSelfReportedDurationEvidence([card.text, card.preview, card.finalResponseText, card.runningRecordText, terminalCards.length === 1 ? sampleText : ''].filter(Boolean).join('\n'));
|
||||
const evidences = [traceEvidence, textEvidence].filter((item) => item && Number.isFinite(item.seconds));
|
||||
if (!evidences.length) continue;
|
||||
const strongest = evidences.sort((a, b) => b.seconds - a.seconds)[0];
|
||||
const tolerance = Math.max(slackSeconds, Math.ceil(strongest.seconds * 0.05));
|
||||
if (strongest.seconds > cardSeconds + tolerance) {
|
||||
findings.push({
|
||||
sampleIndex,
|
||||
timestamp: sample.timestamp || sample.collectedAt || sample.time || null,
|
||||
pageRole: card.pageRole || sample.pageRole || sample.role || sample.contextRole || null,
|
||||
pageId: card.pageId || sample.pageId || sample.contextId || null,
|
||||
sessionId: card.sessionId || sample.sessionId || sample.workbenchSessionId || null,
|
||||
traceId: card.traceId || strongest.traceId || null,
|
||||
status: card.status || card.state || card.phase || null,
|
||||
cardTotalElapsedSeconds: cardSeconds,
|
||||
expectedElapsedSeconds: strongest.seconds,
|
||||
deltaSeconds: strongest.seconds - cardSeconds,
|
||||
toleranceSeconds: tolerance,
|
||||
evidenceKind: strongest.kind,
|
||||
evidencePreview: strongest.preview,
|
||||
cardPreview: card.preview || compactOneLine(cardText || ''),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return findings.slice(0, 100);
|
||||
}
|
||||
|
||||
function traceTimingRowsForSample(sample, timelineItem) {
|
||||
const rows = [];
|
||||
const seen = new Set();
|
||||
const appendRowsFromText = (text, source, baseIndex, meta = {}) => {
|
||||
for (const extracted of extractTraceRowsFromText(text, source, baseIndex, meta)) {
|
||||
const key = [extracted.pageRole || '', extracted.pageId || '', extracted.rowIndex, extracted.preview].join('|');
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
rows.push(extracted);
|
||||
}
|
||||
};
|
||||
for (const candidate of traceRowCandidateArrays(sample, timelineItem)) {
|
||||
const array = Array.isArray(candidate.rows) ? candidate.rows : [];
|
||||
array.forEach((item, index) => {
|
||||
if (typeof item === 'string') {
|
||||
appendRowsFromText(item, candidate.source, index, candidate.meta || {});
|
||||
return;
|
||||
}
|
||||
if (!item || typeof item !== 'object') return;
|
||||
const text = objectText(item);
|
||||
if (!text) return;
|
||||
appendRowsFromText(text, candidate.source, Number.isFinite(Number(item.index)) ? Number(item.index) : index, {
|
||||
...(candidate.meta || {}),
|
||||
pageRole: item.pageRole || item.role || candidate.meta?.pageRole || null,
|
||||
pageId: item.pageId || item.contextId || candidate.meta?.pageId || null,
|
||||
sessionId: item.sessionId || item.workbenchSessionId || candidate.meta?.sessionId || null,
|
||||
traceId: item.traceId || item.trace_id || candidate.meta?.traceId || null,
|
||||
});
|
||||
});
|
||||
}
|
||||
if (!rows.length) {
|
||||
appendRowsFromText(sampleVisibleText(sample, timelineItem), 'visible-text', 0, {
|
||||
pageRole: sample.pageRole || sample.role || sample.contextRole || null,
|
||||
pageId: sample.pageId || sample.contextId || null,
|
||||
sessionId: sample.sessionId || sample.workbenchSessionId || null,
|
||||
});
|
||||
}
|
||||
rows.sort((a, b) => (a.rowIndex ?? 0) - (b.rowIndex ?? 0));
|
||||
return rows;
|
||||
}
|
||||
|
||||
function extractTraceRowsFromText(text, source, baseIndex, meta = {}) {
|
||||
const result = [];
|
||||
const normalized = String(text || '').replace(/\r/g, '\n');
|
||||
if (!normalized.trim()) return result;
|
||||
const lines = normalized.split('\n').map((line) => line.trim()).filter(Boolean);
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const line = lines[index];
|
||||
if (!traceLineLooksRelevant(line)) continue;
|
||||
const nextLine = index + 1 < lines.length && !/^\d{1,2}:\d{2}:\d{2}\b/.test(lines[index + 1]) ? lines[index + 1] : '';
|
||||
const preview = compactOneLine(nextLine ? line + ' ' + nextLine : line);
|
||||
result.push(normalizeTraceTimingRow(preview, source, Number(baseIndex || 0) * 1000 + index, meta));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeTraceTimingRow(text, source, rowIndex, meta = {}) {
|
||||
const preview = compactOneLine(text).slice(0, 240);
|
||||
return {
|
||||
source,
|
||||
rowIndex,
|
||||
preview,
|
||||
pageRole: meta.pageRole || null,
|
||||
pageId: meta.pageId || null,
|
||||
sessionId: meta.sessionId || null,
|
||||
traceId: meta.traceId || parseTraceRowTraceId(preview),
|
||||
clockSeconds: parseTraceRowClockSeconds(preview),
|
||||
totalSeconds: parseTraceRowTotalSeconds(preview),
|
||||
projectedSeq: parseTraceRowProjectedSeq(preview),
|
||||
isCompletion: /轮次完成|turn\s+completed|completed\s+turn/i.test(preview),
|
||||
};
|
||||
}
|
||||
|
||||
function traceRowCandidateArrays(sample, timelineItem) {
|
||||
const candidates = [];
|
||||
const pushArray = (rows, source, meta = {}) => {
|
||||
if (Array.isArray(rows) && rows.length) candidates.push({ rows, source, meta });
|
||||
};
|
||||
const directSources = [
|
||||
[sample?.traceRows, 'sample.traceRows'],
|
||||
[sample?.eventRows, 'sample.eventRows'],
|
||||
[sample?.activityRows, 'sample.activityRows'],
|
||||
[sample?.timelineRows, 'sample.timelineRows'],
|
||||
[sample?.dom?.traceRows, 'sample.dom.traceRows'],
|
||||
[sample?.dom?.eventRows, 'sample.dom.eventRows'],
|
||||
[sample?.dom?.activityRows, 'sample.dom.activityRows'],
|
||||
[sample?.dom?.timelineRows, 'sample.dom.timelineRows'],
|
||||
[timelineItem?.traceRows, 'timeline.traceRows'],
|
||||
[timelineItem?.eventRows, 'timeline.eventRows'],
|
||||
[timelineItem?.activityRows, 'timeline.activityRows'],
|
||||
[timelineItem?.rows, 'timeline.rows'],
|
||||
[timelineItem?.events, 'timeline.events'],
|
||||
];
|
||||
for (const [rows, source] of directSources) pushArray(rows, source, {});
|
||||
if (!candidates.length) collectNamedTraceArrays(sample, candidates, 'sample', 0);
|
||||
if (!candidates.length) collectNamedTraceArrays(timelineItem, candidates, 'timeline', 0);
|
||||
return candidates;
|
||||
}
|
||||
|
||||
function collectNamedTraceArrays(value, candidates, path, depth) {
|
||||
if (!value || depth > 5) return;
|
||||
if (Array.isArray(value)) {
|
||||
const pathLooksTrace = /trace|timeline|activity|event|log|record/i.test(path);
|
||||
const valueLooksTrace = value.slice(0, 5).some((item) => traceLineLooksRelevant(typeof item === 'string' ? item : objectText(item)));
|
||||
if (pathLooksTrace || valueLooksTrace) candidates.push({ rows: value, source: path, meta: {} });
|
||||
return;
|
||||
}
|
||||
if (typeof value !== 'object') return;
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (!child) continue;
|
||||
if (Array.isArray(child)) {
|
||||
const childPath = path + '.' + key;
|
||||
const pathLooksTrace = /trace|timeline|activity|event|log|record/i.test(key);
|
||||
const valueLooksTrace = child.slice(0, 5).some((item) => traceLineLooksRelevant(typeof item === 'string' ? item : objectText(item)));
|
||||
if (pathLooksTrace || valueLooksTrace) candidates.push({ rows: child, source: childPath, meta: {} });
|
||||
continue;
|
||||
}
|
||||
if (typeof child === 'object' && /dom|trace|timeline|activity|event|log|record|page|card|message|panel|diagnostic/i.test(key)) {
|
||||
collectNamedTraceArrays(child, candidates, path + '.' + key, depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function traceLineLooksRelevant(text) {
|
||||
const value = String(text || '').trim();
|
||||
if (!value) return false;
|
||||
if (/^\d{1,2}:\d{2}:\d{2}\b/.test(value)) return true;
|
||||
if (/\btotal=\d/.test(value)) return true;
|
||||
if (/轮次完成（总耗时/.test(value)) return true;
|
||||
if (/\bseq(?:uence)?[=:]\s*\d+/i.test(value)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function parseTraceRowClockSeconds(text) {
|
||||
const match = String(text || '').match(/^\s*(\d{1,2}):(\d{2}):(\d{2})\b/);
|
||||
if (!match) return null;
|
||||
return Number(match[1]) * 3600 + Number(match[2]) * 60 + Number(match[3]);
|
||||
}
|
||||
|
||||
function parseTraceRowTotalSeconds(text) {
|
||||
const value = String(text || '');
|
||||
const totalMatch = value.match(/\btotal=([0-9:.]+)/i);
|
||||
if (totalMatch) return parseTraceDurationSeconds(totalMatch[1]);
|
||||
const completionMatch = value.match(/总耗时\s*([0-9:.]+)/);
|
||||
if (completionMatch) return parseTraceDurationSeconds(completionMatch[1]);
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseTraceDurationSeconds(value) {
|
||||
const text = String(value || '').trim();
|
||||
if (!text) return null;
|
||||
const parts = text.split(':').map((part) => Number(part));
|
||||
if (parts.some((part) => !Number.isFinite(part))) return null;
|
||||
if (parts.length === 3) return parts[0] * 3600 + parts[1] * 60 + parts[2];
|
||||
if (parts.length === 2) return parts[0] * 60 + parts[1];
|
||||
if (parts.length === 1) return parts[0];
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseTraceRowProjectedSeq(text) {
|
||||
const value = String(text || '');
|
||||
const match = value.match(/\b(?:projected[_-]?seq|seq(?:uence)?|event[_-]?seq)\s*[=:]\s*(\d+)/i);
|
||||
return match ? Number(match[1]) : null;
|
||||
}
|
||||
|
||||
function parseTraceRowTraceId(text) {
|
||||
const match = String(text || '').match(/\b(?:trace_id=|traceId[:=]?\s*)(trc_[a-z0-9_-]+|[a-f0-9]{16,})\b/i);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
function traceRowGroupKey(row) {
|
||||
return [row.pageRole || '', row.pageId || '', row.sessionId || '', row.traceId || 'unknown'].join('|');
|
||||
}
|
||||
|
||||
function traceRowMatchesCard(row, card, terminalCardCount) {
|
||||
if (!row) return false;
|
||||
if (row.traceId && card.traceId) return row.traceId === card.traceId;
|
||||
if (row.sessionId && card.sessionId) return row.sessionId === card.sessionId;
|
||||
if (terminalCardCount === 1) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function maxTraceDurationEvidence(rows) {
|
||||
const finiteTotals = rows.map((row) => Number(row.totalSeconds)).filter(Number.isFinite);
|
||||
const finiteClocks = rows.map((row) => Number(row.clockSeconds)).filter(Number.isFinite);
|
||||
const evidences = [];
|
||||
if (finiteTotals.length) {
|
||||
const maxTotal = Math.max(...finiteTotals);
|
||||
const source = rows.find((row) => Number(row.totalSeconds) === maxTotal);
|
||||
evidences.push({ kind: 'trace-total', seconds: maxTotal, traceId: source?.traceId || null, preview: source?.preview || '' });
|
||||
}
|
||||
if (finiteClocks.length >= 2) {
|
||||
const minClock = Math.min(...finiteClocks);
|
||||
const maxClock = Math.max(...finiteClocks);
|
||||
const span = maxClock - minClock;
|
||||
if (span >= 0 && span < 43200) evidences.push({ kind: 'trace-clock-span', seconds: span, traceId: null, preview: 'visible trace row clock span' });
|
||||
}
|
||||
if (!evidences.length) return null;
|
||||
evidences.sort((a, b) => b.seconds - a.seconds);
|
||||
return evidences[0];
|
||||
}
|
||||
|
||||
function maxSelfReportedDurationEvidence(text) {
|
||||
const value = String(text || '');
|
||||
const lines = value.split(/\n+/).map((line) => line.trim()).filter(Boolean);
|
||||
let best = null;
|
||||
for (const line of lines) {
|
||||
if (!/耗时|用时|duration|elapsed/i.test(line)) continue;
|
||||
const seconds = parseSelfReportedRoundDurationSeconds(line);
|
||||
if (!Number.isFinite(seconds)) continue;
|
||||
if (!best || seconds > best.seconds) {
|
||||
best = { kind: 'final-response-duration', seconds, preview: compactOneLine(line).slice(0, 240) };
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function parseSelfReportedRoundDurationSeconds(text) {
|
||||
const value = String(text || '');
|
||||
const clock = value.match(/(\d{1,2}:\d{2}:\d{2}|\d{1,2}:\d{2})/);
|
||||
if (clock) return parseTraceDurationSeconds(clock[1]);
|
||||
const hour = value.match(/(?:约|大约|around|about)?\s*(\d+(?:\.\d+)?)\s*(?:小时|hour|hours|hr|hrs)/i);
|
||||
const minute = value.match(/(?:约|大约|around|about)?\s*(\d+(?:\.\d+)?)\s*(?:分钟|分|min|mins|minute|minutes)/i);
|
||||
const second = value.match(/(?:约|大约|around|about)?\s*(\d+(?:\.\d+)?)\s*(?:秒|sec|secs|second|seconds)/i);
|
||||
let total = 0;
|
||||
if (hour) total += Number(hour[1]) * 3600;
|
||||
if (minute) total += Number(minute[1]) * 60;
|
||||
if (second) total += Number(second[1]);
|
||||
return total > 0 ? total : null;
|
||||
}
|
||||
|
||||
function sampleVisibleText(sample, timelineItem) {
|
||||
const chunks = [];
|
||||
for (const source of [sample?.visibleText, sample?.text, sample?.innerText, sample?.dom?.visibleText, sample?.dom?.text, timelineItem?.visibleText, timelineItem?.text, timelineItem?.message, timelineItem?.summary]) {
|
||||
if (typeof source === 'string' && source.trim()) chunks.push(source);
|
||||
}
|
||||
return chunks.join('\n');
|
||||
}
|
||||
|
||||
function objectText(value) {
|
||||
if (!value || typeof value !== 'object') return typeof value === 'string' ? value : '';
|
||||
const keys = ['text', 'innerText', 'visibleText', 'label', 'title', 'summary', 'message', 'content', 'body', 'preview', 'description'];
|
||||
const chunks = [];
|
||||
for (const key of keys) {
|
||||
const part = value[key];
|
||||
if (typeof part === 'string' && part.trim()) chunks.push(part);
|
||||
}
|
||||
return chunks.join('\n');
|
||||
}
|
||||
|
||||
function compactOneLine(value) {
|
||||
return String(value || '').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function buildCodeAgentCardTimingMetrics(samples, timeline, turnTiming) {
|
||||
const missingElapsed = [];
|
||||
const missingRecentUpdate = [];
|
||||
const cardRows = [];
|
||||
for (let index = 0; index < (Array.isArray(samples) ? samples : []).length; index += 1) {
|
||||
const sample = samples[index];
|
||||
const timelineItem = timeline[index] || {};
|
||||
for (const card of codeAgentCardsForSample(sample)) {
|
||||
const text = codeAgentCardText(card);
|
||||
const totalElapsedValues = parseTotalElapsedSeconds(text).filter(Number.isFinite);
|
||||
const recentUpdateValues = parseRecentUpdateSeconds(text).filter(Number.isFinite);
|
||||
const terminal = isCodeAgentCardTerminal(card);
|
||||
const row = {
|
||||
...ref(sample),
|
||||
promptIndex: timelineItem.promptIndex ?? 0,
|
||||
source: card.source ?? "turn",
|
||||
status: card.status ?? null,
|
||||
messageId: card.messageId ?? null,
|
||||
traceId: card.traceId ?? firstTraceId([text]),
|
||||
totalElapsedSeconds: totalElapsedValues.length > 0 ? Math.max(...totalElapsedValues) : null,
|
||||
recentUpdateSeconds: recentUpdateValues.length > 0 ? Math.max(...recentUpdateValues) : null,
|
||||
terminal,
|
||||
textHash: card.textHash ?? sha256(text),
|
||||
textPreview: limitText(text, 180),
|
||||
valuesRedacted: true
|
||||
};
|
||||
cardRows.push(row);
|
||||
if (row.totalElapsedSeconds === null) missingElapsed.push(row);
|
||||
if (!terminal && row.recentUpdateSeconds === null) missingRecentUpdate.push(row);
|
||||
}
|
||||
}
|
||||
const roundCompletion = buildRoundCompletionMetrics(samples, timeline, turnTiming);
|
||||
return {
|
||||
summary: {
|
||||
cardSampleCount: cardRows.length,
|
||||
terminalCardSampleCount: cardRows.filter((item) => item.terminal).length,
|
||||
runningCardSampleCount: cardRows.filter((item) => !item.terminal).length,
|
||||
missingElapsedCount: missingElapsed.length,
|
||||
missingRecentUpdateCount: missingRecentUpdate.length,
|
||||
roundCompletionEventCount: roundCompletion.events.length,
|
||||
roundCompletionElapsedMismatchCount: roundCompletion.elapsedMismatches.length,
|
||||
roundCompletionFinalResponseMissingCount: roundCompletion.finalResponseMissing.length,
|
||||
roundCompletionPostTimingChangeCount: roundCompletion.postCompletionTimingChanges.length,
|
||||
roundCompletionPostRecentUpdateVisibleCount: roundCompletion.postCompletionRecentUpdateVisible.length,
|
||||
elapsedMismatchToleranceSeconds: alertThresholds.turnTimingSampleSlackSeconds,
|
||||
},
|
||||
cardSamples: cardRows.slice(0, 200),
|
||||
missingElapsed: missingElapsed.slice(0, 200),
|
||||
missingRecentUpdate: missingRecentUpdate.slice(0, 200),
|
||||
roundCompletion,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function codeAgentCardsForSample(sample) {
|
||||
const turnCards = (Array.isArray(sample?.turns) ? sample.turns : [])
|
||||
.map((item) => ({ ...item, source: item?.source || "turn" }))
|
||||
.filter(isCodeAgentCardLike);
|
||||
if (turnCards.length > 0) return turnCards;
|
||||
return (Array.isArray(sample?.messages) ? sample.messages : [])
|
||||
.map((item) => ({ ...item, source: item?.source || "message" }))
|
||||
.filter(isCodeAgentCardLike);
|
||||
}
|
||||
|
||||
function isCodeAgentCardLike(item) {
|
||||
const text = codeAgentCardText(item);
|
||||
const role = String(item?.dataRole || item?.role || "").toLowerCase();
|
||||
if (/agent|assistant/u.test(role)) return true;
|
||||
return /Code Agent|运行记录|耗时|最近\s*(?:\d+|一|两|三)|轮次完成|trace_id=trc_/iu.test(text);
|
||||
}
|
||||
|
||||
function codeAgentCardText(item) {
|
||||
return [
|
||||
item?.durationText,
|
||||
item?.activityText,
|
||||
item?.text,
|
||||
item?.textPreview,
|
||||
item?.status
|
||||
].map((value) => String(value || "")).filter((value) => value.trim().length > 0).join("\n");
|
||||
}
|
||||
|
||||
function isCodeAgentCardTerminal(item) {
|
||||
if (isTerminalTurnStatus(item?.status)) return true;
|
||||
const text = codeAgentCardText(item);
|
||||
return isTerminalTraceText(text) || /轮次完成|轮次失败|轮次取消|已记录|completed|failed|canceled|cancelled|blocked/iu.test(text);
|
||||
}
|
||||
|
||||
function buildRoundCompletionMetrics(samples, timeline, turnTiming) {
|
||||
const events = [];
|
||||
for (let index = 0; index < (Array.isArray(samples) ? samples : []).length; index += 1) {
|
||||
const sample = samples[index];
|
||||
const timelineItem = timeline[index] || {};
|
||||
for (const event of roundCompletionEventsForSample(sample, timelineItem)) events.push(event);
|
||||
}
|
||||
const elapsedMismatches = [];
|
||||
const finalResponseMissing = [];
|
||||
const postCompletionTimingChanges = [];
|
||||
const postCompletionRecentUpdateVisible = [];
|
||||
for (const event of events) {
|
||||
const sampleIndex = samples.findIndex((sample) => sample?.seq === event.seq && sample?.pageRole === event.pageRole && sample?.pageId === event.pageId);
|
||||
const sameSample = sampleIndex >= 0 ? samples[sampleIndex] : null;
|
||||
const sameTimelineItem = sampleIndex >= 0 ? timeline[sampleIndex] || {} : {};
|
||||
const cards = sameSample ? cardMetricItemsForCompletion(sameSample, sameTimelineItem, event) : [];
|
||||
const bestCard = cards.filter((card) => Number.isFinite(Number(card.totalElapsedSeconds)))[0] || null;
|
||||
if (Number.isFinite(Number(event.elapsedSeconds)) && bestCard) {
|
||||
const delta = Math.abs(Number(bestCard.totalElapsedSeconds) - Number(event.elapsedSeconds));
|
||||
if (delta > alertThresholds.turnTimingSampleSlackSeconds) {
|
||||
elapsedMismatches.push({
|
||||
...eventRef(event),
|
||||
cardTotalElapsedSeconds: Number(bestCard.totalElapsedSeconds),
|
||||
completionElapsedSeconds: Number(event.elapsedSeconds),
|
||||
deltaSeconds: Number(delta.toFixed(3)),
|
||||
toleranceSeconds: alertThresholds.turnTimingSampleSlackSeconds,
|
||||
cardTraceId: bestCard.traceId ?? null,
|
||||
cardMessageId: bestCard.messageId ?? null,
|
||||
valuesRedacted: true
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!hasFinalResponseAfterCompletion(samples, timeline, event)) {
|
||||
finalResponseMissing.push({
|
||||
...eventRef(event),
|
||||
completionElapsedSeconds: event.elapsedSeconds,
|
||||
valuesRedacted: true
|
||||
});
|
||||
}
|
||||
const postTiming = detectPostCompletionTimingChanges(turnTiming, event);
|
||||
postCompletionTimingChanges.push(...postTiming.timingChanges);
|
||||
postCompletionRecentUpdateVisible.push(...postTiming.recentUpdateVisible);
|
||||
}
|
||||
return {
|
||||
events: dedupeRoundCompletionRows(events).slice(0, 200),
|
||||
elapsedMismatches: dedupeRoundCompletionRows(elapsedMismatches).slice(0, 200),
|
||||
finalResponseMissing: dedupeRoundCompletionRows(finalResponseMissing).slice(0, 200),
|
||||
postCompletionTimingChanges: dedupeRoundCompletionRows(postCompletionTimingChanges).slice(0, 200),
|
||||
postCompletionRecentUpdateVisible: dedupeRoundCompletionRows(postCompletionRecentUpdateVisible).slice(0, 200),
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function roundCompletionEventsForSample(sample, timelineItem) {
|
||||
const rows = [];
|
||||
for (const item of Array.isArray(sample?.traceRows) ? sample.traceRows : []) {
|
||||
const text = String(item?.text || item?.textPreview || "");
|
||||
if (!/轮次完成/iu.test(text)) continue;
|
||||
const elapsedValues = parseTotalElapsedSeconds(text).filter(Number.isFinite);
|
||||
const attributed = inferRoundCompletionCard(sample, text);
|
||||
rows.push({
|
||||
...ref(sample),
|
||||
promptIndex: timelineItem.promptIndex ?? 0,
|
||||
traceId: item?.traceId ?? firstTraceId([text]) ?? attributed?.traceId ?? null,
|
||||
messageId: item?.messageId ?? attributed?.messageId ?? null,
|
||||
attributed: Boolean(item?.traceId || item?.messageId || attributed?.traceId || attributed?.messageId),
|
||||
traceRowIndex: item?.index ?? null,
|
||||
elapsedSeconds: elapsedValues.length > 0 ? Math.max(...elapsedValues) : null,
|
||||
textHash: item?.textHash ?? sha256(text),
|
||||
preview: limitText(text, 180),
|
||||
valuesRedacted: true
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function inferRoundCompletionCard(sample, text) {
|
||||
const cards = codeAgentCardsForSample(sample)
|
||||
.filter((card) => isCodeAgentCardTerminal(card))
|
||||
.filter((card) => card?.traceId || card?.messageId);
|
||||
const textHash = sha256(String(text || ""));
|
||||
const direct = cards.filter((card) => {
|
||||
const cardText = codeAgentCardText(card);
|
||||
return card?.textHash === textHash || cardText.includes(String(text || "").slice(0, 80)) || /轮次完成/iu.test(cardText);
|
||||
});
|
||||
const candidates = direct.length > 0 ? direct : cards;
|
||||
if (candidates.length !== 1) return null;
|
||||
const card = candidates[0];
|
||||
return { traceId: card.traceId ?? null, messageId: card.messageId ?? card.id ?? null };
|
||||
}
|
||||
|
||||
function cardMetricItemsForCompletion(sample, timelineItem, event) {
|
||||
if (!event.traceId && !event.messageId) return [];
|
||||
return turnMetricItems(sample, timelineItem)
|
||||
.filter((item) => item.promptIndex === event.promptIndex)
|
||||
.filter((item) => !event.traceId || !item.traceId || item.traceId === event.traceId)
|
||||
.filter((item) => !event.messageId || !item.messageId || item.messageId === event.messageId)
|
||||
.sort((left, right) => {
|
||||
const leftTraceMatch = event.traceId && left.traceId === event.traceId ? 0 : 1;
|
||||
const rightTraceMatch = event.traceId && right.traceId === event.traceId ? 0 : 1;
|
||||
const leftMessageMatch = event.messageId && left.messageId === event.messageId ? 0 : 1;
|
||||
const rightMessageMatch = event.messageId && right.messageId === event.messageId ? 0 : 1;
|
||||
return leftTraceMatch - rightTraceMatch || leftMessageMatch - rightMessageMatch || String(left.source || "").localeCompare(String(right.source || ""));
|
||||
});
|
||||
}
|
||||
|
||||
function hasFinalResponseAfterCompletion(samples, timeline, event) {
|
||||
if (!event.traceId && !event.messageId) return true;
|
||||
const eventTsMs = Date.parse(String(event.ts || ""));
|
||||
for (let index = 0; index < (Array.isArray(samples) ? samples : []).length; index += 1) {
|
||||
const sample = samples[index];
|
||||
const tsMs = Date.parse(String(sample?.ts || ""));
|
||||
if (Number.isFinite(eventTsMs) && Number.isFinite(tsMs) && tsMs < eventTsMs) continue;
|
||||
if (sample?.pageRole !== event.pageRole) continue;
|
||||
const sampleSession = sample?.routeSessionId || sample?.activeSessionId || null;
|
||||
const eventSession = event.routeSessionId || event.activeSessionId || null;
|
||||
if (sampleSession && eventSession && sampleSession !== eventSession) continue;
|
||||
const promptIndex = timeline[index]?.promptIndex ?? 0;
|
||||
if (event.promptIndex && promptIndex !== event.promptIndex) continue;
|
||||
if (sampleHasVisibleFinalResponse(sample, event)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function sampleHasVisibleFinalResponse(sample, event = {}) {
|
||||
for (const group of [sample?.messages, sample?.turns]) {
|
||||
if (!Array.isArray(group)) continue;
|
||||
for (const item of group) {
|
||||
if (event.traceId && item?.traceId && item.traceId !== event.traceId) continue;
|
||||
if (event.messageId && item?.messageId && item.messageId !== event.messageId) continue;
|
||||
const text = normalizedText([item?.text, item?.textPreview].filter(Boolean).join(" "));
|
||||
if (text.length < 24) continue;
|
||||
if (isDiagnosticText(text)) continue;
|
||||
if (isFinalResultText(text)) return true;
|
||||
if (/运行记录/iu.test(text) && /(?:已完成|完成|新增|实现|验证|测试|结果|README|文件|summary)/iu.test(text)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function detectPostCompletionTimingChanges(turnTiming, event) {
|
||||
const timingChanges = [];
|
||||
const recentUpdateVisible = [];
|
||||
if (!event.traceId && !event.messageId) return { timingChanges, recentUpdateVisible };
|
||||
const rows = Array.isArray(turnTiming?.rows) ? turnTiming.rows : [];
|
||||
const columns = Array.isArray(turnTiming?.columns) ? turnTiming.columns : [];
|
||||
const eventTsMs = Date.parse(String(event.ts || ""));
|
||||
for (const column of columns) {
|
||||
if (column.pageRole && event.pageRole && column.pageRole !== event.pageRole) continue;
|
||||
if (event.traceId && column.traceId && column.traceId !== event.traceId) continue;
|
||||
if (event.messageId && column.messageId && column.messageId !== event.messageId) continue;
|
||||
if (event.promptIndex && column.promptIndex && column.promptIndex !== event.promptIndex && column.lastPromptIndex !== event.promptIndex) continue;
|
||||
let previousTotal = null;
|
||||
let previousRecent = null;
|
||||
for (const row of rows) {
|
||||
const rowTsMs = Date.parse(String(row.ts || ""));
|
||||
if (Number.isFinite(eventTsMs) && Number.isFinite(rowTsMs) && rowTsMs < eventTsMs) continue;
|
||||
if (row.pageRole && event.pageRole && row.pageRole !== event.pageRole) continue;
|
||||
const cell = row.cells?.[column.id];
|
||||
if (!cell) continue;
|
||||
if (event.traceId && cell.traceId && cell.traceId !== event.traceId) continue;
|
||||
if (event.messageId && cell.messageId && cell.messageId !== event.messageId) continue;
|
||||
const total = Number(cell.totalElapsedSeconds);
|
||||
if (Number.isFinite(total)) {
|
||||
if (previousTotal && Math.abs(total - previousTotal.value) > alertThresholds.turnTimingSampleSlackSeconds) {
|
||||
timingChanges.push({
|
||||
...eventRef(event),
|
||||
columnId: column.id,
|
||||
columnLabel: column.label,
|
||||
metric: "totalElapsedSeconds",
|
||||
fromSeq: previousTotal.seq,
|
||||
fromTs: previousTotal.ts,
|
||||
fromValue: previousTotal.value,
|
||||
toSeq: row.seq ?? null,
|
||||
toTs: row.ts ?? null,
|
||||
toValue: total,
|
||||
delta: Number((total - previousTotal.value).toFixed(3)),
|
||||
toleranceSeconds: alertThresholds.turnTimingSampleSlackSeconds,
|
||||
traceId: cell.traceId ?? column.traceId ?? event.traceId ?? null,
|
||||
messageId: cell.messageId ?? column.messageId ?? null,
|
||||
valuesRedacted: true
|
||||
});
|
||||
}
|
||||
previousTotal = { value: total, seq: row.seq ?? null, ts: row.ts ?? null };
|
||||
}
|
||||
const recent = Number(cell.recentUpdateSeconds);
|
||||
if (Number.isFinite(recent)) {
|
||||
recentUpdateVisible.push({
|
||||
...eventRef(event),
|
||||
columnId: column.id,
|
||||
columnLabel: column.label,
|
||||
metric: "recentUpdateSeconds",
|
||||
seq: row.seq ?? null,
|
||||
ts: row.ts ?? null,
|
||||
value: recent,
|
||||
traceId: cell.traceId ?? column.traceId ?? event.traceId ?? null,
|
||||
messageId: cell.messageId ?? column.messageId ?? null,
|
||||
valuesRedacted: true
|
||||
});
|
||||
if (previousRecent && recent !== previousRecent.value) {
|
||||
timingChanges.push({
|
||||
...eventRef(event),
|
||||
columnId: column.id,
|
||||
columnLabel: column.label,
|
||||
metric: "recentUpdateSeconds",
|
||||
fromSeq: previousRecent.seq,
|
||||
fromTs: previousRecent.ts,
|
||||
fromValue: previousRecent.value,
|
||||
toSeq: row.seq ?? null,
|
||||
toTs: row.ts ?? null,
|
||||
toValue: recent,
|
||||
delta: Number((recent - previousRecent.value).toFixed(3)),
|
||||
traceId: cell.traceId ?? column.traceId ?? event.traceId ?? null,
|
||||
messageId: cell.messageId ?? column.messageId ?? null,
|
||||
valuesRedacted: true
|
||||
});
|
||||
}
|
||||
previousRecent = { value: recent, seq: row.seq ?? null, ts: row.ts ?? null };
|
||||
}
|
||||
}
|
||||
}
|
||||
return { timingChanges, recentUpdateVisible };
|
||||
}
|
||||
|
||||
function eventRef(event) {
|
||||
return {
|
||||
seq: event?.seq ?? null,
|
||||
sampleGroupSeq: event?.sampleGroupSeq ?? null,
|
||||
ts: event?.ts ?? null,
|
||||
pageRole: event?.pageRole ?? null,
|
||||
pageId: event?.pageId ?? null,
|
||||
routeSessionId: event?.routeSessionId ?? null,
|
||||
activeSessionId: event?.activeSessionId ?? null,
|
||||
promptIndex: event?.promptIndex ?? null,
|
||||
traceId: event?.traceId ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function dedupeRoundCompletionRows(rows) {
|
||||
const result = [];
|
||||
const seen = new Set();
|
||||
for (const row of Array.isArray(rows) ? rows : []) {
|
||||
const key = [
|
||||
row?.seq ?? row?.fromSeq ?? "",
|
||||
row?.toSeq ?? "",
|
||||
row?.pageRole ?? "",
|
||||
row?.promptIndex ?? "",
|
||||
row?.traceId ?? "",
|
||||
row?.metric ?? "",
|
||||
row?.textHash ?? "",
|
||||
row?.columnId ?? ""
|
||||
].join("|");
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
result.push(row);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function turnMetricItems(sample, timelineItem) {
|
||||
const promptIndex = timelineItem.promptIndex ?? 0;
|
||||
const pageRole = sample?.pageRole || "control";
|
||||
@@ -5067,6 +5883,11 @@ function renderMarkdown(report) {
|
||||
const loadingSummary = loading.summary || {};
|
||||
const sessionRailTitles = report.sampleMetrics?.sessionRailTitles || {};
|
||||
const sessionRailTitleSummary = sessionRailTitles.summary || {};
|
||||
const codeAgentCardTiming = report.sampleMetrics?.codeAgentCardTiming || {};
|
||||
const codeAgentCardTimingSummary = codeAgentCardTiming.summary || {};
|
||||
const roundCompletion = codeAgentCardTiming.roundCompletion || {};
|
||||
const traceOrder = report.sampleMetrics?.traceOrder || {};
|
||||
const traceOrderSummary = traceOrder.summary || {};
|
||||
const alertSummary = report.runtimeAlerts?.summary || {};
|
||||
const httpAlertLines = Array.isArray(report.runtimeAlerts?.networkHttpErrorsByPath) && report.runtimeAlerts.networkHttpErrorsByPath.length > 0
|
||||
? report.runtimeAlerts.networkHttpErrorsByPath.slice(0, 40).map((item) => "- HTTP " + (item.status ?? "-") + " " + item.method + " " + item.urlPath + " count=" + item.count + " prompts=" + (item.promptIndexes?.join(",") || "-") + " first=" + (item.firstAt || "-") + " last=" + (item.lastAt || "-")).join("\n")
|
||||
@@ -5089,6 +5910,33 @@ function renderMarkdown(report) {
|
||||
const roundLines = Array.isArray(report.sampleMetrics?.rounds) && report.sampleMetrics.rounds.length > 0
|
||||
? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " loadingSamples=" + (item.loadingSamples ?? 0) + " maxLoading=" + (item.maxLoadingCount ?? 0) + " loadingOwners=" + (item.loadingOwnerCount ?? 0) + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " totalDecrease=" + (item.turnTimingTotalElapsedDecreaseCount ?? 0) + " totalForwardJump=" + (item.turnTimingTotalElapsedForwardJumpCount ?? 0) + " totalForwardJumpMax=" + (item.turnTimingTotalElapsedForwardJumpMaxSeconds ?? 0) + " terminalGrowth=" + (item.turnTimingTerminalElapsedGrowthCount ?? 0) + " terminalGrowthMax=" + (item.turnTimingTerminalElapsedGrowthMaxSeconds ?? 0) + " recentJump=" + (item.turnTimingRecentUpdateJumpCount ?? 0) + " recentSawtoothJump=" + (item.turnTimingRecentUpdateSawtoothJumpCount ?? item.turnTimingRecentUpdateJumpCount ?? 0) + " recentStep=" + (item.turnTimingRecentUpdateStepCount ?? 0) + " recentMaxIncrease=" + (item.turnTimingRecentUpdateMaxIncreaseSeconds ?? "-") + " recentMaxExcess=" + (item.turnTimingRecentUpdateMaxExcessSeconds ?? 0) + " recentReset=" + (item.turnTimingRecentUpdateResetCount ?? 0) + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n")
|
||||
: "- 无轮次指标。";
|
||||
const cardMissingElapsedLines = Array.isArray(codeAgentCardTiming.missingElapsed) && codeAgentCardTiming.missingElapsed.length > 0
|
||||
? codeAgentCardTiming.missingElapsed.slice(0, 80).map((item) => "- #" + (item.seq ?? "-") + " " + (item.ts || "-") + " prompt=" + (item.promptIndex ?? "-") + " role=" + (item.pageRole || "-") + " status=" + (item.status || "-") + " traceId=" + (item.traceId || "-") + " messageId=" + (item.messageId || "-") + " preview=" + escapeMarkdownCell(item.textPreview || "")).join("\n")
|
||||
: "- 未观察到 Code Agent 卡片缺少耗时。";
|
||||
const cardMissingRecentLines = Array.isArray(codeAgentCardTiming.missingRecentUpdate) && codeAgentCardTiming.missingRecentUpdate.length > 0
|
||||
? codeAgentCardTiming.missingRecentUpdate.slice(0, 80).map((item) => "- #" + (item.seq ?? "-") + " " + (item.ts || "-") + " prompt=" + (item.promptIndex ?? "-") + " role=" + (item.pageRole || "-") + " status=" + (item.status || "-") + " traceId=" + (item.traceId || "-") + " messageId=" + (item.messageId || "-") + " total=" + (item.totalElapsedSeconds ?? "-") + " preview=" + escapeMarkdownCell(item.textPreview || "")).join("\n")
|
||||
: "- 未观察到未终态 Code Agent 卡片缺少最近更新。";
|
||||
const roundCompletionLines = Array.isArray(roundCompletion.events) && roundCompletion.events.length > 0
|
||||
? roundCompletion.events.slice(0, 80).map((item) => "- #" + (item.seq ?? "-") + " " + (item.ts || "-") + " prompt=" + (item.promptIndex ?? "-") + " role=" + (item.pageRole || "-") + " traceId=" + (item.traceId || "-") + " completionElapsed=" + (item.elapsedSeconds ?? "-") + " preview=" + escapeMarkdownCell(item.preview || "")).join("\n")
|
||||
: "- 未观察到“轮次完成（总耗时 ...）”trace 行。";
|
||||
const roundCompletionMismatchLines = Array.isArray(roundCompletion.elapsedMismatches) && roundCompletion.elapsedMismatches.length > 0
|
||||
? roundCompletion.elapsedMismatches.slice(0, 80).map((item) => "- #" + (item.seq ?? "-") + " " + (item.ts || "-") + " prompt=" + (item.promptIndex ?? "-") + " traceId=" + (item.traceId || "-") + " completion=" + (item.completionElapsedSeconds ?? "-") + " card=" + (item.cardTotalElapsedSeconds ?? "-") + " delta=" + (item.deltaSeconds ?? "-") + " tolerance=" + (item.toleranceSeconds ?? "-")).join("\n")
|
||||
: "- 未观察到轮次完成耗时与卡片耗时不一致。";
|
||||
const roundCompletionFinalMissingLines = Array.isArray(roundCompletion.finalResponseMissing) && roundCompletion.finalResponseMissing.length > 0
|
||||
? roundCompletion.finalResponseMissing.slice(0, 80).map((item) => "- #" + (item.seq ?? "-") + " " + (item.ts || "-") + " prompt=" + (item.promptIndex ?? "-") + " role=" + (item.pageRole || "-") + " traceId=" + (item.traceId || "-") + " completionElapsed=" + (item.completionElapsedSeconds ?? "-")).join("\n")
|
||||
: "- 未观察到轮次完成后 final response 缺失。";
|
||||
const roundCompletionPostTimingLines = Array.isArray(roundCompletion.postCompletionTimingChanges) && roundCompletion.postCompletionTimingChanges.length > 0
|
||||
? roundCompletion.postCompletionTimingChanges.slice(0, 80).map((item) => "- " + (item.columnLabel || item.columnId || "-") + " " + (item.metric || "-") + " " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " completionSeq=" + (item.seq ?? "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " traceId=" + (item.traceId || "-")).join("\n")
|
||||
: "- 未观察到轮次完成后耗时/最近更新继续变化。";
|
||||
const durationUnderreportedLines = Array.isArray(codeAgentCardTiming.durationUnderreported) && codeAgentCardTiming.durationUnderreported.length > 0
|
||||
? codeAgentCardTiming.durationUnderreported.slice(0, 80).map((item) => "- sample=" + (item.sampleIndex ?? "-") + " " + (item.timestamp || "-") + " role=" + (item.pageRole || "-") + " status=" + (item.status || "-") + " traceId=" + (item.traceId || "-") + " card=" + (item.cardTotalElapsedSeconds ?? "-") + "s expected=" + (item.expectedElapsedSeconds ?? "-") + "s delta=" + (item.deltaSeconds ?? "-") + "s evidence=" + (item.evidenceKind || "-") + " preview=" + escapeMarkdownCell(item.evidencePreview || item.cardPreview || "")).join("\n")
|
||||
: "- 未观察到 Code Agent 卡片耗时低于 trace/final-response 证据。";
|
||||
const traceOrderAnomalyLines = Array.isArray(traceOrder.orderAnomalies) && traceOrder.orderAnomalies.length > 0
|
||||
? traceOrder.orderAnomalies.slice(0, 80).map((item) => "- sample=" + (item.sampleIndex ?? "-") + " " + (item.timestamp || "-") + " role=" + (item.pageRole || "-") + " traceId=" + (item.traceId || "-") + " rows=" + (item.previousRowIndex ?? "-") + "->" + (item.currentRowIndex ?? "-") + " reasons=" + (Array.isArray(item.reasons) ? item.reasons.join(",") : "-") + " total=" + (item.previousTotalSeconds ?? "-") + "->" + (item.currentTotalSeconds ?? "-") + " clock=" + (item.previousClockSeconds ?? "-") + "->" + (item.currentClockSeconds ?? "-") + " preview=" + escapeMarkdownCell((item.previousPreview || "") + " / " + (item.currentPreview || ""))).join("\n")
|
||||
: "- 未观察到可见 trace 行顺序非单调。";
|
||||
const traceCompletionNotLastLines = Array.isArray(traceOrder.completionNotLast) && traceOrder.completionNotLast.length > 0
|
||||
? traceOrder.completionNotLast.slice(0, 80).map((item) => "- sample=" + (item.sampleIndex ?? "-") + " " + (item.timestamp || "-") + " role=" + (item.pageRole || "-") + " traceId=" + (item.traceId || "-") + " rows=" + (item.completionRowIndex ?? "-") + "->" + (item.laterRowIndex ?? "-") + " total=" + (item.completionTotalSeconds ?? "-") + "->" + (item.laterTotalSeconds ?? "-") + " completion=" + escapeMarkdownCell(item.completionPreview || "") + " later=" + escapeMarkdownCell(item.laterPreview || "")).join("\n")
|
||||
: "- 未观察到 completion 行后还有同 trace 后续行。";
|
||||
const loadingSegmentLines = Array.isArray(loading.segments) && loading.segments.length > 0
|
||||
? loading.segments.slice(0, 80).map((item) => "- observedDuration=" + (item.durationSeconds ?? 0) + "s upperBound=" + (item.upperBoundSeconds ?? item.durationSeconds ?? 0) + "s endedGap=" + (item.endedGapSeconds ?? "-") + "s samples=" + (item.sampleCount ?? 0) + " countMax=" + (item.maxCount ?? 0) + " owners=" + (item.ownerCount ?? 0) + " seq=" + (item.firstSeq ?? "-") + ".." + (item.lastSeq ?? "-") + " ts=" + (item.firstAt || "-") + ".." + (item.lastAt || "-") + " endedAt=" + (item.endedAt || (item.ongoing ? "ongoing" : "-")) + " ownerLabels=" + ((Array.isArray(item.owners) ? item.owners : []).slice(0, 6).map((owner) => (owner.ownerKind || "-") + ":" + (owner.ownerLabel || "-") + "x" + (owner.count ?? 0)).join(",") || "-")).join("\n")
|
||||
: "- 未观察到“加载中”可见区间。";
|
||||
@@ -5161,7 +6009,48 @@ function renderMarkdown(report) {
|
||||
+ "- turnTimingRecentUpdateMaxIncreaseSeconds: " + (metricSummary.turnTimingRecentUpdateMaxIncreaseSeconds ?? "-") + "\n"
|
||||
+ "- turnTimingRecentUpdateMaxExcessSeconds: " + (metricSummary.turnTimingRecentUpdateMaxExcessSeconds ?? 0) + "\n"
|
||||
+ "- turnTimingRecentUpdateResetCount: " + (metricSummary.turnTimingRecentUpdateResetCount ?? 0) + "\n\n"
|
||||
+ "- codeAgentCardSampleCount: " + (metricSummary.codeAgentCardSampleCount ?? 0) + "\n"
|
||||
+ "- codeAgentCardMissingElapsedCount: " + (metricSummary.codeAgentCardMissingElapsedCount ?? 0) + "\n"
|
||||
+ "- codeAgentCardMissingRecentUpdateCount: " + (metricSummary.codeAgentCardMissingRecentUpdateCount ?? 0) + "\n"
|
||||
+ "- codeAgentCardDurationUnderreportedCount: " + (metricSummary.codeAgentCardDurationUnderreportedCount ?? 0) + "\n"
|
||||
+ "- traceRowCount: " + (metricSummary.traceRowCount ?? 0) + "\n"
|
||||
+ "- traceRowOrderAnomalyCount: " + (metricSummary.traceRowOrderAnomalyCount ?? 0) + "\n"
|
||||
+ "- traceRowCompletionNotLastCount: " + (metricSummary.traceRowCompletionNotLastCount ?? 0) + "\n"
|
||||
+ "- roundCompletionEventCount: " + (metricSummary.roundCompletionEventCount ?? 0) + "\n"
|
||||
+ "- roundCompletionElapsedMismatchCount: " + (metricSummary.roundCompletionElapsedMismatchCount ?? 0) + "\n"
|
||||
+ "- roundCompletionFinalResponseMissingCount: " + (metricSummary.roundCompletionFinalResponseMissingCount ?? 0) + "\n"
|
||||
+ "- roundCompletionPostTimingChangeCount: " + (metricSummary.roundCompletionPostTimingChangeCount ?? 0) + "\n\n"
|
||||
+ "### Rounds\n\n" + roundLines + "\n\n"
|
||||
+ "### Code Agent card timing display\n\n"
|
||||
+ "- cardSampleCount: " + (codeAgentCardTimingSummary.cardSampleCount ?? 0) + "\n"
|
||||
+ "- runningCardSampleCount: " + (codeAgentCardTimingSummary.runningCardSampleCount ?? 0) + "\n"
|
||||
+ "- terminalCardSampleCount: " + (codeAgentCardTimingSummary.terminalCardSampleCount ?? 0) + "\n"
|
||||
+ "- missingElapsedCount: " + (codeAgentCardTimingSummary.missingElapsedCount ?? 0) + "\n"
|
||||
+ "- missingRecentUpdateCount: " + (codeAgentCardTimingSummary.missingRecentUpdateCount ?? 0) + "\n"
|
||||
+ "- durationUnderreportedCount: " + (codeAgentCardTimingSummary.durationUnderreportedCount ?? 0) + "\n"
|
||||
+ "- policy: Code Agent 卡片无论终态/非终态都必须显示耗时；非终态必须显示最近更新。该 analyzer 只报告采样到的页面表现，不做下游 repair。\n\n"
|
||||
+ "#### Missing elapsed samples\n\n" + cardMissingElapsedLines + "\n\n"
|
||||
+ "#### Missing recent update samples\n\n" + cardMissingRecentLines + "\n\n"
|
||||
+ "#### Duration underreported samples\n\n" + durationUnderreportedLines + "\n\n"
|
||||
+ "### Trace row visual order\n\n"
|
||||
+ "- traceRowCount: " + (traceOrderSummary.traceRowCount ?? 0) + "\n"
|
||||
+ "- orderAnomalyCount: " + (traceOrderSummary.orderAnomalyCount ?? 0) + "\n"
|
||||
+ "- completionNotLastCount: " + (traceOrderSummary.completionNotLastCount ?? 0) + "\n"
|
||||
+ "- policy: 可见 trace 行在同一 trace 内必须按 total/时钟/projected seq 单调展示；completion 行不得出现在同 trace 后续行之前。\n\n"
|
||||
+ "#### Trace order anomalies\n\n" + traceOrderAnomalyLines + "\n\n"
|
||||
+ "#### Completion row not last samples\n\n" + traceCompletionNotLastLines + "\n\n"
|
||||
+ "### Round completion consistency\n\n"
|
||||
+ "- completionEventCount: " + (codeAgentCardTimingSummary.roundCompletionEventCount ?? 0) + "\n"
|
||||
+ "- elapsedMismatchCount: " + (codeAgentCardTimingSummary.roundCompletionElapsedMismatchCount ?? 0) + "\n"
|
||||
+ "- finalResponseMissingCount: " + (codeAgentCardTimingSummary.roundCompletionFinalResponseMissingCount ?? 0) + "\n"
|
||||
+ "- postTimingChangeCount: " + (codeAgentCardTimingSummary.roundCompletionPostTimingChangeCount ?? 0) + "\n"
|
||||
+ "- postRecentUpdateVisibleCount: " + (codeAgentCardTimingSummary.roundCompletionPostRecentUpdateVisibleCount ?? 0) + "\n"
|
||||
+ "- elapsedMismatchToleranceSeconds: " + (codeAgentCardTimingSummary.elapsedMismatchToleranceSeconds ?? "-") + "\n"
|
||||
+ "- policy: 轮次完成（总耗时 ...） 的耗时必须与卡片总耗时一致；完成后 final response 必须可见，耗时/最近更新不得继续跳变。\n\n"
|
||||
+ "#### Round completion events\n\n" + roundCompletionLines + "\n\n"
|
||||
+ "#### Completion elapsed mismatches\n\n" + roundCompletionMismatchLines + "\n\n"
|
||||
+ "#### Final response missing after completion\n\n" + roundCompletionFinalMissingLines + "\n\n"
|
||||
+ "#### Post-completion timing changes\n\n" + roundCompletionPostTimingLines + "\n\n"
|
||||
+ "### Loading visibility: visible 加载中\n\n"
|
||||
+ "- sampleCount: " + (loadingSummary.sampleCount ?? 0) + "\n"
|
||||
+ "- loadingSampleCount: " + (loadingSummary.loadingSampleCount ?? 0) + "\n"
|
||||
|
||||
Reference in New Issue
Block a user