From cbecb8cce1cb4de9a3d1b474f29161b2fbc81e2e Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 21 Jun 2026 21:37:31 +0000 Subject: [PATCH] feat(web-probe): report terminal turn duration growth --- scripts/src/hwlab-node-impl.ts | 3 +- .../hwlab-node-web-observe-runner-source.ts | 60 ++++++++++++++++++- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index e7b2e12f..5ff6b31f 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -7517,12 +7517,13 @@ function renderWebObserveAnalyzeTable(value: Record): string { ]), "", "Turn timing:", - webObserveTable(["ROUNDS", "TURNS", "ROWS", "NON_MONO", "ELAPSED_DEC", "RECENT_JUMP", "MAX_RECENT_STEP"], [[ + webObserveTable(["ROUNDS", "TURNS", "ROWS", "NON_MONO", "ELAPSED_DEC", "TERMINAL_GROWTH", "RECENT_JUMP", "MAX_RECENT_STEP"], [[ webObserveText(roundCount), webObserveText(turnColumnCount), webObserveText(sampleMetrics?.turnTimingRows), webObserveText(sampleMetrics?.turnTimingNonMonotonicCount), webObserveText(sampleMetrics?.turnTimingTotalElapsedDecreaseCount), + webObserveText(sampleMetrics?.turnTimingTerminalElapsedGrowthCount), webObserveText(sampleMetrics?.turnTimingRecentUpdateJumpCount ?? sampleMetrics?.turnTimingRecentUpdateSawtoothJumpCount), webObserveText(sampleMetrics?.turnTimingRecentUpdateMaxIncreaseSeconds), ]]), diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index fac1449e..983c3df2 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -2145,8 +2145,12 @@ function buildSampleMetrics(samples, control) { const turnCells = turnTiming.rows.flatMap((row) => Object.values(row.cells || {})); const turnTimingNonMonotonic = Array.isArray(turnTiming.nonMonotonic) ? turnTiming.nonMonotonic : []; const turnTimingRecentUpdateSawtoothJumps = turnTimingNonMonotonic.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump"); + const turnTimingTerminalElapsedGrowth = Array.isArray(turnTiming.terminalElapsedGrowth) ? turnTiming.terminalElapsedGrowth : []; const turnTimingRecentUpdateResets = Array.isArray(turnTiming.recentUpdateResets) ? turnTiming.recentUpdateResets : []; const turnTimingRecentUpdateSteps = Array.isArray(turnTiming.recentUpdateSteps) ? turnTiming.recentUpdateSteps : []; + const turnTimingTerminalElapsedGrowthDeltas = turnTimingTerminalElapsedGrowth + .map((item) => Number(item.delta)) + .filter((value) => Number.isFinite(value) && value > 0); const turnTimingRecentUpdateLargestSteps = turnTimingRecentUpdateSteps .filter((item) => Number.isFinite(Number(item.delta))) .slice() @@ -2163,6 +2167,7 @@ function buildSampleMetrics(samples, control) { const diagnostics = timeline.filter((item) => item.diagnosticSeen).length; const rounds = buildRoundMetricSummaries(timeline, promptCommands, { nonMonotonic: turnTimingNonMonotonic, + terminalElapsedGrowth: turnTimingTerminalElapsedGrowth, recentUpdateResets: turnTimingRecentUpdateResets, recentUpdateSteps: turnTimingRecentUpdateSteps }); @@ -2181,6 +2186,8 @@ function buildSampleMetrics(samples, control) { turnCellsWithRecentUpdate: turnCells.filter((item) => item.recentUpdateSeconds !== null).length, turnTimingNonMonotonicCount: turnTimingNonMonotonic.length, turnTimingTotalElapsedDecreaseCount: turnTimingNonMonotonic.filter((item) => item.metric === "totalElapsedSeconds").length, + turnTimingTerminalElapsedGrowthCount: turnTimingTerminalElapsedGrowth.length, + turnTimingTerminalElapsedGrowthMaxSeconds: turnTimingTerminalElapsedGrowthDeltas.length > 0 ? Math.max(...turnTimingTerminalElapsedGrowthDeltas) : 0, turnTimingRecentUpdateJumpCount: recentUpdateJumpCount, turnTimingRecentUpdateSawtoothJumpCount: recentUpdateJumpCount, turnTimingRecentUpdateStepCount: turnTimingRecentUpdateSteps.length, @@ -2189,12 +2196,14 @@ function buildSampleMetrics(samples, control) { turnTimingRecentUpdateResetCount: turnTimingRecentUpdateResets.length, turnTimingRecentUpdateDecreaseCount: turnTimingRecentUpdateResets.length, roundsWithTurnTimingNonMonotonic: rounds.filter((item) => item.turnTimingNonMonotonicCount > 0).length, + roundsWithTerminalElapsedGrowth: rounds.filter((item) => item.turnTimingTerminalElapsedGrowthCount > 0).length, roundsWithRecentUpdateJumps: rounds.filter((item) => item.turnTimingRecentUpdateJumpCount > 0).length }, rounds, turnColumns: turnTiming.columns, turnTimingTable: turnTiming.rows, turnTimingNonMonotonic, + turnTimingTerminalElapsedGrowth, turnTimingRecentUpdateSawtoothJumps, turnTimingRecentUpdateSteps, turnTimingRecentUpdateLargestSteps, @@ -2263,6 +2272,7 @@ function buildTurnTimingTable(samples, timeline) { columns, rows, nonMonotonic: timingEvents.anomalies, + terminalElapsedGrowth: timingEvents.terminalElapsedGrowth, recentUpdateResets: timingEvents.recentUpdateResets, recentUpdateSteps: timingEvents.recentUpdateSteps }; @@ -2270,10 +2280,12 @@ function buildTurnTimingTable(samples, timeline) { function detectTurnTimingNonMonotonic(columns, rows) { const anomalies = []; + const terminalElapsedGrowth = []; const recentUpdateResets = []; const recentUpdateSteps = []; for (const column of columns) { const previousByMetric = new Map(); + let previousTerminalTotal = null; for (const row of rows) { const cell = row.cells?.[column.id]; if (!cell) continue; @@ -2302,6 +2314,32 @@ function detectTurnTimingNonMonotonic(columns, rows) { valuesRedacted: true }); } + if (metric === "totalElapsedSeconds" && isTerminalTurnStatus(cell.status)) { + if (previousTerminalTotal && current > previousTerminalTotal.value) { + terminalElapsedGrowth.push({ + columnId: column.id, + columnLabel: column.label, + metric, + anomaly: "terminal-growth", + expectedPattern: "terminal-total-elapsed-sealed", + fromSeq: previousTerminalTotal.seq, + fromTs: previousTerminalTotal.ts, + fromValue: previousTerminalTotal.value, + fromStatus: previousTerminalTotal.status, + toSeq: row.seq ?? null, + toTs: row.ts ?? null, + toValue: current, + toStatus: cell.status ?? null, + delta: current - previousTerminalTotal.value, + traceId: cell.traceId ?? column.traceId ?? null, + messageId: cell.messageId ?? column.messageId ?? null, + promptIndex: cell.promptIndex ?? row.promptIndex ?? null, + source: cell.source ?? column.source ?? null, + valuesRedacted: true + }); + } + previousTerminalTotal = { value: current, seq: row.seq ?? null, ts: row.ts ?? null, status: cell.status ?? null }; + } if (previous && metric === "recentUpdateSeconds") { const elapsedMs = Date.parse(String(row.ts ?? "")) - Date.parse(String(previous.ts ?? "")); const elapsedSeconds = Number.isFinite(elapsedMs) && elapsedMs >= 0 ? elapsedMs / 1000 : null; @@ -2380,7 +2418,12 @@ function detectTurnTimingNonMonotonic(columns, rows) { } } } - return { anomalies, recentUpdateResets, recentUpdateSteps }; + return { anomalies, terminalElapsedGrowth, recentUpdateResets, recentUpdateSteps }; +} + +function isTerminalTurnStatus(value) { + const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); + return ["completed", "failed", "blocked", "timeout", "canceled", "cancelled", "stale", "thread-resume-failed"].includes(status); } function turnMetricItems(sample, timelineItem) { @@ -2482,6 +2525,7 @@ function firstTraceId(texts) { function buildRoundMetricSummaries(timeline, promptCommands, timing = {}) { const rounds = []; const nonMonotonic = Array.isArray(timing.nonMonotonic) ? timing.nonMonotonic : []; + const terminalElapsedGrowth = Array.isArray(timing.terminalElapsedGrowth) ? timing.terminalElapsedGrowth : []; const recentUpdateResets = Array.isArray(timing.recentUpdateResets) ? timing.recentUpdateResets : []; const recentUpdateSteps = Array.isArray(timing.recentUpdateSteps) ? timing.recentUpdateSteps : []; for (let index = 0; index < promptCommands.length; index += 1) { @@ -2490,8 +2534,10 @@ function buildRoundMetricSummaries(timeline, promptCommands, timing = {}) { const totalElapsed = items.map((item) => item.totalElapsedSeconds).filter((value) => value !== null); const recentUpdate = items.map((item) => item.recentUpdateSeconds).filter((value) => value !== null); const timingAnomalies = nonMonotonic.filter((item) => item.promptIndex === promptIndex); + const timingTerminalGrowth = terminalElapsedGrowth.filter((item) => item.promptIndex === promptIndex); const timingResets = recentUpdateResets.filter((item) => item.promptIndex === promptIndex); const timingSteps = recentUpdateSteps.filter((item) => item.promptIndex === promptIndex); + const terminalGrowthDeltas = timingTerminalGrowth.map((item) => Number(item.delta)).filter((value) => Number.isFinite(value) && value > 0); const timingStepDeltas = timingSteps.map((item) => Number(item.delta)).filter((value) => Number.isFinite(value) && value >= 0); const timingStepExcess = timingSteps.map((item) => Number(item.excessiveIncreaseSeconds)).filter((value) => Number.isFinite(value) && value > 0); rounds.push({ @@ -2516,6 +2562,8 @@ function buildRoundMetricSummaries(timeline, promptCommands, timing = {}) { finalTextSamples: items.filter((item) => item.finalResultTextSeen).length, turnTimingNonMonotonicCount: timingAnomalies.length, turnTimingTotalElapsedDecreaseCount: timingAnomalies.filter((item) => item.metric === "totalElapsedSeconds").length, + turnTimingTerminalElapsedGrowthCount: timingTerminalGrowth.length, + turnTimingTerminalElapsedGrowthMaxSeconds: terminalGrowthDeltas.length > 0 ? Math.max(...terminalGrowthDeltas) : 0, turnTimingRecentUpdateJumpCount: timingAnomalies.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump").length, turnTimingRecentUpdateSawtoothJumpCount: timingAnomalies.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump").length, turnTimingRecentUpdateStepCount: timingSteps.length, @@ -2686,6 +2734,10 @@ function renderTurnTimingTable(sampleMetrics) { const nonMonotonicLines = nonMonotonic.length > 0 ? nonMonotonic.slice(0, 80).map((item) => "- " + (item.columnLabel || item.columnId || "-") + " " + item.metric + (item.anomaly ? " " + item.anomaly : "") + " " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " sampleDelta=" + (item.sampleDeltaSeconds ?? "-") + " allowed=" + (item.allowedIncreaseSeconds ?? "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") : "- 未观察到总耗时下降或最近更新异常跳增。"; + const terminalGrowth = Array.isArray(sampleMetrics?.turnTimingTerminalElapsedGrowth) ? sampleMetrics.turnTimingTerminalElapsedGrowth : []; + const terminalGrowthLines = terminalGrowth.length > 0 + ? terminalGrowth.slice(0, 80).map((item) => "- " + (item.columnLabel || item.columnId || "-") + " terminal totalElapsed growth " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " status " + (item.fromStatus || "-") + " -> " + (item.toStatus || "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") + : "- 未观察到 terminal 后总耗时增长。"; const sawtoothJumps = Array.isArray(sampleMetrics?.turnTimingRecentUpdateSawtoothJumps) ? sampleMetrics.turnTimingRecentUpdateSawtoothJumps : nonMonotonic.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump"); @@ -2704,7 +2756,7 @@ function renderTurnTimingTable(sampleMetrics) { const resetLines = recentUpdateResets.length > 0 ? recentUpdateResets.slice(0, 80).map((item) => "- " + (item.columnLabel || item.columnId || "-") + " reset " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " sampleDelta=" + (item.sampleDeltaSeconds ?? "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") : "- 未观察到最近更新归零/回落。"; - return lines.join("\n") + "\n\n列说明：\n" + columnLines + "\n\n异常事件（仅报表暴露，不做下游 repair；最近更新按三角波模型检测异常跳增）：\n" + nonMonotonicLines + "\n\n最近更新 sawtooth jump 事件（预期每秒增长约 1，遇到新活动归零；例如 1 秒 -> 1 分 4 秒应被列入这里）：\n" + sawtoothJumpLines + "\n\n最近更新相邻采样 step（按 delta 降序；用于人工识别一秒跳几十秒/一分钟的瞬态）：\n" + stepLines + "\n\n最近更新 reset 事件（预期三角波归零，不计为异常）：\n" + resetLines; + return lines.join("\n") + "\n\n列说明：\n" + columnLines + "\n\n异常事件（仅报表暴露，不做下游 repair；最近更新按三角波模型检测异常跳增）：\n" + nonMonotonicLines + "\n\nTerminal 后总耗时增长事件（终态 turn 的总耗时应 sealed，不应继续增长）：\n" + terminalGrowthLines + "\n\n最近更新 sawtooth jump 事件（预期每秒增长约 1，遇到新活动归零；例如 1 秒 -> 1 分 4 秒应被列入这里）：\n" + sawtoothJumpLines + "\n\n最近更新相邻采样 step（按 delta 降序；用于人工识别一秒跳几十秒/一分钟的瞬态）：\n" + stepLines + "\n\n最近更新 reset 事件（预期三角波归零，不计为异常）：\n" + resetLines; } function formatMetricCell(value) { @@ -2741,7 +2793,7 @@ function renderMarkdown(report) { ? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n") : "- 无 prompt 网络记录。"; const roundLines = Array.isArray(report.sampleMetrics?.rounds) && report.sampleMetrics.rounds.length > 0 - ? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " totalDecrease=" + (item.turnTimingTotalElapsedDecreaseCount ?? 0) + " recentJump=" + (item.turnTimingRecentUpdateJumpCount ?? 0) + " recentSawtoothJump=" + (item.turnTimingRecentUpdateSawtoothJumpCount ?? item.turnTimingRecentUpdateJumpCount ?? 0) + " recentStep=" + (item.turnTimingRecentUpdateStepCount ?? 0) + " recentMaxIncrease=" + (item.turnTimingRecentUpdateMaxIncreaseSeconds ?? "-") + " recentMaxExcess=" + (item.turnTimingRecentUpdateMaxExcessSeconds ?? 0) + " recentReset=" + (item.turnTimingRecentUpdateResetCount ?? 0) + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n") + ? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " totalDecrease=" + (item.turnTimingTotalElapsedDecreaseCount ?? 0) + " terminalGrowth=" + (item.turnTimingTerminalElapsedGrowthCount ?? 0) + " terminalGrowthMax=" + (item.turnTimingTerminalElapsedGrowthMaxSeconds ?? 0) + " recentJump=" + (item.turnTimingRecentUpdateJumpCount ?? 0) + " recentSawtoothJump=" + (item.turnTimingRecentUpdateSawtoothJumpCount ?? item.turnTimingRecentUpdateJumpCount ?? 0) + " recentStep=" + (item.turnTimingRecentUpdateStepCount ?? 0) + " recentMaxIncrease=" + (item.turnTimingRecentUpdateMaxIncreaseSeconds ?? "-") + " recentMaxExcess=" + (item.turnTimingRecentUpdateMaxExcessSeconds ?? 0) + " recentReset=" + (item.turnTimingRecentUpdateResetCount ?? 0) + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n") : "- 无轮次指标。"; const provenanceLines = Array.isArray(report.pageProvenance?.segments) && report.pageProvenance.segments.length > 0 ? report.pageProvenance.segments.slice(0, 40).map((item) => "- fingerprint=" + (item.assetFingerprint || "-") + " samples=" + item.sampleCount + " seq=" + (item.firstSeq ?? "-") + ".." + (item.lastSeq ?? "-") + " ts=" + (item.firstAt || "-") + ".." + (item.lastAt || "-") + " scripts=" + (item.scriptCount ?? "-") + " styles=" + (item.stylesheetCount ?? "-") + " urlPaths=" + (Array.isArray(item.urlPaths) ? item.urlPaths.slice(0, 4).join(",") : "-")).join("\n") @@ -2772,6 +2824,8 @@ function renderMarkdown(report) { + "- turnTimingRows: " + (metricSummary.turnTimingRows ?? 0) + "\n" + "- turnTimingNonMonotonicCount: " + (metricSummary.turnTimingNonMonotonicCount ?? 0) + "\n" + "- turnTimingTotalElapsedDecreaseCount: " + (metricSummary.turnTimingTotalElapsedDecreaseCount ?? 0) + "\n" + + "- turnTimingTerminalElapsedGrowthCount: " + (metricSummary.turnTimingTerminalElapsedGrowthCount ?? 0) + "\n" + + "- turnTimingTerminalElapsedGrowthMaxSeconds: " + (metricSummary.turnTimingTerminalElapsedGrowthMaxSeconds ?? 0) + "\n" + "- turnTimingRecentUpdateJumpCount: " + (metricSummary.turnTimingRecentUpdateJumpCount ?? 0) + "\n" + "- turnTimingRecentUpdateSawtoothJumpCount: " + (metricSummary.turnTimingRecentUpdateSawtoothJumpCount ?? metricSummary.turnTimingRecentUpdateJumpCount ?? 0) + "\n" + "- turnTimingRecentUpdateStepCount: " + (metricSummary.turnTimingRecentUpdateStepCount ?? 0) + "\n"