feat(web-probe): report terminal turn duration growth

scripts/src/hwlab-node-impl.ts

@@ -7517,12 +7517,13 @@ function renderWebObserveAnalyzeTable(value: Record<string, unknown>): string {
     ]),
     "",
     "Turn timing:",
-    webObserveTable(["ROUNDS", "TURNS", "ROWS", "NON_MONO", "ELAPSED_DEC", "RECENT_JUMP", "MAX_RECENT_STEP"], [[
+    webObserveTable(["ROUNDS", "TURNS", "ROWS", "NON_MONO", "ELAPSED_DEC", "TERMINAL_GROWTH", "RECENT_JUMP", "MAX_RECENT_STEP"], [[
       webObserveText(roundCount),
       webObserveText(turnColumnCount),
       webObserveText(sampleMetrics?.turnTimingRows),
       webObserveText(sampleMetrics?.turnTimingNonMonotonicCount),
       webObserveText(sampleMetrics?.turnTimingTotalElapsedDecreaseCount),
+      webObserveText(sampleMetrics?.turnTimingTerminalElapsedGrowthCount),
       webObserveText(sampleMetrics?.turnTimingRecentUpdateJumpCount ?? sampleMetrics?.turnTimingRecentUpdateSawtoothJumpCount),
       webObserveText(sampleMetrics?.turnTimingRecentUpdateMaxIncreaseSeconds),
     ]]),

scripts/src/hwlab-node-web-observe-runner-source.ts

@@ -2145,8 +2145,12 @@ function buildSampleMetrics(samples, control) {
   const turnCells = turnTiming.rows.flatMap((row) => Object.values(row.cells || {}));
   const turnTimingNonMonotonic = Array.isArray(turnTiming.nonMonotonic) ? turnTiming.nonMonotonic : [];
   const turnTimingRecentUpdateSawtoothJumps = turnTimingNonMonotonic.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump");
+  const turnTimingTerminalElapsedGrowth = Array.isArray(turnTiming.terminalElapsedGrowth) ? turnTiming.terminalElapsedGrowth : [];
   const turnTimingRecentUpdateResets = Array.isArray(turnTiming.recentUpdateResets) ? turnTiming.recentUpdateResets : [];
   const turnTimingRecentUpdateSteps = Array.isArray(turnTiming.recentUpdateSteps) ? turnTiming.recentUpdateSteps : [];
+  const turnTimingTerminalElapsedGrowthDeltas = turnTimingTerminalElapsedGrowth
+    .map((item) => Number(item.delta))
+    .filter((value) => Number.isFinite(value) && value > 0);
   const turnTimingRecentUpdateLargestSteps = turnTimingRecentUpdateSteps
     .filter((item) => Number.isFinite(Number(item.delta)))
     .slice()
@@ -2163,6 +2167,7 @@ function buildSampleMetrics(samples, control) {
   const diagnostics = timeline.filter((item) => item.diagnosticSeen).length;
   const rounds = buildRoundMetricSummaries(timeline, promptCommands, {
     nonMonotonic: turnTimingNonMonotonic,
+    terminalElapsedGrowth: turnTimingTerminalElapsedGrowth,
     recentUpdateResets: turnTimingRecentUpdateResets,
     recentUpdateSteps: turnTimingRecentUpdateSteps
   });
@@ -2181,6 +2186,8 @@ function buildSampleMetrics(samples, control) {
       turnCellsWithRecentUpdate: turnCells.filter((item) => item.recentUpdateSeconds !== null).length,
       turnTimingNonMonotonicCount: turnTimingNonMonotonic.length,
       turnTimingTotalElapsedDecreaseCount: turnTimingNonMonotonic.filter((item) => item.metric === "totalElapsedSeconds").length,
+      turnTimingTerminalElapsedGrowthCount: turnTimingTerminalElapsedGrowth.length,
+      turnTimingTerminalElapsedGrowthMaxSeconds: turnTimingTerminalElapsedGrowthDeltas.length > 0 ? Math.max(...turnTimingTerminalElapsedGrowthDeltas) : 0,
       turnTimingRecentUpdateJumpCount: recentUpdateJumpCount,
       turnTimingRecentUpdateSawtoothJumpCount: recentUpdateJumpCount,
       turnTimingRecentUpdateStepCount: turnTimingRecentUpdateSteps.length,
@@ -2189,12 +2196,14 @@ function buildSampleMetrics(samples, control) {
       turnTimingRecentUpdateResetCount: turnTimingRecentUpdateResets.length,
       turnTimingRecentUpdateDecreaseCount: turnTimingRecentUpdateResets.length,
       roundsWithTurnTimingNonMonotonic: rounds.filter((item) => item.turnTimingNonMonotonicCount > 0).length,
+      roundsWithTerminalElapsedGrowth: rounds.filter((item) => item.turnTimingTerminalElapsedGrowthCount > 0).length,
       roundsWithRecentUpdateJumps: rounds.filter((item) => item.turnTimingRecentUpdateJumpCount > 0).length
     },
     rounds,
     turnColumns: turnTiming.columns,
     turnTimingTable: turnTiming.rows,
     turnTimingNonMonotonic,
+    turnTimingTerminalElapsedGrowth,
     turnTimingRecentUpdateSawtoothJumps,
     turnTimingRecentUpdateSteps,
     turnTimingRecentUpdateLargestSteps,
@@ -2263,6 +2272,7 @@ function buildTurnTimingTable(samples, timeline) {
     columns,
     rows,
     nonMonotonic: timingEvents.anomalies,
+    terminalElapsedGrowth: timingEvents.terminalElapsedGrowth,
     recentUpdateResets: timingEvents.recentUpdateResets,
     recentUpdateSteps: timingEvents.recentUpdateSteps
   };
@@ -2270,10 +2280,12 @@ function buildTurnTimingTable(samples, timeline) {
 
 function detectTurnTimingNonMonotonic(columns, rows) {
   const anomalies = [];
+  const terminalElapsedGrowth = [];
   const recentUpdateResets = [];
   const recentUpdateSteps = [];
   for (const column of columns) {
     const previousByMetric = new Map();
+    let previousTerminalTotal = null;
     for (const row of rows) {
       const cell = row.cells?.[column.id];
       if (!cell) continue;
@@ -2302,6 +2314,32 @@ function detectTurnTimingNonMonotonic(columns, rows) {
             valuesRedacted: true
           });
         }
+        if (metric === "totalElapsedSeconds" && isTerminalTurnStatus(cell.status)) {
+          if (previousTerminalTotal && current > previousTerminalTotal.value) {
+            terminalElapsedGrowth.push({
+              columnId: column.id,
+              columnLabel: column.label,
+              metric,
+              anomaly: "terminal-growth",
+              expectedPattern: "terminal-total-elapsed-sealed",
+              fromSeq: previousTerminalTotal.seq,
+              fromTs: previousTerminalTotal.ts,
+              fromValue: previousTerminalTotal.value,
+              fromStatus: previousTerminalTotal.status,
+              toSeq: row.seq ?? null,
+              toTs: row.ts ?? null,
+              toValue: current,
+              toStatus: cell.status ?? null,
+              delta: current - previousTerminalTotal.value,
+              traceId: cell.traceId ?? column.traceId ?? null,
+              messageId: cell.messageId ?? column.messageId ?? null,
+              promptIndex: cell.promptIndex ?? row.promptIndex ?? null,
+              source: cell.source ?? column.source ?? null,
+              valuesRedacted: true
+            });
+          }
+          previousTerminalTotal = { value: current, seq: row.seq ?? null, ts: row.ts ?? null, status: cell.status ?? null };
+        }
         if (previous && metric === "recentUpdateSeconds") {
           const elapsedMs = Date.parse(String(row.ts ?? "")) - Date.parse(String(previous.ts ?? ""));
           const elapsedSeconds = Number.isFinite(elapsedMs) && elapsedMs >= 0 ? elapsedMs / 1000 : null;
@@ -2380,7 +2418,12 @@ function detectTurnTimingNonMonotonic(columns, rows) {
       }
     }
   }
-  return { anomalies, recentUpdateResets, recentUpdateSteps };
+  return { anomalies, terminalElapsedGrowth, recentUpdateResets, recentUpdateSteps };
+}
+
+function isTerminalTurnStatus(value) {
+  const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-");
+  return ["completed", "failed", "blocked", "timeout", "canceled", "cancelled", "stale", "thread-resume-failed"].includes(status);
 }
 
 function turnMetricItems(sample, timelineItem) {
@@ -2482,6 +2525,7 @@ function firstTraceId(texts) {
 function buildRoundMetricSummaries(timeline, promptCommands, timing = {}) {
   const rounds = [];
   const nonMonotonic = Array.isArray(timing.nonMonotonic) ? timing.nonMonotonic : [];
+  const terminalElapsedGrowth = Array.isArray(timing.terminalElapsedGrowth) ? timing.terminalElapsedGrowth : [];
   const recentUpdateResets = Array.isArray(timing.recentUpdateResets) ? timing.recentUpdateResets : [];
   const recentUpdateSteps = Array.isArray(timing.recentUpdateSteps) ? timing.recentUpdateSteps : [];
   for (let index = 0; index < promptCommands.length; index += 1) {
@@ -2490,8 +2534,10 @@ function buildRoundMetricSummaries(timeline, promptCommands, timing = {}) {
     const totalElapsed = items.map((item) => item.totalElapsedSeconds).filter((value) => value !== null);
     const recentUpdate = items.map((item) => item.recentUpdateSeconds).filter((value) => value !== null);
     const timingAnomalies = nonMonotonic.filter((item) => item.promptIndex === promptIndex);
+    const timingTerminalGrowth = terminalElapsedGrowth.filter((item) => item.promptIndex === promptIndex);
     const timingResets = recentUpdateResets.filter((item) => item.promptIndex === promptIndex);
     const timingSteps = recentUpdateSteps.filter((item) => item.promptIndex === promptIndex);
+    const terminalGrowthDeltas = timingTerminalGrowth.map((item) => Number(item.delta)).filter((value) => Number.isFinite(value) && value > 0);
     const timingStepDeltas = timingSteps.map((item) => Number(item.delta)).filter((value) => Number.isFinite(value) && value >= 0);
     const timingStepExcess = timingSteps.map((item) => Number(item.excessiveIncreaseSeconds)).filter((value) => Number.isFinite(value) && value > 0);
     rounds.push({
@@ -2516,6 +2562,8 @@ function buildRoundMetricSummaries(timeline, promptCommands, timing = {}) {
       finalTextSamples: items.filter((item) => item.finalResultTextSeen).length,
       turnTimingNonMonotonicCount: timingAnomalies.length,
       turnTimingTotalElapsedDecreaseCount: timingAnomalies.filter((item) => item.metric === "totalElapsedSeconds").length,
+      turnTimingTerminalElapsedGrowthCount: timingTerminalGrowth.length,
+      turnTimingTerminalElapsedGrowthMaxSeconds: terminalGrowthDeltas.length > 0 ? Math.max(...terminalGrowthDeltas) : 0,
       turnTimingRecentUpdateJumpCount: timingAnomalies.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump").length,
       turnTimingRecentUpdateSawtoothJumpCount: timingAnomalies.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump").length,
       turnTimingRecentUpdateStepCount: timingSteps.length,
@@ -2686,6 +2734,10 @@ function renderTurnTimingTable(sampleMetrics) {
   const nonMonotonicLines = nonMonotonic.length > 0
     ? nonMonotonic.slice(0, 80).map((item) => "- " + (item.columnLabel || item.columnId || "-") + " " + item.metric + (item.anomaly ? " " + item.anomaly : "") + " " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " sampleDelta=" + (item.sampleDeltaSeconds ?? "-") + " allowed=" + (item.allowedIncreaseSeconds ?? "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n")
     : "- 未观察到总耗时下降或最近更新异常跳增。";
+  const terminalGrowth = Array.isArray(sampleMetrics?.turnTimingTerminalElapsedGrowth) ? sampleMetrics.turnTimingTerminalElapsedGrowth : [];
+  const terminalGrowthLines = terminalGrowth.length > 0
+    ? terminalGrowth.slice(0, 80).map((item) => "- " + (item.columnLabel || item.columnId || "-") + " terminal totalElapsed growth " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " status " + (item.fromStatus || "-") + " -> " + (item.toStatus || "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n")
+    : "- 未观察到 terminal 后总耗时增长。";
   const sawtoothJumps = Array.isArray(sampleMetrics?.turnTimingRecentUpdateSawtoothJumps)
     ? sampleMetrics.turnTimingRecentUpdateSawtoothJumps
     : nonMonotonic.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump");
@@ -2704,7 +2756,7 @@ function renderTurnTimingTable(sampleMetrics) {
   const resetLines = recentUpdateResets.length > 0
     ? recentUpdateResets.slice(0, 80).map((item) => "- " + (item.columnLabel || item.columnId || "-") + " reset " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " sampleDelta=" + (item.sampleDeltaSeconds ?? "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n")
     : "- 未观察到最近更新归零/回落。";
-  return lines.join("\n") + "\n\n列说明：\n" + columnLines + "\n\n异常事件（仅报表暴露，不做下游 repair；最近更新按三角波模型检测异常跳增）：\n" + nonMonotonicLines + "\n\n最近更新 sawtooth jump 事件（预期每秒增长约 1，遇到新活动归零；例如 1 秒 -> 1 分 4 秒应被列入这里）：\n" + sawtoothJumpLines + "\n\n最近更新相邻采样 step（按 delta 降序；用于人工识别一秒跳几十秒/一分钟的瞬态）：\n" + stepLines + "\n\n最近更新 reset 事件（预期三角波归零，不计为异常）：\n" + resetLines;
+  return lines.join("\n") + "\n\n列说明：\n" + columnLines + "\n\n异常事件（仅报表暴露，不做下游 repair；最近更新按三角波模型检测异常跳增）：\n" + nonMonotonicLines + "\n\nTerminal 后总耗时增长事件（终态 turn 的总耗时应 sealed，不应继续增长）：\n" + terminalGrowthLines + "\n\n最近更新 sawtooth jump 事件（预期每秒增长约 1，遇到新活动归零；例如 1 秒 -> 1 分 4 秒应被列入这里）：\n" + sawtoothJumpLines + "\n\n最近更新相邻采样 step（按 delta 降序；用于人工识别一秒跳几十秒/一分钟的瞬态）：\n" + stepLines + "\n\n最近更新 reset 事件（预期三角波归零，不计为异常）：\n" + resetLines;
 }
 
 function formatMetricCell(value) {
@@ -2741,7 +2793,7 @@ function renderMarkdown(report) {
     ? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n")
     : "- 无 prompt 网络记录。";
   const roundLines = Array.isArray(report.sampleMetrics?.rounds) && report.sampleMetrics.rounds.length > 0
-    ? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " totalDecrease=" + (item.turnTimingTotalElapsedDecreaseCount ?? 0) + " recentJump=" + (item.turnTimingRecentUpdateJumpCount ?? 0) + " recentSawtoothJump=" + (item.turnTimingRecentUpdateSawtoothJumpCount ?? item.turnTimingRecentUpdateJumpCount ?? 0) + " recentStep=" + (item.turnTimingRecentUpdateStepCount ?? 0) + " recentMaxIncrease=" + (item.turnTimingRecentUpdateMaxIncreaseSeconds ?? "-") + " recentMaxExcess=" + (item.turnTimingRecentUpdateMaxExcessSeconds ?? 0) + " recentReset=" + (item.turnTimingRecentUpdateResetCount ?? 0) + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n")
+    ? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " totalDecrease=" + (item.turnTimingTotalElapsedDecreaseCount ?? 0) + " terminalGrowth=" + (item.turnTimingTerminalElapsedGrowthCount ?? 0) + " terminalGrowthMax=" + (item.turnTimingTerminalElapsedGrowthMaxSeconds ?? 0) + " recentJump=" + (item.turnTimingRecentUpdateJumpCount ?? 0) + " recentSawtoothJump=" + (item.turnTimingRecentUpdateSawtoothJumpCount ?? item.turnTimingRecentUpdateJumpCount ?? 0) + " recentStep=" + (item.turnTimingRecentUpdateStepCount ?? 0) + " recentMaxIncrease=" + (item.turnTimingRecentUpdateMaxIncreaseSeconds ?? "-") + " recentMaxExcess=" + (item.turnTimingRecentUpdateMaxExcessSeconds ?? 0) + " recentReset=" + (item.turnTimingRecentUpdateResetCount ?? 0) + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n")
     : "- 无轮次指标。";
   const provenanceLines = Array.isArray(report.pageProvenance?.segments) && report.pageProvenance.segments.length > 0
     ? report.pageProvenance.segments.slice(0, 40).map((item) => "- fingerprint=" + (item.assetFingerprint || "-") + " samples=" + item.sampleCount + " seq=" + (item.firstSeq ?? "-") + ".." + (item.lastSeq ?? "-") + " ts=" + (item.firstAt || "-") + ".." + (item.lastAt || "-") + " scripts=" + (item.scriptCount ?? "-") + " styles=" + (item.stylesheetCount ?? "-") + " urlPaths=" + (Array.isArray(item.urlPaths) ? item.urlPaths.slice(0, 4).join(",") : "-")).join("\n")
@@ -2772,6 +2824,8 @@ function renderMarkdown(report) {
     + "- turnTimingRows: " + (metricSummary.turnTimingRows ?? 0) + "\n"
     + "- turnTimingNonMonotonicCount: " + (metricSummary.turnTimingNonMonotonicCount ?? 0) + "\n"
     + "- turnTimingTotalElapsedDecreaseCount: " + (metricSummary.turnTimingTotalElapsedDecreaseCount ?? 0) + "\n"
+    + "- turnTimingTerminalElapsedGrowthCount: " + (metricSummary.turnTimingTerminalElapsedGrowthCount ?? 0) + "\n"
+    + "- turnTimingTerminalElapsedGrowthMaxSeconds: " + (metricSummary.turnTimingTerminalElapsedGrowthMaxSeconds ?? 0) + "\n"
     + "- turnTimingRecentUpdateJumpCount: " + (metricSummary.turnTimingRecentUpdateJumpCount ?? 0) + "\n"
     + "- turnTimingRecentUpdateSawtoothJumpCount: " + (metricSummary.turnTimingRecentUpdateSawtoothJumpCount ?? metricSummary.turnTimingRecentUpdateJumpCount ?? 0) + "\n"
     + "- turnTimingRecentUpdateStepCount: " + (metricSummary.turnTimingRecentUpdateStepCount ?? 0) + "\n"