From 21e71125605c67a98758efca85535f0d955b2155 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 20 Jun 2026 23:45:13 +0000 Subject: [PATCH] fix(web-probe): report recent update step jumps --- .../hwlab-node-web-observe-runner-source.ts | 78 +++++++++++++++++-- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index a681e417..790258b5 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -853,7 +853,7 @@ const report = { await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 }); await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 }); const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]); -console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: runtimeAlerts.summary, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); +console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: runtimeAlerts.summary, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); async function readJson(file) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } @@ -1369,12 +1369,25 @@ function buildSampleMetrics(samples, control) { const turnTimingNonMonotonic = Array.isArray(turnTiming.nonMonotonic) ? turnTiming.nonMonotonic : []; const turnTimingRecentUpdateSawtoothJumps = turnTimingNonMonotonic.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump"); const turnTimingRecentUpdateResets = Array.isArray(turnTiming.recentUpdateResets) ? turnTiming.recentUpdateResets : []; + const turnTimingRecentUpdateSteps = Array.isArray(turnTiming.recentUpdateSteps) ? turnTiming.recentUpdateSteps : []; + const turnTimingRecentUpdateLargestSteps = turnTimingRecentUpdateSteps + .filter((item) => Number.isFinite(Number(item.delta))) + .slice() + .sort((a, b) => Number(b.delta) - Number(a.delta)) + .slice(0, 200); + const turnTimingRecentUpdatePositiveSteps = turnTimingRecentUpdateSteps + .map((item) => Number(item.delta)) + .filter((value) => Number.isFinite(value) && value >= 0); + const turnTimingRecentUpdateExcessSteps = turnTimingRecentUpdateSteps + .map((item) => Number(item.excessiveIncreaseSeconds)) + .filter((value) => Number.isFinite(value) && value > 0); const withTotal = timeline.filter((item) => item.totalElapsedSeconds !== null).length; const withRecent = timeline.filter((item) => item.recentUpdateSeconds !== null).length; const diagnostics = timeline.filter((item) => item.diagnosticSeen).length; const rounds = buildRoundMetricSummaries(timeline, promptCommands, { nonMonotonic: turnTimingNonMonotonic, - recentUpdateResets: turnTimingRecentUpdateResets + recentUpdateResets: turnTimingRecentUpdateResets, + recentUpdateSteps: turnTimingRecentUpdateSteps }); const recentUpdateJumpCount = turnTimingRecentUpdateSawtoothJumps.length; return { @@ -1393,6 +1406,9 @@ function buildSampleMetrics(samples, control) { turnTimingTotalElapsedDecreaseCount: turnTimingNonMonotonic.filter((item) => item.metric === "totalElapsedSeconds").length, turnTimingRecentUpdateJumpCount: recentUpdateJumpCount, turnTimingRecentUpdateSawtoothJumpCount: recentUpdateJumpCount, + turnTimingRecentUpdateStepCount: turnTimingRecentUpdateSteps.length, + turnTimingRecentUpdateMaxIncreaseSeconds: turnTimingRecentUpdatePositiveSteps.length > 0 ? Math.max(...turnTimingRecentUpdatePositiveSteps) : null, + turnTimingRecentUpdateMaxExcessSeconds: turnTimingRecentUpdateExcessSteps.length > 0 ? Math.max(...turnTimingRecentUpdateExcessSteps) : 0, turnTimingRecentUpdateResetCount: turnTimingRecentUpdateResets.length, turnTimingRecentUpdateDecreaseCount: turnTimingRecentUpdateResets.length, roundsWithTurnTimingNonMonotonic: rounds.filter((item) => item.turnTimingNonMonotonicCount > 0).length, @@ -1403,6 +1419,8 @@ function buildSampleMetrics(samples, control) { turnTimingTable: turnTiming.rows, turnTimingNonMonotonic, turnTimingRecentUpdateSawtoothJumps, + turnTimingRecentUpdateSteps, + turnTimingRecentUpdateLargestSteps, turnTimingRecentUpdateResets, timeline }; @@ -1468,13 +1486,15 @@ function buildTurnTimingTable(samples, timeline) { columns, rows, nonMonotonic: timingEvents.anomalies, - recentUpdateResets: timingEvents.recentUpdateResets + recentUpdateResets: timingEvents.recentUpdateResets, + recentUpdateSteps: timingEvents.recentUpdateSteps }; } function detectTurnTimingNonMonotonic(columns, rows) { const anomalies = []; const recentUpdateResets = []; + const recentUpdateSteps = []; for (const column of columns) { const previousByMetric = new Map(); for (const row of rows) { @@ -1509,6 +1529,30 @@ function detectTurnTimingNonMonotonic(columns, rows) { const elapsedMs = Date.parse(String(row.ts ?? "")) - Date.parse(String(previous.ts ?? "")); const elapsedSeconds = Number.isFinite(elapsedMs) && elapsedMs >= 0 ? elapsedMs / 1000 : null; const increase = current - previous.value; + const allowedIncrease = elapsedSeconds === null ? 3 : Math.max(3, elapsedSeconds + 2); + const excessiveIncrease = increase > allowedIncrease ? increase - allowedIncrease : 0; + recentUpdateSteps.push({ + columnId: column.id, + columnLabel: column.label, + metric: "recentUpdateSeconds", + event: increase < 0 ? "reset" : excessiveIncrease > 0 ? "jump" : "increase", + expectedPattern: "sawtooth-increase-or-reset", + fromSeq: previous.seq, + fromTs: previous.ts, + fromValue: previous.value, + toSeq: row.seq ?? null, + toTs: row.ts ?? null, + toValue: current, + delta: increase, + sampleDeltaSeconds: elapsedSeconds, + allowedIncreaseSeconds: allowedIncrease, + excessiveIncreaseSeconds: excessiveIncrease, + traceId: cell.traceId ?? column.traceId ?? null, + messageId: cell.messageId ?? column.messageId ?? null, + promptIndex: cell.promptIndex ?? row.promptIndex ?? null, + source: cell.source ?? column.source ?? null, + valuesRedacted: true + }); if (increase < 0) { recentUpdateResets.push({ columnId: column.id, @@ -1530,8 +1574,7 @@ function detectTurnTimingNonMonotonic(columns, rows) { valuesRedacted: true }); } - const allowedIncrease = elapsedSeconds === null ? 3 : Math.max(3, elapsedSeconds + 2); - if (increase > allowedIncrease) { + if (excessiveIncrease > 0) { anomalies.push({ columnId: column.id, columnLabel: column.label, @@ -1547,6 +1590,7 @@ function detectTurnTimingNonMonotonic(columns, rows) { delta: increase, sampleDeltaSeconds: elapsedSeconds, allowedIncreaseSeconds: allowedIncrease, + excessiveIncreaseSeconds: excessiveIncrease, traceId: cell.traceId ?? column.traceId ?? null, messageId: cell.messageId ?? column.messageId ?? null, promptIndex: cell.promptIndex ?? row.promptIndex ?? null, @@ -1559,7 +1603,7 @@ function detectTurnTimingNonMonotonic(columns, rows) { } } } - return { anomalies, recentUpdateResets }; + return { anomalies, recentUpdateResets, recentUpdateSteps }; } function turnMetricItems(sample, timelineItem) { @@ -1651,6 +1695,7 @@ function buildRoundMetricSummaries(timeline, promptCommands, timing = {}) { const rounds = []; const nonMonotonic = Array.isArray(timing.nonMonotonic) ? timing.nonMonotonic : []; const recentUpdateResets = Array.isArray(timing.recentUpdateResets) ? timing.recentUpdateResets : []; + const recentUpdateSteps = Array.isArray(timing.recentUpdateSteps) ? timing.recentUpdateSteps : []; for (let index = 0; index < promptCommands.length; index += 1) { const promptIndex = index + 1; const items = timeline.filter((item) => item.promptIndex === promptIndex); @@ -1658,6 +1703,9 @@ function buildRoundMetricSummaries(timeline, promptCommands, timing = {}) { const recentUpdate = items.map((item) => item.recentUpdateSeconds).filter((value) => value !== null); const timingAnomalies = nonMonotonic.filter((item) => item.promptIndex === promptIndex); const timingResets = recentUpdateResets.filter((item) => item.promptIndex === promptIndex); + const timingSteps = recentUpdateSteps.filter((item) => item.promptIndex === promptIndex); + const timingStepDeltas = timingSteps.map((item) => Number(item.delta)).filter((value) => Number.isFinite(value) && value >= 0); + const timingStepExcess = timingSteps.map((item) => Number(item.excessiveIncreaseSeconds)).filter((value) => Number.isFinite(value) && value > 0); rounds.push({ promptIndex, promptCommandId: promptCommands[index].commandId, @@ -1682,6 +1730,9 @@ function buildRoundMetricSummaries(timeline, promptCommands, timing = {}) { turnTimingTotalElapsedDecreaseCount: timingAnomalies.filter((item) => item.metric === "totalElapsedSeconds").length, turnTimingRecentUpdateJumpCount: timingAnomalies.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump").length, turnTimingRecentUpdateSawtoothJumpCount: timingAnomalies.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump").length, + turnTimingRecentUpdateStepCount: timingSteps.length, + turnTimingRecentUpdateMaxIncreaseSeconds: timingStepDeltas.length > 0 ? Math.max(...timingStepDeltas) : null, + turnTimingRecentUpdateMaxExcessSeconds: timingStepExcess.length > 0 ? Math.max(...timingStepExcess) : 0, turnTimingRecentUpdateResetCount: timingResets.length, turnTimingRecentUpdateDecreaseCount: timingResets.length }); @@ -1853,11 +1904,19 @@ function renderTurnTimingTable(sampleMetrics) { const sawtoothJumpLines = sawtoothJumps.length > 0 ? sawtoothJumps.slice(0, 80).map((item) => "- " + (item.columnLabel || item.columnId || "-") + " recentUpdate sawtooth-jump " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " sampleDelta=" + (item.sampleDeltaSeconds ?? "-") + " allowed=" + (item.allowedIncreaseSeconds ?? "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") : "- 未观察到最近更新三角波异常跳增。"; + const recentUpdateSteps = Array.isArray(sampleMetrics?.turnTimingRecentUpdateLargestSteps) + ? sampleMetrics.turnTimingRecentUpdateLargestSteps + : Array.isArray(sampleMetrics?.turnTimingRecentUpdateSteps) + ? sampleMetrics.turnTimingRecentUpdateSteps.filter((item) => Number.isFinite(Number(item.delta))).slice().sort((a, b) => Number(b.delta) - Number(a.delta)).slice(0, 200) + : []; + const stepLines = recentUpdateSteps.length > 0 + ? recentUpdateSteps.slice(0, 80).map((item) => "- " + (item.columnLabel || item.columnId || "-") + " recentUpdate step " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " sampleDelta=" + (item.sampleDeltaSeconds ?? "-") + " allowed=" + (item.allowedIncreaseSeconds ?? "-") + " excess=" + (item.excessiveIncreaseSeconds ?? 0) + " event=" + (item.event || "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") + : "- 未观察到最近更新相邻采样 step。"; const recentUpdateResets = Array.isArray(sampleMetrics?.turnTimingRecentUpdateResets) ? sampleMetrics.turnTimingRecentUpdateResets : []; const resetLines = recentUpdateResets.length > 0 ? recentUpdateResets.slice(0, 80).map((item) => "- " + (item.columnLabel || item.columnId || "-") + " reset " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " sampleDelta=" + (item.sampleDeltaSeconds ?? "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") : "- 未观察到最近更新归零/回落。"; - return lines.join("\n") + "\n\n列说明：\n" + columnLines + "\n\n异常事件（仅报表暴露，不做下游 repair；最近更新按三角波模型检测异常跳增）：\n" + nonMonotonicLines + "\n\n最近更新 sawtooth jump 事件（预期每秒增长约 1，遇到新活动归零；例如 1 秒 -> 1 分 4 秒应被列入这里）：\n" + sawtoothJumpLines + "\n\n最近更新 reset 事件（预期三角波归零，不计为异常）：\n" + resetLines; + return lines.join("\n") + "\n\n列说明：\n" + columnLines + "\n\n异常事件（仅报表暴露，不做下游 repair；最近更新按三角波模型检测异常跳增）：\n" + nonMonotonicLines + "\n\n最近更新 sawtooth jump 事件（预期每秒增长约 1，遇到新活动归零；例如 1 秒 -> 1 分 4 秒应被列入这里）：\n" + sawtoothJumpLines + "\n\n最近更新相邻采样 step（按 delta 降序；用于人工识别一秒跳几十秒/一分钟的瞬态）：\n" + stepLines + "\n\n最近更新 reset 事件（预期三角波归零，不计为异常）：\n" + resetLines; } function formatMetricCell(value) { @@ -1894,7 +1953,7 @@ function renderMarkdown(report) { ? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n") : "- 无 prompt 网络记录。"; const roundLines = Array.isArray(report.sampleMetrics?.rounds) && report.sampleMetrics.rounds.length > 0 - ? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " totalDecrease=" + (item.turnTimingTotalElapsedDecreaseCount ?? 0) + " recentJump=" + (item.turnTimingRecentUpdateJumpCount ?? 0) + " recentSawtoothJump=" + (item.turnTimingRecentUpdateSawtoothJumpCount ?? item.turnTimingRecentUpdateJumpCount ?? 0) + " recentReset=" + (item.turnTimingRecentUpdateResetCount ?? 0) + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n") + ? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " totalDecrease=" + (item.turnTimingTotalElapsedDecreaseCount ?? 0) + " recentJump=" + (item.turnTimingRecentUpdateJumpCount ?? 0) + " recentSawtoothJump=" + (item.turnTimingRecentUpdateSawtoothJumpCount ?? item.turnTimingRecentUpdateJumpCount ?? 0) + " recentStep=" + (item.turnTimingRecentUpdateStepCount ?? 0) + " recentMaxIncrease=" + (item.turnTimingRecentUpdateMaxIncreaseSeconds ?? "-") + " recentMaxExcess=" + (item.turnTimingRecentUpdateMaxExcessSeconds ?? 0) + " recentReset=" + (item.turnTimingRecentUpdateResetCount ?? 0) + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n") : "- 无轮次指标。"; const metricLines = Array.isArray(report.sampleMetrics?.timeline) && report.sampleMetrics.timeline.length > 0 ? report.sampleMetrics.timeline.slice(0, 120).map((item) => "- #" + item.seq + " " + item.ts + " prompt=" + item.promptIndex + " totalElapsedSeconds=" + (item.totalElapsedSeconds ?? "-") + " recentUpdateSeconds=" + (item.recentUpdateSeconds ?? "-") + " terminal=" + item.terminalSeen + " finalText=" + item.finalResultTextSeen + " diagnostic=" + item.diagnosticSeen).join("\n") @@ -1921,6 +1980,9 @@ function renderMarkdown(report) { + "- turnTimingTotalElapsedDecreaseCount: " + (metricSummary.turnTimingTotalElapsedDecreaseCount ?? 0) + "\n" + "- turnTimingRecentUpdateJumpCount: " + (metricSummary.turnTimingRecentUpdateJumpCount ?? 0) + "\n" + "- turnTimingRecentUpdateSawtoothJumpCount: " + (metricSummary.turnTimingRecentUpdateSawtoothJumpCount ?? metricSummary.turnTimingRecentUpdateJumpCount ?? 0) + "\n" + + "- turnTimingRecentUpdateStepCount: " + (metricSummary.turnTimingRecentUpdateStepCount ?? 0) + "\n" + + "- turnTimingRecentUpdateMaxIncreaseSeconds: " + (metricSummary.turnTimingRecentUpdateMaxIncreaseSeconds ?? "-") + "\n" + + "- turnTimingRecentUpdateMaxExcessSeconds: " + (metricSummary.turnTimingRecentUpdateMaxExcessSeconds ?? 0) + "\n" + "- turnTimingRecentUpdateResetCount: " + (metricSummary.turnTimingRecentUpdateResetCount ?? 0) + "\n\n" + "### Rounds\n\n" + roundLines + "\n\n" + "### Prompt network\n\n" + promptNetworkLines + "\n\n"