diff --git a/scripts/src/hwlab-node-web-observe-analyzer-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-source.ts index 7aa718a4..bc28187f 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-source.ts @@ -259,6 +259,8 @@ console.log(JSON.stringify({ sampleDeltaSeconds: item.sampleDeltaSeconds ?? null, allowedIncreaseSeconds: item.allowedIncreaseSeconds ?? null, excessiveIncreaseSeconds: item.excessiveIncreaseSeconds ?? null, + timingSourceOfTruth: item.timingSourceOfTruth ?? null, + timingStatus: item.timingStatus ?? null, })), turnTimingElapsedZeroResets: recentWindow.sampleMetrics.turnTimingElapsedZeroResets.slice(0, 8).map((item) => ({ columnLabel: item.columnLabel ?? item.columnId ?? null, @@ -275,6 +277,8 @@ console.log(JSON.stringify({ toValue: item.toValue ?? null, delta: item.delta ?? null, anomaly: item.anomaly ?? null, + timingSourceOfTruth: item.timingSourceOfTruth ?? null, + timingStatus: item.timingStatus ?? null, })), turnTimingTotalElapsedForwardJumps: recentWindow.sampleMetrics.turnTimingTotalElapsedForwardJumps.slice(0, 8).map((item) => ({ columnLabel: item.columnLabel ?? item.columnId ?? null, @@ -294,12 +298,14 @@ console.log(JSON.stringify({ allowedIncreaseSeconds: item.allowedIncreaseSeconds ?? null, excessiveIncreaseSeconds: item.excessiveIncreaseSeconds ?? null, anomaly: item.anomaly ?? null, + timingSourceOfTruth: item.timingSourceOfTruth ?? null, + timingStatus: item.timingStatus ?? null, })), pagePerformanceSlowApi: recentWindow.pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream !== true && Number(item.overBudgetCount ?? item.overFiveSecondCount ?? 0) > 0).slice(0, 5).map((item) => ({ path: item.path, route: item.route, sampleCount: item.sampleCount, p95Ms: item.p95Ms, maxMs: item.maxMs, overBudgetCount: item.overBudgetCount, budgetMs: item.budgetMs, overFiveSecondCount: item.overFiveSecondCount, slowSamples: item.slowSamples })), archivePagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream !== true && Number(item.overBudgetCount ?? item.overFiveSecondCount ?? 0) > 0).slice(0, 8).map((item) => ({ path: item.path, route: item.route, sampleCount: item.sampleCount, p95Ms: item.p95Ms, maxMs: item.maxMs, overBudgetCount: item.overBudgetCount, budgetMs: item.budgetMs, overFiveSecondCount: item.overFiveSecondCount, slowSamples: item.slowSamples })), pagePerformancePartialApi: recentWindow.pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream !== true && Number(item.partialOverBudgetCount ?? item.partialOverFiveSecondCount ?? 0) > 0).slice(0, 5).map((item) => ({ path: item.path, route: item.route, sampleCount: item.sampleCount, completeTimingSampleCount: item.completeTimingSampleCount, partialTimingSampleCount: item.partialTimingSampleCount, partialOverBudgetCount: item.partialOverBudgetCount, budgetMs: item.partialBudgetMs, partialOverFiveSecondCount: item.partialOverFiveSecondCount, partialSamples: item.partialSamples })), pagePerformanceSseStreams: recentWindow.pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream === true).slice(0, 5).map((item) => ({ path: item.path, route: item.route, sampleCount: item.sampleCount, streamOpenSampleCount: item.streamOpenSampleCount, streamOpenP95Ms: item.streamOpenP95Ms, streamOpenMaxMs: item.streamOpenMaxMs, streamOpenOverBudgetCount: item.streamOpenOverBudgetCount, streamOpenBudgetMs: item.streamOpenBudgetMs, streamOpenOverFiveSecondCount: item.streamOpenOverFiveSecondCount, streamLifetimeOverFiveSecondCount: item.streamLifetimeOverFiveSecondCount, slowSamples: item.slowSamples })), - findings: prioritizeFindings(recentWindow.findings).slice(0, 8).map((item) => ({ kind: item.id ?? item.kind ?? item.code, severity: item.severity, count: item.count ?? item.sampleCount ?? null, summary: String(item.summary ?? item.message ?? "").slice(0, 180) })), + findings: prioritizeFindings(recentWindow.findings).slice(0, 8).map((item) => ({ kind: item.id ?? item.kind ?? item.code, severity: item.severity, count: item.count ?? item.sampleCount ?? null, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, summary: String(item.summary ?? item.message ?? "").slice(0, 180) })), traceOrderAnomalies: (recentWindow.sampleMetrics?.traceOrder?.orderAnomalies || []).slice(0, 8).map((item) => ({ sampleSeq: item.sampleSeq ?? null, sampleIndex: item.sampleIndex ?? null, @@ -345,6 +351,8 @@ console.log(JSON.stringify({ cardTotalElapsedSeconds: item.cardTotalElapsedSeconds ?? null, deltaSeconds: item.deltaSeconds ?? null, toleranceSeconds: item.toleranceSeconds ?? null, + timingSourceOfTruth: item.timingSourceOfTruth ?? null, + timingStatus: item.timingStatus ?? null, })), codeAgentCardDurationUnderreported: (recentWindow.sampleMetrics?.codeAgentCardTiming?.durationUnderreported || []).slice(0, 8).map((item) => ({ sampleIndex: item.sampleIndex ?? null, @@ -355,6 +363,8 @@ console.log(JSON.stringify({ expectedElapsedSeconds: item.expectedElapsedSeconds ?? null, deltaSeconds: item.deltaSeconds ?? null, evidenceKind: item.evidenceKind ?? null, + timingSourceOfTruth: item.timingSourceOfTruth ?? null, + timingStatus: item.timingStatus ?? null, evidencePreview: String(item.evidencePreview || "").slice(0, 120), })), codeAgentCardDurationMismatches: (recentWindow.sampleMetrics?.codeAgentCardTiming?.durationMismatches || []).slice(0, 8).map((item) => ({ @@ -369,6 +379,8 @@ console.log(JSON.stringify({ deltaSeconds: item.deltaSeconds ?? null, evidenceKind: item.evidenceKind ?? null, exactEvidence: item.exactEvidence === true, + timingSourceOfTruth: item.timingSourceOfTruth ?? null, + timingStatus: item.timingStatus ?? null, evidencePreview: String(item.evidencePreview || "").slice(0, 120), })), valuesRedacted: true, @@ -2747,6 +2759,9 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN id: "code-agent-card-duration-underreported", severity: "amber", summary: "completed Code Agent card total elapsed is shorter than trace/final-response duration evidence; timing mismatch is a non-blocking alert", + timingSourceOfTruth: "trace-completion-total-or-final-response-duration", + timingStatus: timingStatusFromRows(cardTiming.durationUnderreported, "business-turn-completed"), + timingAlert: true, count: cardTimingSummary.durationUnderreportedCount, samples: (cardTiming.durationUnderreported || []).slice(0, 20), }); @@ -2756,6 +2771,9 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN id: "code-agent-card-duration-mismatch", severity: "amber", summary: "completed Code Agent card total elapsed does not match sealed completion/final-response timing evidence; timing mismatch is a non-blocking alert", + timingSourceOfTruth: "trace-completion-total-or-final-response-duration", + timingStatus: timingStatusFromRows(cardTiming.durationMismatches, "business-turn-completed"), + timingAlert: true, count: cardTimingSummary.durationMismatchCount, samples: (cardTiming.durationMismatches || []).slice(0, 20), }); @@ -2780,7 +2798,7 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN samples: (traceOrder.completionNotLast || []).slice(0, 20), }); } - if (Number(cardTimingSummary.roundCompletionElapsedMismatchCount ?? 0) > 0) findings.push({ id: "round-completion-elapsed-mismatch", severity: "amber", summary: "Trace row 轮次完成（总耗时 ...） does not match the visible Code Agent card total elapsed time within YAML timing slack; timing mismatch is a non-blocking alert", count: cardTimingSummary.roundCompletionElapsedMismatchCount, toleranceSeconds: cardTimingSummary.elapsedMismatchToleranceSeconds, samples: (roundCompletion.elapsedMismatches || []).slice(0, 20) }); + if (Number(cardTimingSummary.roundCompletionElapsedMismatchCount ?? 0) > 0) findings.push({ id: "round-completion-elapsed-mismatch", severity: "amber", summary: "Trace row 轮次完成（总耗时 ...） does not match the visible Code Agent card total elapsed time within YAML timing slack; timing mismatch is a non-blocking alert", timingSourceOfTruth: "trace-round-completion-total", timingStatus: timingStatusFromRows(roundCompletion.elapsedMismatches, "business-turn-completed"), timingAlert: true, count: cardTimingSummary.roundCompletionElapsedMismatchCount, toleranceSeconds: cardTimingSummary.elapsedMismatchToleranceSeconds, samples: (roundCompletion.elapsedMismatches || []).slice(0, 20) }); if (Number(cardTimingSummary.roundCompletionFinalResponseMissingCount ?? 0) > 0) findings.push({ id: "round-completion-final-response-missing", severity: "red", summary: "Trace row showed 轮次完成, but no final response was visible in the Code Agent card afterward", count: cardTimingSummary.roundCompletionFinalResponseMissingCount, samples: (roundCompletion.finalResponseMissing || []).slice(0, 20) }); if (Number(cardTimingSummary.roundCompletionPostTimingChangeCount ?? 0) > 0) findings.push({ id: "round-completion-post-timing-change", severity: "amber", summary: "After 轮次完成, card total elapsed or 最近更新 continued changing; terminal timing alert is non-blocking", count: cardTimingSummary.roundCompletionPostTimingChangeCount, samples: (roundCompletion.postCompletionTimingChanges || []).slice(0, 20) }); if (Number(cardTimingSummary.roundCompletionPostRecentUpdateVisibleCount ?? 0) > 0) findings.push({ id: "round-completion-recent-update-still-visible", severity: "info", summary: "最近更新 was still visible after 轮次完成; inspect whether terminal cards should hide activity age or keep it sealed", count: cardTimingSummary.roundCompletionPostRecentUpdateVisibleCount, samples: (roundCompletion.postCompletionRecentUpdateVisible || []).slice(0, 20) }); @@ -2824,15 +2842,15 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN const promptSteerRounds = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.steerUsed === true) : []; if (promptSteerRounds.length > 0) findings.push({ id: "prompt-routed-to-steer", severity: "amber", summary: "sendPrompt was submitted through /v1/agent/chat/steer; verify the previous turn was truly in-flight and not an unsealed terminal failure", count: promptSteerRounds.length, rounds: promptSteerRounds.slice(0, 10) }); const elapsedZeroResets = Array.isArray(sampleMetrics?.turnTimingElapsedZeroResets) ? sampleMetrics.turnTimingElapsedZeroResets : []; - if (elapsedZeroResets.length > 0) findings.push({ id: "turn-timing-total-elapsed-zero-reset", severity: "amber", summary: "Code Agent total elapsed jumped from a non-zero value back to 0 seconds; timing reset is a non-blocking alert", count: elapsedZeroResets.length, samples: elapsedZeroResets.slice(0, 20) }); + if (elapsedZeroResets.length > 0) findings.push({ id: "turn-timing-total-elapsed-zero-reset", severity: "amber", summary: "Code Agent total elapsed jumped from a non-zero value back to 0 seconds; timing reset is a non-blocking alert", timingSourceOfTruth: "dom-card-total-elapsed-sequence", timingStatus: timingStatusFromRows(elapsedZeroResets), timingAlert: true, count: elapsedZeroResets.length, samples: elapsedZeroResets.slice(0, 20) }); const elapsedDecreases = Array.isArray(sampleMetrics?.turnTimingNonMonotonic) ? sampleMetrics.turnTimingNonMonotonic.filter((item) => item.metric === "totalElapsedSeconds" && item.anomaly !== "zero-reset") : []; - if (elapsedDecreases.length > 0) findings.push({ id: "turn-timing-total-elapsed-decrease", severity: "amber", summary: "Code Agent total elapsed decreased between adjacent samples; timing decrease is a non-blocking alert", count: elapsedDecreases.length, samples: elapsedDecreases.slice(0, 20) }); + if (elapsedDecreases.length > 0) findings.push({ id: "turn-timing-total-elapsed-decrease", severity: "amber", summary: "Code Agent total elapsed decreased between adjacent samples; timing decrease is a non-blocking alert", timingSourceOfTruth: "dom-card-total-elapsed-sequence", timingStatus: timingStatusFromRows(elapsedDecreases), timingAlert: true, count: elapsedDecreases.length, samples: elapsedDecreases.slice(0, 20) }); const elapsedForwardJumps = Array.isArray(sampleMetrics?.turnTimingTotalElapsedForwardJumps) ? sampleMetrics.turnTimingTotalElapsedForwardJumps : []; - if (elapsedForwardJumps.length > 0) findings.push({ id: "turn-timing-total-elapsed-forward-jump", severity: "amber", summary: "Code Agent total elapsed jumped forward faster than browser sample interval; timing jump is a non-blocking alert", count: elapsedForwardJumps.length, samples: elapsedForwardJumps.slice(0, 20) }); + if (elapsedForwardJumps.length > 0) findings.push({ id: "turn-timing-total-elapsed-forward-jump", severity: "amber", summary: "Code Agent total elapsed jumped forward faster than browser sample interval; timing jump is a non-blocking alert", timingSourceOfTruth: "dom-card-total-elapsed-sequence", timingStatus: timingStatusFromRows(elapsedForwardJumps), timingAlert: true, count: elapsedForwardJumps.length, samples: elapsedForwardJumps.slice(0, 20) }); const terminalElapsedGrowth = Array.isArray(sampleMetrics?.turnTimingTerminalElapsedGrowth) ? sampleMetrics.turnTimingTerminalElapsedGrowth : []; - if (terminalElapsedGrowth.length > 0) findings.push({ id: "turn-timing-terminal-elapsed-growth", severity: "amber", summary: "terminal Code Agent card total elapsed changed after terminal status; terminal timing alert is non-blocking", count: terminalElapsedGrowth.length, samples: terminalElapsedGrowth.slice(0, 20) }); + if (terminalElapsedGrowth.length > 0) findings.push({ id: "turn-timing-terminal-elapsed-growth", severity: "amber", summary: "terminal Code Agent card total elapsed changed after terminal status; terminal timing alert is non-blocking", timingSourceOfTruth: "terminal-card-total-elapsed-seal", timingStatus: timingStatusFromRows(terminalElapsedGrowth, "business-turn-completed"), timingAlert: true, count: terminalElapsedGrowth.length, samples: terminalElapsedGrowth.slice(0, 20) }); const recentUpdateSawtoothJumps = Array.isArray(sampleMetrics?.turnTimingRecentUpdateSawtoothJumps) ? sampleMetrics.turnTimingRecentUpdateSawtoothJumps : Array.isArray(sampleMetrics?.turnTimingNonMonotonic) @@ -2841,7 +2859,7 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN if (recentUpdateSawtoothJumps.length > 0) findings.push({ id: "turn-timing-recent-update-sawtooth-jump", severity: "amber", summary: "最近更新 value jumped faster than sample interval; expected sawtooth increase-or-reset", count: recentUpdateSawtoothJumps.length, samples: recentUpdateSawtoothJumps.slice(0, 20) }); const severeTimeoutRounds = Array.isArray(sampleMetrics?.rounds) ? sampleMetrics.rounds.filter((item) => Number(item.maxTotalElapsedSeconds) > alertThresholds.turnElapsedSevereTimeoutSeconds) : []; const severeTimeoutSamples = Array.isArray(sampleMetrics?.timeline) ? sampleMetrics.timeline.filter((item) => Number(item.totalElapsedSeconds) > alertThresholds.turnElapsedSevereTimeoutSeconds) : []; - if (severeTimeoutRounds.length > 0 || severeTimeoutSamples.length > 0) findings.push({ id: "turn-elapsed-severe-timeout", severity: "amber", summary: "turn total elapsed exceeded the YAML-configured elapsed alert threshold; timing is a non-blocking alert unless the turn fails to complete or breaks multi-round continuity", thresholdSeconds: alertThresholds.turnElapsedSevereTimeoutSeconds, count: Math.max(severeTimeoutRounds.length, severeTimeoutSamples.length), rounds: severeTimeoutRounds.slice(0, 20), samples: severeTimeoutSamples.slice(0, 20) }); + if (severeTimeoutRounds.length > 0 || severeTimeoutSamples.length > 0) findings.push({ id: "turn-elapsed-severe-timeout", severity: "amber", summary: "turn total elapsed exceeded the YAML-configured elapsed alert threshold; timing is a non-blocking alert unless the turn fails to complete or breaks multi-round continuity", timingSourceOfTruth: "dom-card-total-elapsed-yaml-threshold", timingStatus: timingStatusFromRows([...severeTimeoutRounds, ...severeTimeoutSamples], "observer-timeout"), timingAlert: true, thresholdSeconds: alertThresholds.turnElapsedSevereTimeoutSeconds, count: Math.max(severeTimeoutRounds.length, severeTimeoutSamples.length), rounds: severeTimeoutRounds.slice(0, 20), samples: severeTimeoutSamples.slice(0, 20) }); const loadingSummary = sampleMetrics?.loading?.summary || {}; const visibleLoadingSlowSeconds = alertThresholds.visibleLoadingSlowMs / 1000; if (Number(loadingSummary.longestContinuousSeconds ?? 0) > visibleLoadingSlowSeconds) findings.push({ id: "page-loading-visible-over-budget", severity: "red", summary: "visible 加载中 stayed on screen longer than configured YAML budget; fix real loading latency instead of revealing incomplete content early", count: loadingSummary.overBudgetSegmentCount ?? loadingSummary.overFiveSecondSegmentCount ?? 1, longestContinuousSeconds: loadingSummary.longestContinuousSeconds, budgetSeconds: visibleLoadingSlowSeconds, segments: sampleMetrics.loading.segments.slice(0, 20), owners: sampleMetrics.loading.owners.slice(0, 20) }); diff --git a/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts index f90a8a25..6aff730c 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts @@ -143,6 +143,9 @@ function detectTurnTimingNonMonotonic(columns, rows) { const event = { columnId: column.id, columnLabel: column.label, + ...timingFindingMeta("dom-card-total-elapsed-sequence", timingStatusFromTurnStatus(cell.status ?? previous.status), { + cardElapsedSource: cell.source ?? column.source ?? "dom-card", + }), metric, anomaly, expectedPattern: anomaly === "zero-reset" ? "total-elapsed-should-not-return-to-zero" : "total-elapsed-monotonic", @@ -186,6 +189,9 @@ function detectTurnTimingNonMonotonic(columns, rows) { totalElapsedForwardJumps.push({ columnId: column.id, columnLabel: column.label, + ...timingFindingMeta("dom-card-total-elapsed-sequence", timingStatusFromTurnStatus(cell.status ?? previous.status), { + cardElapsedSource: cell.source ?? column.source ?? "dom-card", + }), metric, anomaly: "forward-jump", expectedPattern: "total-elapsed-increase-should-match-browser-sample-interval", @@ -218,6 +224,9 @@ function detectTurnTimingNonMonotonic(columns, rows) { terminalElapsedGrowth.push({ columnId: column.id, columnLabel: column.label, + ...timingFindingMeta("terminal-card-total-elapsed-seal", timingStatusFromTurnStatus(cell.status, "business-turn-completed"), { + cardElapsedSource: cell.source ?? column.source ?? "dom-card", + }), metric, anomaly: "terminal-growth", expectedPattern: "terminal-total-elapsed-sealed", @@ -507,6 +516,10 @@ function buildCodeAgentCardDurationUnderreportedMetrics(samples, timeline) { if (strongest.seconds > cardSeconds + tolerance) { findings.push({ sampleIndex, + ...timingFindingMeta(strongest.kind, timingStatusFromTurnStatus(card.status || card.state || card.phase, "business-turn-completed"), { + cardElapsedSource: "dom-card-total-elapsed", + expectedElapsedSource: strongest.kind, + }), timestamp: sample.timestamp || sample.collectedAt || sample.time || null, pageRole: card.pageRole || sample.pageRole || sample.role || sample.contextRole || null, pageId: card.pageId || sample.pageId || sample.contextId || null, @@ -557,6 +570,10 @@ function buildCodeAgentCardDurationMismatchMetrics(samples, timeline) { if (!underreported && !overreported) continue; findings.push({ sampleIndex, + ...timingFindingMeta(strongest.kind, timingStatusFromTurnStatus(card.status || card.state || card.phase, "business-turn-completed"), { + cardElapsedSource: "dom-card-total-elapsed", + expectedElapsedSource: strongest.kind, + }), timestamp: sample.timestamp || sample.collectedAt || sample.time || sample.ts || null, pageRole: card.pageRole || sample.pageRole || sample.role || sample.contextRole || null, pageId: card.pageId || sample.pageId || sample.contextId || null, @@ -993,6 +1010,38 @@ function isCodeAgentCardTerminal(item) { return isTerminalTraceText(text) || /轮次完成|轮次失败|轮次取消|已记录|completed|failed|canceled|cancelled|blocked/iu.test(text); } +function timingFindingMeta(sourceOfTruth, status, extra = {}) { + return { + timingSourceOfTruth: sourceOfTruth, + timingStatus: status || "non-blocking-timing-alert", + timingAlert: true, + blocking: false, + ...extra, + }; +} + +function timingStatusFromTurnStatus(status, fallback = "observer-timeout") { + const normalized = String(status ?? "").trim().toLowerCase().replace(/_/gu, "-"); + if (isSuccessfulTurnStatus(normalized)) return "business-turn-completed"; + if (isTerminalTurnStatus(normalized)) return "scenario-incomplete"; + if (isActiveTurnStatus(normalized)) return "observer-timeout"; + return fallback; +} + +function timingStatusFromRows(rows, fallback = "observer-timeout") { + const list = Array.isArray(rows) ? rows : []; + if (list.some((item) => isSuccessfulTurnStatus(item?.status ?? item?.toStatus ?? item?.fromStatus))) return "business-turn-completed"; + if (list.some((item) => Number(item?.finalTextSamples ?? 0) > 0)) return "business-turn-completed"; + if (list.some((item) => isTerminalTurnStatus(item?.status ?? item?.toStatus ?? item?.fromStatus))) return "scenario-incomplete"; + if (list.some((item) => isActiveTurnStatus(item?.status ?? item?.toStatus ?? item?.fromStatus))) return "observer-timeout"; + return fallback; +} + +function isSuccessfulTurnStatus(status) { + const normalized = String(status ?? "").trim().toLowerCase().replace(/_/gu, "-"); + return ["completed", "complete", "succeeded", "success", "done", "ok", "passed"].includes(normalized); +} + function isActiveTurnStatus(value) { const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); return ["pending", "running", "queued", "admitted", "dispatching", "in-progress", "inprogress", "executing", "progress", "thinking", "working", "active", "streaming", "created", "started"].includes(status); @@ -1022,6 +1071,10 @@ function buildRoundCompletionMetrics(samples, timeline, turnTiming) { if (delta > elapsedMismatchToleranceSeconds) { elapsedMismatches.push({ ...eventRef(event), + ...timingFindingMeta("trace-round-completion-total", timingStatusFromTurnStatus(bestCard.status, "business-turn-completed"), { + cardElapsedSource: bestCard.source ?? "dom-card", + completionElapsedSource: "trace-round-completion", + }), cardTotalElapsedSeconds: Number(bestCard.totalElapsedSeconds), completionElapsedSeconds: Number(event.elapsedSeconds), deltaSeconds: Number(delta.toFixed(3)), @@ -1072,6 +1125,8 @@ function roundCompletionEventsForSample(sample, timelineItem) { attributed: Boolean(item?.traceId || item?.messageId || attributed?.traceId || attributed?.messageId), traceRowIndex: item?.rowIndex ?? item?.index ?? null, elapsedSeconds: elapsedValues.length > 0 ? Math.max(...elapsedValues) : null, + timingSourceOfTruth: "trace-round-completion-total", + timingStatus: "business-turn-completed", textHash: item?.textHash ?? sha256(text), preview: limitText(text, 180), valuesRedacted: true @@ -2078,19 +2133,19 @@ function renderTurnTimingTable(sampleMetrics) { const columnLines = columns.map((column) => "- " + formatTurnColumnDisplayLabel(column) + ": pageRole=" + (column.pageRole || "-") + " pageId=" + (column.pageId || "-") + " source=" + (column.source || "-") + " prompt=" + (column.promptIndex ?? "-") + " lastPrompt=" + (column.lastPromptIndex ?? "-") + " firstSeq=" + (column.firstSeq ?? "-") + " lastSeq=" + (column.lastSeq ?? "-") + " traceId=" + (column.traceId || "-") + " messageId=" + (column.messageId || "-")).join("\n"); const nonMonotonic = Array.isArray(sampleMetrics?.turnTimingNonMonotonic) ? sampleMetrics.turnTimingNonMonotonic : []; const nonMonotonicLines = nonMonotonic.length > 0 - ? nonMonotonic.slice(0, 80).map((item) => "- " + formatTurnEventDisplayLabel(item) + " " + item.metric + (item.anomaly ? " " + item.anomaly : "") + " " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " sampleDelta=" + (item.sampleDeltaSeconds ?? "-") + " allowed=" + (item.allowedIncreaseSeconds ?? "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") + ? nonMonotonic.slice(0, 80).map((item) => "- " + formatTurnEventDisplayLabel(item) + " " + item.metric + (item.anomaly ? " " + item.anomaly : "") + " " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " sampleDelta=" + (item.sampleDeltaSeconds ?? "-") + " allowed=" + (item.allowedIncreaseSeconds ?? "-") + formatTimingMeta(item) + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") : "- 未观察到总耗时下降或最近更新异常跳增。"; const terminalGrowth = Array.isArray(sampleMetrics?.turnTimingTerminalElapsedGrowth) ? sampleMetrics.turnTimingTerminalElapsedGrowth : []; const terminalGrowthLines = terminalGrowth.length > 0 - ? terminalGrowth.slice(0, 80).map((item) => "- " + formatTurnEventDisplayLabel(item) + " terminal totalElapsed growth " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " status " + (item.fromStatus || "-") + " -> " + (item.toStatus || "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") + ? terminalGrowth.slice(0, 80).map((item) => "- " + formatTurnEventDisplayLabel(item) + " terminal totalElapsed growth " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " status " + (item.fromStatus || "-") + " -> " + (item.toStatus || "-") + formatTimingMeta(item) + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") : "- 未观察到 terminal 后总耗时增长。"; const elapsedZeroResets = Array.isArray(sampleMetrics?.turnTimingElapsedZeroResets) ? sampleMetrics.turnTimingElapsedZeroResets : []; const elapsedZeroResetLines = elapsedZeroResets.length > 0 - ? elapsedZeroResets.slice(0, 80).map((item) => "- " + formatTurnEventDisplayLabel(item) + " totalElapsed zero-reset " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") + ? elapsedZeroResets.slice(0, 80).map((item) => "- " + formatTurnEventDisplayLabel(item) + " totalElapsed zero-reset " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + formatTimingMeta(item) + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") : "- 未观察到总耗时从非零跳回 0 秒。"; const totalElapsedForwardJumps = Array.isArray(sampleMetrics?.turnTimingTotalElapsedForwardJumps) ? sampleMetrics.turnTimingTotalElapsedForwardJumps : []; const totalElapsedForwardJumpLines = totalElapsedForwardJumps.length > 0 - ? totalElapsedForwardJumps.slice(0, 80).map((item) => "- " + formatTurnEventDisplayLabel(item) + " totalElapsed forward-jump " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " sampleDelta=" + (item.sampleDeltaSeconds ?? "-") + " allowed=" + (item.allowedIncreaseSeconds ?? "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") + ? totalElapsedForwardJumps.slice(0, 80).map((item) => "- " + formatTurnEventDisplayLabel(item) + " totalElapsed forward-jump " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " sampleDelta=" + (item.sampleDeltaSeconds ?? "-") + " allowed=" + (item.allowedIncreaseSeconds ?? "-") + formatTimingMeta(item) + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " ts " + (item.fromTs || "-") + " -> " + (item.toTs || "-") + " traceId=" + (item.traceId || "-")).join("\n") : "- 未观察到总耗时超出采样间隔的异常前跳。"; const sawtoothJumps = Array.isArray(sampleMetrics?.turnTimingRecentUpdateSawtoothJumps) ? sampleMetrics.turnTimingRecentUpdateSawtoothJumps @@ -2113,7 +2168,7 @@ function renderTurnTimingTable(sampleMetrics) { const disclosureLine = disclosure?.truncated ? "表格披露：已按 head/tail 有界输出，totalRows=" + disclosure.totalRows + " includedRows=" + disclosure.includedRows + " omittedRows=" + disclosure.omittedRows + "；异常计数在截断前基于全量采样计算。" : "表格披露：完整输出当前分析窗口的采样点。"; - return disclosureLine + "\n\n" + lines.join("\n") + "\n\n列说明：\n" + columnLines + "\n\n异常事件（仅报表暴露，不做下游 repair；最近更新按三角波模型检测异常跳增）：\n" + nonMonotonicLines + "\n\nTerminal 后总耗时增长事件（终态 turn 的总耗时应 sealed，不应继续增长）：\n" + terminalGrowthLines + "\n\n总耗时归零跳变事件（例如已显示真实耗时后又变成 0 秒）：\n" + elapsedZeroResetLines + "\n\n总耗时异常前跳事件（预期按采样间隔近似递增；例如 14 秒 -> 1137 秒应被列入这里）：\n" + totalElapsedForwardJumpLines + "\n\n最近更新 sawtooth jump 事件（预期每秒增长约 1，遇到新活动归零；例如 1 秒 -> 1 分 4 秒应被列入这里）：\n" + sawtoothJumpLines + "\n\n最近更新相邻采样 step（按 delta 降序；用于人工识别一秒跳几十秒/一分钟的瞬态）：\n" + stepLines + "\n\n最近更新 reset 事件（预期三角波归零，不计为异常）：\n" + resetLines; + return disclosureLine + "\n\n" + lines.join("\n") + "\n\n列说明：\n" + columnLines + "\n\n时间源说明：totalElapsed 异常以 DOM Code Agent card 采样序列为观察源；completion/final-response 可用时作为 sealed source-of-truth；status=business-turn-completed 表示 timing 仅为非阻塞告警，status=observer-timeout 表示观察等待超预算，status=scenario-incomplete 表示业务回合失败或未完成。\n\n异常事件（仅报表暴露，不做下游 repair；最近更新按三角波模型检测异常跳增）：\n" + nonMonotonicLines + "\n\nTerminal 后总耗时增长事件（终态 turn 的总耗时应 sealed，不应继续增长）：\n" + terminalGrowthLines + "\n\n总耗时归零跳变事件（例如已显示真实耗时后又变成 0 秒）：\n" + elapsedZeroResetLines + "\n\n总耗时异常前跳事件（预期按采样间隔近似递增；例如 14 秒 -> 1137 秒应被列入这里）：\n" + totalElapsedForwardJumpLines + "\n\n最近更新 sawtooth jump 事件（预期每秒增长约 1，遇到新活动归零；例如 1 秒 -> 1 分 4 秒应被列入这里）：\n" + sawtoothJumpLines + "\n\n最近更新相邻采样 step（按 delta 降序；用于人工识别一秒跳几十秒/一分钟的瞬态）：\n" + stepLines + "\n\n最近更新 reset 事件（预期三角波归零，不计为异常）：\n" + resetLines; } function formatTurnColumnDisplayLabel(column) { @@ -2146,6 +2201,12 @@ function escapeMarkdownCell(value) { return String(value ?? "-").replace(/\|/gu, "\\|"); } +function formatTimingMeta(item) { + const source = item?.timingSourceOfTruth || item?.expectedElapsedSource || item?.evidenceKind || "-"; + const status = item?.timingStatus || "-"; + return " source=" + source + " status=" + status; +} + function renderMarkdown(report) { const findingLines = report.findings.length === 0 ? "- 无红灯项。" : report.findings.map((item) => "- " + item.severity + ": " + item.id + " - " + item.summary).join("\n"); const commandLines = report.commandTimeline.length === 0 ? "- 无控制命令。" : report.commandTimeline.map((item) => "- " + item.ts + " " + item.phase + " " + item.type + " " + item.commandId + " " + (item.afterUrl || "")).join("\n"); @@ -2208,7 +2269,7 @@ function renderMarkdown(report) { ? roundCompletion.events.slice(0, 80).map((item) => "- #" + (item.seq ?? "-") + " " + (item.ts || "-") + " prompt=" + (item.promptIndex ?? "-") + " role=" + (item.pageRole || "-") + " traceId=" + (item.traceId || "-") + " completionElapsed=" + (item.elapsedSeconds ?? "-") + " preview=" + escapeMarkdownCell(item.preview || "")).join("\n") : "- 未观察到“轮次完成（总耗时 ...）”trace 行。"; const roundCompletionMismatchLines = Array.isArray(roundCompletion.elapsedMismatches) && roundCompletion.elapsedMismatches.length > 0 - ? roundCompletion.elapsedMismatches.slice(0, 80).map((item) => "- #" + (item.seq ?? "-") + " " + (item.ts || "-") + " prompt=" + (item.promptIndex ?? "-") + " traceId=" + (item.traceId || "-") + " completion=" + (item.completionElapsedSeconds ?? "-") + " card=" + (item.cardTotalElapsedSeconds ?? "-") + " delta=" + (item.deltaSeconds ?? "-") + " tolerance=" + (item.toleranceSeconds ?? "-")).join("\n") + ? roundCompletion.elapsedMismatches.slice(0, 80).map((item) => "- #" + (item.seq ?? "-") + " " + (item.ts || "-") + " prompt=" + (item.promptIndex ?? "-") + " traceId=" + (item.traceId || "-") + " completion=" + (item.completionElapsedSeconds ?? "-") + " card=" + (item.cardTotalElapsedSeconds ?? "-") + " delta=" + (item.deltaSeconds ?? "-") + " tolerance=" + (item.toleranceSeconds ?? "-") + formatTimingMeta(item)).join("\n") : "- 未观察到轮次完成耗时与卡片耗时不一致。"; const roundCompletionFinalMissingLines = Array.isArray(roundCompletion.finalResponseMissing) && roundCompletion.finalResponseMissing.length > 0 ? roundCompletion.finalResponseMissing.slice(0, 80).map((item) => "- #" + (item.seq ?? "-") + " " + (item.ts || "-") + " prompt=" + (item.promptIndex ?? "-") + " role=" + (item.pageRole || "-") + " traceId=" + (item.traceId || "-") + " completionElapsed=" + (item.completionElapsedSeconds ?? "-")).join("\n") @@ -2217,10 +2278,10 @@ function renderMarkdown(report) { ? roundCompletion.postCompletionTimingChanges.slice(0, 80).map((item) => "- " + (item.columnLabel || item.columnId || "-") + " " + (item.metric || "-") + " " + (item.fromValue ?? "-") + " -> " + (item.toValue ?? "-") + " delta=" + (item.delta ?? "-") + " completionSeq=" + (item.seq ?? "-") + " seq " + (item.fromSeq ?? "-") + " -> " + (item.toSeq ?? "-") + " traceId=" + (item.traceId || "-")).join("\n") : "- 未观察到轮次完成后耗时/最近更新继续变化。"; const durationUnderreportedLines = Array.isArray(codeAgentCardTiming.durationUnderreported) && codeAgentCardTiming.durationUnderreported.length > 0 - ? codeAgentCardTiming.durationUnderreported.slice(0, 80).map((item) => "- sample=" + (item.sampleIndex ?? "-") + " " + (item.timestamp || "-") + " role=" + (item.pageRole || "-") + " status=" + (item.status || "-") + " traceId=" + (item.traceId || "-") + " card=" + (item.cardTotalElapsedSeconds ?? "-") + "s expected=" + (item.expectedElapsedSeconds ?? "-") + "s delta=" + (item.deltaSeconds ?? "-") + "s evidence=" + (item.evidenceKind || "-") + " preview=" + escapeMarkdownCell(item.evidencePreview || item.cardPreview || "")).join("\n") + ? codeAgentCardTiming.durationUnderreported.slice(0, 80).map((item) => "- sample=" + (item.sampleIndex ?? "-") + " " + (item.timestamp || "-") + " role=" + (item.pageRole || "-") + " status=" + (item.status || "-") + " traceId=" + (item.traceId || "-") + " card=" + (item.cardTotalElapsedSeconds ?? "-") + "s expected=" + (item.expectedElapsedSeconds ?? "-") + "s delta=" + (item.deltaSeconds ?? "-") + "s evidence=" + (item.evidenceKind || "-") + formatTimingMeta(item) + " preview=" + escapeMarkdownCell(item.evidencePreview || item.cardPreview || "")).join("\n") : "- 未观察到 Code Agent 卡片耗时低于 trace/final-response 证据。"; const durationMismatchLines = Array.isArray(codeAgentCardTiming.durationMismatches) && codeAgentCardTiming.durationMismatches.length > 0 - ? codeAgentCardTiming.durationMismatches.slice(0, 80).map((item) => "- sample=" + (item.sampleIndex ?? "-") + " " + (item.timestamp || "-") + " role=" + (item.pageRole || "-") + " status=" + (item.status || "-") + " traceId=" + (item.traceId || "-") + " direction=" + (item.direction || "-") + " card=" + (item.cardTotalElapsedSeconds ?? "-") + "s expected=" + (item.expectedElapsedSeconds ?? "-") + "s signedDelta=" + (item.signedDeltaSeconds ?? "-") + "s delta=" + (item.deltaSeconds ?? "-") + "s tolerance=" + (item.toleranceSeconds ?? "-") + "s evidence=" + (item.evidenceKind || "-") + " exact=" + String(item.exactEvidence === true) + " preview=" + escapeMarkdownCell(item.evidencePreview || item.cardPreview || "")).join("\n") + ? codeAgentCardTiming.durationMismatches.slice(0, 80).map((item) => "- sample=" + (item.sampleIndex ?? "-") + " " + (item.timestamp || "-") + " role=" + (item.pageRole || "-") + " status=" + (item.status || "-") + " traceId=" + (item.traceId || "-") + " direction=" + (item.direction || "-") + " card=" + (item.cardTotalElapsedSeconds ?? "-") + "s expected=" + (item.expectedElapsedSeconds ?? "-") + "s signedDelta=" + (item.signedDeltaSeconds ?? "-") + "s delta=" + (item.deltaSeconds ?? "-") + "s tolerance=" + (item.toleranceSeconds ?? "-") + "s evidence=" + (item.evidenceKind || "-") + " exact=" + String(item.exactEvidence === true) + formatTimingMeta(item) + " preview=" + escapeMarkdownCell(item.evidencePreview || item.cardPreview || "")).join("\n") : "- 未观察到 Code Agent 卡片耗时与 completion/final-response 封口证据不一致。"; const traceOrderAnomalyLines = Array.isArray(traceOrder.orderAnomalies) && traceOrder.orderAnomalies.length > 0 ? traceOrder.orderAnomalies.slice(0, 80).map((item) => "- sample=" + (item.sampleIndex ?? "-") + " " + (item.timestamp || "-") + " role=" + (item.pageRole || "-") + " traceId=" + (item.traceId || "-") + " rows=" + (item.previousRowIndex ?? "-") + "->" + (item.currentRowIndex ?? "-") + " reasons=" + (Array.isArray(item.reasons) ? item.reasons.join(",") : "-") + " total=" + (item.previousTotalSeconds ?? "-") + "->" + (item.currentTotalSeconds ?? "-") + " clock=" + (item.previousClockSeconds ?? "-") + "->" + (item.currentClockSeconds ?? "-") + " preview=" + escapeMarkdownCell((item.previousPreview || "") + " / " + (item.currentPreview || ""))).join("\n") @@ -2340,7 +2401,7 @@ function renderMarkdown(report) { + "- missingRecentUpdateCount: " + (codeAgentCardTimingSummary.missingRecentUpdateCount ?? 0) + "\n" + "- durationUnderreportedCount: " + (codeAgentCardTimingSummary.durationUnderreportedCount ?? 0) + "\n" + "- durationMismatchCount: " + (codeAgentCardTimingSummary.durationMismatchCount ?? 0) + "\n" - + "- policy: Code Agent 卡片无论终态/非终态都必须显示耗时；非终态必须显示最近更新。该 analyzer 只报告采样到的页面表现，不做下游 repair。\n\n" + + "- policy: Code Agent 卡片无论终态/非终态都必须显示耗时；非终态必须显示最近更新。completion/final-response 可用时是 sealed source-of-truth；DOM card elapsed 是显示源。该 analyzer 只报告采样到的页面表现，不做下游 repair。\n\n" + "#### Missing elapsed samples\n\n" + cardMissingElapsedLines + "\n\n" + "#### Missing recent update samples\n\n" + cardMissingRecentLines + "\n\n" + "#### Duration underreported samples\n\n" + durationUnderreportedLines + "\n\n" @@ -2359,7 +2420,7 @@ function renderMarkdown(report) { + "- postTimingChangeCount: " + (codeAgentCardTimingSummary.roundCompletionPostTimingChangeCount ?? 0) + "\n" + "- postRecentUpdateVisibleCount: " + (codeAgentCardTimingSummary.roundCompletionPostRecentUpdateVisibleCount ?? 0) + "\n" + "- elapsedMismatchToleranceSeconds: " + (codeAgentCardTimingSummary.elapsedMismatchToleranceSeconds ?? "-") + "\n" - + "- policy: 轮次完成（总耗时 ...） 的耗时必须与卡片总耗时一致；完成后 final response 必须可见，耗时/最近更新不得继续跳变。\n\n" + + "- policy: 轮次完成（总耗时 ...） 是 trace sealed source-of-truth；卡片总耗时必须与它一致。完成后 final response 必须可见，耗时/最近更新不得继续跳变。\n\n" + "#### Round completion events\n\n" + roundCompletionLines + "\n\n" + "#### Completion elapsed mismatches\n\n" + roundCompletionMismatchLines + "\n\n" + "#### Final response missing after completion\n\n" + roundCompletionFinalMissingLines + "\n\n" diff --git a/scripts/src/hwlab-node-web-observe-render.ts b/scripts/src/hwlab-node-web-observe-render.ts index 3b65ae3c..b2e07c93 100644 --- a/scripts/src/hwlab-node-web-observe-render.ts +++ b/scripts/src/hwlab-node-web-observe-render.ts @@ -355,10 +355,11 @@ function renderWebObserveCollectTable(value: Record): string { if (jsonFindings.length > 0) { lines.push( "Findings:", - webObserveTable(["KIND", "SEVERITY", "COUNT", "ROOT_CAUSE", "SUMMARY"], jsonFindings.map((item) => [ + webObserveTable(["KIND", "SEVERITY", "COUNT", "TIMING", "ROOT_CAUSE", "SUMMARY"], jsonFindings.map((item) => [ webObserveShort(webObserveText(item.kind ?? item.id ?? item.code), 48), item.severity ?? item.level, item.count ?? item.sampleCount, + webObserveShort(webObserveFindingTiming(item), 42), webObserveShort(webObserveText(item.rootCause ?? item.rootCauseStatus), 48), webObserveShort(webObserveText(item.summary ?? item.message), 96), ])), @@ -556,13 +557,14 @@ function renderWebObserveProjectCollectTable(value: Record, col ]) : [["-", "-", "-", "-", "-", "-", "-"]]), "", "Findings:", - webObserveTable(["SEVERITY", "ID", "COUNT", "ROOT_CAUSE", "SUMMARY"], findings.length > 0 ? findings.map((item) => [ + webObserveTable(["SEVERITY", "ID", "COUNT", "TIMING", "ROOT_CAUSE", "SUMMARY"], findings.length > 0 ? findings.map((item) => [ webObserveShort(webObserveText(item.severity ?? item.level), 12), webObserveShort(webObserveText(item.id ?? item.kind ?? item.code), 48), item.count ?? item.sampleCount, + webObserveShort(webObserveFindingTiming(item), 42), webObserveShort(webObserveText(item.rootCause ?? item.rootCauseStatus), 48), webObserveShort(webObserveText(item.summary ?? item.message), 96), - ]) : [["-", "-", "-", "-", "-"]]), + ]) : [["-", "-", "-", "-", "-", "-"]]), "", "Disclosure:", " collect project views are rendered from existing samples/control/analysis artifacts; they do not create a new sampling source.", @@ -600,6 +602,14 @@ function webObserveShort(value: string, maxLength: number): string { return `${value.slice(0, maxLength - 1)}~`; } +function webObserveFindingTiming(item: Record): string { + const status = item.timingStatus ?? null; + const source = item.timingSourceOfTruth ?? item.expectedElapsedSource ?? item.evidenceKind ?? null; + const parts = [status, source].filter((value) => value !== null && value !== undefined && value !== ""); + if (parts.length === 0) return "-"; + return parts.map((value) => webObserveText(value)).join("/"); +} + function record(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; } diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 591ece55..65a39ea1 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -2903,6 +2903,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou status: "blocked", observerId: null, elapsedMs: 0, + businessStatus: quickVerifyBusinessStatus("quick-verify-account-secret-missing", 0, null, null, 0, maxSeconds), steps: [{ phase: "quick-verify-account-env", ok: false, result: accountEnv.summary }], failure: "quick-verify-account-secret-missing", findingCount: findings.length, @@ -2951,6 +2952,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou status: "blocked", observerId, elapsedMs: elapsedMs(), + businessStatus: quickVerifyBusinessStatus("observe-start-failed", 0, null, null, elapsedMs(), maxSeconds), steps, failure: "observe-start-failed", findingCount: findings.length, @@ -3118,6 +3120,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou const blockingFindings = findings.filter(isQuickVerifyBlockingFinding); const analysisWarnings = analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."]; const ok = record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0; + const businessStatus = quickVerifyBusinessStatus(null, promptIndex, turnSummary, traceFrame, elapsedMs(), maxSeconds); printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() }); return recordQuickVerify(state, { ok, @@ -3127,6 +3130,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou status: ok ? "analyzed" : "blocked", observerId, elapsedMs: elapsedMs(), + businessStatus, stateDir: indexEntry?.stateDir ?? null, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, @@ -3308,6 +3312,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: { && record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0; + const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, numberAt(state.cicd, "targetValidation.maxSeconds")); return { ok: recoveredWaitFailure, runId: input.runId, @@ -3316,6 +3321,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: { status: recoveredWaitFailure ? "analyzed" : "blocked", observerId: input.observerId, elapsedMs: input.elapsedMs ?? null, + businessStatus, stateDir: indexEntry?.stateDir ?? null, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, @@ -3347,6 +3353,7 @@ function recordQuickVerify(state: SentinelCicdState, payload: Record | null, + traceFrame: Record | null, + elapsedMs: unknown, + budgetSeconds: number, +): Record { + const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame); + const elapsed = typeof elapsedMs === "number" && Number.isFinite(elapsedMs) ? elapsedMs : null; + const budgetMs = Math.max(0, budgetSeconds) * 1000; + const budgetExceeded = elapsed !== null && budgetMs > 0 && elapsed > budgetMs; + const observerTimeout = budgetExceeded || (failure !== null && isRecoverableQuickVerifyWaitFailure(failure)); + const status = durableBusinessTurn + ? "business-turn-completed" + : observerTimeout + ? "observer-timeout" + : "scenario-incomplete"; + return { + status, + durableBusinessTurn, + observerTimeout, + scenarioComplete: durableBusinessTurn, + failure: failure ?? null, + promptIndex, + elapsedMs: elapsed, + budgetSeconds, + sourceOfTruth: durableBusinessTurn ? "turn-summary-or-trace-frame" : observerTimeout ? "runner-wait-budget" : "control-findings", + valuesRedacted: true, + }; +} + function isRecoverableQuickVerifyWaitFailure(failure: string): boolean { return failure === "quick-verify-wait-chunk-timeout" || failure === "quick-verify-timeout-over-budget" @@ -4969,6 +5009,7 @@ function renderMaintenanceResult(result: Record): string { const serviceHealth = record(result.serviceHealth); const maintenance = record(result.maintenance); const quickVerify = record(result.quickVerify); + const quickVerifyBusiness = record(quickVerify.businessStatus); const planned = record(result.planned); const blocker = record(result.blocker); const next = record(result.next); @@ -4987,7 +5028,7 @@ function renderMaintenanceResult(result: Record): string { ? table(["ACTION", "RELEASE", "REASON", "QUICK_VERIFY"], [[planned.action, planned.releaseId, planned.reason, planned.quickVerify]]) : "MAINTENANCE\n-", "", - Object.keys(quickVerify).length === 0 ? "QUICK_VERIFY\n-" : table(["OK", "RUN", "SCENARIO", "OBSERVER", "REPORT", "FINDINGS"], [[quickVerify.ok, quickVerify.runId, quickVerify.scenarioId, quickVerify.observerId, quickVerify.reportJsonSha256, quickVerify.findingCount]]), + Object.keys(quickVerify).length === 0 ? "QUICK_VERIFY\n-" : table(["OK", "BUSINESS", "RUN", "SCENARIO", "OBSERVER", "REPORT", "FINDINGS"], [[quickVerify.ok, quickVerifyBusiness.status ?? "-", quickVerify.runId, quickVerify.scenarioId, quickVerify.observerId, quickVerify.reportJsonSha256, quickVerify.findingCount]]), "", Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]), "", @@ -5269,6 +5310,7 @@ function renderControlPlaneResult(result: Record): string { const argoApply = record(result.argoApply); const blocker = record(result.blocker); const targetValidation = record(result.targetValidation); + const targetValidationBusiness = record(targetValidation.businessStatus); const next = record(result.next); const warnings = Array.isArray(result.warnings) ? result.warnings : []; return [ @@ -5286,9 +5328,10 @@ function renderControlPlaneResult(result: Record): string { "", Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]), "", - Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[ + Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "BUSINESS", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[ targetValidation.ok, targetValidation.status, + targetValidationBusiness.status ?? "-", targetValidation.scenarioId, targetValidation.runId, targetValidation.observerId, diff --git a/scripts/src/hwlab-node/web-observe-render.ts b/scripts/src/hwlab-node/web-observe-render.ts index bccbee94..d252664c 100644 --- a/scripts/src/hwlab-node/web-observe-render.ts +++ b/scripts/src/hwlab-node/web-observe-render.ts @@ -697,13 +697,14 @@ export function renderWebObserveAnalyzeTable(value: Record): st webObserveTable(consoleAlertGroups.length > 0 ? ["COUNT", "TYPE", "STATUS", "PATH", "LAST", "TRACES"] : ["TS", "TYPE", "STATUS", "PATH", "TRACE", "TEXT"], consoleAlertRows.length > 0 ? consoleAlertRows : [["-", "-", "-", "-", "-", "-"]]), "", "Findings:", - webObserveTable(["KIND", "SEVERITY", "COUNT", "ROOT_CAUSE", "SUMMARY"], findings.length > 0 ? findings.map((item) => [ + webObserveTable(["KIND", "SEVERITY", "COUNT", "TIMING", "ROOT_CAUSE", "SUMMARY"], findings.length > 0 ? findings.map((item) => [ webObserveShort(webObserveText(item.kind ?? item.id ?? item.code), 32), webObserveText(item.severity ?? item.level), webObserveText(item.count ?? item.sampleCount), + webObserveShort(webObserveFindingTiming(item), 42), webObserveShort(webObserveText(item.rootCause ?? item.rootCauseStatus), 48), webObserveShort(webObserveText(item.summary ?? item.message), 96), - ]) : [["-", "-", "-", "-", "-"]]), + ]) : [["-", "-", "-", "-", "-", "-"]]), "", "Archive red findings:", webObserveTable(["KIND", "COUNT", "SUMMARY"], archiveRedFindings.length > 0 ? archiveRedFindings.map((item) => [ @@ -987,6 +988,14 @@ export function webObserveCell(value: unknown, maxLength = 96): string { return `${compact.slice(0, Math.max(1, maxLength - 1))}...`; } +function webObserveFindingTiming(item: Record): string { + const status = item.timingStatus ?? null; + const source = item.timingSourceOfTruth ?? item.expectedElapsedSource ?? item.evidenceKind ?? null; + const parts = [status, source].filter((value) => value !== null && value !== undefined && value !== ""); + if (parts.length === 0) return "-"; + return parts.map((value) => webObserveText(value)).join("/"); +} + export function webObserveTable(headers: string[], rows: unknown[][]): string { const stringRows = rows.map((row) => row.map((value) => webObserveCell(value))); const widths = headers.map((header, index) => Math.max(header.length, ...stringRows.map((row) => row[index]?.length ?? 0))); @@ -1388,8 +1397,8 @@ export function renderWebObserveAnalyzeResult(result: Record): findings.length === 0 ? "FINDINGS\n-" : webObserveTable( - ["FINDING", "SEVERITY", "COUNT", "SUMMARY"], - findings.map((item) => [item.id ?? item.kind ?? item.code, item.severity ?? item.level, item.count ?? item.sampleCount, item.summary ?? item.message]), + ["FINDING", "SEVERITY", "COUNT", "TIMING", "SUMMARY"], + findings.map((item) => [item.id ?? item.kind ?? item.code, item.severity ?? item.level, item.count ?? item.sampleCount, webObserveFindingTiming(item), item.summary ?? item.message]), ), "", "REPORTS", diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index 77a616fe..c7faf4af 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -1784,7 +1784,7 @@ export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOption "const slimSlowSample = (item) => { const v = objectOrNull(item) || {}; return { ts: v.ts ?? null, seq: v.seq ?? null, path: clip(v.path ?? v.rawPath, 96), initiatorType: clip(v.initiatorType, 24), durationMs: v.durationMs ?? null, requestToResponseStartMs: v.requestToResponseStartMs ?? v.streamOpenMs ?? null, responseTransferMs: v.responseTransferMs ?? null, nextHopProtocol: clip(v.nextHopProtocol, 24), timingStatus: clip(v.timingStatus, 16), serverTimingNames: Array.isArray(v.serverTimingNames) ? v.serverTimingNames.slice(0, 4).map((x) => clip(x, 32)) : [], otelTraceId: clip(v.otelTraceId, 32) }; };", "const slimSlowApi = (item) => { const v = objectOrNull(item) || {}; return { path: clip(v.path ?? v.route, 96), route: clip(v.route ?? v.path, 96), sampleCount: v.sampleCount ?? null, p95Ms: v.p95Ms ?? v.p95 ?? null, maxMs: v.maxMs ?? v.max ?? null, budgetMs: v.budgetMs ?? null, overBudgetCount: v.overBudgetCount ?? null, overFiveSecondCount: v.overFiveSecondCount ?? null, slowSamples: Array.isArray(v.slowSamples) ? v.slowSamples.slice(0, 3).map(slimSlowSample) : [] }; };", "const evidenceSummary = (value) => { const v = objectOrNull(value); if (!v) return null; const out = {}; for (const key of ['http404Count','responseErrorCount','requestFailedCount','statuses','afterProjectedSeqs','sinceSeqs','traceIds','maxFallbackRatio','maxFallbackTitleCount','overThresholdSampleCount','majorityFallbackSampleCount']) if (v[key] !== undefined && v[key] !== null) out[key] = Array.isArray(v[key]) ? v[key].slice(0,6) : v[key]; const text = Object.keys(out).length > 0 ? JSON.stringify(out) : null; return text ? clip(text, 220) : null; };", - "const slimFinding = (item) => { const v = objectOrNull(item) || {}; return { kind: clip(v.kind ?? v.id ?? v.code, 48), code: clip(v.code ?? v.id ?? v.kind, 48), severity: clip(v.severity ?? v.level, 24), level: clip(v.level ?? v.severity, 24), count: v.count ?? v.sampleCount ?? null, sampleCount: v.sampleCount ?? v.count ?? null, summary: clip(v.summary ?? v.message, 180), message: clip(v.message ?? v.summary, 180), rootCause: clip(v.rootCause, 120), rootCauseStatus: clip(v.rootCauseStatus, 80), rootCauseConfidence: clip(v.rootCauseConfidence, 40), nextAction: clip(v.nextAction, 220), evidenceSummary: evidenceSummary(v.evidence) ?? clip(v.evidenceSummary, 220) }; };", + "const slimFinding = (item) => { const v = objectOrNull(item) || {}; return { kind: clip(v.kind ?? v.id ?? v.code, 48), code: clip(v.code ?? v.id ?? v.kind, 48), severity: clip(v.severity ?? v.level, 24), level: clip(v.level ?? v.severity, 24), count: v.count ?? v.sampleCount ?? null, sampleCount: v.sampleCount ?? v.count ?? null, timingSourceOfTruth: clip(v.timingSourceOfTruth ?? v.expectedElapsedSource ?? v.evidenceKind, 80), timingStatus: clip(v.timingStatus, 48), timingAlert: v.timingAlert === true, summary: clip(v.summary ?? v.message, 180), message: clip(v.message ?? v.summary, 180), rootCause: clip(v.rootCause, 120), rootCauseStatus: clip(v.rootCauseStatus, 80), rootCauseConfidence: clip(v.rootCauseConfidence, 40), nextAction: clip(v.nextAction, 220), evidenceSummary: evidenceSummary(v.evidence) ?? clip(v.evidenceSummary, 220) }; };", "const slimProjectManagement = (value) => { const v = objectOrNull(value); if (!v) return null; const s = objectOrNull(v.summary) || v; return { summary: { enabled: s.enabled === true, projectSampleCount: s.projectSampleCount ?? null, mdtodoSampleCount: s.mdtodoSampleCount ?? null, latestPageKind: clip(s.latestPageKind, 48), latestPath: clip(s.latestPath, 96), latestSourceCount: s.latestSourceCount ?? null, latestFileCount: s.latestFileCount ?? null, latestTaskCount: s.latestTaskCount ?? null, latestSelectedTaskRefHash: clip(s.latestSelectedTaskRefHash, 80), launchCommandCount: s.launchCommandCount ?? null, launchSuccessCount: s.launchSuccessCount ?? null, launchFailureCount: s.launchFailureCount ?? null, launchWithOtelTraceHeaderCount: s.launchWithOtelTraceHeaderCount ?? null, projectApiResponseCount: s.projectApiResponseCount ?? null, projectApiFailureCount: s.projectApiFailureCount ?? null, projectApiSlowPathCount: s.projectApiSlowPathCount ?? null, slowApiBudgetMs: s.slowApiBudgetMs ?? null }, commands: takeTail(v.commands, 8).map((item) => { const row = objectOrNull(item) || {}; return { ts: row.ts ?? null, phase: clip(row.phase, 16), type: clip(row.type, 32), commandId: clip(row.commandId, 80), launchStatus: row.launchStatus ?? null, sessionId: clip(row.sessionId, 80), workbenchUrl: clip(row.workbenchUrl, 120), otelTraceId: clip(row.otelTraceId, 32), selectedTaskRefHash: clip(row.selectedTaskRefHash, 80) }; }), samples: takeTail(v.samples, 8).map((item) => { const row = objectOrNull(item) || {}; return { seq: row.seq ?? null, ts: row.ts ?? null, pageRole: clip(row.pageRole, 24), path: clip(row.path, 96), pageKind: clip(row.pageKind, 48), sourceCount: row.sourceCount ?? null, fileCount: row.fileCount ?? null, taskCount: row.taskCount ?? null, selectedTaskRefHash: clip(row.selectedTaskRefHash, 80), launchButtonEnabled: row.launchButtonEnabled === true, workbenchLinkCount: row.workbenchLinkCount ?? null }; }), projectApiByPath: takeHead(v.projectApiByPath, 8).map(slimNetworkGroup), valuesRedacted: true }; };", "const slimDomGroup = (item) => { const v = objectOrNull(item) || {}; return { count: v.count ?? null, firstAt: v.firstAt ?? null, lastAt: v.lastAt ?? null, text: clip(v.text ?? v.preview, 180) }; };", "const slimNetworkGroup = (item) => { const v = objectOrNull(item) || {}; return { count: v.count ?? null, method: clip(v.method, 12), status: v.status ?? null, path: clip(v.path ?? v.urlPath, 96), firstAt: v.firstAt ?? null, lastAt: v.lastAt ?? null, promptIndexes: Array.isArray(v.promptIndexes) ? v.promptIndexes.slice(0, 6) : [], failureKinds: Array.isArray(v.failureKinds) ? v.failureKinds.slice(0, 4).map((x) => clip(x, 48)) : [] }; };", @@ -2062,8 +2062,8 @@ export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOption " projectManagement: compact.projectManagement ? { summary: compact.projectManagement.summary ?? compact.projectManagement, samples: Array.isArray(compact.projectManagement.samples) ? compact.projectManagement.samples.slice(-3) : [], commands: Array.isArray(compact.projectManagement.commands) ? compact.projectManagement.commands.slice(-3) : [], launchCommands: Array.isArray(compact.projectManagement.launchCommands) ? compact.projectManagement.launchCommands.slice(-3) : [], projectApiByPath: Array.isArray(compact.projectManagement.projectApiByPath) ? compact.projectManagement.projectApiByPath.slice(0, 3) : [], slowProjectApiPerformance: Array.isArray(compact.projectManagement.slowProjectApiPerformance) ? compact.projectManagement.slowProjectApiPerformance.slice(0, 2) : [], valuesRedacted: true } : null,", " toolFindings: Array.isArray(compact.toolFindings) ? compact.toolFindings.slice(0, 8) : [],", " commandState: compact.commandState ?? null,", - " findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [],", - " archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [],", + " findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, timingAlert: item.timingAlert === true, summary: clip(item.summary ?? item.message, 120) })) : [],", + " archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, timingAlert: item.timingAlert === true, summary: clip(item.summary ?? item.message, 120) })) : [],", " httpErrorGroups: Array.isArray(compact.httpErrorGroups) ? compact.httpErrorGroups.slice(0, 2).map((item) => ({ count: item.count ?? null, method: item.method ?? null, status: item.status ?? null, path: item.path ?? null, lastAt: item.lastAt ?? null })) : [],", " requestFailedGroups: Array.isArray(compact.requestFailedGroups) ? compact.requestFailedGroups.slice(0, 2).map((item) => ({ count: item.count ?? null, method: item.method ?? null, path: item.path ?? null, failureKinds: item.failureKinds ?? null })) : [],", " commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3).map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, durationMs: item.durationMs ?? null, sampleSeq: item.sampleSeq ?? null, beforePath: item.beforePath ?? null, afterPath: item.afterPath ?? null, message: clip(item.message ?? item.failureKind ?? item.name, 120) })) : [],", @@ -2083,7 +2083,7 @@ export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOption " output = compactOutput(ultratiny);", " }", " if (Buffer.byteLength(output, 'utf8') > compactStdoutLimitBytes) {", - " output = compactOutput({ ok: compact.ok, counts: compact.counts ?? null, jsonlScope: compact.jsonlScope ?? null, analysisWindow: compact.analysisWindow ?? null, archiveSummary: compact.archiveSummary ? { redFindingCount: compact.archiveSummary.redFindingCount ?? null, findingCount: compact.archiveSummary.findingCount ?? null, sampleCount: compact.archiveSummary.sampleMetrics?.sampleCount ?? null, slowPathCount: compact.archiveSummary.pagePerformance?.slowPathCount ?? null } : null, projectManagement: compact.projectManagement ? { summary: compact.projectManagement.summary ?? compact.projectManagement, samples: Array.isArray(compact.projectManagement.samples) ? compact.projectManagement.samples.slice(-2) : [], commands: Array.isArray(compact.projectManagement.commands) ? compact.projectManagement.commands.slice(-2) : [], launchCommands: Array.isArray(compact.projectManagement.launchCommands) ? compact.projectManagement.launchCommands.slice(-2) : [], projectApiByPath: Array.isArray(compact.projectManagement.projectApiByPath) ? compact.projectManagement.projectApiByPath.slice(0, 2) : [], valuesRedacted: true } : null, toolFindings: Array.isArray(compact.toolFindings) ? compact.toolFindings.slice(0, 8) : [], commandState: compact.commandState ? { pendingCount: compact.commandState.pendingCount ?? null, processingCount: compact.commandState.processingCount ?? null, abandonedCount: compact.commandState.abandonedCount ?? null, failedCount: compact.commandState.failedCount ?? null } : null, findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3).map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, durationMs: item.durationMs ?? null, sampleSeq: item.sampleSeq ?? null, beforePath: item.beforePath ?? null, afterPath: item.afterPath ?? null, message: clip(item.message ?? item.failureKind ?? item.name, 120) })) : [], archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [], reportJsonPath: compact.reportJsonPath ?? reportJsonPath, reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: compact.reportMdPath ?? reportMdPath, reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, ultratiny: true, hardFallback: true, compactStdoutLimitBytes }, valuesRedacted: true });", + " output = compactOutput({ ok: compact.ok, counts: compact.counts ?? null, jsonlScope: compact.jsonlScope ?? null, analysisWindow: compact.analysisWindow ?? null, archiveSummary: compact.archiveSummary ? { redFindingCount: compact.archiveSummary.redFindingCount ?? null, findingCount: compact.archiveSummary.findingCount ?? null, sampleCount: compact.archiveSummary.sampleMetrics?.sampleCount ?? null, slowPathCount: compact.archiveSummary.pagePerformance?.slowPathCount ?? null } : null, projectManagement: compact.projectManagement ? { summary: compact.projectManagement.summary ?? compact.projectManagement, samples: Array.isArray(compact.projectManagement.samples) ? compact.projectManagement.samples.slice(-2) : [], commands: Array.isArray(compact.projectManagement.commands) ? compact.projectManagement.commands.slice(-2) : [], launchCommands: Array.isArray(compact.projectManagement.launchCommands) ? compact.projectManagement.launchCommands.slice(-2) : [], projectApiByPath: Array.isArray(compact.projectManagement.projectApiByPath) ? compact.projectManagement.projectApiByPath.slice(0, 2) : [], valuesRedacted: true } : null, toolFindings: Array.isArray(compact.toolFindings) ? compact.toolFindings.slice(0, 8) : [], commandState: compact.commandState ? { pendingCount: compact.commandState.pendingCount ?? null, processingCount: compact.commandState.processingCount ?? null, abandonedCount: compact.commandState.abandonedCount ?? null, failedCount: compact.commandState.failedCount ?? null } : null, findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, timingAlert: item.timingAlert === true, summary: clip(item.summary ?? item.message, 120) })) : [], archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, timingSourceOfTruth: item.timingSourceOfTruth ?? null, timingStatus: item.timingStatus ?? null, timingAlert: item.timingAlert === true, summary: clip(item.summary ?? item.message, 120) })) : [], commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3).map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, durationMs: item.durationMs ?? null, sampleSeq: item.sampleSeq ?? null, beforePath: item.beforePath ?? null, afterPath: item.afterPath ?? null, message: clip(item.message ?? item.failureKind ?? item.name, 120) })) : [], archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [], reportJsonPath: compact.reportJsonPath ?? reportJsonPath, reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: compact.reportMdPath ?? reportMdPath, reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, ultratiny: true, hardFallback: true, compactStdoutLimitBytes }, valuesRedacted: true });", " }", " }", "}", @@ -2189,7 +2189,7 @@ export function recoverWebObserveAnalyzeFromArtifacts(options: NodeWebProbeObser "const clip = (value, limit = 160) => value === null || value === undefined ? null : String(value).slice(0, limit);", "const numberish = (...values) => { for (const value of values) { const n = Number(value); if (Number.isFinite(n)) return value; } return null; };", "const evidenceSummary = (value) => { const v = objectOrNull(value); if (!v) return null; const out = {}; for (const key of ['http404Count','responseErrorCount','requestFailedCount','statuses','afterProjectedSeqs','sinceSeqs','traceIds','maxFallbackRatio','maxFallbackTitleCount','overThresholdSampleCount','majorityFallbackSampleCount']) if (v[key] !== undefined && v[key] !== null) out[key] = Array.isArray(v[key]) ? v[key].slice(0,6) : v[key]; const text = Object.keys(out).length > 0 ? JSON.stringify(out) : null; return text ? clip(text, 220) : null; };", - "const slimFinding = (item) => { const v = objectOrNull(item) || {}; return { kind: clip(v.kind ?? v.id ?? v.code, 48), code: clip(v.code ?? v.id ?? v.kind, 48), severity: clip(v.severity ?? v.level, 24), level: clip(v.level ?? v.severity, 24), count: v.count ?? v.sampleCount ?? null, sampleCount: v.sampleCount ?? v.count ?? null, summary: clip(v.summary ?? v.message, 180), message: clip(v.message ?? v.summary, 180), rootCause: clip(v.rootCause, 120), rootCauseStatus: clip(v.rootCauseStatus, 80), rootCauseConfidence: clip(v.rootCauseConfidence, 40), nextAction: clip(v.nextAction, 220), evidenceSummary: evidenceSummary(v.evidence) ?? clip(v.evidenceSummary, 220) }; };", + "const slimFinding = (item) => { const v = objectOrNull(item) || {}; return { kind: clip(v.kind ?? v.id ?? v.code, 48), code: clip(v.code ?? v.id ?? v.kind, 48), severity: clip(v.severity ?? v.level, 24), level: clip(v.level ?? v.severity, 24), count: v.count ?? v.sampleCount ?? null, sampleCount: v.sampleCount ?? v.count ?? null, timingSourceOfTruth: clip(v.timingSourceOfTruth ?? v.expectedElapsedSource ?? v.evidenceKind, 80), timingStatus: clip(v.timingStatus, 48), timingAlert: v.timingAlert === true, summary: clip(v.summary ?? v.message, 180), message: clip(v.message ?? v.summary, 180), rootCause: clip(v.rootCause, 120), rootCauseStatus: clip(v.rootCauseStatus, 80), rootCauseConfidence: clip(v.rootCauseConfidence, 40), nextAction: clip(v.nextAction, 220), evidenceSummary: evidenceSummary(v.evidence) ?? clip(v.evidenceSummary, 220) }; };", "const slimRound = (item) => { const v = objectOrNull(item) || {}; return { promptIndex: v.promptIndex ?? null, promptTextHash: clip(v.promptTextHash, 80), sampleCount: v.sampleCount ?? null, firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, lastTotalElapsedSeconds: v.lastTotalElapsedSeconds ?? null, lastRecentUpdateSeconds: v.lastRecentUpdateSeconds ?? null, loadingSamples: v.loadingSamples ?? null, maxLoadingCount: v.maxLoadingCount ?? null, loadingOwnerCount: v.loadingOwnerCount ?? null, diagnosticSamples: v.diagnosticSamples ?? null, terminalSamples: v.terminalSamples ?? null, finalTextSamples: v.finalTextSamples ?? null, turnTimingTotalElapsedZeroResetCount: v.turnTimingTotalElapsedZeroResetCount ?? null, turnTimingTotalElapsedForwardJumpCount: v.turnTimingTotalElapsedForwardJumpCount ?? null, turnTimingTotalElapsedForwardJumpMaxSeconds: v.turnTimingTotalElapsedForwardJumpMaxSeconds ?? null, turnTimingRecentUpdateJumpCount: v.turnTimingRecentUpdateJumpCount ?? null, turnTimingRecentUpdateMaxIncreaseSeconds: v.turnTimingRecentUpdateMaxIncreaseSeconds ?? null }; };", "const slimTurnColumn = (item) => { const v = objectOrNull(item) || {}; return { label: clip(v.label, 24), source: clip(v.source, 48), pageRole: clip(v.pageRole, 24), pageId: clip(v.pageId, 32), pageEpoch: v.pageEpoch ?? null, promptIndex: v.promptIndex ?? null, lastPromptIndex: v.lastPromptIndex ?? null, firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, traceId: clip(v.traceId, 64), messageId: clip(v.messageId, 64) }; };", "const slimGroup = (item) => { const v = objectOrNull(item) || {}; return { count: v.count ?? null, method: clip(v.method, 12), status: v.status ?? null, path: clip(v.path ?? v.urlPath ?? v.route, 96), firstAt: v.firstAt ?? null, lastAt: v.lastAt ?? null, text: clip(v.text ?? v.preview, 180), failureKinds: arr(v.failureKinds).slice(0, 4).map((x) => clip(x, 48)), traceIds: arr(v.traceIds).slice(0, 3).map((x) => clip(x, 64)) }; };",