diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 3fc14316..b091543d 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -9244,7 +9244,7 @@ function renderWebObserveAnalyzeTable(value: Record): string { ]]), "", "Rounds:", - webObserveTable(["ROUND", "SAMPLES", "LOADING", "MAX_LOAD", "TOTAL_LAST", "RECENT_LAST", "DIAG", "TERMINAL", "PROMPT_HASH"], rounds.length > 0 ? rounds.map((item) => [ + webObserveTable(["ROUND", "SAMPLES", "LOADING", "MAX_LOAD", "TOTAL_LAST", "RECENT_LAST", "CARD_DIAG", "TERMINAL", "PROMPT_HASH"], rounds.length > 0 ? rounds.map((item) => [ webObserveText(item.promptIndex), webObserveText(item.sampleCount), webObserveText(item.loadingSamples), @@ -10039,7 +10039,7 @@ function renderWebObserveAnalyzeResult(result: Record): Record< rounds.length === 0 ? "ROUNDS\n-" : webObserveTable( - ["ROUND", "COMMAND", "SAMPLES", "TOTAL_MAX", "TOTAL_LAST", "RECENT_MAX", "RECENT_LAST", "DIAG", "TERM", "NONMONO", "ELAPSED_JUMP", "JUMP"], + ["ROUND", "COMMAND", "SAMPLES", "TOTAL_MAX", "TOTAL_LAST", "RECENT_MAX", "RECENT_LAST", "CARD_DIAG", "TERM", "NONMONO", "ELAPSED_JUMP", "JUMP"], rounds.map((item) => [ item.promptIndex, item.promptCommandId, @@ -10388,6 +10388,10 @@ const slimFindingSample=(value)=>{ diffKind:short(value.diffKind??value.kind??value.type??value.metric??value.anomaly,48), fromValue:value.fromValue??value.beforeValue??null, toValue:value.toValue??value.afterValue??null, + fromDigest:short(value.fromDigest,80), + toDigest:short(value.toDigest,80), + previousSeq:value.previousSeq??null, + previousTs:short(value.previousTs,32), delta:value.delta??null, sampleDeltaSeconds:value.sampleDeltaSeconds??null, allowedIncreaseSeconds:value.allowedIncreaseSeconds??null, @@ -10395,12 +10399,12 @@ const slimFindingSample=(value)=>{ promptIndex:value.promptIndex??null, event:short(value.event??value.anomaly,48), expectedPattern:short(value.expectedPattern,96), - control:compactMetric(controlCount), - observer:compactMetric(observerCount), + control:compactMetric(controlCount??(value.fromMessageCount!==undefined||value.fromTraceRowCount!==undefined?compactProjectionPair(value.fromMessageCount,value.fromTraceRowCount):undefined)), + observer:compactMetric(observerCount??(value.toMessageCount!==undefined||value.toTraceRowCount!==undefined?compactProjectionPair(value.toMessageCount,value.toTraceRowCount):undefined)), sessionId:short(value.sessionId??value.routeSessionId??value.activeSessionId??value.controlSessionId??value.observerSessionId??refValue.routeSessionId??refValue.activeSessionId,64), path:short(value.path??value.urlPath??value.url??value.controlPath??value.observerPath??refValue.url,120), trace:short(trace,120), - detail:short(value.detail??(value.messageTextDigestDiff?'messageDigest '+String(value.controlMessageDigest??'-')+' != '+String(value.observerMessageDigest??'-'):undefined)??value.summary??value.message??value.preview??value.expectedPattern,180), + detail:short(value.detail??(value.fromDigest||value.toDigest?'digest '+String(value.fromDigest??'-')+' -> '+String(value.toDigest??'-'):undefined)??(value.messageTextDigestDiff?'messageDigest '+String(value.controlMessageDigest??'-')+' != '+String(value.observerMessageDigest??'-'):undefined)??value.summary??value.message??value.preview??value.expectedPattern,180), valuesRedacted:true }; }; diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index e4fd9bfd..eccec717 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -2847,7 +2847,22 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN const pageKey = samplePageKey(sample); const previous = previousDigestByPage.get(pageKey); const next = digestSample(sample); - if (previous && previous.digest !== next && !commandedPromptSeqs.has(sample?.seq) && !nearCommand(sample, commandTimes, alertThresholds.uncommandedStateChangeCommandWindowMs)) uncommandedChanges.push(ref(sample)); + if (previous && previous.digest !== next && !commandedPromptSeqs.has(sample?.seq) && !nearCommand(sample, commandTimes, alertThresholds.uncommandedStateChangeCommandWindowMs)) { + uncommandedChanges.push({ + ...ref(sample), + previousSeq: previous.sample?.seq ?? null, + previousTs: previous.sample?.timestamp ?? previous.sample?.ts ?? null, + fromDigest: previous.digest, + toDigest: next, + fromMessageCount: previous.sample?.messageCount ?? previous.sample?.messages?.length ?? null, + toMessageCount: sample?.messageCount ?? sample?.messages?.length ?? null, + fromTraceRowCount: Array.isArray(previous.sample?.traceRows) ? previous.sample.traceRows.length : previous.sample?.traceRowCount ?? null, + toTraceRowCount: Array.isArray(sample?.traceRows) ? sample.traceRows.length : sample?.traceRowCount ?? null, + fromPath: previous.sample?.path ?? previous.sample?.urlPath ?? null, + toPath: sample?.path ?? sample?.urlPath ?? null, + detail: "visible digest changed without nearby command", + }); + } previousDigestByPage.set(pageKey, { digest: next, sample }); } if (uncommandedChanges.length > 0) findings.push({ id: "uncommanded-visible-state-change", severity: "amber", summary: "visible message/trace digest changed without a nearby command", count: uncommandedChanges.length, samples: uncommandedChanges.slice(0, 20) }); @@ -2916,6 +2931,10 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) }); const elapsedZeroResets = Array.isArray(sampleMetrics?.turnTimingElapsedZeroResets) ? sampleMetrics.turnTimingElapsedZeroResets : []; if (elapsedZeroResets.length > 0) findings.push({ id: "turn-timing-total-elapsed-zero-reset", severity: "red", summary: "Code Agent total elapsed jumped from a non-zero value back to 0 seconds", count: elapsedZeroResets.length, samples: elapsedZeroResets.slice(0, 20) }); + const elapsedDecreases = Array.isArray(sampleMetrics?.turnTimingNonMonotonic) + ? sampleMetrics.turnTimingNonMonotonic.filter((item) => item.metric === "totalElapsedSeconds" && item.anomaly !== "zero-reset") + : []; + if (elapsedDecreases.length > 0) findings.push({ id: "turn-timing-total-elapsed-decrease", severity: "red", summary: "Code Agent total elapsed decreased between adjacent samples; total elapsed must be monotonic per turn", count: elapsedDecreases.length, samples: elapsedDecreases.slice(0, 20) }); const elapsedForwardJumps = Array.isArray(sampleMetrics?.turnTimingTotalElapsedForwardJumps) ? sampleMetrics.turnTimingTotalElapsedForwardJumps : []; if (elapsedForwardJumps.length > 0) findings.push({ id: "turn-timing-total-elapsed-forward-jump", severity: "red", summary: "Code Agent total elapsed jumped forward faster than browser sample interval", count: elapsedForwardJumps.length, samples: elapsedForwardJumps.slice(0, 20) }); const terminalElapsedGrowth = Array.isArray(sampleMetrics?.turnTimingTerminalElapsedGrowth) ? sampleMetrics.turnTimingTerminalElapsedGrowth : [];