mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(web-probe): expose timing decrease findings (#828)

Browse Source 
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-24 15:53:12 +08:00
committed by GitHub
parent 0256989ee0
commit 9c5ee846a3
2 changed files with 29 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-impl.ts
+9 -5
View File
@@ -9244,7 +9244,7 @@ function renderWebObserveAnalyzeTable(value: Record<string, unknown>): string {
]]),
"",
"Rounds:",
webObserveTable(["ROUND", "SAMPLES", "LOADING", "MAX_LOAD", "TOTAL_LAST", "RECENT_LAST", "DIAG", "TERMINAL", "PROMPT_HASH"], rounds.length > 0 ? rounds.map((item) => [
webObserveTable(["ROUND", "SAMPLES", "LOADING", "MAX_LOAD", "TOTAL_LAST", "RECENT_LAST", "CARD_DIAG", "TERMINAL", "PROMPT_HASH"], rounds.length > 0 ? rounds.map((item) => [
webObserveText(item.promptIndex),
webObserveText(item.sampleCount),
webObserveText(item.loadingSamples),
@@ -10039,7 +10039,7 @@ function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<
rounds.length === 0
? "ROUNDS\n-"
: webObserveTable(
["ROUND", "COMMAND", "SAMPLES", "TOTAL_MAX", "TOTAL_LAST", "RECENT_MAX", "RECENT_LAST", "DIAG", "TERM", "NONMONO", "ELAPSED_JUMP", "JUMP"],
["ROUND", "COMMAND", "SAMPLES", "TOTAL_MAX", "TOTAL_LAST", "RECENT_MAX", "RECENT_LAST", "CARD_DIAG", "TERM", "NONMONO", "ELAPSED_JUMP", "JUMP"],
rounds.map((item) => [
item.promptIndex,
item.promptCommandId,
@@ -10388,6 +10388,10 @@ const slimFindingSample=(value)=>{
diffKind:short(value.diffKind??value.kind??value.type??value.metric??value.anomaly,48),
fromValue:value.fromValue??value.beforeValue??null,
toValue:value.toValue??value.afterValue??null,
fromDigest:short(value.fromDigest,80),
toDigest:short(value.toDigest,80),
previousSeq:value.previousSeq??null,
previousTs:short(value.previousTs,32),
delta:value.delta??null,
sampleDeltaSeconds:value.sampleDeltaSeconds??null,
allowedIncreaseSeconds:value.allowedIncreaseSeconds??null,
@@ -10395,12 +10399,12 @@ const slimFindingSample=(value)=>{
promptIndex:value.promptIndex??null,
event:short(value.event??value.anomaly,48),
expectedPattern:short(value.expectedPattern,96),
control:compactMetric(controlCount),
observer:compactMetric(observerCount),
control:compactMetric(controlCount??(value.fromMessageCount!==undefined||value.fromTraceRowCount!==undefined?compactProjectionPair(value.fromMessageCount,value.fromTraceRowCount):undefined)),
observer:compactMetric(observerCount??(value.toMessageCount!==undefined||value.toTraceRowCount!==undefined?compactProjectionPair(value.toMessageCount,value.toTraceRowCount):undefined)),
sessionId:short(value.sessionId??value.routeSessionId??value.activeSessionId??value.controlSessionId??value.observerSessionId??refValue.routeSessionId??refValue.activeSessionId,64),
path:short(value.path??value.urlPath??value.url??value.controlPath??value.observerPath??refValue.url,120),
trace:short(trace,120),
detail:short(value.detail??(value.messageTextDigestDiff?'messageDigest '+String(value.controlMessageDigest??'-')+' != '+String(value.observerMessageDigest??'-'):undefined)??value.summary??value.message??value.preview??value.expectedPattern,180),
detail:short(value.detail??(value.fromDigest||value.toDigest?'digest '+String(value.fromDigest??'-')+' -> '+String(value.toDigest??'-'):undefined)??(value.messageTextDigestDiff?'messageDigest '+String(value.controlMessageDigest??'-')+' != '+String(value.observerMessageDigest??'-'):undefined)??value.summary??value.message??value.preview??value.expectedPattern,180),
valuesRedacted:true
};
};
scripts/src/hwlab-node-web-observe-runner-source.ts
+20 -1
View File
@@ -2847,7 +2847,22 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN
const pageKey = samplePageKey(sample);
const previous = previousDigestByPage.get(pageKey);
const next = digestSample(sample);
if (previous && previous.digest !== next && !commandedPromptSeqs.has(sample?.seq) && !nearCommand(sample, commandTimes, alertThresholds.uncommandedStateChangeCommandWindowMs)) uncommandedChanges.push(ref(sample));
if (previous && previous.digest !== next && !commandedPromptSeqs.has(sample?.seq) && !nearCommand(sample, commandTimes, alertThresholds.uncommandedStateChangeCommandWindowMs)) {
uncommandedChanges.push({
...ref(sample),
previousSeq: previous.sample?.seq ?? null,
previousTs: previous.sample?.timestamp ?? previous.sample?.ts ?? null,
fromDigest: previous.digest,
toDigest: next,
fromMessageCount: previous.sample?.messageCount ?? previous.sample?.messages?.length ?? null,
toMessageCount: sample?.messageCount ?? sample?.messages?.length ?? null,
fromTraceRowCount: Array.isArray(previous.sample?.traceRows) ? previous.sample.traceRows.length : previous.sample?.traceRowCount ?? null,
toTraceRowCount: Array.isArray(sample?.traceRows) ? sample.traceRows.length : sample?.traceRowCount ?? null,
fromPath: previous.sample?.path ?? previous.sample?.urlPath ?? null,
toPath: sample?.path ?? sample?.urlPath ?? null,
detail: "visible digest changed without nearby command",
});
}
previousDigestByPage.set(pageKey, { digest: next, sample });
}
if (uncommandedChanges.length > 0) findings.push({ id: "uncommanded-visible-state-change", severity: "amber", summary: "visible message/trace digest changed without a nearby command", count: uncommandedChanges.length, samples: uncommandedChanges.slice(0, 20) });
@@ -2916,6 +2931,10 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN
if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) });
const elapsedZeroResets = Array.isArray(sampleMetrics?.turnTimingElapsedZeroResets) ? sampleMetrics.turnTimingElapsedZeroResets : [];
if (elapsedZeroResets.length > 0) findings.push({ id: "turn-timing-total-elapsed-zero-reset", severity: "red", summary: "Code Agent total elapsed jumped from a non-zero value back to 0 seconds", count: elapsedZeroResets.length, samples: elapsedZeroResets.slice(0, 20) });
const elapsedDecreases = Array.isArray(sampleMetrics?.turnTimingNonMonotonic)
? sampleMetrics.turnTimingNonMonotonic.filter((item) => item.metric === "totalElapsedSeconds" && item.anomaly !== "zero-reset")
: [];
if (elapsedDecreases.length > 0) findings.push({ id: "turn-timing-total-elapsed-decrease", severity: "red", summary: "Code Agent total elapsed decreased between adjacent samples; total elapsed must be monotonic per turn", count: elapsedDecreases.length, samples: elapsedDecreases.slice(0, 20) });
const elapsedForwardJumps = Array.isArray(sampleMetrics?.turnTimingTotalElapsedForwardJumps) ? sampleMetrics.turnTimingTotalElapsedForwardJumps : [];
if (elapsedForwardJumps.length > 0) findings.push({ id: "turn-timing-total-elapsed-forward-jump", severity: "red", summary: "Code Agent total elapsed jumped forward faster than browser sample interval", count: elapsedForwardJumps.length, samples: elapsedForwardJumps.slice(0, 20) });
const terminalElapsedGrowth = Array.isArray(sampleMetrics?.turnTimingTerminalElapsedGrowth) ? sampleMetrics.turnTimingTerminalElapsedGrowth : [];