diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 896589a2..ad186129 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -180,6 +180,10 @@ lanes: longLivedStreamOpenSlowMs: 10000 visibleLoadingSlowMs: 10000 turnTimingSampleSlackSeconds: 3 + uncommandedStateChangeCommandWindowMs: 10000 + scrollJumpCommandWindowMs: 8000 + scrollJumpFromY: 250 + scrollJumpToY: 40 sessionRailFallbackRatio: 0.5 tektonDir: tekton-v03 argoApplicationFile: application-v03.yaml diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 3cd40ac2..0625c968 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -136,6 +136,10 @@ export interface HwlabRuntimeWebProbeAlertThresholdsSpec { readonly longLivedStreamOpenSlowMs: number; readonly visibleLoadingSlowMs: number; readonly turnTimingSampleSlackSeconds: number; + readonly uncommandedStateChangeCommandWindowMs: number; + readonly scrollJumpCommandWindowMs: number; + readonly scrollJumpFromY: number; + readonly scrollJumpToY: number; readonly sessionRailFallbackRatio: number; } @@ -720,6 +724,10 @@ function webProbeAlertThresholdsConfig(value: unknown, path: string): HwlabRunti longLivedStreamOpenSlowMs: positiveNumberField(raw, "longLivedStreamOpenSlowMs", path), visibleLoadingSlowMs: positiveNumberField(raw, "visibleLoadingSlowMs", path), turnTimingSampleSlackSeconds: positiveNumberField(raw, "turnTimingSampleSlackSeconds", path), + uncommandedStateChangeCommandWindowMs: positiveNumberField(raw, "uncommandedStateChangeCommandWindowMs", path), + scrollJumpCommandWindowMs: positiveNumberField(raw, "scrollJumpCommandWindowMs", path), + scrollJumpFromY: positiveNumberField(raw, "scrollJumpFromY", path), + scrollJumpToY: positiveNumberField(raw, "scrollJumpToY", path), sessionRailFallbackRatio, }; } diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 6054553a..ef085bf4 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -1958,6 +1958,10 @@ function parseAlertThresholds(value) { longLivedStreamOpenSlowMs: requiredPositiveThreshold(raw, "longLivedStreamOpenSlowMs"), visibleLoadingSlowMs: requiredPositiveThreshold(raw, "visibleLoadingSlowMs"), turnTimingSampleSlackSeconds: requiredPositiveThreshold(raw, "turnTimingSampleSlackSeconds"), + uncommandedStateChangeCommandWindowMs: requiredPositiveThreshold(raw, "uncommandedStateChangeCommandWindowMs"), + scrollJumpCommandWindowMs: requiredPositiveThreshold(raw, "scrollJumpCommandWindowMs"), + scrollJumpFromY: requiredPositiveThreshold(raw, "scrollJumpFromY"), + scrollJumpToY: requiredPositiveThreshold(raw, "scrollJumpToY"), sessionRailFallbackRatio, source: "yaml-env", }; @@ -2299,6 +2303,10 @@ function parseAlertThresholds(value) { longLivedStreamOpenSlowMs: requiredPositiveThreshold(raw, "longLivedStreamOpenSlowMs"), visibleLoadingSlowMs: requiredPositiveThreshold(raw, "visibleLoadingSlowMs"), turnTimingSampleSlackSeconds: requiredPositiveThreshold(raw, "turnTimingSampleSlackSeconds"), + uncommandedStateChangeCommandWindowMs: requiredPositiveThreshold(raw, "uncommandedStateChangeCommandWindowMs"), + scrollJumpCommandWindowMs: requiredPositiveThreshold(raw, "scrollJumpCommandWindowMs"), + scrollJumpFromY: requiredPositiveThreshold(raw, "scrollJumpFromY"), + scrollJumpToY: requiredPositiveThreshold(raw, "scrollJumpToY"), sessionRailFallbackRatio, source: "yaml-env", }; @@ -2565,7 +2573,7 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN for (let i = 1; i < samples.length; i += 1) { const prev = digestSample(samples[i - 1]); const next = digestSample(samples[i]); - if (prev !== next && !commandedPromptSeqs.has(samples[i]?.seq) && !nearCommand(samples[i], commandTimes, 10000)) uncommandedChanges.push(ref(samples[i])); + if (prev !== next && !commandedPromptSeqs.has(samples[i]?.seq) && !nearCommand(samples[i], commandTimes, alertThresholds.uncommandedStateChangeCommandWindowMs)) uncommandedChanges.push(ref(samples[i])); } if (uncommandedChanges.length > 0) findings.push({ id: "uncommanded-visible-state-change", severity: "amber", summary: "visible message/trace digest changed without a nearby command", count: uncommandedChanges.length, samples: uncommandedChanges.slice(0, 20) }); const finalFlicker = detectFinalFlicker(samples); @@ -2576,7 +2584,7 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN for (let i = 1; i < samples.length; i += 1) { const prevY = Number(samples[i - 1]?.scroll?.y ?? 0); const nextY = Number(samples[i]?.scroll?.y ?? 0); - if (prevY > 250 && nextY < 40 && !nearCommand(samples[i], commandTimes, 8000)) scrollJumps.push({ from: ref(samples[i - 1]), to: ref(samples[i]) }); + if (prevY > alertThresholds.scrollJumpFromY && nextY < alertThresholds.scrollJumpToY && !nearCommand(samples[i], commandTimes, alertThresholds.scrollJumpCommandWindowMs)) scrollJumps.push({ from: ref(samples[i - 1]), to: ref(samples[i]) }); } if (scrollJumps.length > 0) findings.push({ id: "scroll-jump-top", severity: "amber", summary: "scroll position jumped near top without nearby command", count: scrollJumps.length, samples: scrollJumps.slice(0, 10) }); const traceTerminal = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.some((row) => isTerminalTraceText((row.status || "") + " " + (row.textPreview || ""))));