From 59fe4a4d98e36b725b0e12b42d25ebec014fc0d4 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Wed, 24 Jun 2026 07:48:56 +0800 Subject: [PATCH] fix(web-probe): require authoritative prompt submit evidence (#805) Co-authored-by: Codex --- .../hwlab-node-web-observe-runner-source.ts | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 27f9ef38..e4fd9bfd 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -1090,7 +1090,7 @@ async function sendPrompt(text) { await page.waitForTimeout(500); if (chatResponse?.waitError) { const sideEffect = await waitForPromptSideEffect(beforeEvidence, 5000); - if (sideEffect.submitted) { + if (sideEffectHasAuthoritativePromptSubmission(sideEffect)) { return { beforeUrl, afterUrl: currentPageUrl(), @@ -1100,7 +1100,17 @@ async function sendPrompt(text) { pageId }; } - throw new Error("sendPrompt did not observe POST /v1/agent/chat response after submit: " + (chatResponse.waitError.message || chatResponse.waitError.name || "timeout")); + const error = new Error("sendPrompt did not observe POST /v1/agent/chat response or an authoritative new turn after submit: " + (chatResponse.waitError.message || chatResponse.waitError.name || "timeout")); + error.details = { + beforeUrl, + afterUrl: currentPageUrl(), + textHash: sha256Text(text), + textBytes: Buffer.byteLength(text), + chatSubmit: { status: null, statusText: null, urlPath: "/v1/agent/chat", waitError: chatResponse.waitError, sideEffectObserved: false, sideEffect }, + pageId, + valuesRedacted: true + }; + throw error; } const chatStatus = chatResponse.status(); if (chatStatus < 200 || chatStatus >= 300) { @@ -1146,6 +1156,14 @@ async function sendPrompt(text) { }; } +function sideEffectHasAuthoritativePromptSubmission(sideEffect) { + return Boolean( + (Array.isArray(sideEffect?.newRunIds) && sideEffect.newRunIds.length > 0) + || (Array.isArray(sideEffect?.newTraceIds) && sideEffect.newTraceIds.length > 0) + || Number(sideEffect?.messageCountDelta || 0) > 0 + ); +} + async function waitForPromptSideEffect(beforeEvidence, timeoutMs) { await page.waitForFunction((before) => { const current = (() => { @@ -1154,37 +1172,49 @@ async function waitForPromptSideEffect(beforeEvidence, timeoutMs) { const traceIds = Array.from(new Set(text.match(/trc_[A-Za-z0-9_:-]+/gu) || [])).slice(-20); const running = /Trace running|最近\s*\d+\s*(?:秒|分钟|分|小时)前|Code Agent\s*耗时/iu.test(text); const executionError = /AgentRun error|agentrun:error:|provider-stream-disconnected|provider-unavailable/iu.test(text); + const visible = (element) => { + const rect = element.getBoundingClientRect(); + const style = window.getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && style.visibility !== "hidden" && style.display !== "none"; + }; + const messageCount = Array.from(document.querySelectorAll('article.message-card, .message-card[data-message-id], article[data-message-id]')).filter(visible).length; const textBytes = new TextEncoder().encode(text).length; - return { runIds, traceIds, running, executionError, textBytes }; + return { runIds, traceIds, running, executionError, messageCount, textBytes }; })(); const beforeRuns = Array.isArray(before?.runIds) ? before.runIds : []; const beforeTraces = Array.isArray(before?.traceIds) ? before.traceIds : []; const newRun = current.runIds.some((id) => !beforeRuns.includes(id)); const newTrace = current.traceIds.some((id) => !beforeTraces.includes(id)); - const meaningfulChange = Math.abs(current.textBytes - Number(before?.textBytes || 0)) > 20 && (current.running || current.executionError); - return newRun || newTrace || meaningfulChange; + const newMessage = current.messageCount > Number(before?.messageCount || 0); + return newRun || newTrace || newMessage; }, beforeEvidence || {}, { timeout: timeoutMs }).catch(() => null); const after = await promptSideEffectSnapshot(); const beforeRuns = Array.isArray(beforeEvidence?.runIds) ? beforeEvidence.runIds : []; const beforeTraces = Array.isArray(beforeEvidence?.traceIds) ? beforeEvidence.traceIds : []; const newRunIds = after.runIds.filter((id) => !beforeRuns.includes(id)); const newTraceIds = after.traceIds.filter((id) => !beforeTraces.includes(id)); - const meaningfulChange = Math.abs(after.textBytes - Number(beforeEvidence?.textBytes || 0)) > 20 && (after.running || after.executionError); - return { ...after, newRunIds, newTraceIds, submitted: newRunIds.length > 0 || newTraceIds.length > 0 || meaningfulChange, valuesRedacted: true }; + const messageCountDelta = Math.max(0, Number(after.messageCount || 0) - Number(beforeEvidence?.messageCount || 0)); + return { ...after, newRunIds, newTraceIds, messageCountDelta, submitted: newRunIds.length > 0 || newTraceIds.length > 0 || messageCountDelta > 0, valuesRedacted: true }; } async function promptSideEffectSnapshot() { return page.evaluate(() => { const text = document.body?.innerText || ""; + const visible = (element) => { + const rect = element.getBoundingClientRect(); + const style = window.getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && style.visibility !== "hidden" && style.display !== "none"; + }; return { runIds: Array.from(new Set(text.match(/run_[A-Za-z0-9_:-]+/gu) || [])).slice(-20), traceIds: Array.from(new Set(text.match(/trc_[A-Za-z0-9_:-]+/gu) || [])).slice(-20), running: /Trace running|最近\s*\d+\s*(?:秒|分钟|分|小时)前|Code Agent\s*耗时/iu.test(text), executionError: /AgentRun error|agentrun:error:|provider-stream-disconnected|provider-unavailable/iu.test(text), + messageCount: Array.from(document.querySelectorAll('article.message-card, .message-card[data-message-id], article[data-message-id]')).filter(visible).length, textBytes: new TextEncoder().encode(text).length, valuesRedacted: true }; - }).catch(() => ({ runIds: [], traceIds: [], running: false, executionError: false, textBytes: 0, valuesRedacted: true })); + }).catch(() => ({ runIds: [], traceIds: [], running: false, executionError: false, messageCount: 0, textBytes: 0, valuesRedacted: true })); } async function selectProvider(provider) {