From e4427478f70e208e3d408ee2cb08b4301690601f Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Thu, 25 Jun 2026 04:13:16 +0800 Subject: [PATCH] fix: require active composer action for web observe steer (#850) Co-authored-by: Codex --- .../hwlab-node-web-observe-runner-source.ts | 105 ++++++++++++++---- 1 file changed, 85 insertions(+), 20 deletions(-) diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 99503fb7..a7e0428c 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -342,7 +342,7 @@ async function processCommand(command) { case "goto": return withObserverSync(await gotoTarget(command.path || command.url || targetPath), "goto"); case "newSession": return withObserverSync(await createSessionFromUi(), "newSession"); case "sendPrompt": return withObserverSync(await sendPrompt(String(command.text || "")), "sendPrompt"); - case "steer": return withObserverSync(await sendPrompt(String(command.text || "")), "steer"); + case "steer": return withObserverSync(await sendPrompt(String(command.text || ""), { expectedAction: "steer", responsePath: "/v1/agent/chat/steer", noActiveReason: "steer-no-active-turn" }), "steer"); case "cancel": return withObserverSync(await cancelRunningTurn(), "cancel"); case "selectProvider": return withObserverSync(await selectProvider(String(command.provider || command.value || command.text || "")), "selectProvider"); case "clickSession": return withObserverSync(await clickSession(String(command.sessionId || command.value || "")), "clickSession"); @@ -1048,8 +1048,9 @@ async function workbenchSessionSnapshot(targetPage = page) { }).catch(() => null); } -async function sendPrompt(text) { +async function sendPrompt(text, options = {}) { if (text.trim().length === 0) throw new Error("sendPrompt requires non-empty text"); + const responsePath = options.responsePath || "/v1/agent/chat"; const beforeUrl = currentPageUrl(); const beforeEvidence = await promptSideEffectSnapshot(); const primaryEditor = page.locator("#command-input").last(); @@ -1057,17 +1058,9 @@ async function sendPrompt(text) { ? primaryEditor : page.locator('textarea, [role="textbox"], [contenteditable="true"], input[type="text"]').last(); await editor.waitFor({ state: "visible", timeout: 15000 }); - const tag = await editor.evaluate((element) => element.tagName.toLowerCase()).catch(() => ""); - const editable = await editor.evaluate((element) => element.getAttribute("contenteditable") === "true").catch(() => false); - if (tag === "textarea" || tag === "input") await editor.fill(text); - else if (editable) { - await editor.click(); - await page.keyboard.insertText(text); - } else { - await editor.click(); - await page.keyboard.insertText(text); - } - const primarySubmit = page.locator('#command-submit, [data-testid="command-submit"], [data-testid="composer-submit"], [data-testid="send-command"]').last(); + await fillComposerEditor(editor, text); + const primarySubmitSelector = '#command-send, #command-submit, [data-testid="command-submit"], [data-testid="composer-submit"], [data-testid="send-command"]'; + const primarySubmit = page.locator(primarySubmitSelector).last(); const submit = await primarySubmit.isVisible().catch(() => false) ? primarySubmit : page.locator([ @@ -1079,11 +1072,45 @@ async function sendPrompt(text) { '[aria-label*="发送"]' ].join(", ")).last(); await submit.waitFor({ state: "visible", timeout: 15000 }); + if (options.expectedAction) { + await page.waitForFunction(({ selector, expected }) => { + const element = document.querySelector(selector); + return element?.getAttribute("data-action") === expected || element?.getAttribute("disabled") !== null; + }, { selector: primarySubmitSelector, expected: options.expectedAction }, { timeout: 1000 }).catch(() => null); + const composer = await composerButtonState(submit); + if (composer.action !== options.expectedAction || composer.disabled === true) { + await clearComposerEditor(editor).catch(() => {}); + const sideEffect = await waitForPromptSideEffect(beforeEvidence, 200).catch(() => promptSideEffectSnapshot()); + return { + beforeUrl, + afterUrl: currentPageUrl(), + textHash: sha256Text(text), + textBytes: Buffer.byteLength(text), + submitted: false, + blocked: true, + degradedReason: options.noActiveReason || "composer-action-mismatch", + composer, + chatSubmit: { + status: null, + statusText: null, + urlPath: responsePath, + responseObserved: false, + sideEffectObserved: sideEffectHasAuthoritativePromptSubmission(sideEffect), + sideEffect, + expectedAction: options.expectedAction, + actualAction: composer.action, + valuesRedacted: true + }, + pageId, + valuesRedacted: true + }; + } + } const chatResponsePromise = page.waitForResponse((response) => { const request = response.request(); if (request.method().toUpperCase() !== "POST") return false; try { - return new URL(response.url()).pathname === "/v1/agent/chat"; + return new URL(response.url()).pathname === responsePath; } catch { return false; } @@ -1099,17 +1126,17 @@ async function sendPrompt(text) { afterUrl: currentPageUrl(), textHash: sha256Text(text), textBytes: Buffer.byteLength(text), - chatSubmit: { status: null, statusText: null, urlPath: "/v1/agent/chat", waitError: chatResponse.waitError, sideEffectObserved: true, sideEffect }, + chatSubmit: { status: null, statusText: null, urlPath: responsePath, waitError: chatResponse.waitError, sideEffectObserved: true, sideEffect }, pageId }; } - const error = new Error("sendPrompt did not observe POST /v1/agent/chat response or an authoritative new turn after submit: " + (chatResponse.waitError.message || chatResponse.waitError.name || "timeout")); + const error = new Error("sendPrompt did not observe POST " + responsePath + " response or an authoritative new turn after submit: " + (chatResponse.waitError.message || chatResponse.waitError.name || "timeout")); error.details = { beforeUrl, afterUrl: currentPageUrl(), textHash: sha256Text(text), textBytes: Buffer.byteLength(text), - chatSubmit: { status: null, statusText: null, urlPath: "/v1/agent/chat", waitError: chatResponse.waitError, sideEffectObserved: false, sideEffect }, + chatSubmit: { status: null, statusText: null, urlPath: responsePath, waitError: chatResponse.waitError, sideEffectObserved: false, sideEffect }, pageId, valuesRedacted: true }; @@ -1117,7 +1144,7 @@ async function sendPrompt(text) { } const chatStatus = chatResponse.status(); if (chatStatus < 200 || chatStatus >= 300) { - throw new Error("sendPrompt observed POST /v1/agent/chat HTTP " + chatStatus + " " + chatResponse.statusText()); + throw new Error("sendPrompt observed POST " + responsePath + " HTTP " + chatStatus + " " + chatResponse.statusText()); } let chatPayload = null; let chatPayloadError = null; @@ -1126,11 +1153,11 @@ async function sendPrompt(text) { } catch (error) { chatPayloadError = errorSummary(error); } - let chatUrlPath = "/v1/agent/chat"; + let chatUrlPath = responsePath; try { chatUrlPath = new URL(chatResponse.url()).pathname; } catch { - chatUrlPath = "/v1/agent/chat"; + chatUrlPath = responsePath; } const payloadText = chatPayload ? JSON.stringify(chatPayload) : ""; const traceId = payloadText.match(/\btrc_[A-Za-z0-9_-]+\b/u)?.[0] || null; @@ -1159,6 +1186,44 @@ async function sendPrompt(text) { }; } +async function fillComposerEditor(editor, text) { + const tag = await editor.evaluate((element) => element.tagName.toLowerCase()).catch(() => ""); + const editable = await editor.evaluate((element) => element.getAttribute("contenteditable") === "true").catch(() => false); + if (tag === "textarea" || tag === "input") await editor.fill(text); + else if (editable) { + await editor.click(); + await page.keyboard.insertText(text); + } else { + await editor.click(); + await page.keyboard.insertText(text); + } +} + +async function clearComposerEditor(editor) { + const tag = await editor.evaluate((element) => element.tagName.toLowerCase()).catch(() => ""); + const editable = await editor.evaluate((element) => element.getAttribute("contenteditable") === "true").catch(() => false); + if (tag === "textarea" || tag === "input") { + await editor.fill(""); + return; + } + if (editable) { + await editor.click(); + await page.keyboard.press(process.platform === "darwin" ? "Meta+A" : "Control+A").catch(() => {}); + await page.keyboard.press("Backspace").catch(() => {}); + } +} + +async function composerButtonState(button) { + return button.evaluate((element) => ({ + action: element.getAttribute("data-action") || null, + disabled: element.hasAttribute("disabled") || element.getAttribute("aria-disabled") === "true", + text: (element.textContent || "").trim().slice(0, 80), + title: element.getAttribute("title") || null, + ariaLabel: element.getAttribute("aria-label") || null, + testId: element.getAttribute("data-testid") || null, + })).catch((error) => ({ action: null, disabled: null, error: errorSummary(error), valuesRedacted: true })); +} + function sideEffectHasAuthoritativePromptSubmission(sideEffect) { return Boolean( (Array.isArray(sideEffect?.newRunIds) && sideEffect.newRunIds.length > 0)