From 999a593373ffbc63532b503f9714dac9def58512 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Jun 2026 01:44:21 +0000 Subject: [PATCH] fix: tighten sentinel quick verify waits --- .../hwlab-node-web-observe-runner-source.ts | 49 ++++-- scripts/src/hwlab-node-web-sentinel-cicd.ts | 151 ++++++++++++++---- 2 files changed, 161 insertions(+), 39 deletions(-) diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 3a72844d..e930f862 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -343,7 +343,13 @@ async function processCommand(command) { case "preflight": return preflightSummary(); case "goto": return withObserverSync(await gotoTarget(command.path || command.url || targetPath), "goto"); case "newSession": return withObserverSync(await createSessionFromUi(), "newSession"); - case "sendPrompt": return withObserverSync(await sendPrompt(String(command.text || "")), "sendPrompt"); + case "sendPrompt": return withObserverSync(await sendPrompt(String(command.text || ""), { + expectedAction: "turn", + responsePath: "/v1/agent/chat", + alternateResponsePaths: ["/v1/agent/chat/steer"], + noActiveReason: "send-no-turn-composer", + throwOnActionMismatch: true, + }), "sendPrompt"); case "steer": return withObserverSync(await sendPrompt(String(command.text || ""), { expectedAction: "steer", responsePath: "/v1/agent/chat/steer", noActiveReason: "steer-no-active-turn" }), "steer"); case "cancel": return withObserverSync(await cancelRunningTurn(), "cancel"); case "selectProvider": return withObserverSync(await selectProvider(String(command.provider || command.value || command.text || "")), "selectProvider"); @@ -1168,7 +1174,7 @@ async function sendPrompt(text, options = {}) { if (composer.action !== options.expectedAction || composer.disabled === true) { await clearComposerEditor(editor).catch(() => {}); const sideEffect = await waitForPromptSideEffect(beforeEvidence, 200).catch(() => promptSideEffectSnapshot()); - return { + const blocked = { beforeUrl, afterUrl: currentPageUrl(), textHash: sha256Text(text), @@ -1191,13 +1197,20 @@ async function sendPrompt(text, options = {}) { pageId, valuesRedacted: true }; + if (options.throwOnActionMismatch === true) { + const error = new Error("sendPrompt composer action mismatch: expected " + options.expectedAction + " actual " + (composer.action || "null")); + error.details = blocked; + throw error; + } + return blocked; } } + const acceptedResponsePaths = [responsePath, ...(Array.isArray(options.alternateResponsePaths) ? options.alternateResponsePaths : [])]; const chatResponsePromise = page.waitForResponse((response) => { const request = response.request(); if (request.method().toUpperCase() !== "POST") return false; try { - return new URL(response.url()).pathname === responsePath; + return acceptedResponsePaths.includes(new URL(response.url()).pathname); } catch { return false; } @@ -1230,8 +1243,14 @@ async function sendPrompt(text, options = {}) { throw error; } const chatStatus = chatResponse.status(); + let chatUrlPath = responsePath; + try { + chatUrlPath = new URL(chatResponse.url()).pathname; + } catch { + chatUrlPath = responsePath; + } if (chatStatus < 200 || chatStatus >= 300) { - throw new Error("sendPrompt observed POST " + responsePath + " HTTP " + chatStatus + " " + chatResponse.statusText()); + throw new Error("sendPrompt observed POST " + chatUrlPath + " HTTP " + chatStatus + " " + chatResponse.statusText()); } let chatPayload = null; let chatPayloadError = null; @@ -1240,12 +1259,6 @@ async function sendPrompt(text, options = {}) { } catch (error) { chatPayloadError = errorSummary(error); } - let chatUrlPath = responsePath; - try { - chatUrlPath = new URL(chatResponse.url()).pathname; - } catch { - chatUrlPath = responsePath; - } const payloadText = chatPayload ? JSON.stringify(chatPayload) : ""; const traceId = payloadText.match(/\btrc_[A-Za-z0-9_-]+\b/u)?.[0] || null; const otelTraceId = typeof chatPayload?.otelTrace?.traceId === "string" && /^[0-9a-f]{32}$/u.test(chatPayload.otelTrace.traceId) @@ -2496,6 +2509,9 @@ async function sampleOnePage(targetPage, { reason, groupSeq, pageRole, targetPag const routeSessionMatch = url.match(/\/workbench\/sessions\/([^/?#]+)/u); const activeSession = document.querySelector('[data-active="true"][data-session-id], [aria-selected="true"][data-session-id], .active[data-session-id]'); const activeSessionId = activeSession ? activeSession.getAttribute("data-session-id") : null; + const commandInput = document.querySelector("#command-input"); + const commandSubmit = document.querySelector('#command-send, #command-submit, [data-testid="command-submit"], [data-testid="composer-submit"], [data-testid="send-command"]'); + const composerWarning = document.querySelector(".composer-warning"); const messageSelector = 'article.message-card, .message-card[data-message-id], article[data-message-id]'; const stableTraceSelector = 'li.trace-render-row[data-row-id], li.trace-render-row[data-testid="trace-render-row"], [data-testid="trace-render-row"][data-row-id]'; const fallbackTraceSelector = '[data-testid*="trace" i], [class*="trace" i], [data-trace-id], [data-testid*="event" i]'; @@ -2573,6 +2589,19 @@ async function sampleOnePage(targetPage, { reason, groupSeq, pageRole, targetPag sessionRail, diagnostics, turns, + composer: { + inputPresent: visible(commandInput), + inputDisabled: Boolean(commandInput?.disabled) || commandInput?.getAttribute("aria-disabled") === "true", + warningPresent: visible(composerWarning), + warningText: trim(composerWarning?.textContent || "", 160), + submitPresent: visible(commandSubmit), + submitDisabled: Boolean(commandSubmit?.disabled) || commandSubmit?.getAttribute("aria-disabled") === "true", + submitAction: commandSubmit?.getAttribute("data-action") || null, + submitText: trim(commandSubmit?.textContent || "", 80), + submitTestId: commandSubmit?.getAttribute("data-testid") || null, + activeStatus: activeSession?.getAttribute("data-status") || null, + valuesRedacted: true + }, projectManagement: collectProjectManagement(), pageProvenance: { url: location.href, diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 3a90ace8..4d160551 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -1842,49 +1842,50 @@ function runChildCli(args: string[], timeoutSeconds: number, input?: string): Ch function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: string, promptIndex: number, deadline: number, pollIntervalMs: number): Record { const observations: Record[] = []; + const indexEntry = readLocalObserveIndex(observerId); + if (indexEntry === null) { + return { + ok: false, + failure: "observe-index-entry-missing", + round: promptIndex, + observerId, + valuesRedacted: true, + }; + } const pollSleepMs = Math.max(250, Math.min(5000, Math.trunc(pollIntervalMs))); while (Date.now() < deadline) { - const view = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 20)); - const rows = Array.isArray(record(view.collect).rows) ? record(view.collect).rows.map(record) : []; - const row = rows.find((item) => Number(item.round) === promptIndex) ?? null; - const status = typeof row?.status === "string" ? row.status : null; - const finalResponse = record(row?.finalResponse); - observations.push({ - ok: view.ok, + const waitMs = Math.max(1000, Math.min(55_000, deadline - Date.now())); + const script = quickVerifyPromptWaitScript(indexEntry.stateDir, promptIndex, waitMs, pollSleepMs); + const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 5000 }); + const payload = parseJsonObject(result.stdout); + if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record)); + const status = typeof payload?.status === "string" ? payload.status : null; + const terminalPayload = { round: promptIndex, status, - traceId: row?.traceId ?? null, - finalResponseEmpty: finalResponse.empty === true, - lastSeq: row?.lastSeq ?? null, - lastTs: row?.lastTs ?? null, + traceId: payload?.traceId ?? null, + finalResponseEmpty: payload?.finalResponseEmpty === true, + composerReadyForTurn: payload?.composerReadyForTurn === true, + composerAction: typeof payload?.composerAction === "string" ? payload.composerAction : null, + observations: observations.slice(-6), + waitResult: compactCommand(result), valuesRedacted: true, - }); - if (isQuickVerifyTurnSuccessful(status)) { + }; + if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-wait-chunk-timeout") { return { - ok: true, - round: promptIndex, - status, - traceId: row?.traceId ?? null, - finalResponseEmpty: finalResponse.empty === true, - observations: observations.slice(-6), - valuesRedacted: true, + ok: false, + failure: text(payload?.failure ?? "quick-verify-artifact-wait-failed"), + ...terminalPayload, }; } + if (payload.ok === true) return { ok: true, ...terminalPayload }; if (isQuickVerifyTurnTerminal(status)) { return { ok: false, failure: "observe-turn-terminal-non-success", - round: promptIndex, - status, - traceId: row?.traceId ?? null, - finalResponseEmpty: finalResponse.empty === true, - observations: observations.slice(-6), - valuesRedacted: true, + ...terminalPayload, }; } - const sleepMs = Math.min(pollSleepMs, Math.max(0, deadline - Date.now())); - if (sleepMs <= 0) break; - runCommand(["sleep", String(sleepMs / 1000)], repoRoot, { timeoutMs: sleepMs + 1000 }); } return { ok: false, @@ -1896,6 +1897,98 @@ function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: stri }; } +function quickVerifyPromptWaitScript(stateDir: string, promptIndex: number, timeoutMs: number, pollSleepMs: number): string { + return [ + "set -eu", + `state_dir=${shellQuote(stateDir)}`, + `prompt_index=${shellQuote(String(promptIndex))}`, + `timeout_ms=${shellQuote(String(Math.max(1, Math.trunc(timeoutMs))))}`, + `poll_ms=${shellQuote(String(Math.max(250, Math.trunc(pollSleepMs))))}`, + "test -d \"$state_dir\" || { printf '{\"ok\":false,\"failure\":\"state-dir-missing\",\"stateDir\":\"%s\",\"valuesRedacted\":true}\\n' \"$state_dir\"; exit 0; }", + "node - \"$state_dir\" \"$prompt_index\" \"$timeout_ms\" \"$poll_ms\" <<'NODE'", + "const fs = require('node:fs');", + "const path = require('node:path');", + "const dir = process.argv[2];", + "const promptIndex = Number(process.argv[3]);", + "const timeoutMs = Number(process.argv[4]);", + "const pollMs = Number(process.argv[5]);", + "const startedAt = Date.now();", + "const short = (value, limit = 160) => String(value || '').replace(/\\s+/gu, ' ').trim().slice(0, limit);", + "const textOf = (value) => String(value?.text || value?.textPreview || value?.preview || '');", + "const arr = (value) => Array.isArray(value) ? value : [];", + "const unique = (values) => Array.from(new Set(values.filter(Boolean)));", + "const numOrNull = (value) => { const n = Number(value); return Number.isFinite(n) ? n : null; };", + "const tsMs = (value) => { const ms = Date.parse(String(value || '')); return Number.isFinite(ms) ? ms : null; };", + "const readJson = (rel) => { try { return JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); } catch { return null; } };", + "const readJsonl = (rel) => { try { return fs.readFileSync(path.join(dir, rel), 'utf8').split(/\\r?\\n/u).filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean); } catch { return []; } };", + "const readDone = (id) => id ? readJson(path.join('commands', 'done', `${id}.json`)) : null;", + "function sessionIdFromUrl(value) { const match = String(value || '').match(/\\/workbench\\/sessions\\/(ses_[A-Za-z0-9_-]+)/u); return match ? match[1] : null; }", + "function commandSessionId(item) { const done = readDone(item?.commandId); return item?.sessionId || item?.detail?.sessionId || item?.input?.sessionId || item?.result?.sessionId || done?.result?.sessionId || done?.result?.observer?.sessionId || sessionIdFromUrl(item?.afterUrl) || sessionIdFromUrl(item?.detail?.afterUrl) || sessionIdFromUrl(done?.result?.afterUrl) || null; }", + "function firstTraceId(value) { const match = String(value || '').match(/\\btrc_[A-Za-z0-9_-]+\\b/u); return match ? match[0] : null; }", + "function commandTraceId(item) { const done = readDone(item?.commandId); return item?.traceId || item?.detail?.chatSubmit?.traceId || item?.detail?.traceId || item?.input?.traceId || item?.result?.chatSubmit?.traceId || done?.result?.chatSubmit?.traceId || done?.result?.traceId || null; }", + "function itemTraceId(item) { return item?.traceId || firstTraceId(textOf(item)) || null; }", + "function completedNewSessionIdsBefore(control, ts) { const limit = tsMs(ts); return control.filter((item) => item.type === 'newSession' && item.phase === 'completed').filter((item) => limit === null || tsMs(item.ts) === null || tsMs(item.ts) <= limit).map(commandSessionId).filter(Boolean); }", + "function authoritativeSessionIdForPrompts(control, prompts) { const ids = completedNewSessionIdsBefore(control, prompts[0]?.firstTs || null); return ids.slice(-1)[0] || unique(prompts.map((item) => item.sessionId))[0] || null; }", + "function promptCommands(control) {", + " const map = new Map();", + " for (const item of control) {", + " if (item.type !== 'sendPrompt' || !['started', 'completed', 'failed'].includes(item.phase)) continue;", + " const id = item.commandId || item.seq || String(map.size + 1);", + " const existing = map.get(id) || {};", + " map.set(id, { ...existing, ...item, input: { ...(existing.input || {}), ...(item.input || {}) }, sessionId: existing.sessionId || commandSessionId(item), traceId: existing.traceId || commandTraceId(item), firstTs: existing.firstTs || item.ts, lastTs: item.ts });", + " }", + " const prompts = Array.from(map.values()).filter((item) => tsMs(item.firstTs) !== null).sort((a, b) => tsMs(a.firstTs) - tsMs(b.firstTs));", + " const sessionId = authoritativeSessionIdForPrompts(control, prompts);", + " if (!sessionId) return prompts;", + " const scoped = prompts.filter((item) => item.sessionId === sessionId);", + " return scoped.length > 0 ? scoped : prompts;", + "}", + "function segmentFor(samples, prompts, index) { const start = tsMs(prompts[index]?.firstTs); const end = index + 1 < prompts.length ? tsMs(prompts[index + 1].firstTs) : Infinity; return samples.filter((sample) => { const ms = tsMs(sample.ts); return ms !== null && ms >= start && ms < end; }); }", + "function entryGroups(sample) { return [...arr(sample.turns).map((item) => ({ group: 'turn', item })), ...arr(sample.traceRows).map((item) => ({ group: 'traceRow', item })), ...arr(sample.messages).map((item) => ({ group: 'message', item }))]; }", + "function traceIdsFromSamples(items) { const ids = []; for (const sample of items) for (const entry of entryGroups(sample)) { const id = itemTraceId(entry.item); if (id) ids.push(id); } return unique(ids); }", + "function chooseTraceId(segment, prompt) { const promptTraceId = commandTraceId(prompt) || prompt?.traceId || null; const ids = traceIdsFromSamples(segment); if (promptTraceId && (ids.length === 0 || ids.includes(promptTraceId))) return promptTraceId; return ids.slice(-1)[0] || promptTraceId || null; }", + "function traceEntries(items, traceId) { const entries = []; for (const sample of items) for (const entry of entryGroups(sample)) { const text = textOf(entry.item); const id = itemTraceId(entry.item); if (!traceId || id === traceId || text.includes(traceId)) entries.push({ ...entry, sample, text }); } return entries; }", + "function normalizeLifecycleStatus(value) { const raw = String(value || '').trim().toLowerCase(); if (/^(canceled|cancelled)$/u.test(raw)) return 'canceled'; if (/^(failed|failure|error)$/u.test(raw)) return 'failed'; if (/^(completed|complete|succeeded|success|terminal)$/u.test(raw)) return 'completed'; if (/^(running|admitting|queued|pending|in_progress|in-progress)$/u.test(raw)) return 'running'; return null; }", + "function statusFor(items, traceId) { const entries = traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) }))); const lastTurn = entries.filter((entry) => entry.group === 'turn').slice(-1)[0]?.item || null; const turnStatus = normalizeLifecycleStatus(lastTurn?.status); if (turnStatus) return turnStatus; const lastMessage = entries.filter((entry) => entry.group === 'message').slice(-1)[0]?.item || null; return normalizeLifecycleStatus(lastMessage?.status) || 'unknown'; }", + "function cleanFinalResponseText(value) { const raw = String(value || '').trim(); if (!raw) return ''; if (/^(completed|failed|canceled|cancelled|轮次完成|轮次失败|轮次取消|已记录)$/iu.test(raw.replace(/\\s+/gu, ' '))) return ''; if (/^(admitted|run|ok|error)\\s+/iu.test(raw)) return ''; return raw; }", + "function finalResponseEmpty(items, traceId) { if (!/^(completed|failed|canceled)$/u.test(statusFor(items, traceId))) return true; const entries = (traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) })))).slice().reverse(); for (const entry of entries) { const role = String(entry.item?.role || entry.item?.dataRole || entry.item?.messageRole || '').toLowerCase(); if (entry.group === 'message' && role && !/assistant|agent|system/u.test(role)) continue; const text = cleanFinalResponseText(entry.item?.finalResponse?.text || entry.item?.finalResponse?.preview || entry.text); if (text && !/^Code Agent\\s*耗时/iu.test(text)) return false; } return true; }", + "function rowFor() {", + " const control = readJsonl('control.jsonl');", + " const samples = readJsonl('samples.jsonl');", + " const prompts = promptCommands(control);", + " const prompt = prompts[promptIndex - 1] || null;", + " if (!prompt) return { ok: true, round: promptIndex, status: null, traceId: null, finalResponseEmpty: true, lastSeq: null, lastTs: null, valuesRedacted: true };", + " const segment = segmentFor(samples, prompts, promptIndex - 1);", + " const traceId = chooseTraceId(segment, prompt);", + " const status = statusFor(segment, traceId);", + " const sampleForTrace = traceId ? segment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || null : null;", + " const lastSample = sampleForTrace || segment.slice(-1)[0] || null;", + " const composerSample = segment.filter((sample) => sample.pageRole === 'control').slice(-1)[0] || lastSample;", + " const composer = composerSample?.composer || {};", + " const composerReadyForTurn = composer.inputPresent === true && composer.inputDisabled !== true && composer.submitPresent === true && composer.warningPresent !== true && composer.submitAction === 'turn';", + " return { ok: true, round: promptIndex, status, traceId, finalResponseEmpty: finalResponseEmpty(segment, traceId), lastSeq: numOrNull(lastSample?.seq), lastTs: lastSample?.ts || null, composerReadyForTurn, composerAction: composer.submitAction || null, source: 'observe-artifact-wait-script', valuesRedacted: true };", + "}", + "function norm(value) { return String(value || '').trim().toLowerCase().replace(/_/gu, '-'); }", + "function successful(value) { return ['completed', 'succeeded', 'success'].includes(norm(value)); }", + "function terminal(value) { return ['completed', 'succeeded', 'success', 'failed', 'error', 'blocked', 'timeout', 'canceled', 'cancelled', 'terminal'].includes(norm(value)); }", + "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", + "(async () => {", + " const observations = [];", + " while (Date.now() - startedAt <= timeoutMs) {", + " const row = rowFor();", + " observations.push(row);", + " if (successful(row.status) && row.composerReadyForTurn === true) { console.log(JSON.stringify({ ok: true, ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", + " if (!successful(row.status) && terminal(row.status)) { console.log(JSON.stringify({ ok: false, failure: 'observe-turn-terminal-non-success', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", + " await sleep(Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - startedAt))));", + " }", + " const row = rowFor();", + " observations.push(row);", + " console.log(JSON.stringify({ ok: false, failure: 'quick-verify-wait-chunk-timeout', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true }));", + "})().catch((error) => { console.log(JSON.stringify({ ok: false, failure: 'quick-verify-artifact-wait-script-error', error: error instanceof Error ? error.message : String(error), valuesRedacted: true })); });", + "NODE", + ].join("\n"); +} + function isQuickVerifyTurnSuccessful(value: string | null): boolean { const status = normalizeQuickVerifyStatus(value); return status === "completed" || status === "succeeded" || status === "success";