fix(web-probe): surface command failures during observe (#723)

Co-authored-by: Codex <codex@noreply.local>
2026-06-23 07:44:05 +08:00
parent 6c9dbc6a34
commit 6157b57e08
2 changed files with 116 additions and 12 deletions
@@ -273,6 +273,7 @@ async function drainOneCommand() {
  const command = JSON.parse(raw);
  const id = safeId(command.id || name.replace(/[.]json$/u, ""));
  command.id = id;
+  const stopCommandSampler = startCommandActiveSampler(command);
  try {
    const result = await processCommand(command);
    const done = { ok: true, commandId: id, type: command.type, completedAt: new Date().toISOString(), result: sanitize(result) };
@@ -280,16 +281,51 @@ async function drainOneCommand() {
    await appendJsonl(files.control, controlRecord(command, "completed", done.result));
    await unlink(processing).catch(() => {});
  } catch (error) {
-    const failed = { ok: false, commandId: id, type: command.type, failedAt: new Date().toISOString(), error: errorSummary(error) };
+    const failureSample = await samplePage("command-failed", { refreshObserver: false, screenshot: false })
+      .then(() => ({ ok: true, sampleSeq, valuesRedacted: true }))
+      .catch((sampleError) => ({ ok: false, error: errorSummary(sampleError), valuesRedacted: true }));
+    const failed = { ok: false, commandId: id, type: command.type, failedAt: new Date().toISOString(), error: errorSummary(error), failureSample };
    await writeFile(path.join(dirs.commandsFailed, id + ".json"), JSON.stringify(failed, null, 2) + "\n", { mode: 0o600 });
-    await appendJsonl(files.control, controlRecord(command, "failed", failed.error));
+    await appendJsonl(files.control, controlRecord(command, "failed", { error: failed.error, failureSample }));
    await unlink(processing).catch(() => {});
  } finally {
+    stopCommandSampler();
    activeCommandId = null;
    await writeHeartbeat({ status: terminalStatus });
  }
 }

+function startCommandActiveSampler(command) {
+  const intervalMs = Math.max(1000, Number(sampleIntervalMs) || 5000);
+  let stopped = false;
+  let timer = null;
+  let inFlight = false;
+  const schedule = () => {
+    if (stopped) return;
+    timer = setTimeout(tick, intervalMs);
+    if (timer && typeof timer.unref === "function") timer.unref();
+  };
+  const tick = () => {
+    if (stopped) return;
+    if (inFlight) {
+      schedule();
+      return;
+    }
+    inFlight = true;
+    samplePage("command-active", { refreshObserver: false, screenshot: false })
+      .catch((error) => appendJsonl(files.errors, eventRecord("command-active-sample-error", { commandId: command.id, commandType: command.type, error: errorSummary(error) })))
+      .finally(() => {
+        inFlight = false;
+        schedule();
+      });
+  };
+  schedule();
+  return () => {
+    stopped = true;
+    if (timer) clearTimeout(timer);
+  };
+}
+
 async function processCommand(command) {
  commandSeq += 1;
  activeCommandId = command.id;
@@ -894,8 +930,22 @@ function isWorkbenchPathname(value) {
 async function createSessionFromUi() {
  const beforeUrl = currentPageUrl();
  const before = await workbenchSessionSnapshot();
+  const readinessBeforeClick = await workbenchReadinessSnapshot(page);
  const create = page.locator("#session-create").first();
  await create.waitFor({ state: "visible", timeout: 15000 });
+  const createButtonState = await create.evaluate((element) => {
+    const rect = element.getBoundingClientRect();
+    const style = window.getComputedStyle(element);
+    return {
+      tag: element.tagName.toLowerCase(),
+      id: element.id || null,
+      disabled: Boolean(element.disabled),
+      ariaDisabled: element.getAttribute("aria-disabled") || null,
+      visible: rect.width > 0 && rect.height > 0 && style.visibility !== "hidden" && style.display !== "none",
+      rect: { width: Math.round(rect.width), height: Math.round(rect.height) },
+      valuesRedacted: true
+    };
+  }).catch((error) => ({ error: errorSummary(error), valuesRedacted: true }));
  const createResponsePromise = page.waitForResponse((response) => {
    const request = response.request();
    if (request.method().toUpperCase() !== "POST") return false;
@@ -907,7 +957,11 @@ async function createSessionFromUi() {
  }, { timeout: 45000 }).catch((error) => ({ waitError: errorSummary(error) }));
  await create.click();
  const createResponse = await createResponsePromise;
-  if (createResponse?.waitError) throw new Error("newSession did not observe POST /v1/agent/sessions response after click: " + (createResponse.waitError.message || createResponse.waitError.name || "timeout"));
+  if (createResponse?.waitError) {
+    const error = new Error("newSession did not observe POST /v1/agent/sessions response after click: " + (createResponse.waitError.message || createResponse.waitError.name || "timeout"));
+    error.details = { beforeUrl, afterUrl: currentPageUrl(), before, readinessBeforeClick, createButtonState, waitError: createResponse.waitError, pageId, valuesRedacted: true };
+    throw error;
+  }
  const createStatus = createResponse.status();
  let createPayload = null;
  let createPayloadError = null;
@@ -1273,14 +1327,14 @@ async function preflightSummary() {
  return { currentUrl: currentPageUrl(), title: await page.title().catch(() => null), pageId, auth: publicAuth(auth) };
 }

-async function samplePage(reason) {
-  await maybeRefreshObserverPage(reason);
+async function samplePage(reason, options = {}) {
+  if (options?.refreshObserver !== false) await maybeRefreshObserverPage(reason);
  const groupSeq = sampleSeq + 1;
  if (page && !page.isClosed()) await sampleOnePage(page, { reason, groupSeq, pageRole: "control", targetPageId: pageId });
  if (observerPage && !observerPage.isClosed()) {
    await sampleOnePage(observerPage, { reason, groupSeq, pageRole: "observer", targetPageId: observerPageId }).catch((error) => appendJsonl(files.errors, eventRecord("observer-sample-error", { pageRole: "observer", pageId: observerPageId, error: errorSummary(error) })));
  }
-  if (screenshotIntervalMs > 0 && Date.now() - lastScreenshotAtMs >= screenshotIntervalMs) {
+  if (options?.screenshot !== false && screenshotIntervalMs > 0 && Date.now() - lastScreenshotAtMs >= screenshotIntervalMs) {
    await captureScreenshot("checkpoint", "jpeg").catch((error) => appendJsonl(files.errors, eventRecord("screenshot-error", { pageRole: "control", pageId, error: errorSummary(error) })));
  }
  await writeHeartbeat({ status: terminalStatus });
@@ -1934,7 +1988,8 @@ const runnerErrors = errors.slice(-8).map((item) => {
    })),
  };
 });
-const findings = buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance);
+const commandFailures = summarizeCommandFailures(control);
+const findings = buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures);
 if (jsonlReadIssues.length > 0) findings.unshift({ id: "jsonl-read-issues", severity: "red", summary: "observer analyzer hit JSONL read/parse issues", count: jsonlReadIssues.length, issues: jsonlReadIssues.slice(0, 20) });
 const recentWindow = buildRecentAnalysisWindow({ samples, control, network, consoleEvents, errors, manifest });
 const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl }));
@@ -1955,6 +2010,7 @@ const report = {
  promptNetwork,
  runtimeAlerts,
  runnerErrors,
+  commandFailures,
  findings,
  windows: { recent: recentWindow },
  readIssues: jsonlReadIssues,
@@ -1997,6 +2053,7 @@ console.log(JSON.stringify({
  promptNetwork: recentWindow.promptNetwork.summary,
  runtimeAlerts: recentWindow.runtimeAlerts.summary,
  runnerErrors,
+  commandFailures: commandFailures.slice(-8),
  httpErrorGroups: recentWindow.runtimeAlerts.networkHttpErrorsByPath.slice(0, 8).map((item) => ({
    method: item.method ?? null,
    status: item.status ?? null,
@@ -2345,8 +2402,31 @@ function compactSamplePageProvenance(value) {
  };
 }

-function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance) {
+function summarizeCommandFailures(control) {
+  return control.filter((item) => item?.phase === "failed").map((item) => {
+    const detail = item?.detail && typeof item.detail === "object" ? item.detail : {};
+    const error = detail?.error && typeof detail.error === "object" ? detail.error : detail;
+    return {
+      ts: item.ts ?? null,
+      commandId: item.commandId ?? null,
+      type: item.type ?? item.input?.type ?? null,
+      source: item.source ?? null,
+      durationMs: detail.durationMs ?? null,
+      beforePath: urlPath(detail.beforeUrl || item.beforeUrl),
+      afterPath: urlPath(detail.afterUrl || item.afterUrl),
+      name: error?.name ?? null,
+      message: limitText(error?.message ?? detail?.message ?? "", 240),
+      failureKind: error?.failureKind ?? detail?.failureKind ?? null,
+      failureSampleOk: detail?.failureSample?.ok === true,
+      sampleSeq: detail?.failureSample?.sampleSeq ?? null,
+      valuesRedacted: true
+    };
+  });
+}
+
+function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures = []) {
  const findings = [];
+  if (commandFailures.length > 0) findings.push({ id: "observer-command-failed", severity: "red", summary: "observer control commands failed; analyze must surface command failure instead of hiding it in command artifacts", count: commandFailures.length, commands: commandFailures.slice(0, 20) });
  const commandTimes = control.filter((item) => item.phase === "completed" || item.phase === "started").map((item) => Date.parse(item.ts)).filter(Number.isFinite);
  const routeSessions = new Set(samples.map((item) => item.routeSessionId).filter(Boolean));
  const activeSessions = new Set(samples.map((item) => item.activeSessionId).filter(Boolean));
@@ -4788,6 +4868,9 @@ function escapeMarkdownCell(value) {
 function renderMarkdown(report) {
  const findingLines = report.findings.length === 0 ? "- 无红灯项。" : report.findings.map((item) => "- " + item.severity + ": " + item.id + " - " + item.summary).join("\n");
  const commandLines = report.commandTimeline.length === 0 ? "- 无控制命令。" : report.commandTimeline.map((item) => "- " + item.ts + " " + item.phase + " " + item.type + " " + item.commandId + " " + (item.afterUrl || "")).join("\n");
+  const commandFailureLines = Array.isArray(report.commandFailures) && report.commandFailures.length > 0
+    ? report.commandFailures.slice(0, 80).map((item) => "- " + (item.ts || "-") + " type=" + (item.type || "-") + " commandId=" + (item.commandId || "-") + " durationMs=" + (item.durationMs ?? "-") + " sampleSeq=" + (item.sampleSeq ?? "-") + " path=" + (item.beforePath || "-") + "->" + (item.afterPath || "-") + " message=" + escapeMarkdownCell(item.message || item.failureKind || item.name || "-")).join("\n")
+    : "- 无失败控制命令。";
  const transitionLines = report.transitions.length === 0 ? "- 无状态变化。" : report.transitions.slice(0, 80).map((item) => "- #" + item.seq + " " + item.ts + " messages=" + item.messageCount + " traceRows=" + item.traceRowCount + " route=" + (item.routeSessionId || "-") + " active=" + (item.activeSessionId || "-")).join("\n");
  const metricSummary = report.sampleMetrics?.summary || {};
  const loading = report.sampleMetrics?.loading || {};
@@ -4855,6 +4938,7 @@ function renderMarkdown(report) {
    + "- console: " + (report.counts.console ?? 0) + "\n"
    + "- errors: " + report.counts.errors + "\n\n"
    + "## Findings\n\n" + findingLines + "\n\n"
+    + "## Command failures\n\n" + commandFailureLines + "\n\n"
    + "## Sample metrics\n\n"
    + "- sampleCount: " + (metricSummary.sampleCount ?? 0) + "\n"
    + "- withTotalElapsed: " + (metricSummary.withTotalElapsed ?? 0) + "\n"