fix(web-probe): surface command failures during observe (#723)

Co-authored-by: Codex <codex@noreply.local>

scripts/src/hwlab-node-impl.ts

@@ -8025,7 +8025,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
     "const readJsonlTail = (path, limit) => readText(path).split(/\\r?\\n/).filter(Boolean).slice(-limit).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean);",
     "const mergeArrays = (...values) => { const out = []; const seen = new Set(); for (const value of values) { if (!Array.isArray(value)) continue; for (const item of value) { const key = JSON.stringify([item?.id ?? item?.kind ?? item?.code ?? item?.columnLabel ?? item?.traceId ?? null, item?.path ?? item?.urlPath ?? null, item?.method ?? null, item?.status ?? null, item?.summary ?? item?.message ?? item?.fromSeq ?? item?.firstAt ?? null, item?.toSeq ?? item?.lastAt ?? null]); if (seen.has(key)) continue; seen.add(key); out.push(item); } } return out; };",
     "const clip = (value, limit = 160) => value === null || value === undefined ? null : String(value).slice(0, limit);",
-    "const findingRank = (item) => { const id = String(item?.id ?? item?.kind ?? item?.code ?? ''); if (id === 'page-performance-slow-same-origin-api') return 0; if (id === 'session-rail-title-fallback-majority') return 1; if (id.startsWith('turn-timing-total-elapsed')) return 2; if (id.startsWith('turn-timing-recent-update')) return 3; if (id.includes('runtime-execution') || id.includes('prompt-chat-submit-failed')) return 4; return 10; };",
+    "const findingRank = (item) => { const id = String(item?.id ?? item?.kind ?? item?.code ?? ''); if (id === 'observer-command-failed') return 0; if (id === 'page-performance-slow-same-origin-api') return 1; if (id === 'session-rail-title-fallback-majority') return 2; if (id.startsWith('turn-timing-total-elapsed')) return 3; if (id.startsWith('turn-timing-recent-update')) return 4; if (id.includes('runtime-execution') || id.includes('prompt-chat-submit-failed')) return 5; return 10; };",
     "const severityRank = (item) => { const severity = String(item?.severity ?? item?.level ?? '').toLowerCase(); if (severity === 'red') return 0; if (severity === 'amber' || severity === 'warning') return 1; if (severity === 'info') return 3; return 2; };",
     "const sortFindings = (items) => (Array.isArray(items) ? items : []).slice().sort((a, b) => findingRank(a) - findingRank(b) || severityRank(a) - severityRank(b));",
     "const stdoutJson = readJson(stdoutPath);",
@@ -8045,6 +8045,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
     "const slimConsoleSample = (item) => { const v = objectOrNull(item) || {}; return { ts: v.ts ?? null, type: clip(v.type, 24), status: v.status ?? null, path: clip(v.path ?? v.urlPath, 96), traceId: clip(v.traceId, 64), text: clip(v.text ?? v.preview, 180) }; };",
     "const slimRunnerError = (item) => { const v = objectOrNull(item) || {}; const readiness = objectOrNull(v.lastReadiness); return { ts: v.ts ?? null, type: clip(v.type, 32), commandId: clip(v.commandId, 80), sampleSeq: v.sampleSeq ?? null, message: clip(v.message, 240), retry: clip(v.retry, 24), retryExhausted: v.retryExhausted === true, lastError: clip(v.lastError, 180), attemptCount: v.attemptCount ?? null, lastFailureKind: clip(v.lastFailureKind, 48), lastReadinessReason: clip(v.lastReadinessReason, 48), lastReadiness: readiness ? { reason: clip(readiness.reason, 48), path: clip(readiness.path, 96), readyState: clip(readiness.readyState, 24), workbenchShellVisible: readiness.workbenchShellVisible === true, sessionCreateVisible: readiness.sessionCreateVisible === true, commandInputPresent: readiness.commandInputPresent === true, activeTabPresent: readiness.activeTabPresent === true, warningPresent: readiness.warningPresent === true, loginVisible: readiness.loginVisible === true, bodyTextHash: clip(readiness.bodyTextHash, 80) } : null }; };",
     "const slimRunnerErrorFromJsonl = (item) => { const v = objectOrNull(item) || {}; const error = objectOrNull(v.error) || {}; const attempts = Array.isArray(error.attempts) ? error.attempts : []; const lastAttempt = attempts.length > 0 ? objectOrNull(attempts[attempts.length - 1]) || {} : {}; const rawReadiness = objectOrNull(lastAttempt.readiness) || objectOrNull(error.navigationReadiness); const readiness = objectOrNull(rawReadiness?.snapshot) || rawReadiness; return { ts: v.ts ?? null, type: v.type ?? null, commandId: v.commandId ?? null, sampleSeq: v.sampleSeq ?? null, message: error.message ?? v.message ?? null, attemptCount: attempts.length, lastFailureKind: lastAttempt.failureKind ?? null, lastReadinessReason: rawReadiness?.reason ?? readiness?.reason ?? null, lastReadiness: readiness ?? null }; };",
+    "const slimCommandFailure = (item) => { const v = objectOrNull(item) || {}; return { ts: v.ts ?? null, commandId: clip(v.commandId, 80), type: clip(v.type, 32), source: clip(v.source, 24), durationMs: v.durationMs ?? null, beforePath: clip(v.beforePath, 80), afterPath: clip(v.afterPath, 80), name: clip(v.name, 48), failureKind: clip(v.failureKind, 48), sampleSeq: v.sampleSeq ?? null, failureSampleOk: v.failureSampleOk === true, message: clip(v.message, 240) }; };",
     "const slimJump = (item) => { const v = objectOrNull(item) || {}; return { columnLabel: v.columnLabel ?? null, promptIndex: v.promptIndex ?? null, fromSeq: v.fromSeq ?? null, toSeq: v.toSeq ?? null, fromValue: v.fromValue ?? null, toValue: v.toValue ?? null, delta: v.delta ?? null, sampleDeltaSeconds: v.sampleDeltaSeconds ?? null, allowedIncreaseSeconds: v.allowedIncreaseSeconds ?? null, traceId: v.traceId ?? null }; };",
     "const slimLoadingOwner = (item) => { const v = objectOrNull(item) || {}; return { ownerKey: clip(v.ownerKey, 120), ownerKind: clip(v.ownerKind, 32), ownerLabel: clip(v.ownerLabel, 120), sampleCount: v.sampleCount ?? null, occurrenceCount: v.occurrenceCount ?? null, maxSimultaneousCount: v.maxSimultaneousCount ?? null, longestContinuousSeconds: v.longestContinuousSeconds ?? null, firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, promptIndexes: Array.isArray(v.promptIndexes) ? v.promptIndexes.slice(0, 8) : [] }; };",
     "const slimLoadingSegment = (item) => { const v = objectOrNull(item) || {}; return { firstAt: v.firstAt ?? null, lastAt: v.lastAt ?? null, endedAt: v.endedAt ?? null, firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, durationSeconds: v.durationSeconds ?? null, upperBoundSeconds: v.upperBoundSeconds ?? null, endedGapSeconds: v.endedGapSeconds ?? null, sampleCount: v.sampleCount ?? null, maxCount: v.maxCount ?? null, ownerCount: v.ownerCount ?? null, ongoing: v.ongoing === true, owners: Array.isArray(v.owners) ? v.owners.slice(0, 6).map((owner) => ({ ownerKind: clip(owner?.ownerKind, 32), ownerLabel: clip(owner?.ownerLabel, 120), count: owner?.count ?? null })) : [] }; };",
@@ -8087,8 +8088,11 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
     "const sourceSlowApi = Array.isArray(source?.pagePerformanceSlowApi) ? source.pagePerformanceSlowApi.filter((item) => !isLongLivedApi(item) && Number(item?.overFiveSecondCount ?? 0) > 0) : (Array.isArray(fullPagePerformance?.sameOriginApiByPath) ? fullPagePerformance.sameOriginApiByPath.filter((item) => !isLongLivedApi(item) && Number(item?.overFiveSecondCount ?? 0) > 0) : []);",
     "const archiveSlowApi = firstNonEmptyArray(source?.archivePagePerformanceSlowApi, Array.isArray(fullArchivePagePerformance?.sameOriginApiByPath) ? fullArchivePagePerformance.sameOriginApiByPath : []).filter((item) => !isLongLivedApi(item) && Number(item?.overFiveSecondCount ?? 0) > 0);",
     "const sourceSseStreams = Array.isArray(source?.pagePerformanceSseStreams) ? source.pagePerformanceSseStreams : (Array.isArray(fullPagePerformance?.sameOriginApiByPath) ? fullPagePerformance.sameOriginApiByPath.filter((item) => isLongLivedApi(item)) : []);",
-    "const combinedFindings = sortFindings(firstArray(source?.findings, fullRecentWindow?.findings, fullSource?.findings));",
-    "const archiveRedFindings = sortFindings(firstArray(source?.archiveSummary?.redFindings, fullSource?.archiveSummary?.redFindings, fullSource?.findings).filter((item) => String(item?.severity ?? item?.level ?? '').toLowerCase() === 'red'));",
+    "const combinedFindingSource = firstNonEmptyArray(source?.findings, fullSource?.findings, fullRecentWindow?.findings, source?.archiveSummary?.redFindings, fullSource?.archiveSummary?.redFindings);",
+    "const combinedFindings = sortFindings(combinedFindingSource);",
+    "const archiveRedFindings = sortFindings(firstNonEmptyArray(source?.archiveSummary?.redFindings, fullSource?.archiveSummary?.redFindings, fullSource?.findings).filter((item) => String(item?.severity ?? item?.level ?? '').toLowerCase() === 'red'));",
+    "const allFindingsForCommands = [...firstArray(source?.findings), ...firstArray(fullSource?.findings), ...firstArray(fullRecentWindow?.findings), ...firstArray(source?.archiveSummary?.redFindings), ...firstArray(fullSource?.archiveSummary?.redFindings)];",
+    "const commandFailuresFromFindings = [...allFindingsForCommands, ...archiveRedFindings].flatMap((item) => Array.isArray(item?.commands) ? item.commands : []);",
     "const srcPromptNetwork = objectOrNull(source?.promptNetwork);",
     "const promptNetwork = srcPromptNetwork ? { promptSegments: srcPromptNetwork.promptSegments ?? null } : null;",
     "const runnerErrorsFromJsonl = readJsonlTail(reportJsonPath.replace(/\\/analysis\\/report\\.json$/u, '/errors.jsonl'), 8).filter((item) => item?.type === 'runner-error').map(slimRunnerErrorFromJsonl);",
@@ -8114,6 +8118,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
     "  consoleAlertGroups: takeHead(firstArray(source.consoleAlertGroups, fullRecentWindow?.runtimeAlerts?.consoleAlertsByPath, fullSource?.consoleAlertGroups, fullSource?.runtimeAlerts?.consoleAlertsByPath), 5).map(slimConsoleGroup),",
     "  consoleAlertSamples: takeHead(firstArray(source.consoleAlertSamples, fullRecentWindow?.runtimeAlerts?.consoleAlerts, fullSource?.consoleAlertSamples, fullSource?.runtimeAlerts?.consoleAlerts), 5).map(slimConsoleSample),",
     "  runnerErrors: takeTail(firstNonEmptyArray(source.runnerErrors, fullSource?.runnerErrors, runnerErrorsFromJsonl), 8).map(slimRunnerError),",
+    "  commandFailures: takeTail(firstNonEmptyArray(source.commandFailures, fullSource?.commandFailures, commandFailuresFromFindings), 8).map(slimCommandFailure),",
     "  turnTimingRecentUpdateJumps: takeHead(firstArray(source.turnTimingRecentUpdateJumps, srcMetrics?.turnTimingRecentUpdateJumps, srcMetrics?.turnTimingRecentUpdateSawtoothJumps, fullRecentWindow?.sampleMetrics?.turnTimingRecentUpdateSawtoothJumps, fullRecentWindow?.turnTimingRecentUpdateJumps, fullSource?.sampleMetrics?.turnTimingRecentUpdateSawtoothJumps, fullSource?.turnTimingRecentUpdateJumps), 5).map(slimJump),",
     "  turnTimingElapsedZeroResets: takeHead(firstArray(source.turnTimingElapsedZeroResets, srcMetrics?.turnTimingElapsedZeroResets, fullRecentWindow?.sampleMetrics?.turnTimingElapsedZeroResets, fullRecentWindow?.turnTimingElapsedZeroResets, fullSource?.sampleMetrics?.turnTimingElapsedZeroResets, fullSource?.turnTimingElapsedZeroResets), 5).map(slimJump),",
     "  turnTimingTotalElapsedForwardJumps: takeHead(firstArray(source.turnTimingTotalElapsedForwardJumps, srcMetrics?.turnTimingTotalElapsedForwardJumps, fullRecentWindow?.sampleMetrics?.turnTimingTotalElapsedForwardJumps, fullRecentWindow?.turnTimingTotalElapsedForwardJumps, fullSource?.sampleMetrics?.turnTimingTotalElapsedForwardJumps, fullSource?.turnTimingTotalElapsedForwardJumps), 5).map(slimJump),",
@@ -8195,6 +8200,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
     "    consoleAlertGroups: Array.isArray(compact.consoleAlertGroups) ? compact.consoleAlertGroups.slice(0, 3) : [],",
     "    consoleAlertSamples: Array.isArray(compact.consoleAlertSamples) ? compact.consoleAlertSamples.slice(0, 2) : [],",
     "    runnerErrors: Array.isArray(compact.runnerErrors) ? compact.runnerErrors.slice(-4) : [],",
+    "    commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-4) : [],",
     "    turnTimingRecentUpdateJumps: Array.isArray(compact.turnTimingRecentUpdateJumps) ? compact.turnTimingRecentUpdateJumps.slice(0, 4) : [],",
     "    turnTimingElapsedZeroResets: Array.isArray(compact.turnTimingElapsedZeroResets) ? compact.turnTimingElapsedZeroResets.slice(0, 4) : [],",
     "    turnTimingTotalElapsedForwardJumps: Array.isArray(compact.turnTimingTotalElapsedForwardJumps) ? compact.turnTimingTotalElapsedForwardJumps.slice(0, 4) : [],",
@@ -8246,6 +8252,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
     "      httpErrorGroups: Array.isArray(compact.httpErrorGroups) ? compact.httpErrorGroups.slice(0, 2) : [],",
     "      requestFailedGroups: Array.isArray(compact.requestFailedGroups) ? compact.requestFailedGroups.slice(0, 2) : [],",
     "      consoleAlertGroups: Array.isArray(compact.consoleAlertGroups) ? compact.consoleAlertGroups.slice(0, 2) : [],",
+    "      commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3) : [],",
     "      turnTimingRecentUpdateJumps: Array.isArray(compact.turnTimingRecentUpdateJumps) ? compact.turnTimingRecentUpdateJumps.slice(0, 3) : [],",
     "      turnTimingTotalElapsedForwardJumps: Array.isArray(compact.turnTimingTotalElapsedForwardJumps) ? compact.turnTimingTotalElapsedForwardJumps.slice(0, 3) : [],",
     "      reportJsonPath: compact.reportJsonPath ?? reportJsonPath,",
@@ -8278,6 +8285,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
     "        archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [],",
     "        httpErrorGroups: Array.isArray(compact.httpErrorGroups) ? compact.httpErrorGroups.slice(0, 2).map((item) => ({ count: item.count ?? null, method: item.method ?? null, status: item.status ?? null, path: item.path ?? null, lastAt: item.lastAt ?? null })) : [],",
     "        requestFailedGroups: Array.isArray(compact.requestFailedGroups) ? compact.requestFailedGroups.slice(0, 2).map((item) => ({ count: item.count ?? null, method: item.method ?? null, path: item.path ?? null, failureKinds: item.failureKinds ?? null })) : [],",
+    "        commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3).map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, durationMs: item.durationMs ?? null, sampleSeq: item.sampleSeq ?? null, beforePath: item.beforePath ?? null, afterPath: item.afterPath ?? null, message: clip(item.message ?? item.failureKind ?? item.name, 120) })) : [],",
     "        pagePerformanceSlowApi: Array.isArray(compact.pagePerformanceSlowApi) ? compact.pagePerformanceSlowApi.slice(0, 2).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [],",
     "        archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [],",
     "        pagePerformanceSseStreams: Array.isArray(compact.pagePerformanceSseStreams) ? compact.pagePerformanceSseStreams.slice(0, 2).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, streamOpenP95Ms: item.streamOpenP95Ms ?? null, streamOpenMaxMs: item.streamOpenMaxMs ?? null, streamOpenOverFiveSecondCount: item.streamOpenOverFiveSecondCount ?? null, streamLifetimeOverFiveSecondCount: item.streamLifetimeOverFiveSecondCount ?? null })) : [],",
@@ -8291,7 +8299,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
     "      output = compactOutput(ultratiny);",
     "    }",
     "    if (Buffer.byteLength(output, 'utf8') > compactStdoutLimitBytes) {",
-    "      output = compactOutput({ ok: compact.ok, counts: compact.counts ?? null, analysisWindow: compact.analysisWindow ?? null, archiveSummary: compact.archiveSummary ?? null, archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [], reportJsonPath: compact.reportJsonPath ?? reportJsonPath, reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: compact.reportMdPath ?? reportMdPath, reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, ultratiny: true, hardFallback: true, compactStdoutLimitBytes }, valuesRedacted: true });",
+    "      output = compactOutput({ ok: compact.ok, counts: compact.counts ?? null, analysisWindow: compact.analysisWindow ?? null, archiveSummary: compact.archiveSummary ?? null, findings: Array.isArray(compact.findings) ? compact.findings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], archiveRedFindings: Array.isArray(compact.archiveRedFindings) ? compact.archiveRedFindings.slice(0, 4).map((item) => ({ kind: item.kind ?? item.code ?? null, severity: item.severity ?? item.level ?? null, count: item.count ?? item.sampleCount ?? null, summary: clip(item.summary ?? item.message, 120) })) : [], commandFailures: Array.isArray(compact.commandFailures) ? compact.commandFailures.slice(-3).map((item) => ({ ts: item.ts ?? null, type: item.type ?? null, commandId: item.commandId ?? null, durationMs: item.durationMs ?? null, sampleSeq: item.sampleSeq ?? null, beforePath: item.beforePath ?? null, afterPath: item.afterPath ?? null, message: clip(item.message ?? item.failureKind ?? item.name, 120) })) : [], archivePagePerformanceSlowApi: Array.isArray(compact.archivePagePerformanceSlowApi) ? compact.archivePagePerformanceSlowApi.slice(0, 4).map((item) => ({ path: item.path ?? item.route ?? null, sampleCount: item.sampleCount ?? null, p95Ms: item.p95Ms ?? null, maxMs: item.maxMs ?? null, overFiveSecondCount: item.overFiveSecondCount ?? null })) : [], reportJsonPath: compact.reportJsonPath ?? reportJsonPath, reportJsonSha256: compact.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: compact.reportMdPath ?? reportMdPath, reportMdSha256: compact.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(compact.analyzer ?? {}), compactStdoutLimited: true, ultratiny: true, hardFallback: true, compactStdoutLimitBytes }, valuesRedacted: true });",
     "    }",
     "  }",
     "}",
@@ -8367,6 +8375,7 @@ function renderWebObserveAnalyzeTable(value: Record<string, unknown>): string {
   const consoleAlertGroups = Array.isArray(analysis?.consoleAlertGroups) ? analysis.consoleAlertGroups.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
   const consoleAlertSamples = Array.isArray(analysis?.consoleAlertSamples) ? analysis.consoleAlertSamples.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
   const runnerErrors = Array.isArray(analysis?.runnerErrors) ? analysis.runnerErrors.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8) : [];
+  const commandFailures = Array.isArray(analysis?.commandFailures) ? analysis.commandFailures.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8) : [];
   const recentUpdateJumps = Array.isArray(analysis?.turnTimingRecentUpdateJumps) ? analysis.turnTimingRecentUpdateJumps.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
   const elapsedZeroResets = Array.isArray(analysis?.turnTimingElapsedZeroResets) ? analysis.turnTimingElapsedZeroResets.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
   const elapsedForwardJumps = Array.isArray(analysis?.turnTimingTotalElapsedForwardJumps) ? analysis.turnTimingTotalElapsedForwardJumps.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
@@ -8646,6 +8655,17 @@ function renderWebObserveAnalyzeTable(value: Record<string, unknown>): string {
       webObserveShort(webObserveText(item.message || item.lastError), 96),
     ]) : [["-", "-", "-", "-", "-", "-", "-", "-"]]),
     "",
+    "Command failures:",
+    webObserveTable(["TS", "TYPE", "COMMAND", "DURATION", "SAMPLE", "PATH", "MESSAGE"], commandFailures.length > 0 ? commandFailures.map((item) => [
+      webObserveShort(webObserveText(item.ts), 24),
+      webObserveShort(webObserveText(item.type), 18),
+      webObserveShort(webObserveText(item.commandId), 28),
+      webObserveText(item.durationMs),
+      webObserveText(item.failureSampleOk === true ? item.sampleSeq : "-"),
+      webObserveShort([webObserveText(item.beforePath), webObserveText(item.afterPath)].filter((part) => part !== "-" && part !== "").join("->") || "-", 52),
+      webObserveShort(webObserveText(item.message || item.failureKind || item.name), 96),
+    ]) : [["-", "-", "-", "-", "-", "-", "-"]]),
+    "",
     "HTTP error groups:",
     webObserveTable(["COUNT", "METHOD", "STATUS", "PATH", "LAST", "FAILURE"], httpErrorRows.length > 0 ? httpErrorRows : [["-", "-", "-", "-", "-", "-"]]),
     "",

scripts/src/hwlab-node-web-observe-runner-source.ts

@@ -273,6 +273,7 @@ async function drainOneCommand() {
   const command = JSON.parse(raw);
   const id = safeId(command.id || name.replace(/[.]json$/u, ""));
   command.id = id;
+  const stopCommandSampler = startCommandActiveSampler(command);
   try {
     const result = await processCommand(command);
     const done = { ok: true, commandId: id, type: command.type, completedAt: new Date().toISOString(), result: sanitize(result) };
@@ -280,16 +281,51 @@ async function drainOneCommand() {
     await appendJsonl(files.control, controlRecord(command, "completed", done.result));
     await unlink(processing).catch(() => {});
   } catch (error) {
-    const failed = { ok: false, commandId: id, type: command.type, failedAt: new Date().toISOString(), error: errorSummary(error) };
+    const failureSample = await samplePage("command-failed", { refreshObserver: false, screenshot: false })
+      .then(() => ({ ok: true, sampleSeq, valuesRedacted: true }))
+      .catch((sampleError) => ({ ok: false, error: errorSummary(sampleError), valuesRedacted: true }));
+    const failed = { ok: false, commandId: id, type: command.type, failedAt: new Date().toISOString(), error: errorSummary(error), failureSample };
     await writeFile(path.join(dirs.commandsFailed, id + ".json"), JSON.stringify(failed, null, 2) + "\n", { mode: 0o600 });
-    await appendJsonl(files.control, controlRecord(command, "failed", failed.error));
+    await appendJsonl(files.control, controlRecord(command, "failed", { error: failed.error, failureSample }));
     await unlink(processing).catch(() => {});
   } finally {
+    stopCommandSampler();
     activeCommandId = null;
     await writeHeartbeat({ status: terminalStatus });
   }
 }
 
+function startCommandActiveSampler(command) {
+  const intervalMs = Math.max(1000, Number(sampleIntervalMs) || 5000);
+  let stopped = false;
+  let timer = null;
+  let inFlight = false;
+  const schedule = () => {
+    if (stopped) return;
+    timer = setTimeout(tick, intervalMs);
+    if (timer && typeof timer.unref === "function") timer.unref();
+  };
+  const tick = () => {
+    if (stopped) return;
+    if (inFlight) {
+      schedule();
+      return;
+    }
+    inFlight = true;
+    samplePage("command-active", { refreshObserver: false, screenshot: false })
+      .catch((error) => appendJsonl(files.errors, eventRecord("command-active-sample-error", { commandId: command.id, commandType: command.type, error: errorSummary(error) })))
+      .finally(() => {
+        inFlight = false;
+        schedule();
+      });
+  };
+  schedule();
+  return () => {
+    stopped = true;
+    if (timer) clearTimeout(timer);
+  };
+}
+
 async function processCommand(command) {
   commandSeq += 1;
   activeCommandId = command.id;
@@ -894,8 +930,22 @@ function isWorkbenchPathname(value) {
 async function createSessionFromUi() {
   const beforeUrl = currentPageUrl();
   const before = await workbenchSessionSnapshot();
+  const readinessBeforeClick = await workbenchReadinessSnapshot(page);
   const create = page.locator("#session-create").first();
   await create.waitFor({ state: "visible", timeout: 15000 });
+  const createButtonState = await create.evaluate((element) => {
+    const rect = element.getBoundingClientRect();
+    const style = window.getComputedStyle(element);
+    return {
+      tag: element.tagName.toLowerCase(),
+      id: element.id || null,
+      disabled: Boolean(element.disabled),
+      ariaDisabled: element.getAttribute("aria-disabled") || null,
+      visible: rect.width > 0 && rect.height > 0 && style.visibility !== "hidden" && style.display !== "none",
+      rect: { width: Math.round(rect.width), height: Math.round(rect.height) },
+      valuesRedacted: true
+    };
+  }).catch((error) => ({ error: errorSummary(error), valuesRedacted: true }));
   const createResponsePromise = page.waitForResponse((response) => {
     const request = response.request();
     if (request.method().toUpperCase() !== "POST") return false;
@@ -907,7 +957,11 @@ async function createSessionFromUi() {
   }, { timeout: 45000 }).catch((error) => ({ waitError: errorSummary(error) }));
   await create.click();
   const createResponse = await createResponsePromise;
-  if (createResponse?.waitError) throw new Error("newSession did not observe POST /v1/agent/sessions response after click: " + (createResponse.waitError.message || createResponse.waitError.name || "timeout"));
+  if (createResponse?.waitError) {
+    const error = new Error("newSession did not observe POST /v1/agent/sessions response after click: " + (createResponse.waitError.message || createResponse.waitError.name || "timeout"));
+    error.details = { beforeUrl, afterUrl: currentPageUrl(), before, readinessBeforeClick, createButtonState, waitError: createResponse.waitError, pageId, valuesRedacted: true };
+    throw error;
+  }
   const createStatus = createResponse.status();
   let createPayload = null;
   let createPayloadError = null;
@@ -1273,14 +1327,14 @@ async function preflightSummary() {
   return { currentUrl: currentPageUrl(), title: await page.title().catch(() => null), pageId, auth: publicAuth(auth) };
 }
 
-async function samplePage(reason) {
-  await maybeRefreshObserverPage(reason);
+async function samplePage(reason, options = {}) {
+  if (options?.refreshObserver !== false) await maybeRefreshObserverPage(reason);
   const groupSeq = sampleSeq + 1;
   if (page && !page.isClosed()) await sampleOnePage(page, { reason, groupSeq, pageRole: "control", targetPageId: pageId });
   if (observerPage && !observerPage.isClosed()) {
     await sampleOnePage(observerPage, { reason, groupSeq, pageRole: "observer", targetPageId: observerPageId }).catch((error) => appendJsonl(files.errors, eventRecord("observer-sample-error", { pageRole: "observer", pageId: observerPageId, error: errorSummary(error) })));
   }
-  if (screenshotIntervalMs > 0 && Date.now() - lastScreenshotAtMs >= screenshotIntervalMs) {
+  if (options?.screenshot !== false && screenshotIntervalMs > 0 && Date.now() - lastScreenshotAtMs >= screenshotIntervalMs) {
     await captureScreenshot("checkpoint", "jpeg").catch((error) => appendJsonl(files.errors, eventRecord("screenshot-error", { pageRole: "control", pageId, error: errorSummary(error) })));
   }
   await writeHeartbeat({ status: terminalStatus });
@@ -1934,7 +1988,8 @@ const runnerErrors = errors.slice(-8).map((item) => {
     })),
   };
 });
-const findings = buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance);
+const commandFailures = summarizeCommandFailures(control);
+const findings = buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures);
 if (jsonlReadIssues.length > 0) findings.unshift({ id: "jsonl-read-issues", severity: "red", summary: "observer analyzer hit JSONL read/parse issues", count: jsonlReadIssues.length, issues: jsonlReadIssues.slice(0, 20) });
 const recentWindow = buildRecentAnalysisWindow({ samples, control, network, consoleEvents, errors, manifest });
 const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl }));
@@ -1955,6 +2010,7 @@ const report = {
   promptNetwork,
   runtimeAlerts,
   runnerErrors,
+  commandFailures,
   findings,
   windows: { recent: recentWindow },
   readIssues: jsonlReadIssues,
@@ -1997,6 +2053,7 @@ console.log(JSON.stringify({
   promptNetwork: recentWindow.promptNetwork.summary,
   runtimeAlerts: recentWindow.runtimeAlerts.summary,
   runnerErrors,
+  commandFailures: commandFailures.slice(-8),
   httpErrorGroups: recentWindow.runtimeAlerts.networkHttpErrorsByPath.slice(0, 8).map((item) => ({
     method: item.method ?? null,
     status: item.status ?? null,
@@ -2345,8 +2402,31 @@ function compactSamplePageProvenance(value) {
   };
 }
 
-function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance) {
+function summarizeCommandFailures(control) {
+  return control.filter((item) => item?.phase === "failed").map((item) => {
+    const detail = item?.detail && typeof item.detail === "object" ? item.detail : {};
+    const error = detail?.error && typeof detail.error === "object" ? detail.error : detail;
+    return {
+      ts: item.ts ?? null,
+      commandId: item.commandId ?? null,
+      type: item.type ?? item.input?.type ?? null,
+      source: item.source ?? null,
+      durationMs: detail.durationMs ?? null,
+      beforePath: urlPath(detail.beforeUrl || item.beforeUrl),
+      afterPath: urlPath(detail.afterUrl || item.afterUrl),
+      name: error?.name ?? null,
+      message: limitText(error?.message ?? detail?.message ?? "", 240),
+      failureKind: error?.failureKind ?? detail?.failureKind ?? null,
+      failureSampleOk: detail?.failureSample?.ok === true,
+      sampleSeq: detail?.failureSample?.sampleSeq ?? null,
+      valuesRedacted: true
+    };
+  });
+}
+
+function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures = []) {
   const findings = [];
+  if (commandFailures.length > 0) findings.push({ id: "observer-command-failed", severity: "red", summary: "observer control commands failed; analyze must surface command failure instead of hiding it in command artifacts", count: commandFailures.length, commands: commandFailures.slice(0, 20) });
   const commandTimes = control.filter((item) => item.phase === "completed" || item.phase === "started").map((item) => Date.parse(item.ts)).filter(Number.isFinite);
   const routeSessions = new Set(samples.map((item) => item.routeSessionId).filter(Boolean));
   const activeSessions = new Set(samples.map((item) => item.activeSessionId).filter(Boolean));
@@ -4788,6 +4868,9 @@ function escapeMarkdownCell(value) {
 function renderMarkdown(report) {
   const findingLines = report.findings.length === 0 ? "- 无红灯项。" : report.findings.map((item) => "- " + item.severity + ": " + item.id + " - " + item.summary).join("\n");
   const commandLines = report.commandTimeline.length === 0 ? "- 无控制命令。" : report.commandTimeline.map((item) => "- " + item.ts + " " + item.phase + " " + item.type + " " + item.commandId + " " + (item.afterUrl || "")).join("\n");
+  const commandFailureLines = Array.isArray(report.commandFailures) && report.commandFailures.length > 0
+    ? report.commandFailures.slice(0, 80).map((item) => "- " + (item.ts || "-") + " type=" + (item.type || "-") + " commandId=" + (item.commandId || "-") + " durationMs=" + (item.durationMs ?? "-") + " sampleSeq=" + (item.sampleSeq ?? "-") + " path=" + (item.beforePath || "-") + "->" + (item.afterPath || "-") + " message=" + escapeMarkdownCell(item.message || item.failureKind || item.name || "-")).join("\n")
+    : "- 无失败控制命令。";
   const transitionLines = report.transitions.length === 0 ? "- 无状态变化。" : report.transitions.slice(0, 80).map((item) => "- #" + item.seq + " " + item.ts + " messages=" + item.messageCount + " traceRows=" + item.traceRowCount + " route=" + (item.routeSessionId || "-") + " active=" + (item.activeSessionId || "-")).join("\n");
   const metricSummary = report.sampleMetrics?.summary || {};
   const loading = report.sampleMetrics?.loading || {};
@@ -4855,6 +4938,7 @@ function renderMarkdown(report) {
     + "- console: " + (report.counts.console ?? 0) + "\n"
     + "- errors: " + report.counts.errors + "\n\n"
     + "## Findings\n\n" + findingLines + "\n\n"
+    + "## Command failures\n\n" + commandFailureLines + "\n\n"
     + "## Sample metrics\n\n"
     + "- sampleCount: " + (metricSummary.sampleCount ?? 0) + "\n"
     + "- withTotalElapsed: " + (metricSummary.withTotalElapsed ?? 0) + "\n"