diff --git a/scripts/src/hwlab-node-web-observe-analyzer-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-source.ts index e1fe7fb1..c721a9e0 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-source.ts @@ -2223,6 +2223,218 @@ function compactApiDomLagForOutput(report) { }; } +function detectWorkbenchInPlaceProjectionLag(samples, network) { + const terminalTraceMissing = detectWorkbenchTerminalTraceMissing(samples); + const terminalApiDomLag = detectWorkbenchTerminalApiDomLag(samples, network); + return { + terminalTraceMissing, + terminalApiDomLag, + valuesRedacted: true + }; +} + +function detectWorkbenchTerminalTraceMissing(samples) { + const rows = []; + for (const sample of Array.isArray(samples) ? samples : []) { + if (!isWorkbenchPathSample(sample)) continue; + const terminalTraceIds = workbenchTerminalTraceIdsFromDom(sample); + for (const traceId of terminalTraceIds) { + const traceRows = workbenchTraceRowsForTrace(sample, traceId); + if (traceRows.length > 0) continue; + rows.push({ + ...ref(sample), + traceId, + messageCount: Array.isArray(sample.messages) ? sample.messages.length : 0, + turnCount: Array.isArray(sample.turns) ? sample.turns.length : 0, + traceRowCount: 0, + finalMessageVisible: workbenchFinalMessageVisible(sample, traceId), + terminalTurnVisible: workbenchTerminalTurnVisible(sample, traceId), + detail: "terminal turn/message was visible for this trace, but the same in-place Workbench sample still had zero trace rows", + valuesRedacted: true + }); + } + } + return rows; +} + +function detectWorkbenchTerminalApiDomLag(samples, network) { + const windowMs = 30_000; + const budgetMs = Number.isFinite(Number(alertThresholds.sameOriginApiSlowMs)) ? Number(alertThresholds.sameOriginApiSlowMs) : 10_000; + const sampleRows = (Array.isArray(samples) ? samples : []) + .filter(isWorkbenchPathSample) + .map((sample) => ({ sample, tsMs: Date.parse(sample?.ts || ""), pageKey: samplePageKey(sample) })) + .filter((item) => Number.isFinite(item.tsMs)) + .sort((a, b) => a.tsMs - b.tsMs); + const rowsByPage = new Map(); + for (const row of sampleRows) { + const rows = rowsByPage.get(row.pageKey) || []; + rows.push(row); + rowsByPage.set(row.pageKey, rows); + } + const terminalEvents = (Array.isArray(network) ? network : []) + .map(compactWorkbenchTerminalApiEvent) + .filter((item) => item !== null); + const overBudget = []; + for (const event of terminalEvents) { + const pageSamples = rowsByPage.get(event.pageKey) || []; + const firstAfter = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event); + const resolved = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event, (row) => workbenchSampleHasTerminalProjection(row.sample, event)); + const deltaMs = resolved ? Math.max(0, Math.round(resolved.tsMs - event.tsMs)) : null; + const unresolved = !resolved; + const exceedsBudget = unresolved || (Number.isFinite(deltaMs) && deltaMs > budgetMs); + if (!exceedsBudget) continue; + overBudget.push({ + ts: event.ts, + pageRole: event.pageRole, + pageId: event.pageId, + routeKind: event.routeKind, + method: event.method, + path: event.path, + traceIds: event.traceIds.slice(0, 6), + sessionIds: event.sessionIds.slice(0, 4), + terminalEvidenceCount: event.terminalEvidenceCount, + traceEventLikeCount: event.traceEventLikeCount, + finalTextFieldCount: event.finalTextFieldCount, + budgetMs, + windowMs, + resolvedDeltaMs: deltaMs, + unresolvedWithinWindow: unresolved, + firstAfterSample: compactWorkbenchProjectionSample(firstAfter?.sample, event), + resolvedSample: compactWorkbenchProjectionSample(resolved?.sample, event), + valuesRedacted: true + }); + } + return { + summary: { + terminalEventCount: terminalEvents.length, + overBudgetCount: overBudget.length, + budgetMs, + windowMs, + valuesRedacted: true + }, + overBudget, + terminalEvents: terminalEvents.slice(-20), + valuesRedacted: true + }; +} + +function compactWorkbenchTerminalApiEvent(item) { + if (!item || item.type !== "response" || item.observerInitiated === true) return null; + const summary = objectValue(item.bodySummary); + if (!summary || Number(summary.terminalEvidenceCount ?? 0) <= 0) return null; + const parsed = parseApiDomLagUrl(item.url); + if (!String(parsed.path || "").startsWith("/v1/workbench/") && parsed.path !== "/v1/agent/chat" && parsed.path !== "/v1/agent/chat/steer") return null; + const tsMs = Date.parse(item.ts || ""); + if (!Number.isFinite(tsMs)) return null; + return { + ts: item.ts, + tsMs, + pageRole: item.pageRole ?? null, + pageId: item.pageId ?? null, + pageKey: String(item.pageRole || "control") + ":" + String(item.pageId || "default"), + method: String(item.method || "GET").toUpperCase(), + path: parsed.path, + routeKind: summary.pathKind ?? apiDomLagRouteKind(parsed.path), + traceIds: uniqueSorted([...(Array.isArray(summary.traceIds) ? summary.traceIds : []), parsed.traceId].filter(Boolean)).slice(0, 12), + sessionIds: uniqueSorted([...(Array.isArray(summary.sessionIds) ? summary.sessionIds : []), parsed.sessionId].filter(Boolean)).slice(0, 12), + terminalEvidenceCount: Number(summary.terminalEvidenceCount ?? 0), + terminalStatusCount: Number(summary.terminalStatusCount ?? 0), + terminalTextCount: Number(summary.terminalTextCount ?? 0), + traceEventLikeCount: Number(summary.traceEventLikeCount ?? 0), + finalTextFieldCount: Number(summary.finalTextFieldCount ?? 0), + valuesRedacted: true + }; +} + +function firstWorkbenchSampleAfter(rows, startMs, endMs, event, predicate = null) { + for (const row of rows || []) { + if (row.tsMs < startMs) continue; + if (row.tsMs > endMs) break; + if (!workbenchSampleMatchesTerminalEvent(row.sample, event)) continue; + if (typeof predicate === "function" && !predicate(row)) continue; + return row; + } + return null; +} + +function workbenchSampleMatchesTerminalEvent(sample, event) { + if (!sample || !event) return false; + if (event.sessionIds.length > 0) { + const sessionIds = new Set([sample.routeSessionId, sample.activeSessionId].filter(Boolean).map(String)); + if (!event.sessionIds.some((id) => sessionIds.has(id))) return false; + } + if (event.traceIds.length > 0) { + const traces = sampleTraceIds(sample); + if (traces.size > 0 && !event.traceIds.some((id) => traces.has(id))) return false; + } + return true; +} + +function workbenchSampleHasTerminalProjection(sample, event) { + const traceIds = event.traceIds.length > 0 ? event.traceIds : Array.from(sampleTraceIds(sample)); + if (traceIds.length === 0) return false; + return traceIds.some((traceId) => workbenchFinalMessageVisible(sample, traceId) && workbenchTraceRowsForTrace(sample, traceId).length > 0); +} + +function compactWorkbenchProjectionSample(sample, event = null) { + if (!sample) return null; + const eventTraceIds = event?.traceIds && event.traceIds.length > 0 ? event.traceIds : Array.from(sampleTraceIds(sample)); + const visibleTraceIds = eventTraceIds.slice(0, 6); + return { + ...ref(sample), + messageCount: Array.isArray(sample.messages) ? sample.messages.length : 0, + turnCount: Array.isArray(sample.turns) ? sample.turns.length : 0, + traceRowCount: Array.isArray(sample.traceRows) ? sample.traceRows.length : 0, + traceIds: visibleTraceIds, + finalMessageVisible: visibleTraceIds.some((traceId) => workbenchFinalMessageVisible(sample, traceId)), + terminalTurnVisible: visibleTraceIds.some((traceId) => workbenchTerminalTurnVisible(sample, traceId)), + valuesRedacted: true + }; +} + +function isWorkbenchPathSample(sample) { + return /\/workbench(?:\/|$)/u.test(String(sample?.path || sample?.url || "")); +} + +function workbenchTerminalTraceIdsFromDom(sample) { + const ids = new Set(); + for (const groupName of ["turns", "messages"]) { + const group = Array.isArray(sample?.[groupName]) ? sample[groupName] : []; + for (const item of group) { + const traceId = stringOrNull(item?.traceId); + if (!traceId) continue; + if (workbenchDomItemIsTerminal(item)) ids.add(traceId); + } + } + return Array.from(ids).sort(); +} + +function workbenchTraceRowsForTrace(sample, traceId) { + const rows = Array.isArray(sample?.traceRows) ? sample.traceRows : []; + return rows.filter((item) => !traceId || item?.traceId === traceId); +} + +function workbenchFinalMessageVisible(sample, traceId) { + const messages = Array.isArray(sample?.messages) ? sample.messages : []; + return messages.some((item) => (!traceId || item?.traceId === traceId) && workbenchDomItemLooksFinal(item)); +} + +function workbenchTerminalTurnVisible(sample, traceId) { + const turns = Array.isArray(sample?.turns) ? sample.turns : []; + return turns.some((item) => (!traceId || item?.traceId === traceId) && workbenchDomItemIsTerminal(item)); +} + +function workbenchDomItemIsTerminal(item) { + const status = String(item?.status || "").toLowerCase(); + if (/^(completed|complete|succeeded|success|failed|failure|error|canceled|cancelled|done)$/u.test(status)) return true; + return isTerminalTraceText([item?.status, item?.textPreview, item?.text, item?.durationText, item?.activityText].filter(Boolean).join(" ")); +} + +function workbenchDomItemLooksFinal(item) { + const text = [item?.status, item?.textPreview, item?.text].filter(Boolean).join(" "); + return workbenchDomItemIsTerminal(item) && isFinalResultText(text); +} + function promptCommandHasAuthoritativeSubmitSideEffect(control, promptRound) { const commandId = stringOrNull(promptRound?.promptCommandId); if (!commandId) return false; @@ -2341,6 +2553,31 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN const traceTerminal = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.some((row) => isTerminalTraceText((row.status || "") + " " + (row.textPreview || "")))); const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0); if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) }); + const workbenchInPlaceProjectionLag = detectWorkbenchInPlaceProjectionLag(samples, network); + if (workbenchInPlaceProjectionLag.terminalTraceMissing.length > 0) findings.push({ + id: "workbench-terminal-trace-not-hydrated-in-place", + severity: "red", + summary: "Workbench rendered a terminal turn/message in-place while the same trace still had no visible run-record trace rows", + count: workbenchInPlaceProjectionLag.terminalTraceMissing.length, + samples: workbenchInPlaceProjectionLag.terminalTraceMissing.slice(0, 20), + rootCause: "workbench_trace_projection_not_hydrated_in_place", + rootCauseStatus: "confirmed-from-dom-samples", + rootCauseConfidence: "high", + valuesRedacted: true + }); + if (workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.length > 0) findings.push({ + id: "workbench-terminal-api-dom-not-refreshed-in-place", + severity: "red", + summary: "Workbench REST returned terminal/final trace evidence but the same in-place page did not render terminal message plus trace rows within the configured budget", + count: workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.length, + budgetMs: workbenchInPlaceProjectionLag.terminalApiDomLag.summary.budgetMs, + windowMs: workbenchInPlaceProjectionLag.terminalApiDomLag.summary.windowMs, + samples: workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.slice(0, 20), + rootCause: "workbench_rest_terminal_projection_dom_lag", + rootCauseStatus: "confirmed-from-network-body-summary-and-dom-samples", + rootCauseConfidence: "high", + valuesRedacted: true + }); const promptFailures = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.chatPostOk === false && !promptCommandHasAuthoritativeSubmitSideEffect(control, item)) : []; if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat or /v1/agent/chat/steer POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) }); const promptSteerRounds = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.steerUsed === true) : []; @@ -3556,6 +3793,7 @@ function prioritizeFindings(findings) { if (id.startsWith("project-management-") || id.startsWith("mdtodo-") || id === "workbench-launch-button-unavailable") return 0; if (id === "page-performance-slow-same-origin-api") return 0; if (id === "session-rail-title-fallback-majority") return 0.5; + if (id.startsWith("workbench-terminal-")) return 0.6; if (id.startsWith("code-agent-card-")) return 0.8; if (id.startsWith("round-completion-")) return 0.9; if (id.startsWith("turn-timing-total-elapsed")) return 1; diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 719c8c8f..8b9336ae 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -234,9 +234,10 @@ function attachPassiveListeners(targetPage, pageRole = "control", targetPageId = }); targetPage.on("response", (response) => { const request = response.request(); - void appendJsonl(files.network, eventRecord("response", { + const base = { pageRole, pageId: targetPageId, + sampleSeq, observerInitiated: false, commandId: activeCommandId, method: request.method(), @@ -245,8 +246,20 @@ function attachPassiveListeners(targetPage, pageRole = "control", targetPageId = status: response.status(), statusText: response.statusText(), fromServiceWorker: response.fromServiceWorker(), - bodyRead: false, - })); + }; + void (async () => { + const bodyFields = await summarizeWorkbenchResponseBody(response, request); + await appendJsonl(files.network, eventRecord("response", { ...base, ...bodyFields })); + })().catch((error) => appendJsonl(files.errors, eventRecord("response-body-summary-error", { + pageRole, + pageId: targetPageId, + sampleSeq, + commandId: activeCommandId, + method: request.method(), + url: safeUrl(response.url()), + error: errorSummary(error), + valuesRedacted: true + }))); }); targetPage.on("requestfailed", (request) => { void appendJsonl(files.network, eventRecord("requestfailed", { @@ -3871,6 +3884,171 @@ function eventRecord(type, data) { return { ts: new Date().toISOString(), type, jobId, pageId: clean.pageId ?? pageId, pageRole: clean.pageRole ?? "control", sampleSeq, commandId: activeCommandId, ...clean }; } +async function summarizeWorkbenchResponseBody(response, request) { + const method = String(request.method() || "GET").toUpperCase(); + const path = safeUrlPath(response.url()) || ""; + const resourceType = String(request.resourceType() || ""); + const status = Number(response.status()); + if (!shouldSummarizeWorkbenchResponseBody({ method, path, resourceType, status })) return { bodyRead: false }; + const headers = response.headers(); + const contentType = String(headers["content-type"] || headers["Content-Type"] || ""); + if (!/json/iu.test(contentType)) return { bodyRead: false, bodyReadSkipped: "non-json", valuesRedacted: true }; + const contentLength = Number(headers["content-length"] || headers["Content-Length"]); + const maxBytes = 512 * 1024; + if (Number.isFinite(contentLength) && contentLength > maxBytes) return { bodyRead: false, bodyReadSkipped: "content-length-too-large", bodyByteCount: contentLength, valuesRedacted: true }; + const text = await response.text(); + const byteCount = Buffer.byteLength(text); + if (byteCount > maxBytes) return { bodyRead: true, bodyReadSkipped: "body-too-large", bodyByteCount: byteCount, bodyHash: sha256Text(text), valuesRedacted: true }; + let parsed = null; + try { + parsed = JSON.parse(text); + } catch (error) { + return { bodyRead: true, bodyReadSkipped: "json-parse-error", bodyByteCount: byteCount, bodyHash: sha256Text(text), bodyParseError: errorSummary(error), valuesRedacted: true }; + } + return { + bodyRead: true, + bodyByteCount: byteCount, + bodyHash: sha256Text(text), + bodySummary: summarizeWorkbenchJsonBody(parsed, path), + valuesRedacted: true + }; +} + +function shouldSummarizeWorkbenchResponseBody({ method, path, resourceType, status }) { + if (method !== "GET" && method !== "POST") return false; + if (!Number.isFinite(status) || status < 200 || status >= 300) return false; + if (resourceType === "eventsource" || path === "/v1/workbench/events") return false; + return path === "/v1/agent/chat" + || path === "/v1/agent/chat/steer" + || path === "/v1/workbench/sessions" + || /^\/v1\/workbench\/sessions\/[^/]+\/messages$/u.test(path) + || /^\/v1\/workbench\/turns\/[^/]+$/u.test(path) + || /^\/v1\/workbench\/traces\/[^/]+\/events$/u.test(path); +} + +function summarizeWorkbenchJsonBody(value, path) { + const traceIds = new Set(); + const sessionIds = new Set(); + const statusCounts = {}; + const counters = { + objectCount: 0, + arrayCount: 0, + traceEventLikeCount: 0, + messageLikeCount: 0, + turnLikeCount: 0, + terminalStatusCount: 0, + runningStatusCount: 0, + terminalTextCount: 0, + finalTextFieldCount: 0, + finalTextByteCount: 0 + }; + + const visit = (node, key = "", parent = null, depth = 0) => { + if (depth > 32 || node === null || node === undefined) return; + if (typeof node === "string") { + collectWorkbenchIdsFromText(node, traceIds, sessionIds); + const normalizedStatus = normalizeWorkbenchStatus(key, node); + if (normalizedStatus) { + statusCounts[normalizedStatus] = (statusCounts[normalizedStatus] || 0) + 1; + if (isWorkbenchTerminalStatus(normalizedStatus)) counters.terminalStatusCount += 1; + if (isWorkbenchRunningStatus(normalizedStatus)) counters.runningStatusCount += 1; + } + if (isWorkbenchTerminalText(node)) counters.terminalTextCount += 1; + if (isLikelyWorkbenchFinalTextField(key, parent, node)) { + counters.finalTextFieldCount += 1; + counters.finalTextByteCount += Buffer.byteLength(node); + } + return; + } + if (typeof node !== "object") return; + if (Array.isArray(node)) { + counters.arrayCount += 1; + for (const item of node) visit(item, key, parent, depth + 1); + return; + } + counters.objectCount += 1; + const record = node; + for (const raw of [record.traceId, record.trace_id, record.id, record.turnId, record.messageId]) { + if (typeof raw === "string") collectWorkbenchIdsFromText(raw, traceIds, sessionIds); + } + for (const raw of [record.sessionId, record.session_id]) { + if (typeof raw === "string") collectWorkbenchIdsFromText(raw, traceIds, sessionIds); + } + const statusValue = record.status ?? record.state ?? record.phase ?? record.result ?? record.lifecycle; + if (typeof statusValue === "string") { + const normalizedStatus = normalizeWorkbenchStatus("status", statusValue); + if (normalizedStatus) { + statusCounts[normalizedStatus] = (statusCounts[normalizedStatus] || 0) + 1; + if (isWorkbenchTerminalStatus(normalizedStatus)) counters.terminalStatusCount += 1; + if (isWorkbenchRunningStatus(normalizedStatus)) counters.runningStatusCount += 1; + } + } + if (record.traceId || record.trace_id || record.turnId || record.turn_id) counters.turnLikeCount += 1; + if (record.messageId || record.message_id || record.role || record.author || record.content || record.text || record.finalResponse) counters.messageLikeCount += 1; + if (record.projectedSeq !== undefined || record.sourceSeq !== undefined || record.eventSeq !== undefined || record.eventKind !== undefined || record.eventTimestamp !== undefined) counters.traceEventLikeCount += 1; + for (const [childKey, childValue] of Object.entries(record)) visit(childValue, childKey, record, depth + 1); + }; + visit(value); + return { + pathKind: workbenchBodyPathKind(path), + traceIds: Array.from(traceIds).sort().slice(0, 12), + sessionIds: Array.from(sessionIds).sort().slice(0, 12), + statusCounts, + ...counters, + terminalEvidenceCount: counters.terminalStatusCount + counters.terminalTextCount, + valuesRedacted: true + }; +} + +function collectWorkbenchIdsFromText(value, traceIds, sessionIds) { + const text = String(value || ""); + for (const match of text.matchAll(/\btrc_[A-Za-z0-9_-]+\b/gu)) traceIds.add(match[0]); + for (const match of text.matchAll(/\bses_[A-Za-z0-9_-]+\b/gu)) sessionIds.add(match[0]); +} + +function normalizeWorkbenchStatus(key, value) { + if (!/status|state|phase|result|lifecycle/iu.test(String(key || ""))) return null; + const text = String(value || "").toLowerCase().replace(/[^a-z0-9_-]+/gu, "-").replace(/^-+|-+$/gu, ""); + if (!text) return null; + if (/^(completed|complete|succeeded|success|finished|done|terminal|sealed)$/u.test(text)) return "completed"; + if (/^(failed|failure|error|errored)$/u.test(text)) return "failed"; + if (/^(canceled|cancelled|aborted|cancel)$/u.test(text)) return "canceled"; + if (/^(running|active|in-progress|in_progress|processing|streaming|executing)$/u.test(text)) return "running"; + if (/^(queued|pending|admitted|created|waiting)$/u.test(text)) return "pending"; + return text.slice(0, 80); +} + +function isWorkbenchTerminalStatus(value) { + return value === "completed" || value === "failed" || value === "canceled"; +} + +function isWorkbenchRunningStatus(value) { + return value === "running" || value === "pending"; +} + +function isWorkbenchTerminalText(value) { + return /轮次完成|轮次失败|轮次取消|已记录|final response|sealed final response|turn completed|turn failed|turn canceled|terminal result|\bcompleted\b|\bfailed\b|\bcanceled\b|\bcancelled\b|\bterminal\b|\bdone\b/iu.test(String(value || "")); +} + +function isLikelyWorkbenchFinalTextField(key, parent, value) { + const text = String(value || "").trim(); + if (!text) return false; + const field = String(key || ""); + if (!/final|assistant|response|content|markdown|text|message|output|result/iu.test(field)) return false; + const parentStatus = normalizeWorkbenchStatus("status", parent?.status ?? parent?.state ?? parent?.phase ?? parent?.result ?? ""); + const parentRole = String(parent?.role ?? parent?.author ?? parent?.dataRole ?? "").toLowerCase(); + return isWorkbenchTerminalStatus(parentStatus) || /assistant|agent|code/iu.test(parentRole) || /final|response|result/iu.test(field); +} + +function workbenchBodyPathKind(path) { + if (path === "/v1/agent/chat" || path === "/v1/agent/chat/steer") return "agent-chat-submit"; + if (path === "/v1/workbench/sessions") return "workbench-sessions"; + if (/^\/v1\/workbench\/sessions\/[^/]+\/messages$/u.test(path)) return "workbench-session-messages"; + if (/^\/v1\/workbench\/turns\/[^/]+$/u.test(path)) return "workbench-turn"; + if (/^\/v1\/workbench\/traces\/[^/]+\/events$/u.test(path)) return "workbench-trace-events"; + return "workbench"; +} + function controlRecord(command, phase, detail) { return { ts: new Date().toISOString(),