diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 5b268f3d..48fe3328 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -686,7 +686,8 @@ const heartbeat = await readJson(path.join(stateDir, "heartbeat.json")); await mkdir(analysisDir, { recursive: true, mode: 0o700 }); const transitions = buildTransitions(samples); const sampleMetrics = buildSampleMetrics(samples, control); -const findings = buildFindings(samples, control, network, errors, sampleMetrics); +const promptNetwork = buildPromptNetworkReport(control, network); +const findings = buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork); const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl })); const report = { ok: findings.filter((item) => item.severity === "red").length === 0, @@ -699,6 +700,7 @@ const report = { commandTimeline, transitions, sampleMetrics, + promptNetwork, findings, artifactSummary: await artifactSummary(artifacts), safety: { offlineOnly: true, browserDriven: false, apiFetch: false, valuesRedacted: true }, @@ -706,7 +708,7 @@ const report = { await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 }); await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 }); const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]); -console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); +console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, promptNetwork: promptNetwork.summary, findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); async function readJson(file) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } @@ -721,7 +723,7 @@ async function readJsonl(file) { } catch { return []; } } -function buildFindings(samples, control, network, errors, sampleMetrics) { +function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork) { const findings = []; const commandTimes = control.filter((item) => item.phase === "completed" || item.phase === "started").map((item) => Date.parse(item.ts)).filter(Number.isFinite); const routeSessions = new Set(samples.map((item) => item.routeSessionId).filter(Boolean)); @@ -749,6 +751,8 @@ function buildFindings(samples, control, network, errors, sampleMetrics) { const traceTerminal = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.some((row) => /completed|failed|canceled|terminal|done/iu.test((row.status || "") + " " + (row.textPreview || "")))); const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0); if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) }); + const promptFailures = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.chatPostOk === false) : []; + if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) }); const naturalApi = network.filter((item) => item.observerInitiated === false && item.type === "response" && /\/v1\/|\/auth\//u.test(String(item.url || ""))); findings.push({ id: "natural-api-dom-lag-baseline", severity: "info", summary: "natural API responses and DOM samples are available for manual lag correlation", naturalApiResponses: naturalApi.length, sampleCount: samples.length }); if (errors.length > 0) findings.push({ id: "browser-console-or-page-errors", severity: "amber", summary: "pageerror/runner errors were captured", count: errors.length, first: errors.slice(0, 5) }); @@ -756,6 +760,94 @@ function buildFindings(samples, control, network, errors, sampleMetrics) { return findings; } +function buildPromptNetworkReport(control, network) { + const promptsById = new Map(); + for (const item of control) { + if (item?.type !== "sendPrompt" || !item.commandId) continue; + const existing = promptsById.get(item.commandId) || { + commandId: item.commandId, + promptIndex: promptsById.size + 1, + promptTextHash: item.input?.textHash ?? null, + promptTextBytes: item.input?.textBytes ?? null, + startedAt: null, + completedAt: null, + failedAt: null, + phase: null + }; + if (!existing.promptTextHash && item.input?.textHash) existing.promptTextHash = item.input.textHash; + if (!existing.promptTextBytes && item.input?.textBytes) existing.promptTextBytes = item.input.textBytes; + if (item.phase === "started") existing.startedAt = item.ts ?? existing.startedAt; + if (item.phase === "completed") existing.completedAt = item.ts ?? existing.completedAt; + if (item.phase === "failed") existing.failedAt = item.ts ?? existing.failedAt; + existing.phase = item.phase ?? existing.phase; + promptsById.set(item.commandId, existing); + } + const prompts = Array.from(promptsById.values()).sort((a, b) => Date.parse(a.startedAt || a.completedAt || a.failedAt || "") - Date.parse(b.startedAt || b.completedAt || b.failedAt || "")); + prompts.forEach((item, index) => { item.promptIndex = index + 1; }); + const chatEvents = network + .filter((item) => String(item?.method || "").toUpperCase() === "POST" && /\/v1\/agent\/chat(?:\?|$)/u.test(String(item?.url || ""))) + .map((item) => ({ + ts: item.ts ?? null, + tsMs: Date.parse(item.ts), + type: item.type ?? null, + status: Number.isFinite(Number(item.status)) ? Number(item.status) : null, + commandId: item.commandId ?? null, + urlPath: urlPath(item.url), + failureKind: item.failureKind ?? null, + errorTextHash: item.errorText ? sha256(item.errorText) : null + })) + .filter((item) => Number.isFinite(item.tsMs)) + .sort((a, b) => a.tsMs - b.tsMs); + const rounds = prompts.map((prompt) => { + const startMs = Date.parse(prompt.startedAt || prompt.completedAt || prompt.failedAt || ""); + const endAnchorMs = Date.parse(prompt.completedAt || prompt.failedAt || prompt.startedAt || ""); + const fromMs = Number.isFinite(startMs) ? startMs - 3000 : Number.NEGATIVE_INFINITY; + const toMs = Number.isFinite(endAnchorMs) ? endAnchorMs + 30000 : Number.POSITIVE_INFINITY; + const events = chatEvents.filter((event) => { + if (event.commandId && prompt.commandId && event.commandId === prompt.commandId) return true; + return event.tsMs >= fromMs && event.tsMs <= toMs; + }); + const responses = events.filter((event) => event.type === "response"); + const failures = events.filter((event) => event.type === "requestfailed"); + const responseStatuses = responses.map((event) => event.status).filter((status) => status !== null); + const chatPostOk = responseStatuses.some((status) => status >= 200 && status < 300) && failures.length === 0; + const failureKind = chatPostOk + ? null + : failures.length > 0 + ? "requestfailed" + : responseStatuses.length === 0 + ? "missing-response" + : "http-status"; + return { + promptIndex: prompt.promptIndex, + promptCommandId: prompt.commandId, + promptTextHash: prompt.promptTextHash, + promptTextBytes: prompt.promptTextBytes, + startedAt: prompt.startedAt, + completedAt: prompt.completedAt, + failedAt: prompt.failedAt, + chatPostOk, + failureKind, + requestCount: events.filter((event) => event.type === "request").length, + responseCount: responses.length, + requestFailedCount: failures.length, + responseStatuses, + firstChatEventAt: events[0]?.ts ?? null, + lastChatEventAt: events[events.length - 1]?.ts ?? null, + events: events.slice(0, 12).map((event) => ({ ts: event.ts, type: event.type, status: event.status, urlPath: event.urlPath, failureKind: event.failureKind, errorTextHash: event.errorTextHash })) + }; + }); + return { + summary: { + promptCount: rounds.length, + chatPostOk: rounds.filter((item) => item.chatPostOk === true).length, + chatPostFailed: rounds.filter((item) => item.chatPostOk === false).length, + chatPostMissing: rounds.filter((item) => item.failureKind === "missing-response").length + }, + rounds + }; +} + function buildSampleMetrics(samples, control) { const promptCommands = control .filter((item) => item.type === "sendPrompt" && item.phase === "completed") @@ -836,6 +928,7 @@ function buildTurnTimingTable(samples, timeline) { lastSeq: sample.seq ?? null, lastTs: sample.ts ?? null, promptIndex: metric.promptIndex ?? null, + lastPromptIndex: metric.promptIndex ?? null, traceId: metric.traceId ?? null, messageId: metric.messageId ?? null, domIndex: metric.domIndex ?? null @@ -845,6 +938,7 @@ function buildTurnTimingTable(samples, timeline) { } else { column.lastSeq = sample.seq ?? null; column.lastTs = sample.ts ?? null; + column.lastPromptIndex = metric.promptIndex ?? column.lastPromptIndex ?? null; if (!column.traceId && metric.traceId) column.traceId = metric.traceId; if (!column.messageId && metric.messageId) column.messageId = metric.messageId; } @@ -882,7 +976,7 @@ function turnMetricItems(sample, timelineItem) { const traceId = turn.traceId || firstTraceId(texts); const messageId = turn.messageId || null; const domIndex = Number.isFinite(Number(turn.index)) ? Number(turn.index) : items.length; - const key = "turn:" + (traceId || messageId || ("dom-index-" + String(domIndex))); + const key = "turn:dom-index-" + String(domIndex); items.push({ key, source: "turn", @@ -1133,7 +1227,7 @@ function renderTurnTimingTable(sampleMetrics) { } lines.push("| " + cells.map(escapeMarkdownCell).join(" | ") + " |"); } - const columnLines = columns.map((column) => "- " + column.label + ": source=" + (column.source || "-") + " prompt=" + (column.promptIndex ?? "-") + " firstSeq=" + (column.firstSeq ?? "-") + " lastSeq=" + (column.lastSeq ?? "-") + " traceId=" + (column.traceId || "-") + " messageId=" + (column.messageId || "-")).join("\n"); + const columnLines = columns.map((column) => "- " + column.label + ": source=" + (column.source || "-") + " prompt=" + (column.promptIndex ?? "-") + " lastPrompt=" + (column.lastPromptIndex ?? "-") + " firstSeq=" + (column.firstSeq ?? "-") + " lastSeq=" + (column.lastSeq ?? "-") + " traceId=" + (column.traceId || "-") + " messageId=" + (column.messageId || "-")).join("\n"); return lines.join("\n") + "\n\n列说明：\n" + columnLines; } @@ -1151,6 +1245,9 @@ function renderMarkdown(report) { const commandLines = report.commandTimeline.length === 0 ? "- 无控制命令。" : report.commandTimeline.map((item) => "- " + item.ts + " " + item.phase + " " + item.type + " " + item.commandId + " " + (item.afterUrl || "")).join("\n"); const transitionLines = report.transitions.length === 0 ? "- 无状态变化。" : report.transitions.slice(0, 80).map((item) => "- #" + item.seq + " " + item.ts + " messages=" + item.messageCount + " traceRows=" + item.traceRowCount + " route=" + (item.routeSessionId || "-") + " active=" + (item.activeSessionId || "-")).join("\n"); const metricSummary = report.sampleMetrics?.summary || {}; + const promptNetworkLines = Array.isArray(report.promptNetwork?.rounds) && report.promptNetwork.rounds.length > 0 + ? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n") + : "- 无 prompt 网络记录。"; const roundLines = Array.isArray(report.sampleMetrics?.rounds) && report.sampleMetrics.rounds.length > 0 ? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n") : "- 无轮次指标。"; @@ -1173,6 +1270,7 @@ function renderMarkdown(report) { + "- diagnostics: " + (metricSummary.diagnostics ?? 0) + "\n" + "- promptSegments: " + (metricSummary.promptSegments ?? 0) + "\n\n" + "### Rounds\n\n" + roundLines + "\n\n" + + "### Prompt network\n\n" + promptNetworkLines + "\n\n" + "### Turn timing table\n\n" + turnTimingTable + "\n\n" + "### Aggregate timeline\n\n" @@ -1189,5 +1287,14 @@ async function fileMeta(file) { function sha256(value) { return "sha256:" + createHash("sha256").update(String(value)).digest("hex"); } + +function urlPath(value) { + try { + const url = new URL(String(value || "http://invalid.local/")); + return url.pathname; + } catch { + return "-"; + } +} `; }