fix(web-probe): report prompt submit network status

This commit is contained in:
Codex
2026-06-20 17:15:13 +00:00
parent 400b612496
commit 567ae5b147
@@ -686,7 +686,8 @@ const heartbeat = await readJson(path.join(stateDir, "heartbeat.json"));
await mkdir(analysisDir, { recursive: true, mode: 0o700 });
const transitions = buildTransitions(samples);
const sampleMetrics = buildSampleMetrics(samples, control);
const findings = buildFindings(samples, control, network, errors, sampleMetrics);
const promptNetwork = buildPromptNetworkReport(control, network);
const findings = buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork);
const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl }));
const report = {
ok: findings.filter((item) => item.severity === "red").length === 0,
@@ -699,6 +700,7 @@ const report = {
commandTimeline,
transitions,
sampleMetrics,
promptNetwork,
findings,
artifactSummary: await artifactSummary(artifacts),
safety: { offlineOnly: true, browserDriven: false, apiFetch: false, valuesRedacted: true },
@@ -706,7 +708,7 @@ const report = {
await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 });
await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 });
const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]);
console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, findings: findings.slice(0, 20), valuesRedacted: true }, null, 2));
console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, promptNetwork: promptNetwork.summary, findings: findings.slice(0, 20), valuesRedacted: true }, null, 2));
async function readJson(file) {
try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; }
@@ -721,7 +723,7 @@ async function readJsonl(file) {
} catch { return []; }
}
function buildFindings(samples, control, network, errors, sampleMetrics) {
function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork) {
const findings = [];
const commandTimes = control.filter((item) => item.phase === "completed" || item.phase === "started").map((item) => Date.parse(item.ts)).filter(Number.isFinite);
const routeSessions = new Set(samples.map((item) => item.routeSessionId).filter(Boolean));
@@ -749,6 +751,8 @@ function buildFindings(samples, control, network, errors, sampleMetrics) {
const traceTerminal = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.some((row) => /completed|failed|canceled|terminal|done/iu.test((row.status || "") + " " + (row.textPreview || ""))));
const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0);
if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) });
const promptFailures = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.chatPostOk === false) : [];
if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) });
const naturalApi = network.filter((item) => item.observerInitiated === false && item.type === "response" && /\/v1\/|\/auth\//u.test(String(item.url || "")));
findings.push({ id: "natural-api-dom-lag-baseline", severity: "info", summary: "natural API responses and DOM samples are available for manual lag correlation", naturalApiResponses: naturalApi.length, sampleCount: samples.length });
if (errors.length > 0) findings.push({ id: "browser-console-or-page-errors", severity: "amber", summary: "pageerror/runner errors were captured", count: errors.length, first: errors.slice(0, 5) });
@@ -756,6 +760,94 @@ function buildFindings(samples, control, network, errors, sampleMetrics) {
return findings;
}
function buildPromptNetworkReport(control, network) {
const promptsById = new Map();
for (const item of control) {
if (item?.type !== "sendPrompt" || !item.commandId) continue;
const existing = promptsById.get(item.commandId) || {
commandId: item.commandId,
promptIndex: promptsById.size + 1,
promptTextHash: item.input?.textHash ?? null,
promptTextBytes: item.input?.textBytes ?? null,
startedAt: null,
completedAt: null,
failedAt: null,
phase: null
};
if (!existing.promptTextHash && item.input?.textHash) existing.promptTextHash = item.input.textHash;
if (!existing.promptTextBytes && item.input?.textBytes) existing.promptTextBytes = item.input.textBytes;
if (item.phase === "started") existing.startedAt = item.ts ?? existing.startedAt;
if (item.phase === "completed") existing.completedAt = item.ts ?? existing.completedAt;
if (item.phase === "failed") existing.failedAt = item.ts ?? existing.failedAt;
existing.phase = item.phase ?? existing.phase;
promptsById.set(item.commandId, existing);
}
const prompts = Array.from(promptsById.values()).sort((a, b) => Date.parse(a.startedAt || a.completedAt || a.failedAt || "") - Date.parse(b.startedAt || b.completedAt || b.failedAt || ""));
prompts.forEach((item, index) => { item.promptIndex = index + 1; });
const chatEvents = network
.filter((item) => String(item?.method || "").toUpperCase() === "POST" && /\/v1\/agent\/chat(?:\?|$)/u.test(String(item?.url || "")))
.map((item) => ({
ts: item.ts ?? null,
tsMs: Date.parse(item.ts),
type: item.type ?? null,
status: Number.isFinite(Number(item.status)) ? Number(item.status) : null,
commandId: item.commandId ?? null,
urlPath: urlPath(item.url),
failureKind: item.failureKind ?? null,
errorTextHash: item.errorText ? sha256(item.errorText) : null
}))
.filter((item) => Number.isFinite(item.tsMs))
.sort((a, b) => a.tsMs - b.tsMs);
const rounds = prompts.map((prompt) => {
const startMs = Date.parse(prompt.startedAt || prompt.completedAt || prompt.failedAt || "");
const endAnchorMs = Date.parse(prompt.completedAt || prompt.failedAt || prompt.startedAt || "");
const fromMs = Number.isFinite(startMs) ? startMs - 3000 : Number.NEGATIVE_INFINITY;
const toMs = Number.isFinite(endAnchorMs) ? endAnchorMs + 30000 : Number.POSITIVE_INFINITY;
const events = chatEvents.filter((event) => {
if (event.commandId && prompt.commandId && event.commandId === prompt.commandId) return true;
return event.tsMs >= fromMs && event.tsMs <= toMs;
});
const responses = events.filter((event) => event.type === "response");
const failures = events.filter((event) => event.type === "requestfailed");
const responseStatuses = responses.map((event) => event.status).filter((status) => status !== null);
const chatPostOk = responseStatuses.some((status) => status >= 200 && status < 300) && failures.length === 0;
const failureKind = chatPostOk
? null
: failures.length > 0
? "requestfailed"
: responseStatuses.length === 0
? "missing-response"
: "http-status";
return {
promptIndex: prompt.promptIndex,
promptCommandId: prompt.commandId,
promptTextHash: prompt.promptTextHash,
promptTextBytes: prompt.promptTextBytes,
startedAt: prompt.startedAt,
completedAt: prompt.completedAt,
failedAt: prompt.failedAt,
chatPostOk,
failureKind,
requestCount: events.filter((event) => event.type === "request").length,
responseCount: responses.length,
requestFailedCount: failures.length,
responseStatuses,
firstChatEventAt: events[0]?.ts ?? null,
lastChatEventAt: events[events.length - 1]?.ts ?? null,
events: events.slice(0, 12).map((event) => ({ ts: event.ts, type: event.type, status: event.status, urlPath: event.urlPath, failureKind: event.failureKind, errorTextHash: event.errorTextHash }))
};
});
return {
summary: {
promptCount: rounds.length,
chatPostOk: rounds.filter((item) => item.chatPostOk === true).length,
chatPostFailed: rounds.filter((item) => item.chatPostOk === false).length,
chatPostMissing: rounds.filter((item) => item.failureKind === "missing-response").length
},
rounds
};
}
function buildSampleMetrics(samples, control) {
const promptCommands = control
.filter((item) => item.type === "sendPrompt" && item.phase === "completed")
@@ -836,6 +928,7 @@ function buildTurnTimingTable(samples, timeline) {
lastSeq: sample.seq ?? null,
lastTs: sample.ts ?? null,
promptIndex: metric.promptIndex ?? null,
lastPromptIndex: metric.promptIndex ?? null,
traceId: metric.traceId ?? null,
messageId: metric.messageId ?? null,
domIndex: metric.domIndex ?? null
@@ -845,6 +938,7 @@ function buildTurnTimingTable(samples, timeline) {
} else {
column.lastSeq = sample.seq ?? null;
column.lastTs = sample.ts ?? null;
column.lastPromptIndex = metric.promptIndex ?? column.lastPromptIndex ?? null;
if (!column.traceId && metric.traceId) column.traceId = metric.traceId;
if (!column.messageId && metric.messageId) column.messageId = metric.messageId;
}
@@ -882,7 +976,7 @@ function turnMetricItems(sample, timelineItem) {
const traceId = turn.traceId || firstTraceId(texts);
const messageId = turn.messageId || null;
const domIndex = Number.isFinite(Number(turn.index)) ? Number(turn.index) : items.length;
const key = "turn:" + (traceId || messageId || ("dom-index-" + String(domIndex)));
const key = "turn:dom-index-" + String(domIndex);
items.push({
key,
source: "turn",
@@ -1133,7 +1227,7 @@ function renderTurnTimingTable(sampleMetrics) {
}
lines.push("| " + cells.map(escapeMarkdownCell).join(" | ") + " |");
}
const columnLines = columns.map((column) => "- " + column.label + ": source=" + (column.source || "-") + " prompt=" + (column.promptIndex ?? "-") + " firstSeq=" + (column.firstSeq ?? "-") + " lastSeq=" + (column.lastSeq ?? "-") + " traceId=" + (column.traceId || "-") + " messageId=" + (column.messageId || "-")).join("\n");
const columnLines = columns.map((column) => "- " + column.label + ": source=" + (column.source || "-") + " prompt=" + (column.promptIndex ?? "-") + " lastPrompt=" + (column.lastPromptIndex ?? "-") + " firstSeq=" + (column.firstSeq ?? "-") + " lastSeq=" + (column.lastSeq ?? "-") + " traceId=" + (column.traceId || "-") + " messageId=" + (column.messageId || "-")).join("\n");
return lines.join("\n") + "\n\n列说明:\n" + columnLines;
}
@@ -1151,6 +1245,9 @@ function renderMarkdown(report) {
const commandLines = report.commandTimeline.length === 0 ? "- 无控制命令。" : report.commandTimeline.map((item) => "- " + item.ts + " " + item.phase + " " + item.type + " " + item.commandId + " " + (item.afterUrl || "")).join("\n");
const transitionLines = report.transitions.length === 0 ? "- 无状态变化。" : report.transitions.slice(0, 80).map((item) => "- #" + item.seq + " " + item.ts + " messages=" + item.messageCount + " traceRows=" + item.traceRowCount + " route=" + (item.routeSessionId || "-") + " active=" + (item.activeSessionId || "-")).join("\n");
const metricSummary = report.sampleMetrics?.summary || {};
const promptNetworkLines = Array.isArray(report.promptNetwork?.rounds) && report.promptNetwork.rounds.length > 0
? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n")
: "- 无 prompt 网络记录。";
const roundLines = Array.isArray(report.sampleMetrics?.rounds) && report.sampleMetrics.rounds.length > 0
? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n")
: "- 无轮次指标。";
@@ -1173,6 +1270,7 @@ function renderMarkdown(report) {
+ "- diagnostics: " + (metricSummary.diagnostics ?? 0) + "\n"
+ "- promptSegments: " + (metricSummary.promptSegments ?? 0) + "\n\n"
+ "### Rounds\n\n" + roundLines + "\n\n"
+ "### Prompt network\n\n" + promptNetworkLines + "\n\n"
+ "### Turn timing table\n\n"
+ turnTimingTable + "\n\n"
+ "### Aggregate timeline\n\n"
@@ -1189,5 +1287,14 @@ async function fileMeta(file) {
function sha256(value) {
return "sha256:" + createHash("sha256").update(String(value)).digest("hex");
}
function urlPath(value) {
try {
const url = new URL(String(value || "http://invalid.local/"));
return url.pathname;
} catch {
return "-";
}
}
`;
}