fix(web-probe): report prompt submit network status
This commit is contained in:
@@ -686,7 +686,8 @@ const heartbeat = await readJson(path.join(stateDir, "heartbeat.json"));
|
||||
await mkdir(analysisDir, { recursive: true, mode: 0o700 });
|
||||
const transitions = buildTransitions(samples);
|
||||
const sampleMetrics = buildSampleMetrics(samples, control);
|
||||
const findings = buildFindings(samples, control, network, errors, sampleMetrics);
|
||||
const promptNetwork = buildPromptNetworkReport(control, network);
|
||||
const findings = buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork);
|
||||
const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl }));
|
||||
const report = {
|
||||
ok: findings.filter((item) => item.severity === "red").length === 0,
|
||||
@@ -699,6 +700,7 @@ const report = {
|
||||
commandTimeline,
|
||||
transitions,
|
||||
sampleMetrics,
|
||||
promptNetwork,
|
||||
findings,
|
||||
artifactSummary: await artifactSummary(artifacts),
|
||||
safety: { offlineOnly: true, browserDriven: false, apiFetch: false, valuesRedacted: true },
|
||||
@@ -706,7 +708,7 @@ const report = {
|
||||
await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 });
|
||||
await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 });
|
||||
const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]);
|
||||
console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, findings: findings.slice(0, 20), valuesRedacted: true }, null, 2));
|
||||
console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, promptNetwork: promptNetwork.summary, findings: findings.slice(0, 20), valuesRedacted: true }, null, 2));
|
||||
|
||||
async function readJson(file) {
|
||||
try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; }
|
||||
@@ -721,7 +723,7 @@ async function readJsonl(file) {
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
function buildFindings(samples, control, network, errors, sampleMetrics) {
|
||||
function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork) {
|
||||
const findings = [];
|
||||
const commandTimes = control.filter((item) => item.phase === "completed" || item.phase === "started").map((item) => Date.parse(item.ts)).filter(Number.isFinite);
|
||||
const routeSessions = new Set(samples.map((item) => item.routeSessionId).filter(Boolean));
|
||||
@@ -749,6 +751,8 @@ function buildFindings(samples, control, network, errors, sampleMetrics) {
|
||||
const traceTerminal = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.some((row) => /completed|failed|canceled|terminal|done/iu.test((row.status || "") + " " + (row.textPreview || ""))));
|
||||
const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0);
|
||||
if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) });
|
||||
const promptFailures = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.chatPostOk === false) : [];
|
||||
if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) });
|
||||
const naturalApi = network.filter((item) => item.observerInitiated === false && item.type === "response" && /\/v1\/|\/auth\//u.test(String(item.url || "")));
|
||||
findings.push({ id: "natural-api-dom-lag-baseline", severity: "info", summary: "natural API responses and DOM samples are available for manual lag correlation", naturalApiResponses: naturalApi.length, sampleCount: samples.length });
|
||||
if (errors.length > 0) findings.push({ id: "browser-console-or-page-errors", severity: "amber", summary: "pageerror/runner errors were captured", count: errors.length, first: errors.slice(0, 5) });
|
||||
@@ -756,6 +760,94 @@ function buildFindings(samples, control, network, errors, sampleMetrics) {
|
||||
return findings;
|
||||
}
|
||||
|
||||
function buildPromptNetworkReport(control, network) {
|
||||
const promptsById = new Map();
|
||||
for (const item of control) {
|
||||
if (item?.type !== "sendPrompt" || !item.commandId) continue;
|
||||
const existing = promptsById.get(item.commandId) || {
|
||||
commandId: item.commandId,
|
||||
promptIndex: promptsById.size + 1,
|
||||
promptTextHash: item.input?.textHash ?? null,
|
||||
promptTextBytes: item.input?.textBytes ?? null,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
failedAt: null,
|
||||
phase: null
|
||||
};
|
||||
if (!existing.promptTextHash && item.input?.textHash) existing.promptTextHash = item.input.textHash;
|
||||
if (!existing.promptTextBytes && item.input?.textBytes) existing.promptTextBytes = item.input.textBytes;
|
||||
if (item.phase === "started") existing.startedAt = item.ts ?? existing.startedAt;
|
||||
if (item.phase === "completed") existing.completedAt = item.ts ?? existing.completedAt;
|
||||
if (item.phase === "failed") existing.failedAt = item.ts ?? existing.failedAt;
|
||||
existing.phase = item.phase ?? existing.phase;
|
||||
promptsById.set(item.commandId, existing);
|
||||
}
|
||||
const prompts = Array.from(promptsById.values()).sort((a, b) => Date.parse(a.startedAt || a.completedAt || a.failedAt || "") - Date.parse(b.startedAt || b.completedAt || b.failedAt || ""));
|
||||
prompts.forEach((item, index) => { item.promptIndex = index + 1; });
|
||||
const chatEvents = network
|
||||
.filter((item) => String(item?.method || "").toUpperCase() === "POST" && /\/v1\/agent\/chat(?:\?|$)/u.test(String(item?.url || "")))
|
||||
.map((item) => ({
|
||||
ts: item.ts ?? null,
|
||||
tsMs: Date.parse(item.ts),
|
||||
type: item.type ?? null,
|
||||
status: Number.isFinite(Number(item.status)) ? Number(item.status) : null,
|
||||
commandId: item.commandId ?? null,
|
||||
urlPath: urlPath(item.url),
|
||||
failureKind: item.failureKind ?? null,
|
||||
errorTextHash: item.errorText ? sha256(item.errorText) : null
|
||||
}))
|
||||
.filter((item) => Number.isFinite(item.tsMs))
|
||||
.sort((a, b) => a.tsMs - b.tsMs);
|
||||
const rounds = prompts.map((prompt) => {
|
||||
const startMs = Date.parse(prompt.startedAt || prompt.completedAt || prompt.failedAt || "");
|
||||
const endAnchorMs = Date.parse(prompt.completedAt || prompt.failedAt || prompt.startedAt || "");
|
||||
const fromMs = Number.isFinite(startMs) ? startMs - 3000 : Number.NEGATIVE_INFINITY;
|
||||
const toMs = Number.isFinite(endAnchorMs) ? endAnchorMs + 30000 : Number.POSITIVE_INFINITY;
|
||||
const events = chatEvents.filter((event) => {
|
||||
if (event.commandId && prompt.commandId && event.commandId === prompt.commandId) return true;
|
||||
return event.tsMs >= fromMs && event.tsMs <= toMs;
|
||||
});
|
||||
const responses = events.filter((event) => event.type === "response");
|
||||
const failures = events.filter((event) => event.type === "requestfailed");
|
||||
const responseStatuses = responses.map((event) => event.status).filter((status) => status !== null);
|
||||
const chatPostOk = responseStatuses.some((status) => status >= 200 && status < 300) && failures.length === 0;
|
||||
const failureKind = chatPostOk
|
||||
? null
|
||||
: failures.length > 0
|
||||
? "requestfailed"
|
||||
: responseStatuses.length === 0
|
||||
? "missing-response"
|
||||
: "http-status";
|
||||
return {
|
||||
promptIndex: prompt.promptIndex,
|
||||
promptCommandId: prompt.commandId,
|
||||
promptTextHash: prompt.promptTextHash,
|
||||
promptTextBytes: prompt.promptTextBytes,
|
||||
startedAt: prompt.startedAt,
|
||||
completedAt: prompt.completedAt,
|
||||
failedAt: prompt.failedAt,
|
||||
chatPostOk,
|
||||
failureKind,
|
||||
requestCount: events.filter((event) => event.type === "request").length,
|
||||
responseCount: responses.length,
|
||||
requestFailedCount: failures.length,
|
||||
responseStatuses,
|
||||
firstChatEventAt: events[0]?.ts ?? null,
|
||||
lastChatEventAt: events[events.length - 1]?.ts ?? null,
|
||||
events: events.slice(0, 12).map((event) => ({ ts: event.ts, type: event.type, status: event.status, urlPath: event.urlPath, failureKind: event.failureKind, errorTextHash: event.errorTextHash }))
|
||||
};
|
||||
});
|
||||
return {
|
||||
summary: {
|
||||
promptCount: rounds.length,
|
||||
chatPostOk: rounds.filter((item) => item.chatPostOk === true).length,
|
||||
chatPostFailed: rounds.filter((item) => item.chatPostOk === false).length,
|
||||
chatPostMissing: rounds.filter((item) => item.failureKind === "missing-response").length
|
||||
},
|
||||
rounds
|
||||
};
|
||||
}
|
||||
|
||||
function buildSampleMetrics(samples, control) {
|
||||
const promptCommands = control
|
||||
.filter((item) => item.type === "sendPrompt" && item.phase === "completed")
|
||||
@@ -836,6 +928,7 @@ function buildTurnTimingTable(samples, timeline) {
|
||||
lastSeq: sample.seq ?? null,
|
||||
lastTs: sample.ts ?? null,
|
||||
promptIndex: metric.promptIndex ?? null,
|
||||
lastPromptIndex: metric.promptIndex ?? null,
|
||||
traceId: metric.traceId ?? null,
|
||||
messageId: metric.messageId ?? null,
|
||||
domIndex: metric.domIndex ?? null
|
||||
@@ -845,6 +938,7 @@ function buildTurnTimingTable(samples, timeline) {
|
||||
} else {
|
||||
column.lastSeq = sample.seq ?? null;
|
||||
column.lastTs = sample.ts ?? null;
|
||||
column.lastPromptIndex = metric.promptIndex ?? column.lastPromptIndex ?? null;
|
||||
if (!column.traceId && metric.traceId) column.traceId = metric.traceId;
|
||||
if (!column.messageId && metric.messageId) column.messageId = metric.messageId;
|
||||
}
|
||||
@@ -882,7 +976,7 @@ function turnMetricItems(sample, timelineItem) {
|
||||
const traceId = turn.traceId || firstTraceId(texts);
|
||||
const messageId = turn.messageId || null;
|
||||
const domIndex = Number.isFinite(Number(turn.index)) ? Number(turn.index) : items.length;
|
||||
const key = "turn:" + (traceId || messageId || ("dom-index-" + String(domIndex)));
|
||||
const key = "turn:dom-index-" + String(domIndex);
|
||||
items.push({
|
||||
key,
|
||||
source: "turn",
|
||||
@@ -1133,7 +1227,7 @@ function renderTurnTimingTable(sampleMetrics) {
|
||||
}
|
||||
lines.push("| " + cells.map(escapeMarkdownCell).join(" | ") + " |");
|
||||
}
|
||||
const columnLines = columns.map((column) => "- " + column.label + ": source=" + (column.source || "-") + " prompt=" + (column.promptIndex ?? "-") + " firstSeq=" + (column.firstSeq ?? "-") + " lastSeq=" + (column.lastSeq ?? "-") + " traceId=" + (column.traceId || "-") + " messageId=" + (column.messageId || "-")).join("\n");
|
||||
const columnLines = columns.map((column) => "- " + column.label + ": source=" + (column.source || "-") + " prompt=" + (column.promptIndex ?? "-") + " lastPrompt=" + (column.lastPromptIndex ?? "-") + " firstSeq=" + (column.firstSeq ?? "-") + " lastSeq=" + (column.lastSeq ?? "-") + " traceId=" + (column.traceId || "-") + " messageId=" + (column.messageId || "-")).join("\n");
|
||||
return lines.join("\n") + "\n\n列说明:\n" + columnLines;
|
||||
}
|
||||
|
||||
@@ -1151,6 +1245,9 @@ function renderMarkdown(report) {
|
||||
const commandLines = report.commandTimeline.length === 0 ? "- 无控制命令。" : report.commandTimeline.map((item) => "- " + item.ts + " " + item.phase + " " + item.type + " " + item.commandId + " " + (item.afterUrl || "")).join("\n");
|
||||
const transitionLines = report.transitions.length === 0 ? "- 无状态变化。" : report.transitions.slice(0, 80).map((item) => "- #" + item.seq + " " + item.ts + " messages=" + item.messageCount + " traceRows=" + item.traceRowCount + " route=" + (item.routeSessionId || "-") + " active=" + (item.activeSessionId || "-")).join("\n");
|
||||
const metricSummary = report.sampleMetrics?.summary || {};
|
||||
const promptNetworkLines = Array.isArray(report.promptNetwork?.rounds) && report.promptNetwork.rounds.length > 0
|
||||
? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n")
|
||||
: "- 无 prompt 网络记录。";
|
||||
const roundLines = Array.isArray(report.sampleMetrics?.rounds) && report.sampleMetrics.rounds.length > 0
|
||||
? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n")
|
||||
: "- 无轮次指标。";
|
||||
@@ -1173,6 +1270,7 @@ function renderMarkdown(report) {
|
||||
+ "- diagnostics: " + (metricSummary.diagnostics ?? 0) + "\n"
|
||||
+ "- promptSegments: " + (metricSummary.promptSegments ?? 0) + "\n\n"
|
||||
+ "### Rounds\n\n" + roundLines + "\n\n"
|
||||
+ "### Prompt network\n\n" + promptNetworkLines + "\n\n"
|
||||
+ "### Turn timing table\n\n"
|
||||
+ turnTimingTable + "\n\n"
|
||||
+ "### Aggregate timeline\n\n"
|
||||
@@ -1189,5 +1287,14 @@ async function fileMeta(file) {
|
||||
function sha256(value) {
|
||||
return "sha256:" + createHash("sha256").update(String(value)).digest("hex");
|
||||
}
|
||||
|
||||
function urlPath(value) {
|
||||
try {
|
||||
const url = new URL(String(value || "http://invalid.local/"));
|
||||
return url.pathname;
|
||||
} catch {
|
||||
return "-";
|
||||
}
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user