mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(web-probe): classify steer prompt submissions (#831)

Browse Source 
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-24 16:42:55 +08:00
committed by GitHub
parent 2fd4ca2fee
commit b366bffe93
2 changed files with 42 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-web-observe-runner-source.ts
+19 -5
View File
@@ -2928,7 +2928,9 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN
const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0);
if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) });
const promptFailures = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.chatPostOk === false) : [];
if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) });
if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat or /v1/agent/chat/steer POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) });
const promptSteerRounds = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.steerUsed === true) : [];
if (promptSteerRounds.length > 0) findings.push({ id: "prompt-routed-to-steer", severity: "amber", summary: "sendPrompt was submitted through /v1/agent/chat/steer; verify the previous turn was truly in-flight and not an unsealed terminal failure", count: promptSteerRounds.length, rounds: promptSteerRounds.slice(0, 10) });
const elapsedZeroResets = Array.isArray(sampleMetrics?.turnTimingElapsedZeroResets) ? sampleMetrics.turnTimingElapsedZeroResets : [];
if (elapsedZeroResets.length > 0) findings.push({ id: "turn-timing-total-elapsed-zero-reset", severity: "red", summary: "Code Agent total elapsed jumped from a non-zero value back to 0 seconds", count: elapsedZeroResets.length, samples: elapsedZeroResets.slice(0, 20) });
const elapsedDecreases = Array.isArray(sampleMetrics?.turnTimingNonMonotonic)
@@ -3452,16 +3454,18 @@ function buildPromptNetworkReport(control, network) {
const prompts = Array.from(promptsById.values()).sort((a, b) => Date.parse(a.startedAt || a.completedAt || a.failedAt || "") - Date.parse(b.startedAt || b.completedAt || b.failedAt || ""));
prompts.forEach((item, index) => { item.promptIndex = index + 1; });
const chatEvents = network
.filter((item) => String(item?.method || "").toUpperCase() === "POST" && /\/v1\/agent\/chat(?:\?|$)/u.test(String(item?.url || "")))
.filter((item) => String(item?.method || "").toUpperCase() === "POST" && promptSubmitModeForUrl(item?.url) !== null)
.map((item) => {
const failureText = item.failureKind ?? item.failure ?? item.errorText ?? null;
const urlPathValue = urlPath(item.url);
return {
ts: item.ts ?? null,
tsMs: Date.parse(item.ts),
type: item.type ?? null,
status: Number.isFinite(Number(item.status)) ? Number(item.status) : null,
commandId: item.commandId ?? null,
urlPath: urlPath(item.url),
urlPath: urlPathValue,
submitMode: promptSubmitModeForUrl(item.url),
failureKind: failureText ? String(failureText) : null,
errorTextHash: failureText ? sha256(failureText) : null
};
@@ -3480,6 +3484,7 @@ function buildPromptNetworkReport(control, network) {
const responses = events.filter((event) => event.type === "response");
const failures = events.filter((event) => event.type === "requestfailed");
const responseStatuses = responses.map((event) => event.status).filter((status) => status !== null);
const submitModes = Array.from(new Set(events.map((event) => event.submitMode).filter(Boolean))).sort();
const chatPostOk = responseStatuses.some((status) => status >= 200 && status < 300) && failures.length === 0;
const failureKind = chatPostOk
? null
@@ -3502,9 +3507,11 @@ function buildPromptNetworkReport(control, network) {
responseCount: responses.length,
requestFailedCount: failures.length,
responseStatuses,
submitModes,
steerUsed: submitModes.includes("steer"),
firstChatEventAt: events[0]?.ts ?? null,
lastChatEventAt: events[events.length - 1]?.ts ?? null,
events: events.slice(0, 12).map((event) => ({ ts: event.ts, type: event.type, status: event.status, urlPath: event.urlPath, failureKind: event.failureKind, errorTextHash: event.errorTextHash }))
events: events.slice(0, 12).map((event) => ({ ts: event.ts, type: event.type, status: event.status, urlPath: event.urlPath, submitMode: event.submitMode, failureKind: event.failureKind, errorTextHash: event.errorTextHash }))
};
});
return {
@@ -3518,6 +3525,13 @@ function buildPromptNetworkReport(control, network) {
};
}
function promptSubmitModeForUrl(value) {
const pathValue = urlPath(value);
if (pathValue === "/v1/agent/chat") return "chat";
if (pathValue === "/v1/agent/chat/steer") return "steer";
return null;
}
function parseDomDiagnosticSummary(text) {
const value = String(text || "");
const traceMatch = value.match(/\b(?:trace_id=)?(trc_[A-Za-z0-9_-]+|[a-f0-9]{16,64})\b/iu);
@@ -6784,7 +6798,7 @@ function renderMarkdown(report) {
? report.runtimeAlerts.consoleAlertsByPath.slice(0, 40).map((item) => "- console " + (item.type || "-") + " status=" + (item.status ?? "-") + " path=" + (item.urlPath || "-") + " count=" + item.count + " prompts=" + (item.promptIndexes?.join(",") || "-") + " traces=" + (item.traceIds?.slice(0, 6).join(",") || "-")).join("\n")
: "- 无 console 分组。";
const promptNetworkLines = Array.isArray(report.promptNetwork?.rounds) && report.promptNetwork.rounds.length > 0
? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n")
? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " modes=" + (Array.isArray(item.submitModes) && item.submitModes.length > 0 ? item.submitModes.join(",") : "-") + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n")
: "- 无 prompt 网络记录。";
const roundLines = Array.isArray(report.sampleMetrics?.rounds) && report.sampleMetrics.rounds.length > 0
? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " loadingSamples=" + (item.loadingSamples ?? 0) + " maxLoading=" + (item.maxLoadingCount ?? 0) + " loadingOwners=" + (item.loadingOwnerCount ?? 0) + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " totalDecrease=" + (item.turnTimingTotalElapsedDecreaseCount ?? 0) + " totalForwardJump=" + (item.turnTimingTotalElapsedForwardJumpCount ?? 0) + " totalForwardJumpMax=" + (item.turnTimingTotalElapsedForwardJumpMaxSeconds ?? 0) + " terminalGrowth=" + (item.turnTimingTerminalElapsedGrowthCount ?? 0) + " terminalGrowthMax=" + (item.turnTimingTerminalElapsedGrowthMaxSeconds ?? 0) + " recentJump=" + (item.turnTimingRecentUpdateJumpCount ?? 0) + " recentSawtoothJump=" + (item.turnTimingRecentUpdateSawtoothJumpCount ?? item.turnTimingRecentUpdateJumpCount ?? 0) + " recentStep=" + (item.turnTimingRecentUpdateStepCount ?? 0) + " recentMaxIncrease=" + (item.turnTimingRecentUpdateMaxIncreaseSeconds ?? "-") + " recentMaxExcess=" + (item.turnTimingRecentUpdateMaxExcessSeconds ?? 0) + " recentReset=" + (item.turnTimingRecentUpdateResetCount ?? 0) + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n")