From b366bffe93a2a3e12a6c2d523e9b1d070ec517aa Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:42:55 +0800 Subject: [PATCH] fix(web-probe): classify steer prompt submissions (#831) Co-authored-by: Codex --- scripts/src/hwlab-node-impl.ts | 30 ++++++++++++++----- .../hwlab-node-web-observe-runner-source.ts | 24 +++++++++++---- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index ad764a4f..4ea2cdb1 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -10277,7 +10277,7 @@ console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir: function nodeWebObserveCollectNodeScript(maxFiles: number, collectFile: string | null, collectFinding: string | null, collectGrep: string | null): string { return `node -e ${shellQuote(` const fs=require('fs'),path=require('path'),crypto=require('crypto'); -const dir=process.argv[1]; const findingFilter=process.argv[3]||''; const selected=(process.argv[2]||'') || (findingFilter ? 'analysis/report.json' : ''); const grepText=process.argv[4]||''; const maxFiles=${maxFiles}; const maxReadBytes=64*1024; const maxJsonlTailBytes=1024*1024; const maxTextPreviewBytes=4096; const maxGrepContextLines=24; const maxGrepLineText=180; +const dir=process.argv[1]; const findingFilter=process.argv[3]||''; let selected=(process.argv[2]||'') || (findingFilter ? 'analysis/report.json' : ''); const grepText=process.argv[4]||''; const maxFiles=${maxFiles}; const maxReadBytes=64*1024; const maxJsonlTailBytes=1024*1024; const maxTextPreviewBytes=4096; const maxGrepContextLines=24; const maxGrepLineText=180; const shaFile=(file)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); const shaText=(value)=>'sha256:'+crypto.createHash('sha256').update(String(value||'')).digest('hex'); const safeRel=(value)=>Boolean(value)&&!path.isAbsolute(value)&&!value.includes('..')&&!value.includes('\\\\')&&value.split('/').every((part)=>part&&part!=='.'&&part!=='..'); @@ -10397,14 +10397,14 @@ const slimFindingSample=(value)=>{ allowedIncreaseSeconds:value.allowedIncreaseSeconds??null, excessiveIncreaseSeconds:value.excessiveIncreaseSeconds??null, promptIndex:value.promptIndex??null, - event:short(value.event??value.anomaly,48), + event:short(value.event??value.anomaly??(value.steerUsed===true?'steer':Array.isArray(value.submitModes)?value.submitModes.join(','):undefined),48), expectedPattern:short(value.expectedPattern,96), control:compactMetric(controlCount??(value.fromMessageCount!==undefined||value.fromTraceRowCount!==undefined?compactProjectionPair(value.fromMessageCount,value.fromTraceRowCount):undefined)), observer:compactMetric(observerCount??(value.toMessageCount!==undefined||value.toTraceRowCount!==undefined?compactProjectionPair(value.toMessageCount,value.toTraceRowCount):undefined)), sessionId:short(value.sessionId??value.routeSessionId??value.activeSessionId??value.controlSessionId??value.observerSessionId??refValue.routeSessionId??refValue.activeSessionId,64), path:short(value.path??value.urlPath??value.url??value.controlPath??value.observerPath??refValue.url,120), trace:short(trace,120), - detail:short(value.detail??(value.durationText!==undefined||value.activityText!==undefined||value.textPreview!==undefined?['duration='+String(value.durationText??'-'),'activity='+String(value.activityText??'-'),'status='+String(value.status??'-'),'preview='+String(value.textPreview??'-')].join(' '):undefined)??(value.fromDigest||value.toDigest?'digest '+String(value.fromDigest??'-')+' -> '+String(value.toDigest??'-'):undefined)??(value.messageTextDigestDiff?'messageDigest '+String(value.controlMessageDigest??'-')+' != '+String(value.observerMessageDigest??'-'):undefined)??value.summary??value.message??value.preview??value.expectedPattern,180), + detail:short(value.detail??(value.durationText!==undefined||value.activityText!==undefined||value.textPreview!==undefined?['duration='+String(value.durationText??'-'),'activity='+String(value.activityText??'-'),'status='+String(value.status??'-'),'preview='+String(value.textPreview??'-')].join(' '):undefined)??(Array.isArray(value.submitModes)||Array.isArray(value.responseStatuses)?['modes='+String(Array.isArray(value.submitModes)?value.submitModes.join(','):'-'),'statuses='+String(Array.isArray(value.responseStatuses)?value.responseStatuses.join(','):'-'),'failure='+String(value.failureKind??'-')].join(' '):undefined)??(value.fromDigest||value.toDigest?'digest '+String(value.fromDigest??'-')+' -> '+String(value.toDigest??'-'):undefined)??(value.messageTextDigestDiff?'messageDigest '+String(value.controlMessageDigest??'-')+' != '+String(value.observerMessageDigest??'-'):undefined)??value.summary??value.message??value.preview??value.expectedPattern,180), valuesRedacted:true }; }; @@ -10412,11 +10412,23 @@ const slimFindingDetail=(parsed,id)=>{ if(!id||!parsed||typeof parsed!=='object'||!Array.isArray(parsed.findings)) return null; const finding=parsed.findings.find((item)=>item&&typeof item==='object'&&String(item.id??item.kind??item.code??'')===id); if(!finding) return {id,found:false,samples:[],valuesRedacted:true}; - const sampleSource=Array.isArray(finding.samples)?'samples':Array.isArray(finding.groups)?'groups':Array.isArray(finding.examples)?'examples':Array.isArray(finding.rows)?'rows':'none'; - const samples=firstArray(finding.samples,finding.groups,finding.examples,finding.rows).slice(0,4).map(slimFindingSample).filter(Boolean); + const sampleSource=Array.isArray(finding.samples)?'samples':Array.isArray(finding.groups)?'groups':Array.isArray(finding.examples)?'examples':Array.isArray(finding.rows)?'rows':Array.isArray(finding.rounds)?'rounds':'none'; + const samples=firstArray(finding.samples,finding.groups,finding.examples,finding.rows,finding.rounds).slice(0,4).map(slimFindingSample).filter(Boolean); return {id:short(finding.id??finding.kind??finding.code,80),found:true,severity:short(finding.severity??finding.level,24),count:finding.count??finding.sampleCount??samples.length,summary:short(finding.summary??finding.message,180),sampleSource,samples,valuesRedacted:true}; }; const slimRunnerError=(item)=>item&&typeof item==='object'?{ts:short(item.ts||item.at,32),type:short(item.type,48),retry:item.retry||item.lastRetryLabel||null,retryExhausted:item.retryExhausted===true,attemptCount:item.attemptCount??(Array.isArray(item.attempts)?item.attempts.length:null),lastFailureKind:short(item.lastFailureKind||item.failureKind,80),lastReadinessReason:short(item.lastReadinessReason,80),message:short(item.message||item.lastError,240)}:null; +const collectFallbackCandidates=(requested)=>{ + const candidates=[]; + if(findingFilter&&requested!=='analysis/report.json') candidates.push('analysis/report.json'); + if(requested==='analysis/report.json'||findingFilter||requested.startsWith('analysis/')){ + try{ + const analysisDir=path.join(dir,'analysis'); + const names=fs.readdirSync(analysisDir).filter((name)=>name.endsWith('.json')).map((name)=>({name,mtime:fs.statSync(path.join(analysisDir,name)).mtimeMs})).sort((a,b)=>b.mtime-a.mtime); + for(const item of names) candidates.push('analysis/'+item.name); + }catch{} + } + return [...new Set(candidates)].filter((item)=>item&&item!==requested&&safeRel(item)); +}; const slimJsonl=(value)=>{ if(!value||typeof value!=='object'||Array.isArray(value)) return {value:short(value)}; const readiness=value.readiness&&typeof value.readiness==='object'?value.readiness:null; @@ -10449,9 +10461,13 @@ const slimJsonl=(value)=>{ }; if(selected){ if(!safeRel(selected)){console.log(JSON.stringify({ok:false,command:'web-probe-observe collect',stateDir:dir,mode:'file',reason:'unsafe-file',file:selected,valuesRedacted:true},null,2)); process.exit(2);} - const file=path.join(dir,selected); const relative=path.relative(dir,file); + let file=path.join(dir,selected); let relative=path.relative(dir,file); if(relative.startsWith('..')||path.isAbsolute(relative)){console.log(JSON.stringify({ok:false,command:'web-probe-observe collect',stateDir:dir,mode:'file',reason:'file-outside-state-dir',file:selected,valuesRedacted:true},null,2)); process.exit(2);} - if(!fs.existsSync(file)||!fs.statSync(file).isFile()){console.log(JSON.stringify({ok:false,command:'web-probe-observe collect',stateDir:dir,mode:'file',reason:'file-not-found',file:selected,valuesRedacted:true},null,2)); process.exit(1);} + if(!fs.existsSync(file)||!fs.statSync(file).isFile()){ + const fallback=collectFallbackCandidates(selected).find((item)=>{const candidate=path.join(dir,item); return fs.existsSync(candidate)&&fs.statSync(candidate).isFile();}); + if(fallback){selected=fallback; file=path.join(dir,selected); relative=path.relative(dir,file);} + } + if(!fs.existsSync(file)||!fs.statSync(file).isFile()){console.log(JSON.stringify({ok:false,command:'web-probe-observe collect',stateDir:dir,mode:'file',reason:'file-not-found',file:selected,fallbackCandidates:collectFallbackCandidates(selected).slice(0,8),valuesRedacted:true},null,2)); process.exit(1);} const st=fs.statSync(file); const raw=fs.readFileSync(file); const truncated=raw.length>maxReadBytes; const isJsonl=selected.endsWith('.jsonl'); const isJson=selected.endsWith('.json'); const tailReadBytes=isJsonl?maxJsonlTailBytes:maxReadBytes; const text=(isJsonl?raw.slice(Math.max(0,raw.length-tailReadBytes)):raw.slice(0,Math.min(raw.length,maxReadBytes))).toString('utf8'); diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 3ff8b05f..efc67c9e 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -2928,7 +2928,9 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0); if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) }); const promptFailures = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.chatPostOk === false) : []; - if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) }); + if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat or /v1/agent/chat/steer POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) }); + const promptSteerRounds = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.steerUsed === true) : []; + if (promptSteerRounds.length > 0) findings.push({ id: "prompt-routed-to-steer", severity: "amber", summary: "sendPrompt was submitted through /v1/agent/chat/steer; verify the previous turn was truly in-flight and not an unsealed terminal failure", count: promptSteerRounds.length, rounds: promptSteerRounds.slice(0, 10) }); const elapsedZeroResets = Array.isArray(sampleMetrics?.turnTimingElapsedZeroResets) ? sampleMetrics.turnTimingElapsedZeroResets : []; if (elapsedZeroResets.length > 0) findings.push({ id: "turn-timing-total-elapsed-zero-reset", severity: "red", summary: "Code Agent total elapsed jumped from a non-zero value back to 0 seconds", count: elapsedZeroResets.length, samples: elapsedZeroResets.slice(0, 20) }); const elapsedDecreases = Array.isArray(sampleMetrics?.turnTimingNonMonotonic) @@ -3452,16 +3454,18 @@ function buildPromptNetworkReport(control, network) { const prompts = Array.from(promptsById.values()).sort((a, b) => Date.parse(a.startedAt || a.completedAt || a.failedAt || "") - Date.parse(b.startedAt || b.completedAt || b.failedAt || "")); prompts.forEach((item, index) => { item.promptIndex = index + 1; }); const chatEvents = network - .filter((item) => String(item?.method || "").toUpperCase() === "POST" && /\/v1\/agent\/chat(?:\?|$)/u.test(String(item?.url || ""))) + .filter((item) => String(item?.method || "").toUpperCase() === "POST" && promptSubmitModeForUrl(item?.url) !== null) .map((item) => { const failureText = item.failureKind ?? item.failure ?? item.errorText ?? null; + const urlPathValue = urlPath(item.url); return { ts: item.ts ?? null, tsMs: Date.parse(item.ts), type: item.type ?? null, status: Number.isFinite(Number(item.status)) ? Number(item.status) : null, commandId: item.commandId ?? null, - urlPath: urlPath(item.url), + urlPath: urlPathValue, + submitMode: promptSubmitModeForUrl(item.url), failureKind: failureText ? String(failureText) : null, errorTextHash: failureText ? sha256(failureText) : null }; @@ -3480,6 +3484,7 @@ function buildPromptNetworkReport(control, network) { const responses = events.filter((event) => event.type === "response"); const failures = events.filter((event) => event.type === "requestfailed"); const responseStatuses = responses.map((event) => event.status).filter((status) => status !== null); + const submitModes = Array.from(new Set(events.map((event) => event.submitMode).filter(Boolean))).sort(); const chatPostOk = responseStatuses.some((status) => status >= 200 && status < 300) && failures.length === 0; const failureKind = chatPostOk ? null @@ -3502,9 +3507,11 @@ function buildPromptNetworkReport(control, network) { responseCount: responses.length, requestFailedCount: failures.length, responseStatuses, + submitModes, + steerUsed: submitModes.includes("steer"), firstChatEventAt: events[0]?.ts ?? null, lastChatEventAt: events[events.length - 1]?.ts ?? null, - events: events.slice(0, 12).map((event) => ({ ts: event.ts, type: event.type, status: event.status, urlPath: event.urlPath, failureKind: event.failureKind, errorTextHash: event.errorTextHash })) + events: events.slice(0, 12).map((event) => ({ ts: event.ts, type: event.type, status: event.status, urlPath: event.urlPath, submitMode: event.submitMode, failureKind: event.failureKind, errorTextHash: event.errorTextHash })) }; }); return { @@ -3518,6 +3525,13 @@ function buildPromptNetworkReport(control, network) { }; } +function promptSubmitModeForUrl(value) { + const pathValue = urlPath(value); + if (pathValue === "/v1/agent/chat") return "chat"; + if (pathValue === "/v1/agent/chat/steer") return "steer"; + return null; +} + function parseDomDiagnosticSummary(text) { const value = String(text || ""); const traceMatch = value.match(/\b(?:trace_id=)?(trc_[A-Za-z0-9_-]+|[a-f0-9]{16,64})\b/iu); @@ -6784,7 +6798,7 @@ function renderMarkdown(report) { ? report.runtimeAlerts.consoleAlertsByPath.slice(0, 40).map((item) => "- console " + (item.type || "-") + " status=" + (item.status ?? "-") + " path=" + (item.urlPath || "-") + " count=" + item.count + " prompts=" + (item.promptIndexes?.join(",") || "-") + " traces=" + (item.traceIds?.slice(0, 6).join(",") || "-")).join("\n") : "- 无 console 分组。"; const promptNetworkLines = Array.isArray(report.promptNetwork?.rounds) && report.promptNetwork.rounds.length > 0 - ? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n") + ? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " modes=" + (Array.isArray(item.submitModes) && item.submitModes.length > 0 ? item.submitModes.join(",") : "-") + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n") : "- 无 prompt 网络记录。"; const roundLines = Array.isArray(report.sampleMetrics?.rounds) && report.sampleMetrics.rounds.length > 0 ? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " loadingSamples=" + (item.loadingSamples ?? 0) + " maxLoading=" + (item.maxLoadingCount ?? 0) + " loadingOwners=" + (item.loadingOwnerCount ?? 0) + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " totalDecrease=" + (item.turnTimingTotalElapsedDecreaseCount ?? 0) + " totalForwardJump=" + (item.turnTimingTotalElapsedForwardJumpCount ?? 0) + " totalForwardJumpMax=" + (item.turnTimingTotalElapsedForwardJumpMaxSeconds ?? 0) + " terminalGrowth=" + (item.turnTimingTerminalElapsedGrowthCount ?? 0) + " terminalGrowthMax=" + (item.turnTimingTerminalElapsedGrowthMaxSeconds ?? 0) + " recentJump=" + (item.turnTimingRecentUpdateJumpCount ?? 0) + " recentSawtoothJump=" + (item.turnTimingRecentUpdateSawtoothJumpCount ?? item.turnTimingRecentUpdateJumpCount ?? 0) + " recentStep=" + (item.turnTimingRecentUpdateStepCount ?? 0) + " recentMaxIncrease=" + (item.turnTimingRecentUpdateMaxIncreaseSeconds ?? "-") + " recentMaxExcess=" + (item.turnTimingRecentUpdateMaxExcessSeconds ?? 0) + " recentReset=" + (item.turnTimingRecentUpdateResetCount ?? 0) + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n")