mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(web-probe): classify steer prompt submissions (#831)

Browse Source 
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-24 16:42:55 +08:00
committed by GitHub
parent 2fd4ca2fee
commit b366bffe93
2 changed files with 42 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-impl.ts
+23 -7
View File
@@ -10277,7 +10277,7 @@ console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir:
function nodeWebObserveCollectNodeScript(maxFiles: number, collectFile: string | null, collectFinding: string | null, collectGrep: string | null): string {
return `node -e ${shellQuote(`
const fs=require('fs'),path=require('path'),crypto=require('crypto');
const dir=process.argv[1]; const findingFilter=process.argv[3]||''; const selected=(process.argv[2]||'') || (findingFilter ? 'analysis/report.json' : ''); const grepText=process.argv[4]||''; const maxFiles=${maxFiles}; const maxReadBytes=64*1024; const maxJsonlTailBytes=1024*1024; const maxTextPreviewBytes=4096; const maxGrepContextLines=24; const maxGrepLineText=180;
const dir=process.argv[1]; const findingFilter=process.argv[3]||''; let selected=(process.argv[2]||'') || (findingFilter ? 'analysis/report.json' : ''); const grepText=process.argv[4]||''; const maxFiles=${maxFiles}; const maxReadBytes=64*1024; const maxJsonlTailBytes=1024*1024; const maxTextPreviewBytes=4096; const maxGrepContextLines=24; const maxGrepLineText=180;
const shaFile=(file)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
const shaText=(value)=>'sha256:'+crypto.createHash('sha256').update(String(value||'')).digest('hex');
const safeRel=(value)=>Boolean(value)&&!path.isAbsolute(value)&&!value.includes('..')&&!value.includes('\\\\')&&value.split('/').every((part)=>part&&part!=='.'&&part!=='..');
@@ -10397,14 +10397,14 @@ const slimFindingSample=(value)=>{
allowedIncreaseSeconds:value.allowedIncreaseSeconds??null,
excessiveIncreaseSeconds:value.excessiveIncreaseSeconds??null,
promptIndex:value.promptIndex??null,
event:short(value.event??value.anomaly,48),
event:short(value.event??value.anomaly??(value.steerUsed===true?'steer':Array.isArray(value.submitModes)?value.submitModes.join(','):undefined),48),
expectedPattern:short(value.expectedPattern,96),
control:compactMetric(controlCount??(value.fromMessageCount!==undefined||value.fromTraceRowCount!==undefined?compactProjectionPair(value.fromMessageCount,value.fromTraceRowCount):undefined)),
observer:compactMetric(observerCount??(value.toMessageCount!==undefined||value.toTraceRowCount!==undefined?compactProjectionPair(value.toMessageCount,value.toTraceRowCount):undefined)),
sessionId:short(value.sessionId??value.routeSessionId??value.activeSessionId??value.controlSessionId??value.observerSessionId??refValue.routeSessionId??refValue.activeSessionId,64),
path:short(value.path??value.urlPath??value.url??value.controlPath??value.observerPath??refValue.url,120),
trace:short(trace,120),
detail:short(value.detail??(value.durationText!==undefined||value.activityText!==undefined||value.textPreview!==undefined?['duration='+String(value.durationText??'-'),'activity='+String(value.activityText??'-'),'status='+String(value.status??'-'),'preview='+String(value.textPreview??'-')].join(' '):undefined)??(value.fromDigest||value.toDigest?'digest '+String(value.fromDigest??'-')+' -> '+String(value.toDigest??'-'):undefined)??(value.messageTextDigestDiff?'messageDigest '+String(value.controlMessageDigest??'-')+' != '+String(value.observerMessageDigest??'-'):undefined)??value.summary??value.message??value.preview??value.expectedPattern,180),
detail:short(value.detail??(value.durationText!==undefined||value.activityText!==undefined||value.textPreview!==undefined?['duration='+String(value.durationText??'-'),'activity='+String(value.activityText??'-'),'status='+String(value.status??'-'),'preview='+String(value.textPreview??'-')].join(' '):undefined)??(Array.isArray(value.submitModes)||Array.isArray(value.responseStatuses)?['modes='+String(Array.isArray(value.submitModes)?value.submitModes.join(','):'-'),'statuses='+String(Array.isArray(value.responseStatuses)?value.responseStatuses.join(','):'-'),'failure='+String(value.failureKind??'-')].join(' '):undefined)??(value.fromDigest||value.toDigest?'digest '+String(value.fromDigest??'-')+' -> '+String(value.toDigest??'-'):undefined)??(value.messageTextDigestDiff?'messageDigest '+String(value.controlMessageDigest??'-')+' != '+String(value.observerMessageDigest??'-'):undefined)??value.summary??value.message??value.preview??value.expectedPattern,180),
valuesRedacted:true
};
};
@@ -10412,11 +10412,23 @@ const slimFindingDetail=(parsed,id)=>{
if(!id||!parsed||typeof parsed!=='object'||!Array.isArray(parsed.findings)) return null;
const finding=parsed.findings.find((item)=>item&&typeof item==='object'&&String(item.id??item.kind??item.code??'')===id);
if(!finding) return {id,found:false,samples:[],valuesRedacted:true};
const sampleSource=Array.isArray(finding.samples)?'samples':Array.isArray(finding.groups)?'groups':Array.isArray(finding.examples)?'examples':Array.isArray(finding.rows)?'rows':'none';
const samples=firstArray(finding.samples,finding.groups,finding.examples,finding.rows).slice(0,4).map(slimFindingSample).filter(Boolean);
const sampleSource=Array.isArray(finding.samples)?'samples':Array.isArray(finding.groups)?'groups':Array.isArray(finding.examples)?'examples':Array.isArray(finding.rows)?'rows':Array.isArray(finding.rounds)?'rounds':'none';
const samples=firstArray(finding.samples,finding.groups,finding.examples,finding.rows,finding.rounds).slice(0,4).map(slimFindingSample).filter(Boolean);
return {id:short(finding.id??finding.kind??finding.code,80),found:true,severity:short(finding.severity??finding.level,24),count:finding.count??finding.sampleCount??samples.length,summary:short(finding.summary??finding.message,180),sampleSource,samples,valuesRedacted:true};
};
const slimRunnerError=(item)=>item&&typeof item==='object'?{ts:short(item.ts||item.at,32),type:short(item.type,48),retry:item.retry||item.lastRetryLabel||null,retryExhausted:item.retryExhausted===true,attemptCount:item.attemptCount??(Array.isArray(item.attempts)?item.attempts.length:null),lastFailureKind:short(item.lastFailureKind||item.failureKind,80),lastReadinessReason:short(item.lastReadinessReason,80),message:short(item.message||item.lastError,240)}:null;
const collectFallbackCandidates=(requested)=>{
const candidates=[];
if(findingFilter&&requested!=='analysis/report.json') candidates.push('analysis/report.json');
if(requested==='analysis/report.json'||findingFilter||requested.startsWith('analysis/')){
try{
const analysisDir=path.join(dir,'analysis');
const names=fs.readdirSync(analysisDir).filter((name)=>name.endsWith('.json')).map((name)=>({name,mtime:fs.statSync(path.join(analysisDir,name)).mtimeMs})).sort((a,b)=>b.mtime-a.mtime);
for(const item of names) candidates.push('analysis/'+item.name);
}catch{}
}
return [...new Set(candidates)].filter((item)=>item&&item!==requested&&safeRel(item));
};
const slimJsonl=(value)=>{
if(!value||typeof value!=='object'||Array.isArray(value)) return {value:short(value)};
const readiness=value.readiness&&typeof value.readiness==='object'?value.readiness:null;
@@ -10449,9 +10461,13 @@ const slimJsonl=(value)=>{
};
if(selected){
if(!safeRel(selected)){console.log(JSON.stringify({ok:false,command:'web-probe-observe collect',stateDir:dir,mode:'file',reason:'unsafe-file',file:selected,valuesRedacted:true},null,2)); process.exit(2);}
const file=path.join(dir,selected); const relative=path.relative(dir,file);
let file=path.join(dir,selected); let relative=path.relative(dir,file);
if(relative.startsWith('..')||path.isAbsolute(relative)){console.log(JSON.stringify({ok:false,command:'web-probe-observe collect',stateDir:dir,mode:'file',reason:'file-outside-state-dir',file:selected,valuesRedacted:true},null,2)); process.exit(2);}
if(!fs.existsSync(file)||!fs.statSync(file).isFile()){console.log(JSON.stringify({ok:false,command:'web-probe-observe collect',stateDir:dir,mode:'file',reason:'file-not-found',file:selected,valuesRedacted:true},null,2)); process.exit(1);}
if(!fs.existsSync(file)||!fs.statSync(file).isFile()){
const fallback=collectFallbackCandidates(selected).find((item)=>{const candidate=path.join(dir,item); return fs.existsSync(candidate)&&fs.statSync(candidate).isFile();});
if(fallback){selected=fallback; file=path.join(dir,selected); relative=path.relative(dir,file);}
}
if(!fs.existsSync(file)||!fs.statSync(file).isFile()){console.log(JSON.stringify({ok:false,command:'web-probe-observe collect',stateDir:dir,mode:'file',reason:'file-not-found',file:selected,fallbackCandidates:collectFallbackCandidates(selected).slice(0,8),valuesRedacted:true},null,2)); process.exit(1);}
const st=fs.statSync(file); const raw=fs.readFileSync(file); const truncated=raw.length>maxReadBytes; const isJsonl=selected.endsWith('.jsonl'); const isJson=selected.endsWith('.json');
const tailReadBytes=isJsonl?maxJsonlTailBytes:maxReadBytes;
const text=(isJsonl?raw.slice(Math.max(0,raw.length-tailReadBytes)):raw.slice(0,Math.min(raw.length,maxReadBytes))).toString('utf8');
scripts/src/hwlab-node-web-observe-runner-source.ts
+19 -5
View File
@@ -2928,7 +2928,9 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN
const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0);
if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) });
const promptFailures = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.chatPostOk === false) : [];
if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) });
if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat or /v1/agent/chat/steer POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) });
const promptSteerRounds = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.steerUsed === true) : [];
if (promptSteerRounds.length > 0) findings.push({ id: "prompt-routed-to-steer", severity: "amber", summary: "sendPrompt was submitted through /v1/agent/chat/steer; verify the previous turn was truly in-flight and not an unsealed terminal failure", count: promptSteerRounds.length, rounds: promptSteerRounds.slice(0, 10) });
const elapsedZeroResets = Array.isArray(sampleMetrics?.turnTimingElapsedZeroResets) ? sampleMetrics.turnTimingElapsedZeroResets : [];
if (elapsedZeroResets.length > 0) findings.push({ id: "turn-timing-total-elapsed-zero-reset", severity: "red", summary: "Code Agent total elapsed jumped from a non-zero value back to 0 seconds", count: elapsedZeroResets.length, samples: elapsedZeroResets.slice(0, 20) });
const elapsedDecreases = Array.isArray(sampleMetrics?.turnTimingNonMonotonic)
@@ -3452,16 +3454,18 @@ function buildPromptNetworkReport(control, network) {
const prompts = Array.from(promptsById.values()).sort((a, b) => Date.parse(a.startedAt || a.completedAt || a.failedAt || "") - Date.parse(b.startedAt || b.completedAt || b.failedAt || ""));
prompts.forEach((item, index) => { item.promptIndex = index + 1; });
const chatEvents = network
.filter((item) => String(item?.method || "").toUpperCase() === "POST" && /\/v1\/agent\/chat(?:\?|$)/u.test(String(item?.url || "")))
.filter((item) => String(item?.method || "").toUpperCase() === "POST" && promptSubmitModeForUrl(item?.url) !== null)
.map((item) => {
const failureText = item.failureKind ?? item.failure ?? item.errorText ?? null;
const urlPathValue = urlPath(item.url);
return {
ts: item.ts ?? null,
tsMs: Date.parse(item.ts),
type: item.type ?? null,
status: Number.isFinite(Number(item.status)) ? Number(item.status) : null,
commandId: item.commandId ?? null,
urlPath: urlPath(item.url),
urlPath: urlPathValue,
submitMode: promptSubmitModeForUrl(item.url),
failureKind: failureText ? String(failureText) : null,
errorTextHash: failureText ? sha256(failureText) : null
};
@@ -3480,6 +3484,7 @@ function buildPromptNetworkReport(control, network) {
const responses = events.filter((event) => event.type === "response");
const failures = events.filter((event) => event.type === "requestfailed");
const responseStatuses = responses.map((event) => event.status).filter((status) => status !== null);
const submitModes = Array.from(new Set(events.map((event) => event.submitMode).filter(Boolean))).sort();
const chatPostOk = responseStatuses.some((status) => status >= 200 && status < 300) && failures.length === 0;
const failureKind = chatPostOk
? null
@@ -3502,9 +3507,11 @@ function buildPromptNetworkReport(control, network) {
responseCount: responses.length,
requestFailedCount: failures.length,
responseStatuses,
submitModes,
steerUsed: submitModes.includes("steer"),
firstChatEventAt: events[0]?.ts ?? null,
lastChatEventAt: events[events.length - 1]?.ts ?? null,
events: events.slice(0, 12).map((event) => ({ ts: event.ts, type: event.type, status: event.status, urlPath: event.urlPath, failureKind: event.failureKind, errorTextHash: event.errorTextHash }))
events: events.slice(0, 12).map((event) => ({ ts: event.ts, type: event.type, status: event.status, urlPath: event.urlPath, submitMode: event.submitMode, failureKind: event.failureKind, errorTextHash: event.errorTextHash }))
};
});
return {
@@ -3518,6 +3525,13 @@ function buildPromptNetworkReport(control, network) {
};
}
function promptSubmitModeForUrl(value) {
const pathValue = urlPath(value);
if (pathValue === "/v1/agent/chat") return "chat";
if (pathValue === "/v1/agent/chat/steer") return "steer";
return null;
}
function parseDomDiagnosticSummary(text) {
const value = String(text || "");
const traceMatch = value.match(/\b(?:trace_id=)?(trc_[A-Za-z0-9_-]+|[a-f0-9]{16,64})\b/iu);
@@ -6784,7 +6798,7 @@ function renderMarkdown(report) {
? report.runtimeAlerts.consoleAlertsByPath.slice(0, 40).map((item) => "- console " + (item.type || "-") + " status=" + (item.status ?? "-") + " path=" + (item.urlPath || "-") + " count=" + item.count + " prompts=" + (item.promptIndexes?.join(",") || "-") + " traces=" + (item.traceIds?.slice(0, 6).join(",") || "-")).join("\n")
: "- 无 console 分组。";
const promptNetworkLines = Array.isArray(report.promptNetwork?.rounds) && report.promptNetwork.rounds.length > 0
? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n")
? report.promptNetwork.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " chatPostOk=" + String(item.chatPostOk) + " modes=" + (Array.isArray(item.submitModes) && item.submitModes.length > 0 ? item.submitModes.join(",") : "-") + " failure=" + (item.failureKind || "-") + " statuses=" + (Array.isArray(item.responseStatuses) && item.responseStatuses.length > 0 ? item.responseStatuses.join(",") : "-") + " firstChat=" + (item.firstChatEventAt || "-") + " lastChat=" + (item.lastChatEventAt || "-")).join("\n")
: "- 无 prompt 网络记录。";
const roundLines = Array.isArray(report.sampleMetrics?.rounds) && report.sampleMetrics.rounds.length > 0
? report.sampleMetrics.rounds.map((item) => "- round " + item.promptIndex + " promptHash=" + (item.promptTextHash || "-") + " samples=" + item.sampleCount + " loadingSamples=" + (item.loadingSamples ?? 0) + " maxLoading=" + (item.maxLoadingCount ?? 0) + " loadingOwners=" + (item.loadingOwnerCount ?? 0) + " totalMax=" + (item.maxTotalElapsedSeconds ?? "-") + " totalLast=" + (item.lastTotalElapsedSeconds ?? "-") + " recentMax=" + (item.maxRecentUpdateSeconds ?? "-") + " recentLast=" + (item.lastRecentUpdateSeconds ?? "-") + " totalDecrease=" + (item.turnTimingTotalElapsedDecreaseCount ?? 0) + " totalForwardJump=" + (item.turnTimingTotalElapsedForwardJumpCount ?? 0) + " totalForwardJumpMax=" + (item.turnTimingTotalElapsedForwardJumpMaxSeconds ?? 0) + " terminalGrowth=" + (item.turnTimingTerminalElapsedGrowthCount ?? 0) + " terminalGrowthMax=" + (item.turnTimingTerminalElapsedGrowthMaxSeconds ?? 0) + " recentJump=" + (item.turnTimingRecentUpdateJumpCount ?? 0) + " recentSawtoothJump=" + (item.turnTimingRecentUpdateSawtoothJumpCount ?? item.turnTimingRecentUpdateJumpCount ?? 0) + " recentStep=" + (item.turnTimingRecentUpdateStepCount ?? 0) + " recentMaxIncrease=" + (item.turnTimingRecentUpdateMaxIncreaseSeconds ?? "-") + " recentMaxExcess=" + (item.turnTimingRecentUpdateMaxExcessSeconds ?? 0) + " recentReset=" + (item.turnTimingRecentUpdateResetCount ?? 0) + " diagnostics=" + item.diagnosticSamples + " terminal=" + item.terminalSamples + " finalText=" + item.finalTextSamples).join("\n")