From 2d699b7cec522f28b4554812c90c42bdce4d2383 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Tue, 23 Jun 2026 15:38:40 +0800 Subject: [PATCH] fix(web-probe): add collect grep drilldown (#738) Co-authored-by: Codex --- scripts/src/hwlab-node-impl.ts | 62 +++++++++++++++++-- .../hwlab-node-web-observe-runner-source.ts | 5 +- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 7415c5aa..96e06229 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -83,6 +83,7 @@ interface NodeWebProbeObserveOptions { maxFiles: number; collectFile: string | null; collectFinding: string | null; + collectGrep: string | null; analyzeArchivePrefix: string | null; full: boolean; stateDir: string | null; @@ -7268,6 +7269,7 @@ function parseNodeWebProbeObserveOptions( "--max-files", "--file", "--finding", + "--grep", "--archive-prefix", "--state-dir", "--job-id", @@ -7288,6 +7290,8 @@ function parseNodeWebProbeObserveOptions( if (collectFile !== null && !isSafeWebObserveCollectFile(collectFile)) throw new Error(`unsafe web-probe observe --file: ${collectFile}`); const collectFinding = optionValue(args, "--finding") ?? null; if (collectFinding !== null && !isSafeWebObserveFindingId(collectFinding)) throw new Error(`unsafe web-probe observe --finding: ${collectFinding}`); + const collectGrep = optionValue(args, "--grep") ?? null; + if (collectGrep !== null && (collectGrep.includes("\0") || collectGrep.length > 200)) throw new Error("unsafe web-probe observe --grep: expected 1-200 non-NUL chars"); const analyzeArchivePrefix = optionValue(args, "--archive-prefix") ?? null; if (analyzeArchivePrefix !== null && !isSafeWebObserveArchivePrefix(analyzeArchivePrefix)) throw new Error(`unsafe web-probe observe --archive-prefix: ${analyzeArchivePrefix}`); if (observeActionRaw !== "start" && stateDir === null && jobId === null) { @@ -7322,6 +7326,7 @@ function parseNodeWebProbeObserveOptions( maxFiles: positiveIntegerOption(args, "--max-files", 80, 5000), collectFile, collectFinding, + collectGrep, analyzeArchivePrefix, full: args.includes("--full"), stateDir, @@ -8193,7 +8198,7 @@ function runNodeWebProbeObserveCollect(options: NodeWebProbeObserveOptions, spec const script = [ "set -eu", nodeWebObserveResolveStateDirShell(options), - nodeWebObserveCollectNodeScript(options.maxFiles, options.collectFile, options.collectFinding), + nodeWebObserveCollectNodeScript(options.maxFiles, options.collectFile, options.collectFinding, options.collectGrep), ].join("\n"); const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); const collect = parseJsonObject(result.stdout); @@ -8206,6 +8211,7 @@ function runNodeWebProbeObserveCollect(options: NodeWebProbeObserveOptions, spec lane: options.lane, workspace: spec.workspace, requestedFile: options.collectFile, + requestedGrep: options.collectGrep, degradedReason: collect === null ? "collect-json-parse-failed" : null, collect, result: collect === null ? compactCommandResultWithStdoutTail(result) : compactCommandResult(result), @@ -8229,6 +8235,8 @@ function renderWebObserveCollectTable(value: Record): string { const files = Array.isArray(collect?.files) ? collect.files.map(record).filter((item): item is Record => item !== null).slice(0, 20) : []; const jsonlTail = Array.isArray(file.jsonlTail) ? file.jsonlTail.map(record).filter((item): item is Record => item !== null).slice(-8) : []; const jsonSummary = record(file.jsonSummary); + const grepSummary = record(file.grep); + const grepLines = Array.isArray(grepSummary?.lines) ? grepSummary.lines.map(record).filter((item): item is Record => item !== null).slice(0, 40) : []; const jsonContentPreview = file.jsonContent === undefined || file.jsonContent === null ? "" : JSON.stringify(file.jsonContent, null, 2); const jsonRunnerErrors = Array.isArray(jsonSummary.runnerErrors) ? jsonSummary.runnerErrors.map(record).filter((item): item is Record => item !== null).slice(-8) : []; const jsonFindings = Array.isArray(jsonSummary.findings) ? jsonSummary.findings.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; @@ -8286,6 +8294,27 @@ function renderWebObserveCollectTable(value: Record): string { "", ); } + if (grepSummary !== null) { + lines.push( + "Grep matches:", + webObserveTable(["PATTERN_SHA", "MATCHES", "RETURNED", "TRUNC"], [[ + webObserveShort(webObserveText(grepSummary.patternSha256), 24), + webObserveText(grepSummary.matchCount), + webObserveText(grepSummary.returnedLineCount), + webObserveText(grepSummary.truncated), + ]]), + webObserveTable(["LINE", "MATCH", "TEXT"], grepLines.length > 0 ? grepLines.map((item) => [ + webObserveText(item.lineNo), + webObserveText(item.match === true), + webObserveShort(webObserveText(item.text), 360), + ]) : [["-", "-", "-"]]), + "Grep context:", + grepLines.length > 0 + ? grepLines.map((item) => "- " + webObserveText(item.lineNo) + (item.match === true ? " * " : " ") + webObserveShort(webObserveText(item.text), 500)).join("\n") + : "-", + "", + ); + } if (jsonContentPreview.length > 0) { lines.push("JSON content:", webObserveShort(jsonContentPreview, 2400), ""); } @@ -9945,13 +9974,34 @@ console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir: `)} "$state_dir" ${shellQuote(node)} ${shellQuote(lane)}`; } -function nodeWebObserveCollectNodeScript(maxFiles: number, collectFile: string | null, collectFinding: string | null): string { +function nodeWebObserveCollectNodeScript(maxFiles: number, collectFile: string | null, collectFinding: string | null, collectGrep: string | null): string { return `node -e ${shellQuote(` const fs=require('fs'),path=require('path'),crypto=require('crypto'); -const dir=process.argv[1]; const selected=process.argv[2]||''; const findingFilter=process.argv[3]||''; const maxFiles=${maxFiles}; const maxReadBytes=64*1024; const maxJsonlTailBytes=1024*1024; const maxTextPreviewBytes=4096; +const dir=process.argv[1]; const selected=process.argv[2]||''; const findingFilter=process.argv[3]||''; const grepText=process.argv[4]||''; const maxFiles=${maxFiles}; const maxReadBytes=64*1024; const maxJsonlTailBytes=1024*1024; const maxTextPreviewBytes=4096; const shaFile=(file)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); +const shaText=(value)=>'sha256:'+crypto.createHash('sha256').update(String(value||'')).digest('hex'); const safeRel=(value)=>Boolean(value)&&!path.isAbsolute(value)&&!value.includes('..')&&!value.includes('\\\\')&&value.split('/').every((part)=>part&&part!=='.'&&part!=='..'); const short=(value,max=180)=>value===undefined||value===null?undefined:String(value).replace(/\\s+/g,' ').slice(0,max); +const grepLines=(text,pattern)=>{ + if(!pattern) return null; + const needle=String(pattern).toLowerCase(); + const lines=String(text||'').split(/\\r?\\n/); + const contexts=[]; let matchCount=0; const emitted=new Set(); + for(let index=0; index=lines.length) continue; + if(emitted.has(lineIndex)) continue; + emitted.add(lineIndex); + contexts.push({lineNo:lineIndex+1,match:offset===0,text:short(lines[lineIndex],260)}); + if(contexts.length>=80) break; + } + if(contexts.length>=80) break; + } + return {patternSha256:shaText(pattern),matchCount,returnedLineCount:contexts.length,truncated:matchCount>0&&contexts.length>=80,lines:contexts,valuesRedacted:true}; +}; const slimObject=(value,maxKeys=24)=>{ if(!value||typeof value!=='object'||Array.isArray(value)) return value??null; const out={}; @@ -10101,12 +10151,14 @@ if(selected){ const st=fs.statSync(file); const raw=fs.readFileSync(file); const truncated=raw.length>maxReadBytes; const isJsonl=selected.endsWith('.jsonl'); const isJson=selected.endsWith('.json'); const tailReadBytes=isJsonl?maxJsonlTailBytes:maxReadBytes; const text=(isJsonl?raw.slice(Math.max(0,raw.length-tailReadBytes)):raw.slice(0,Math.min(raw.length,maxReadBytes))).toString('utf8'); + const fullTextForGrep=grepText?raw.toString('utf8'):''; + const grep=grepText?grepLines(fullTextForGrep,grepText):null; const lines=text.split(/\\r?\\n/).filter(Boolean); if(isJsonl&&truncated&&lines.length>0) lines.shift(); let jsonSummary=null; let jsonContent=null; if(isJson){try{const parsed=JSON.parse(raw.toString('utf8')); jsonContent=raw.length<=4096?slimJsonValue(parsed):null; jsonSummary={topLevelKeys:parsed&&typeof parsed==='object'&&!Array.isArray(parsed)?Object.keys(parsed).slice(0,24):[],counts:slimObject(parsed&&parsed.counts,16),sampleMetrics:slimObject(parsed&&parsed.sampleMetrics,16),runtimeAlerts:slimObject(parsed&&parsed.runtimeAlerts,16),pagePerformance:slimObject(parsed&&parsed.pagePerformance,16),runnerErrors:Array.isArray(parsed&&parsed.runnerErrors)?parsed.runnerErrors.slice(-4).map(slimRunnerError).filter(Boolean):[],findings:Array.isArray(parsed&&parsed.findings)?parsed.findings.slice(0,6).map(slimFinding).filter(Boolean):[],findingDetail:slimFindingDetail(parsed,findingFilter)}}catch(error){jsonSummary={parseError:String(error&&error.message||error).slice(0,160)}}} const jsonlTail=isJsonl?lines.slice(-8).map((line,index)=>{try{return slimJsonl(JSON.parse(line))}catch(error){return {parseError:true,index,lineTail:line.slice(-240),error:String(error&&error.message||error).slice(0,160)}}}):null; - console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',stateDir:dir,mode:'file',file:{path:file,relative,byteCount:st.size,sha256:shaFile(file),truncated,content:isJsonl||isJson||jsonSummary?undefined:text.slice(0,maxTextPreviewBytes),contentTruncated:!isJsonl&&!isJson&&!jsonSummary&&text.length>maxTextPreviewBytes,jsonlTail,jsonSummary,jsonContent,lineCount:lines.length},valuesRedacted:true})); + console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',stateDir:dir,mode:'file',file:{path:file,relative,byteCount:st.size,sha256:shaFile(file),truncated,content:isJsonl||isJson||jsonSummary||grep?undefined:text.slice(0,maxTextPreviewBytes),contentTruncated:!isJsonl&&!isJson&&!jsonSummary&&!grep&&text.length>maxTextPreviewBytes,jsonlTail,jsonSummary,jsonContent,grep,lineCount:lines.length},valuesRedacted:true})); process.exit(0); } const out=[]; const walk=(p)=>{for(const ent of fs.readdirSync(p,{withFileTypes:true})){const full=path.join(p,ent.name); if(ent.isDirectory()) walk(full); else out.push(full); if(out.length>=maxFiles) return;}}; @@ -10115,7 +10167,7 @@ const selectedFiles=out.slice(0,maxFiles); const files=selectedFiles.slice(0,24).map(file=>{const st=fs.statSync(file); return {relative:path.relative(dir,file),byteCount:st.size,sha256:shaFile(file)}}); const totalBytes=selectedFiles.reduce((sum,file)=>sum+fs.statSync(file).size,0); console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',stateDir:dir,fileCount:selectedFiles.length,listedFileCount:files.length,omittedFileCount:Math.max(0,selectedFiles.length-files.length),totalBytes,files,valuesRedacted:true})); -`)} "$state_dir" ${shellQuote(collectFile ?? "")} ${shellQuote(collectFinding ?? "")}`; +`)} "$state_dir" ${shellQuote(collectFile ?? "")} ${shellQuote(collectFinding ?? "")} ${shellQuote(collectGrep ?? "")}`; } function nodeWebObserveWaitCommandShell(commandId: string, waitMs: number): string { diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 9543d4a3..f5f4bb86 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -4808,7 +4808,8 @@ function parseTraceRowTraceId(text) { } function traceRowGroupKey(row) { - return [row.pageRole || '', row.pageId || '', row.sessionId || '', row.traceId || 'unknown'].join('|'); + const identity = row.traceId ? 'trace:' + row.traceId : 'sample:' + (row.sampleIndex ?? '-') + ':' + (row.source || 'unknown'); + return [row.pageRole || '', row.pageId || '', row.sessionId || '', identity].join('|'); } function traceRowMatchesCard(row, card, terminalCardCount) { @@ -4856,7 +4857,7 @@ function maxSelfReportedDurationEvidence(text) { function parseSelfReportedRoundDurationSeconds(text) { const value = String(text || ''); - const clock = value.match(/(\d{1,2}:\d{2}:\d{2}|\d{1,2}:\d{2})/); + const clock = value.match(/(?:耗时|用时|duration|elapsed)[^0-9]{0,24}(\d{1,2}:\d{2}:\d{2}|\d{1,2}:\d{2})/i); if (clock) return parseTraceDurationSeconds(clock[1]); const hour = value.match(/(?:约|大约|around|about)?\s*(\d+(?:\.\d+)?)\s*(?:小时|hour|hours|hr|hrs)/i); const minute = value.match(/(?:约|大约|around|about)?\s*(\d+(?:\.\d+)?)\s*(?:分钟|分|min|mins|minute|minutes)/i);