mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(web-probe): add collect grep drilldown (#738)

Browse Source 
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-23 15:38:40 +08:00
committed by GitHub
parent 2b89905a73
commit 2d699b7cec
2 changed files with 60 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-impl.ts
+57 -5
View File
@@ -83,6 +83,7 @@ interface NodeWebProbeObserveOptions {
maxFiles: number;
collectFile: string | null;
collectFinding: string | null;
collectGrep: string | null;
analyzeArchivePrefix: string | null;
full: boolean;
stateDir: string | null;
@@ -7268,6 +7269,7 @@ function parseNodeWebProbeObserveOptions(
"--max-files",
"--file",
"--finding",
"--grep",
"--archive-prefix",
"--state-dir",
"--job-id",
@@ -7288,6 +7290,8 @@ function parseNodeWebProbeObserveOptions(
if (collectFile !== null && !isSafeWebObserveCollectFile(collectFile)) throw new Error(`unsafe web-probe observe --file: ${collectFile}`);
const collectFinding = optionValue(args, "--finding") ?? null;
if (collectFinding !== null && !isSafeWebObserveFindingId(collectFinding)) throw new Error(`unsafe web-probe observe --finding: ${collectFinding}`);
const collectGrep = optionValue(args, "--grep") ?? null;
if (collectGrep !== null && (collectGrep.includes("\0") || collectGrep.length > 200)) throw new Error("unsafe web-probe observe --grep: expected 1-200 non-NUL chars");
const analyzeArchivePrefix = optionValue(args, "--archive-prefix") ?? null;
if (analyzeArchivePrefix !== null && !isSafeWebObserveArchivePrefix(analyzeArchivePrefix)) throw new Error(`unsafe web-probe observe --archive-prefix: ${analyzeArchivePrefix}`);
if (observeActionRaw !== "start" && stateDir === null && jobId === null) {
@@ -7322,6 +7326,7 @@ function parseNodeWebProbeObserveOptions(
maxFiles: positiveIntegerOption(args, "--max-files", 80, 5000),
collectFile,
collectFinding,
collectGrep,
analyzeArchivePrefix,
full: args.includes("--full"),
stateDir,
@@ -8193,7 +8198,7 @@ function runNodeWebProbeObserveCollect(options: NodeWebProbeObserveOptions, spec
const script = [
"set -eu",
nodeWebObserveResolveStateDirShell(options),
nodeWebObserveCollectNodeScript(options.maxFiles, options.collectFile, options.collectFinding),
nodeWebObserveCollectNodeScript(options.maxFiles, options.collectFile, options.collectFinding, options.collectGrep),
].join("\n");
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const collect = parseJsonObject(result.stdout);
@@ -8206,6 +8211,7 @@ function runNodeWebProbeObserveCollect(options: NodeWebProbeObserveOptions, spec
lane: options.lane,
workspace: spec.workspace,
requestedFile: options.collectFile,
requestedGrep: options.collectGrep,
degradedReason: collect === null ? "collect-json-parse-failed" : null,
collect,
result: collect === null ? compactCommandResultWithStdoutTail(result) : compactCommandResult(result),
@@ -8229,6 +8235,8 @@ function renderWebObserveCollectTable(value: Record<string, unknown>): string {
const files = Array.isArray(collect?.files) ? collect.files.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 20) : [];
const jsonlTail = Array.isArray(file.jsonlTail) ? file.jsonlTail.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8) : [];
const jsonSummary = record(file.jsonSummary);
const grepSummary = record(file.grep);
const grepLines = Array.isArray(grepSummary?.lines) ? grepSummary.lines.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 40) : [];
const jsonContentPreview = file.jsonContent === undefined || file.jsonContent === null ? "" : JSON.stringify(file.jsonContent, null, 2);
const jsonRunnerErrors = Array.isArray(jsonSummary.runnerErrors) ? jsonSummary.runnerErrors.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8) : [];
const jsonFindings = Array.isArray(jsonSummary.findings) ? jsonSummary.findings.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
@@ -8286,6 +8294,27 @@ function renderWebObserveCollectTable(value: Record<string, unknown>): string {
"",
);
}
if (grepSummary !== null) {
lines.push(
"Grep matches:",
webObserveTable(["PATTERN_SHA", "MATCHES", "RETURNED", "TRUNC"], [[
webObserveShort(webObserveText(grepSummary.patternSha256), 24),
webObserveText(grepSummary.matchCount),
webObserveText(grepSummary.returnedLineCount),
webObserveText(grepSummary.truncated),
]]),
webObserveTable(["LINE", "MATCH", "TEXT"], grepLines.length > 0 ? grepLines.map((item) => [
webObserveText(item.lineNo),
webObserveText(item.match === true),
webObserveShort(webObserveText(item.text), 360),
]) : [["-", "-", "-"]]),
"Grep context:",
grepLines.length > 0
? grepLines.map((item) => "- " + webObserveText(item.lineNo) + (item.match === true ? " * " : " ") + webObserveShort(webObserveText(item.text), 500)).join("\n")
: "-",
"",
);
}
if (jsonContentPreview.length > 0) {
lines.push("JSON content:", webObserveShort(jsonContentPreview, 2400), "");
}
@@ -9945,13 +9974,34 @@ console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir:
`)} "$state_dir" ${shellQuote(node)} ${shellQuote(lane)}`;
}
function nodeWebObserveCollectNodeScript(maxFiles: number, collectFile: string | null, collectFinding: string | null): string {
function nodeWebObserveCollectNodeScript(maxFiles: number, collectFile: string | null, collectFinding: string | null, collectGrep: string | null): string {
return `node -e ${shellQuote(`
const fs=require('fs'),path=require('path'),crypto=require('crypto');
const dir=process.argv[1]; const selected=process.argv[2]||''; const findingFilter=process.argv[3]||''; const maxFiles=${maxFiles}; const maxReadBytes=64*1024; const maxJsonlTailBytes=1024*1024; const maxTextPreviewBytes=4096;
const dir=process.argv[1]; const selected=process.argv[2]||''; const findingFilter=process.argv[3]||''; const grepText=process.argv[4]||''; const maxFiles=${maxFiles}; const maxReadBytes=64*1024; const maxJsonlTailBytes=1024*1024; const maxTextPreviewBytes=4096;
const shaFile=(file)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
const shaText=(value)=>'sha256:'+crypto.createHash('sha256').update(String(value||'')).digest('hex');
const safeRel=(value)=>Boolean(value)&&!path.isAbsolute(value)&&!value.includes('..')&&!value.includes('\\\\')&&value.split('/').every((part)=>part&&part!=='.'&&part!=='..');
const short=(value,max=180)=>value===undefined||value===null?undefined:String(value).replace(/\\s+/g,' ').slice(0,max);
const grepLines=(text,pattern)=>{
if(!pattern) return null;
const needle=String(pattern).toLowerCase();
const lines=String(text||'').split(/\\r?\\n/);
const contexts=[]; let matchCount=0; const emitted=new Set();
for(let index=0; index<lines.length; index++){
if(!lines[index].toLowerCase().includes(needle)) continue;
matchCount++;
for(let offset=-2; offset<=2; offset++){
const lineIndex=index+offset;
if(lineIndex<0||lineIndex>=lines.length) continue;
if(emitted.has(lineIndex)) continue;
emitted.add(lineIndex);
contexts.push({lineNo:lineIndex+1,match:offset===0,text:short(lines[lineIndex],260)});
if(contexts.length>=80) break;
}
if(contexts.length>=80) break;
}
return {patternSha256:shaText(pattern),matchCount,returnedLineCount:contexts.length,truncated:matchCount>0&&contexts.length>=80,lines:contexts,valuesRedacted:true};
};
const slimObject=(value,maxKeys=24)=>{
if(!value||typeof value!=='object'||Array.isArray(value)) return value??null;
const out={};
@@ -10101,12 +10151,14 @@ if(selected){
const st=fs.statSync(file); const raw=fs.readFileSync(file); const truncated=raw.length>maxReadBytes; const isJsonl=selected.endsWith('.jsonl'); const isJson=selected.endsWith('.json');
const tailReadBytes=isJsonl?maxJsonlTailBytes:maxReadBytes;
const text=(isJsonl?raw.slice(Math.max(0,raw.length-tailReadBytes)):raw.slice(0,Math.min(raw.length,maxReadBytes))).toString('utf8');
const fullTextForGrep=grepText?raw.toString('utf8'):'';
const grep=grepText?grepLines(fullTextForGrep,grepText):null;
const lines=text.split(/\\r?\\n/).filter(Boolean);
if(isJsonl&&truncated&&lines.length>0) lines.shift();
let jsonSummary=null; let jsonContent=null;
if(isJson){try{const parsed=JSON.parse(raw.toString('utf8')); jsonContent=raw.length<=4096?slimJsonValue(parsed):null; jsonSummary={topLevelKeys:parsed&&typeof parsed==='object'&&!Array.isArray(parsed)?Object.keys(parsed).slice(0,24):[],counts:slimObject(parsed&&parsed.counts,16),sampleMetrics:slimObject(parsed&&parsed.sampleMetrics,16),runtimeAlerts:slimObject(parsed&&parsed.runtimeAlerts,16),pagePerformance:slimObject(parsed&&parsed.pagePerformance,16),runnerErrors:Array.isArray(parsed&&parsed.runnerErrors)?parsed.runnerErrors.slice(-4).map(slimRunnerError).filter(Boolean):[],findings:Array.isArray(parsed&&parsed.findings)?parsed.findings.slice(0,6).map(slimFinding).filter(Boolean):[],findingDetail:slimFindingDetail(parsed,findingFilter)}}catch(error){jsonSummary={parseError:String(error&&error.message||error).slice(0,160)}}}
const jsonlTail=isJsonl?lines.slice(-8).map((line,index)=>{try{return slimJsonl(JSON.parse(line))}catch(error){return {parseError:true,index,lineTail:line.slice(-240),error:String(error&&error.message||error).slice(0,160)}}}):null;
console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',stateDir:dir,mode:'file',file:{path:file,relative,byteCount:st.size,sha256:shaFile(file),truncated,content:isJsonl||isJson||jsonSummary?undefined:text.slice(0,maxTextPreviewBytes),contentTruncated:!isJsonl&&!isJson&&!jsonSummary&&text.length>maxTextPreviewBytes,jsonlTail,jsonSummary,jsonContent,lineCount:lines.length},valuesRedacted:true}));
console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',stateDir:dir,mode:'file',file:{path:file,relative,byteCount:st.size,sha256:shaFile(file),truncated,content:isJsonl||isJson||jsonSummary||grep?undefined:text.slice(0,maxTextPreviewBytes),contentTruncated:!isJsonl&&!isJson&&!jsonSummary&&!grep&&text.length>maxTextPreviewBytes,jsonlTail,jsonSummary,jsonContent,grep,lineCount:lines.length},valuesRedacted:true}));
process.exit(0);
}
const out=[]; const walk=(p)=>{for(const ent of fs.readdirSync(p,{withFileTypes:true})){const full=path.join(p,ent.name); if(ent.isDirectory()) walk(full); else out.push(full); if(out.length>=maxFiles) return;}};
@@ -10115,7 +10167,7 @@ const selectedFiles=out.slice(0,maxFiles);
const files=selectedFiles.slice(0,24).map(file=>{const st=fs.statSync(file); return {relative:path.relative(dir,file),byteCount:st.size,sha256:shaFile(file)}});
const totalBytes=selectedFiles.reduce((sum,file)=>sum+fs.statSync(file).size,0);
console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',stateDir:dir,fileCount:selectedFiles.length,listedFileCount:files.length,omittedFileCount:Math.max(0,selectedFiles.length-files.length),totalBytes,files,valuesRedacted:true}));
`)} "$state_dir" ${shellQuote(collectFile ?? "")} ${shellQuote(collectFinding ?? "")}`;
`)} "$state_dir" ${shellQuote(collectFile ?? "")} ${shellQuote(collectFinding ?? "")} ${shellQuote(collectGrep ?? "")}`;
}
function nodeWebObserveWaitCommandShell(commandId: string, waitMs: number): string {
scripts/src/hwlab-node-web-observe-runner-source.ts
+3 -2
View File
@@ -4808,7 +4808,8 @@ function parseTraceRowTraceId(text) {
}
function traceRowGroupKey(row) {
return [row.pageRole || '', row.pageId || '', row.sessionId || '', row.traceId || 'unknown'].join('|');
const identity = row.traceId ? 'trace:' + row.traceId : 'sample:' + (row.sampleIndex ?? '-') + ':' + (row.source || 'unknown');
return [row.pageRole || '', row.pageId || '', row.sessionId || '', identity].join('|');
}
function traceRowMatchesCard(row, card, terminalCardCount) {
@@ -4856,7 +4857,7 @@ function maxSelfReportedDurationEvidence(text) {
function parseSelfReportedRoundDurationSeconds(text) {
const value = String(text || '');
const clock = value.match(/(\d{1,2}:\d{2}:\d{2}|\d{1,2}:\d{2})/);
const clock = value.match(/(?:耗时|用时|duration|elapsed)[^0-9]{0,24}(\d{1,2}:\d{2}:\d{2}|\d{1,2}:\d{2})/i);
if (clock) return parseTraceDurationSeconds(clock[1]);
const hour = value.match(/(?:约|大约|around|about)?\s*(\d+(?:\.\d+)?)\s*(?:小时|hour|hours|hr|hrs)/i);
const minute = value.match(/(?:约|大约|around|about)?\s*(\d+(?:\.\d+)?)\s*(?:分钟|分|min|mins|minute|minutes)/i);