From 1b24e994f8454528347d662851c5243e95062345 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 2 Jul 2026 03:55:25 +0000 Subject: [PATCH] fix: keep observe collect readable during artifact churn --- scripts/src/hwlab-node-web-observe-collect.ts | 16 +++++++++------- scripts/src/hwlab-node-web-observe-render.ts | 13 +++++++++++++ scripts/src/hwlab-node/web-probe-observe.ts | 3 +++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/scripts/src/hwlab-node-web-observe-collect.ts b/scripts/src/hwlab-node-web-observe-collect.ts index 7d75a64a..a1ca4beb 100644 --- a/scripts/src/hwlab-node-web-observe-collect.ts +++ b/scripts/src/hwlab-node-web-observe-collect.ts @@ -30,12 +30,14 @@ const short=(value,limit=96)=>{const text=String(value||'').replace(/\\s+/gu,' ' const textOf=(value)=>String(value?.text||value?.textPreview||value?.preview||''); const readJson=(rel)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,rel),'utf8'))}catch{return null}}; const readJsonl=(rel)=>{try{return fs.readFileSync(path.join(dir,rel),'utf8').split(/\\r?\\n/u).filter(Boolean).map((line)=>{try{return JSON.parse(line)}catch{return {parseError:true,rawHash:sha(line)}}})}catch{return []}}; -const out=[]; const walk=(p)=>{for(const ent of fs.readdirSync(p,{withFileTypes:true})){const full=path.join(p,ent.name); if(ent.isDirectory()) walk(full); else out.push(full); if(out.length>=maxFiles) return;}}; +const out=[]; const skippedFiles=[]; +const skipFile=(file,reason,error=null)=>skippedFiles.push({path:file,relative:path.relative(dir,file),reason,message:error?.message?short(error.message,160):null,valuesRedacted:true}); +const walk=(p)=>{let entries=[]; try{entries=fs.readdirSync(p,{withFileTypes:true});}catch(error){skipFile(p,'readdir-failed',error); return;} for(const ent of entries){const full=path.join(p,ent.name); if(ent.isDirectory()) walk(full); else out.push(full); if(out.length>=maxFiles) return;}}; walk(dir); -const files=out.slice(0,maxFiles).map(file=>{const st=fs.statSync(file); const hash=crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); return {path:file,relative:path.relative(dir,file),byteCount:st.size,sha256:'sha256:'+hash}}); +const files=out.slice(0,maxFiles).map(file=>{try{const st=fs.statSync(file); if(!st.isFile())return null; const bytes=fs.readFileSync(file); const hash=crypto.createHash('sha256').update(bytes).digest('hex'); return {path:file,relative:path.relative(dir,file),byteCount:st.size,sha256:'sha256:'+hash}}catch(error){skipFile(file,'stat-or-read-failed',error); return null;}}).filter(Boolean); const totalBytes=files.reduce((sum,item)=>sum+item.byteCount,0); if(view==='files'){ - console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,fileCount:files.length,totalBytes,files,valuesRedacted:true},null,2)); + console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,fileCount:files.length,skippedFileCount:skippedFiles.length,totalBytes,files,skippedFiles:skippedFiles.slice(0,20),valuesRedacted:true},null,2)); process.exit(0); } const samples=readJsonl('samples.jsonl'); @@ -629,19 +631,19 @@ if(view==='project-summary'||view==='project-mdtodo-summary'){ const projectMutations=project.mutations.slice(-10).map(compactCommand); const projectLaunches=project.launches.slice(-4).map((item)=>({ts:short(item.ts,24),phase:item.phase??null,commandId:short(item.commandId,24),status:item.status??null,sessionId:short(item.sessionId,28),workbenchUrl:short(item.workbenchUrl,52),otelTraceId:short(item.otelTraceId,28),chatObserved:item.chatObserved??null,chatStatus:item.chatStatus??null,chatTraceId:short(item.chatTraceId,28),workbenchMessageCount:item.workbenchMessageCount??null,workbenchTraceRowCount:item.workbenchTraceRowCount??null,taskHash:short(item.taskHash,24),message:item.message?short(item.message,80):null,valuesRedacted:true})); const projectFindings=project.findings.slice(0,4).map((item)=>({severity:item.severity??item.level??null,id:short(item.id??item.kind??item.code,48),count:item.count??item.sampleCount??null,summary:short(item.summary??item.message,96),valuesRedacted:true})); - console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,summary:projectSummary,sampleRowCount:project.sampleRows.length,commandCount:project.commands.length,mutationCount:project.mutations.length,launchCount:project.launches.length,findingCount:project.findings.length,sampleRows:projectSampleRows,commands:projectCommands,mutations:projectMutations,launches:projectLaunches,findings:projectFindings,sourceFiles:['samples.jsonl','control.jsonl','analysis/report.json'],valuesRedacted:true})); + console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,summary:projectSummary,sampleRowCount:project.sampleRows.length,commandCount:project.commands.length,mutationCount:project.mutations.length,launchCount:project.launches.length,findingCount:project.findings.length,artifactFileCount:files.length,skippedFileCount:skippedFiles.length,skippedFiles:skippedFiles.slice(0,20),sampleRows:projectSampleRows,commands:projectCommands,mutations:projectMutations,launches:projectLaunches,findings:projectFindings,sourceFiles:['samples.jsonl','control.jsonl','analysis/report.json'],valuesRedacted:true})); process.exit(0); } if(view==='turn-summary'){ - console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,turnCount:rows.length,rows:rows.slice(0,80),renderedText:renderTurnSummary(rows),sourceFiles:['samples.jsonl','control.jsonl','analysis/report.json'],valuesRedacted:true},null,2)); + console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,turnCount:rows.length,artifactFileCount:files.length,skippedFileCount:skippedFiles.length,skippedFiles:skippedFiles.slice(0,20),rows:rows.slice(0,80),renderedText:renderTurnSummary(rows),sourceFiles:['samples.jsonl','control.jsonl','analysis/report.json'],valuesRedacted:true},null,2)); process.exit(0); } if(view==='timeline'){ const timeline=timelineView(rows); - console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,...timeline,sourceFiles:['samples.jsonl','control.jsonl','commands/pending/*.json','commands/processing/*.json','commands/done/*.json','commands/failed/*.json','commands/abandoned/*.json'],valuesRedacted:true})); + console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,artifactFileCount:files.length,skippedFileCount:skippedFiles.length,skippedFiles:skippedFiles.slice(0,20),...timeline,sourceFiles:['samples.jsonl','control.jsonl','commands/pending/*.json','commands/processing/*.json','commands/done/*.json','commands/failed/*.json','commands/abandoned/*.json'],valuesRedacted:true})); process.exit(0); } const frame=renderTraceFrame(selectSample(rows),rows); -console.log(JSON.stringify({ok:frame.ok!==false,command:'web-probe-observe collect',view,stateDir:dir,...frame,sourceFiles:['samples.jsonl','control.jsonl','analysis/report.json'],valuesRedacted:true},null,2)); +console.log(JSON.stringify({ok:frame.ok!==false,command:'web-probe-observe collect',view,stateDir:dir,artifactFileCount:files.length,skippedFileCount:skippedFiles.length,skippedFiles:skippedFiles.slice(0,20),...frame,sourceFiles:['samples.jsonl','control.jsonl','analysis/report.json'],valuesRedacted:true},null,2)); `)} "$state_dir"`; } diff --git a/scripts/src/hwlab-node-web-observe-render.ts b/scripts/src/hwlab-node-web-observe-render.ts index d0a77659..d47f9ab5 100644 --- a/scripts/src/hwlab-node-web-observe-render.ts +++ b/scripts/src/hwlab-node-web-observe-render.ts @@ -251,6 +251,7 @@ function renderWebObserveCollectTable(value: Record): string { const result = record(value.result); const file = record(collect?.file); const files = Array.isArray(collect?.files) ? collect.files.map(record).filter((item): item is Record => item !== null).slice(0, 20) : []; + const skippedFiles = Array.isArray(collect?.skippedFiles) ? collect.skippedFiles.map(record).filter((item): item is Record => item !== null).slice(0, 12) : []; const jsonlTail = Array.isArray(file.jsonlTail) ? file.jsonlTail.map(record).filter((item): item is Record => item !== null).slice(-8) : []; const jsonSummary = record(file.jsonSummary); const grepSummary = record(file.grep); @@ -293,6 +294,18 @@ function renderWebObserveCollectTable(value: Record): string { return lines.join("\n"); } + if (skippedFiles.length > 0) { + lines.push( + "Skipped files:", + webObserveTable(["RELATIVE", "REASON", "MESSAGE"], skippedFiles.map((item) => [ + webObserveShort(webObserveText(item.relative ?? item.path), 72), + webObserveShort(webObserveText(item.reason), 28), + webObserveShort(webObserveText(item.message), 96), + ])), + "", + ); + } + if (collect.mode === "file") { lines.push( "File:", diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index 323d1f67..1b1d6534 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -2102,6 +2102,9 @@ function compactObserveCollectForRaw(collect: Record | null): R view: collect.view, stateDir: collect.stateDir, turnCount: collect.turnCount, + artifactFileCount: collect.artifactFileCount ?? collect.fileCount, + skippedFileCount: collect.skippedFileCount, + skippedFiles: Array.isArray(collect.skippedFiles) ? collect.skippedFiles.slice(0, 8) : undefined, anchor: observeRecord(collect.anchor), window: observeRecord(collect.window), counts: observeRecord(collect.counts),