diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 45311314..ce5f9aa9 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -7094,6 +7094,7 @@ function renderWebObserveStatusTable(value: Record): string { const manifest = record(observer?.manifest); const heartbeat = record(observer?.heartbeat); const tails = record(observer?.tails); + const result = record(value.result); const samples = Array.isArray(tails?.samples) ? tails.samples.map(record).filter((item): item is Record => item !== null).slice(-5) : []; const controls = Array.isArray(tails?.control) ? tails.control.map(record).filter((item): item is Record => item !== null).slice(-5) : []; const network = Array.isArray(tails?.network) @@ -7117,6 +7118,17 @@ function renderWebObserveStatusTable(value: Record): string { webObserveShort(`${webObserveText(manifest?.baseUrl)}${webObserveText(manifest?.targetPath) === "-" ? "" : webObserveText(manifest?.targetPath)}`, 52), ]]), "", + ...(value.ok === false ? [ + "Blocked detail:", + webObserveTable(["REASON", "EXIT", "TIMEOUT", "STDOUT", "STDERR"], [[ + webObserveShort(webObserveText(value.degradedReason), 48), + webObserveText(result.exitCode), + webObserveText(result.timedOut), + webObserveShort(webObserveText(result.stdoutTail), 120), + webObserveShort(webObserveText(result.stderr), 120), + ]]), + "", + ] : []), "Recent samples:", webObserveTable(["SEQ", "TS", "PATH", "ROUTE_SESSION", "ACTIVE_SESSION", "MSG", "TRACE"], samples.length > 0 ? samples.map((sample) => [ webObserveText(sample.seq), @@ -7338,6 +7350,7 @@ function withWebObserveAnalyzeRendered(value: Record): Rendered function renderWebObserveAnalyzeTable(value: Record): string { const analysis = record(value.analysis); + const result = record(value.result); const counts = record(analysis?.counts); const sampleMetrics = record(analysis?.sampleMetrics); const runtimeAlerts = record(analysis?.runtimeAlerts); @@ -7349,6 +7362,16 @@ function renderWebObserveAnalyzeTable(value: Record): string { const lines = [ `hwlab nodes web-probe observe analyze (${webObserveText(value.status)})`, "", + ...(value.ok === false ? [ + "Blocked detail:", + webObserveTable(["EXIT", "TIMEOUT", "STDOUT", "STDERR"], [[ + webObserveText(result.exitCode), + webObserveText(result.timedOut), + webObserveShort(webObserveText(result.stdoutTail), 120), + webObserveShort(webObserveText(result.stderr), 120), + ]]), + "", + ] : []), webObserveTable(["ID", "NODE", "LANE", "SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS"], [[ webObserveText(value.id), webObserveText(value.node), @@ -8031,17 +8054,18 @@ const fs=require('fs'),path=require('path'); const dir=process.env.state_dir||process.argv[1]; const node=process.argv[2]; const lane=process.argv[3]; -const tailN=${tailLines}; +const tailN=Math.min(${tailLines},3); const readJson=(name)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,name),'utf8'))}catch{return null}}; const tailJsonl=(name)=>{try{const file=path.join(dir,name); const st=fs.statSync(file); const maxBytes=Math.min(st.size,8*1024*1024); const fd=fs.openSync(file,'r'); try{const buf=Buffer.alloc(maxBytes); fs.readSync(fd,buf,0,maxBytes,st.size-maxBytes); const lines=buf.toString('utf8').split(/\\r?\\n/).filter(Boolean); if(st.size>maxBytes&&lines.length>0) lines.shift(); return lines.slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}});}finally{fs.closeSync(fd)}}catch{return []}}; const short=(value)=>String(value||'').slice(0,160); const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt}:null; -const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,source:item.source,input:item.input,afterUrl:item.afterUrl,detail:item.detail&&typeof item.detail==='object'?{reason:item.detail.reason||null,mark:item.detail.mark||null,stopping:item.detail.stopping===true,pageId:item.detail.pageId||null}:null}); -const compactSample=(item)=>({seq:item.seq,ts:item.ts,reason:item.reason,url:item.url,path:item.path,routeSessionId:item.routeSessionId||null,activeSessionId:item.activeSessionId||null,observerInitiated:item.observerInitiated===true,scroll:item.scroll||null,messageCount:Array.isArray(item.messages)?item.messages.length:0,traceRowCount:Array.isArray(item.traceRows)?item.traceRows.length:0,messages:Array.isArray(item.messages)?item.messages.slice(-3).map((row)=>({index:row.index,tag:row.tag,status:row.status||null,textHash:row.textHash||null,textPreview:short(row.textPreview),textBytes:row.textBytes||0})):[],traceRows:Array.isArray(item.traceRows)?item.traceRows.slice(-3).map((row)=>({index:row.index,tag:row.tag,status:row.status||null,textHash:row.textHash||null,textPreview:short(row.textPreview),textBytes:row.textBytes||0})):[],performanceCount:Array.isArray(item.performance)?item.performance.length:0}); -const compactNetwork=(item)=>({ts:item.ts,type:item.type,method:item.method,url:item.url,resourceType:item.resourceType,status:item.status||null,observerInitiated:item.observerInitiated===true,bodyRead:item.bodyRead===true,commandId:item.commandId||null,failure:item.failure||null}); +const compactHeartbeat=(item)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,updatedAt:item.updatedAt,uptimeMs:item.uptimeMs}:null; +const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId}); +const compactSample=(item)=>({seq:item.seq,ts:item.ts,path:item.path,routeSessionId:item.routeSessionId||null,activeSessionId:item.activeSessionId||null,messageCount:Array.isArray(item.messages)?item.messages.length:0,traceRowCount:Array.isArray(item.traceRows)?item.traceRows.length:0}); +const compactNetwork=(item)=>({ts:item.ts,type:item.type,method:item.method,url:short(item.url),status:item.status||null,failure:item.failure?short(item.failure):null}); const pidText=fs.existsSync(path.join(dir,'pid'))?fs.readFileSync(path.join(dir,'pid'),'utf8').trim():null; let alive=false; if(pidText){try{process.kill(Number(pidText),0); alive=true}catch{}} -console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir:dir,pid:pidText?Number(pidText):null,processAlive:alive,manifest:compactManifest(readJson('manifest.json')),heartbeat:readJson('heartbeat.json'),tails:{control:tailJsonl('control.jsonl').map(compactControl),samples:tailJsonl('samples.jsonl').map(compactSample),network:tailJsonl('network.jsonl').map(compactNetwork),errors:tailJsonl('errors.jsonl'),artifacts:tailJsonl('artifacts.jsonl')},next:{command:'bun scripts/cli.ts hwlab nodes web-probe observe command --node '+node+' --lane '+lane+' --state-dir '+dir+' --type mark --label checkpoint',stop:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+node+' --lane '+lane+' --state-dir '+dir,analyze:'bun scripts/cli.ts hwlab nodes web-probe observe analyze --node '+node+' --lane '+lane+' --state-dir '+dir},valuesRedacted:true},null,2)); +console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir:dir,pid:pidText?Number(pidText):null,processAlive:alive,manifest:compactManifest(readJson('manifest.json')),heartbeat:compactHeartbeat(readJson('heartbeat.json')),tails:{control:tailJsonl('control.jsonl').map(compactControl),samples:tailJsonl('samples.jsonl').map(compactSample),network:tailJsonl('network.jsonl').map(compactNetwork)},next:{command:'bun scripts/cli.ts hwlab nodes web-probe observe command --node '+node+' --lane '+lane+' --state-dir '+dir+' --type mark --label checkpoint',stop:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+node+' --lane '+lane+' --state-dir '+dir,analyze:'bun scripts/cli.ts hwlab nodes web-probe observe analyze --node '+node+' --lane '+lane+' --state-dir '+dir},valuesRedacted:true})); `)} "$state_dir" ${shellQuote(node)} ${shellQuote(lane)}`; } @@ -11214,6 +11238,7 @@ function compactCommandResult(result: CommandResult): Record { command: compactCommand(result.command), exitCode: result.exitCode, stdoutBytes: result.stdout.length, + stdoutTail: result.stdout.trim().slice(-2000), stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), timedOut: result.timedOut, };