diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts index 2a5b951d..e3cb57cf 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts @@ -35,6 +35,8 @@ import { } from "./hwlab-node-web-sentinel-cicd"; import { emitWebProbeSentinelSpan } from "./hwlab-node-web-sentinel-otel"; +const QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS = 55; + function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record = {}): void { const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => { if (typeof value === "string") return [key, short(value)]; @@ -312,7 +314,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result }); printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() }); const indexEntry = readLocalObserveIndex(observerId); - const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, remainingSeconds(deadline, 30)); + const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS); const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30)); const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30)); const controlFindings = quickVerifyControlFindings(null, promptIndex, turnSummary, traceFrame); @@ -510,7 +512,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: { const indexEntry = readLocalObserveIndex(input.observerId); const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId: input.observerId, valuesRedacted: true } - : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, 30); + : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS); const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30); const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30); const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(input.promptIndex, turnSummary, traceFrame); @@ -965,17 +967,21 @@ function boundQuickVerifyRecordText(value: unknown, maxChars: number): string | function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: string, timeoutSeconds: number): Record { if (!isSafeRelativeStateDir(stateDir)) return { ok: false, reason: "unsafe-state-dir", stateDir, valuesRedacted: true }; + const waitMs = Math.max(0, Math.min(50_000, (Math.max(5, timeoutSeconds) * 1000) - 5000)); const script = [ "set -eu", `state_dir=${shellQuote(stateDir)}`, - "node - \"$state_dir\" <<'NODE'", + `wait_ms=${waitMs}`, + "node - \"$state_dir\" \"$wait_ms\" <<'NODE'", "const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');", - "const stateDir=process.argv[2]; const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');", - "const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}}; const jsonBuf=read(reportPath);", + "const stateDir=process.argv[2]; const waitMs=Math.max(0, Math.min(60000, Number(process.argv[3]||0)||0)); const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');", + "const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}};", "const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;", "const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);", "const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};", - "let report=null; try{report=jsonBuf?JSON.parse(jsonBuf.toString('utf8')):null}catch{}", + "const sleep=(ms)=>{try{Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms)}catch{}};", + "let jsonBuf=read(reportPath); let report=null; let reportParseError=null; const deadline=Date.now()+waitMs;", + "while(true){ if(jsonBuf){try{report=JSON.parse(jsonBuf.toString('utf8')); break}catch(err){reportParseError=String(err&&err.message?err.message:err)}} if(Date.now()>=deadline)break; sleep(Math.min(500, Math.max(50, deadline-Date.now()))); jsonBuf=read(reportPath); }", "let artifactCount=0; let screenshot=null;", "function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}", "walk(stateDir);", @@ -985,7 +991,7 @@ function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: st "function pageMemory(page){const eff=rec(page.effectiveMemory); const heap=rec(page.heapUsage); const perf=rec(page.performance); const metrics=rec(perf.metrics); const heapUsed=num(eff.heapUsedMb)??mb(heap.usedSize); const jsHeap=num(eff.jsHeapUsedMb)??mb(metrics.JSHeapUsedSize); const memory=heapUsed??jsHeap; if(memory==null)return null; return {memoryMb:memory,heapUsedMb:heapUsed,jsHeapUsedMb:jsHeap,effectiveHeapUsedMb:num(eff.effectiveHeapUsedMb),effectiveJsHeapUsedMb:num(eff.effectiveJsHeapUsedMb),domNodes:num(eff.domNodes)??num(rec(page.domCounters).nodes)??num(metrics.Nodes)}; }", "function browserProcessPageSeries(){const p=path.join(stateDir,'browser-process.jsonl'); let lines=[]; try{lines=String(read(p)||'').split(/\\r?\\n/).filter(Boolean)}catch{} const map=new Map(); let firstTs=null; for(const line of lines){let row=null; try{row=JSON.parse(line)}catch{} if(!row||row.type!=='browser-process-sample')continue; const ts=row.ts||null; const tsMs=Date.parse(String(ts||'')); if(!ts||!Number.isFinite(tsMs))continue; for(const page of arr(row.pages)){const memory=pageMemory(rec(page)); if(!memory)continue; if(firstTs==null)firstTs=tsMs; const role=clip(page.pageRole||'page',32); const id=clip(page.pageId||`${role}-${page.pageEpoch??map.size}`,80); const key=`${role}:${id}`; const current=map.get(key)||{key,pageRole:role,pageId:id,label:`${role} ${String(id).slice(0,10)}`,url:clip(page.url,180),points:[],valuesRedacted:true}; current.points.push({ts,elapsedSeconds:Math.max(0,Math.round((tsMs-firstTs)/1000)),elapsedMinutes:Math.round(Math.max(0,(tsMs-firstTs)/60000)*100)/100,...memory,valuesRedacted:true}); map.set(key,current);} } return Array.from(map.values()).filter((item)=>item.points.length>0); }", "const browserPageSeries=browserProcessPageSeries();", - "console.log(JSON.stringify({ok:!!report,reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,browserProcess:{source:'analysis-browser-process-jsonl',unit:'MB',metric:'page-heap-used',pageCount:browserPageSeries.length,sampleCount:browserPageSeries.reduce((sum,item)=>sum+item.points.length,0),pageSeries:browserPageSeries,valuesRedacted:true},valuesRedacted:true}));", + "console.log(JSON.stringify({ok:!!report,reason:report?null:(jsonBuf?'report-json-parse-failed':'report-json-missing'),reportReadWaitMs:waitMs,reportParseError:clip(reportParseError,220),reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,browserProcess:{source:'analysis-browser-process-jsonl',unit:'MB',metric:'page-heap-used',pageCount:browserPageSeries.length,sampleCount:browserPageSeries.reduce((sum,item)=>sum+item.points.length,0),pageSeries:browserPageSeries,valuesRedacted:true},valuesRedacted:true}));", "NODE", ].join("\n"); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); diff --git a/scripts/src/hwlab-node-web-sentinel-p5.ts b/scripts/src/hwlab-node-web-sentinel-p5.ts index e7a1c69c..a838c139 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5.ts @@ -34,6 +34,8 @@ import { } from "./hwlab-node-web-sentinel-cicd"; import { metricNames, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe"; +const SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS = 55; + export function runSentinelMaintenance(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel maintenance ${options.action}`; const serviceHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); @@ -234,16 +236,16 @@ export function runSentinelReport(state: SentinelCicdState, options: Extract 0 ? body : report; if (options.full) return rendered(report.ok && body.ok !== false, command, JSON.stringify(rawPayload, null, 2)); if (options.raw) { - const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, 55)); + const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS)); return rendered(report.ok && body.ok !== false, command, JSON.stringify(compactSentinelReportRawPayload(state, body, report, artifactSummary), null, 2)); } if (options.view === "summary") { - const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, 55)); + const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS)); const payload = compactSentinelReportRawPayload(state, body, report, artifactSummary); return rendered(report.ok && body.ok !== false, command, renderSentinelReportSummary(payload, state)); } if (options.view === "findings") { - const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, 55)); + const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS)); const payload = compactSentinelReportRawPayload(state, body, report, artifactSummary); return rendered(report.ok && body.ok !== false, command, renderSentinelReportFindings(payload)); } @@ -306,6 +308,9 @@ function compactSentinelReportRawPayload( findings: visibleFindings.slice(0, 12).map(compactSentinelReportFinding), artifactSummary: Object.keys(artifact).length === 0 ? null : { ok: artifact.ok === true, + reason: artifact.reason ?? null, + reportReadWaitMs: artifact.reportReadWaitMs ?? null, + reportParseError: artifact.reportParseError ?? null, reportOk: artifact.reportOk === true ? true : artifact.reportOk === false ? false : null, reportJsonPath: artifact.reportJsonPath ?? null, reportJsonSha256: artifact.reportJsonSha256 ?? null, @@ -387,23 +392,27 @@ function readSentinelReportArtifactSummary(state: SentinelCicdState, body: Recor const run = record(body.run); const stateDir = stringAtNullable(run, "state_dir") ?? stringAtNullable(run, "stateDir"); if (stateDir === null || !isSafeSentinelReportStateDir(stateDir)) return null; + const waitMs = Math.max(0, Math.min(50_000, (Math.max(5, timeoutSeconds) * 1000) - 5000)); const script = [ "set -eu", `state_dir=${shellQuote(stateDir)}`, - "node - \"$state_dir\" <<'NODE'", + `wait_ms=${waitMs}`, + "node - \"$state_dir\" \"$wait_ms\" <<'NODE'", "const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');", - "const stateDir=process.argv[2]; const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');", - "const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}}; const jsonBuf=read(reportPath);", + "const stateDir=process.argv[2]; const waitMs=Math.max(0, Math.min(60000, Number(process.argv[3]||0)||0)); const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');", + "const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}};", "const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;", "const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);", "const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};", - "let report=null; try{report=jsonBuf?JSON.parse(jsonBuf.toString('utf8')):null}catch{}", + "const sleep=(ms)=>{try{Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms)}catch{}};", + "let jsonBuf=read(reportPath); let report=null; let reportParseError=null; const deadline=Date.now()+waitMs;", + "while(true){ if(jsonBuf){try{report=JSON.parse(jsonBuf.toString('utf8')); break}catch(err){reportParseError=String(err&&err.message?err.message:err)}} if(Date.now()>=deadline)break; sleep(Math.min(500, Math.max(50, deadline-Date.now()))); jsonBuf=read(reportPath); }", "let artifactCount=0; let screenshot=null;", "function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}", "walk(stateDir);", "const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,rootCauseSignals:compactRootCauseSignals(v.rootCauseSignals),blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});", "const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});", - "console.log(JSON.stringify({ok:!!report,reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));", + "console.log(JSON.stringify({ok:!!report,reason:report?null:(jsonBuf?'report-json-parse-failed':'report-json-missing'),reportReadWaitMs:waitMs,reportParseError:clip(reportParseError,220),reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));", "NODE", ].join("\n"); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.max(5, Math.min(timeoutSeconds, 55)) * 1000 });