From e0199e5515ab1e01691a7d3fcac9e07b2da40507 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 6 Jul 2026 23:48:33 +0000 Subject: [PATCH] fix: improve sentinel observer evidence --- ...de-web-observe-analyzer-findings-source.ts | 27 +++++- ...lab-node-web-observe-analyzer-io-source.ts | 7 ++ ...hwlab-node-web-sentinel-p5-observe.test.ts | 31 ++++++ .../src/hwlab-node-web-sentinel-p5-observe.ts | 95 +++++++++++++++++-- scripts/src/hwlab-node-web-sentinel-p5.ts | 14 ++- .../src/hwlab-node-web-sentinel-service.ts | 12 +++ 6 files changed, 171 insertions(+), 15 deletions(-) diff --git a/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts index 851bdba5..f0a71e78 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts @@ -13,6 +13,7 @@ export function nodeWebObserveAnalyzerFindingsSource(): string { summary: "observer control commands failed; analyze must surface the first failed command with phase/status/stdout/stderr evidence.", count: commandFailures.length, firstCommandFailure, + evidenceSummary: firstCommandFailure ? "command=" + (firstCommandFailure.commandId || "-") + " type=" + (firstCommandFailure.type || "-") + " phase=" + (firstCommandFailure.phase || "-") + " status=" + (firstCommandFailure.status || "-") + " resultOk=" + (firstCommandFailure.resultOk ?? "-") + " failedReason=" + (firstCommandFailure.failedReason || firstCommandFailure.failureKind || firstCommandFailure.message || "-") : "commandFailures>0", commands: compactCommandFailures.slice(0, 20) }); findings.push(...buildFrontendFreezeFindings(errors, control)); @@ -429,18 +430,36 @@ export function nodeWebObserveAnalyzerFindingsSource(): string { function compactObserverCommandFailure(item) { const result = item && typeof item.result === "object" ? item.result : {}; const output = item && typeof item.output === "object" ? item.output : {}; + const error = item && typeof item.error === "object" ? item.error : result && typeof result.error === "object" ? result.error : {}; const stdout = item?.stdoutPreview ?? item?.stdoutTail ?? result.stdoutPreview ?? result.stdoutTail ?? output.stdoutPreview ?? output.stdoutTail ?? null; const stderr = item?.stderrPreview ?? item?.stderrTail ?? result.stderrPreview ?? result.stderrTail ?? output.stderrPreview ?? output.stderrTail ?? null; + const commandId = item?.commandId ?? item?.id ?? item?.command_id ?? null; + const phase = item?.phase ?? item?.statusPhase ?? (item?.failedAt ? "failed" : item?.abandonedAt ? "abandoned" : item?.completedAt ? "completed" : null); + const resultOk = item?.resultOk ?? item?.ok ?? result.ok ?? null; + const status = item?.status ?? item?.resultStatus ?? result.status ?? (phase === "failed" || resultOk === false ? "failed" : phase === "abandoned" ? "abandoned" : phase === "completed" ? "completed" : null); + const failureKind = item?.failureKind ?? result.failureKind ?? error.failureKind ?? null; + const message = item?.message ?? result.message ?? error.message ?? item?.reason ?? result.reason ?? null; + const failedReason = item?.failedReason ?? item?.reason ?? result.failedReason ?? result.reason ?? failureKind ?? message ?? null; + const artifactBucket = item?.artifactBucket ?? (phase === "failed" ? "failed" : phase === "abandoned" ? "abandoned" : phase === "completed" ? "done" : null); + const artifactRef = item?.artifactRef ?? item?.artifactPath ?? item?.path ?? (commandId && artifactBucket ? "commands/" + artifactBucket + "/" + commandId + ".json" : null); return { - commandId: item?.commandId ?? item?.id ?? item?.command_id ?? null, + commandId, type: item?.type ?? item?.commandType ?? item?.command_type ?? null, - phase: item?.phase ?? item?.statusPhase ?? null, - status: item?.status ?? item?.resultStatus ?? null, + phase, + status, + resultOk: resultOk === true ? true : resultOk === false ? false : null, + failureKind: failureKind == null ? null : String(failureKind).slice(0, 160), + failedReason: failedReason == null ? null : String(failedReason).slice(0, 240), + message: message == null ? null : String(message).slice(0, 240), exitCode: item?.exitCode ?? result.exitCode ?? null, timedOut: item?.timedOut === true || result.timedOut === true, stdoutPreview: stdout == null ? null : String(stdout).slice(0, 240), stderrPreview: stderr == null ? null : String(stderr).slice(0, 240), - artifactRef: item?.artifactRef ?? item?.artifactPath ?? item?.path ?? null, + artifactBucket, + artifactRef, + failedAt: item?.failedAt ?? null, + completedAt: item?.completedAt ?? null, + abandonedAt: item?.abandonedAt ?? null, valuesRedacted: true }; } diff --git a/scripts/src/hwlab-node-web-observe-analyzer-io-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-io-source.ts index 4e05c0bc..c9531c66 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-io-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-io-source.ts @@ -838,6 +838,8 @@ function summarizeCommandFailures(control) { ts: item.ts ?? null, commandId: item.commandId ?? null, type: item.type ?? item.input?.type ?? null, + phase: "failed", + status: "failed", source: item.source ?? null, durationMs: detail.durationMs ?? null, beforePath: urlPath(detail.beforeUrl || item.beforeUrl), @@ -845,6 +847,11 @@ function summarizeCommandFailures(control) { name: error?.name ?? null, message: limitText(error?.message ?? detail?.message ?? "", 240), failureKind: error?.failureKind ?? detail?.failureKind ?? null, + failedReason: limitText(error?.message ?? detail?.message ?? error?.failureKind ?? detail?.failureKind ?? "observer-command-failed", 240), + resultOk: false, + artifactBucket: "failed", + artifactRef: item.commandId ? "commands/failed/" + item.commandId + ".json" : null, + failedAt: item.ts ?? null, failureSampleOk: detail?.failureSample?.ok === true, sampleSeq: detail?.failureSample?.sampleSeq ?? null, valuesRedacted: true diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts index cec190b4..79794782 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts @@ -43,6 +43,35 @@ test("quick verify cleanup downgrades abandoned graceful stop when force stop co assert.equal(findings[0]?.severity, "warning"); }); +test("quick verify cleanup reads force-stop evidence from top-level CLI payload", () => { + const step = { + phase: "observe-stop-after-terminal", + ok: false, + payload: { + ok: true, + command: "web-probe-observe force-stop", + forced: true, + reason: "graceful-stop-not-consumed", + aliveBefore: true, + aliveAfter: false, + termSent: true, + killSent: true, + pendingAbandoned: 1, + processingAbandoned: 0, + valuesRedacted: true, + }, + valuesRedacted: true, + }; + + const cleanup = classifyQuickVerifyCleanupStep(step); + const findings = quickVerifyCleanupFindings(step); + assert.equal(cleanup.cleanupClass, "bounded-force-stop"); + assert.equal(cleanup.forceStopOk, true); + assert.equal(cleanup.aliveAfter, false); + assert.deepEqual(cleanup.evidenceSources, ["payload"]); + assert.equal(findings[0]?.id, "observer-graceful-stop-timeout-force-stop-succeeded"); +}); + test("quick verify cleanup emits process-alive code when force stop leaves observer alive", () => { const step = forceStopStep({ aliveBefore: true, @@ -76,6 +105,8 @@ test("quick verify cleanup emits evidence-gap code when stopped state cannot be const findings = quickVerifyCleanupFindings(step); assert.equal(cleanup.cleanupClass, "observer-stop-failed"); assert.equal(cleanup.aliveAfter, null); + assert.deepEqual(cleanup.missingEvidenceFields, ["aliveAfter"]); assert.equal(findings.length, 1); assert.equal(findings[0]?.id, "observer-cleanup-evidence-missing"); + assert.match(String(findings[0]?.evidenceSummary), /missing=aliveAfter/u); }); diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts index 36553458..9911e9d6 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts @@ -683,19 +683,40 @@ export function classifyQuickVerifyCleanupStep(cleanupStep: Record 0; + const aliveBefore = firstBooleanFromCleanupSources(sources, "aliveBefore"); + const aliveAfter = firstBooleanFromCleanupSources(sources, "aliveAfter"); + const termSent = firstBooleanFromCleanupSources(sources, "termSent"); + const killSent = firstBooleanFromCleanupSources(sources, "killSent"); + const pendingAbandoned = firstNumberFromCleanupSources(sources, "pendingAbandoned"); + const processingAbandoned = firstNumberFromCleanupSources(sources, "processingAbandoned"); const forceStopOk = forceStopAttempted && aliveAfter === false; const gracefulStopOk = !forceStopAttempted && (cleanupStep.ok === true || payload.ok === true); const cleanupClass = gracefulStopOk @@ -703,6 +724,15 @@ export function classifyQuickVerifyCleanupStep(cleanupStep: Record value !== null) + : []; return { cleanupClass, phase: stringAtNullable(cleanupStep, "phase"), @@ -717,6 +747,8 @@ export function classifyQuickVerifyCleanupStep(cleanupStep: Record): Record return {}; } +function cleanupEvidenceSources(entries: readonly (readonly [string, Record])[]): readonly { name: string; value: Record }[] { + return entries + .filter((entry) => Object.keys(entry[1]).length > 0) + .map(([name, value]) => ({ name, value })); +} + +function cleanupEvidenceSourceNames(sources: readonly { name: string; value: Record }[], keys: readonly string[]): string[] { + const names: string[] = []; + for (const source of sources) { + if (keys.some((key) => source.value[key] !== undefined && source.value[key] !== null)) names.push(source.name); + } + return [...new Set(names)]; +} + +function firstStringFromCleanupSources(sources: readonly { value: Record }[], keys: readonly string[]): string | null { + for (const source of sources) { + for (const key of keys) { + const value = stringAtNullable(source.value, key); + if (value !== null) return value; + } + } + return null; +} + +function firstBooleanFromCleanupSources(sources: readonly { value: Record }[], key: string): boolean | null { + for (const source of sources) { + const value = booleanAtNullable(source.value, key); + if (value !== null) return value; + } + return null; +} + +function firstNumberFromCleanupSources(sources: readonly { value: Record }[], key: string): number | null { + for (const source of sources) { + const value = numberAtNullable(source.value, key); + if (value !== null) return value; + } + return null; +} + function quickVerifyCleanupFromSteps(steps: readonly Record[]): Record | null { for (const step of steps.slice().reverse()) { const phase = stringAtNullable(step, "phase"); @@ -765,6 +837,7 @@ function formatQuickVerifyCleanupLine(cleanup: Record | null): ["processingAbandoned", cleanup.processingAbandoned], ["gracefulStopOk", cleanup.gracefulStopOk], ["forceStopOk", cleanup.forceStopOk], + ["missing", Array.isArray(cleanup.missingEvidenceFields) && cleanup.missingEvidenceFields.length > 0 ? cleanup.missingEvidenceFields.join(",") : null], ]; return fields.map(([key, value]) => `${key}=${value ?? "-"}`).join(" "); } @@ -1263,6 +1336,8 @@ function compactQuickVerifyCleanupEvidence(value: unknown): Record typeof value === "string").slice(0, 6) : [], + missingEvidenceFields: Array.isArray(item.missingEvidenceFields) ? item.missingEvidenceFields.filter((value): value is string => typeof value === "string").slice(0, 8) : [], valuesRedacted: true, }; } diff --git a/scripts/src/hwlab-node-web-sentinel-p5.ts b/scripts/src/hwlab-node-web-sentinel-p5.ts index 6bb3c8fc..cb74de50 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5.ts @@ -456,6 +456,10 @@ function hasCommandFailureEvidence(item: Record): boolean { return stringAtNullable(first, "type") !== null || stringAtNullable(first, "phase") !== null || stringAtNullable(first, "status") !== null + || stringAtNullable(first, "failedReason") !== null + || stringAtNullable(first, "failureKind") !== null + || stringAtNullable(first, "message") !== null + || stringAtNullable(first, "artifactRef") !== null || stringAtNullable(first, "stdoutPreview") !== null || stringAtNullable(first, "stderrPreview") !== null || numberAtNullable(first, "exitCode") !== null; @@ -616,11 +620,19 @@ function compactSentinelReportCommandFailure(value: unknown): Recordbuf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;", "const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);", "const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};", - "const compactCommandFailure=(value)=>{const v=rec(value); if(Object.keys(v).length===0)return null; return {commandId:clip(v.commandId,80),type:clip(v.type,48),phase:clip(v.phase,80),status:clip(v.status,80),exitCode:Number.isFinite(Number(v.exitCode))?Number(v.exitCode):null,timedOut:v.timedOut===true?true:v.timedOut===false?false:null,stdoutPreview:clip(v.stdoutPreview,160),stderrPreview:clip(v.stderrPreview,160),artifactRef:clip(v.artifactRef,160),valuesRedacted:true};};", + "const compactCommandFailure=(value)=>{const v=rec(value); if(Object.keys(v).length===0)return null; return {commandId:clip(v.commandId,80),type:clip(v.type,48),phase:clip(v.phase,80),status:clip(v.status,80),resultOk:v.resultOk===true?true:v.resultOk===false?false:null,failureKind:clip(v.failureKind,160),failedReason:clip(v.failedReason,240),message:clip(v.message,240),exitCode:Number.isFinite(Number(v.exitCode))?Number(v.exitCode):null,timedOut:v.timedOut===true?true:v.timedOut===false?false:null,stdoutPreview:clip(v.stdoutPreview,160),stderrPreview:clip(v.stderrPreview,160),artifactBucket:clip(v.artifactBucket,48),artifactRef:clip(v.artifactRef,160),failedAt:clip(v.failedAt,48),completedAt:clip(v.completedAt,48),abandonedAt:clip(v.abandonedAt,48),valuesRedacted:true};};", "const sleep=(ms)=>{try{Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms)}catch{}};", "let jsonBuf=read(reportPath); let report=null; let reportParseError=null; const deadline=Date.now()+waitMs;", "while(true){ if(jsonBuf){try{report=JSON.parse(jsonBuf.toString('utf8')); break}catch(err){reportParseError=String(err&&err.message?err.message:err)}} if(Date.now()>=deadline)break; sleep(Math.min(500, Math.max(50, deadline-Date.now()))); jsonBuf=read(reportPath); }", diff --git a/scripts/src/hwlab-node-web-sentinel-service.ts b/scripts/src/hwlab-node-web-sentinel-service.ts index 1577e202..80aa8f64 100644 --- a/scripts/src/hwlab-node-web-sentinel-service.ts +++ b/scripts/src/hwlab-node-web-sentinel-service.ts @@ -1896,6 +1896,10 @@ function hasCommandFailureEvidence(item: Record): boolean { return stringOrNull(first.type) !== null || stringOrNull(first.phase) !== null || stringOrNull(first.status) !== null + || stringOrNull(first.failedReason) !== null + || stringOrNull(first.failureKind) !== null + || stringOrNull(first.message) !== null + || stringOrNull(first.artifactRef) !== null || stringOrNull(first.stdoutPreview) !== null || stringOrNull(first.stderrPreview) !== null || numberOrNull(first.exitCode) !== null; @@ -2061,11 +2065,19 @@ function compactCommandFailureEvidence(value: unknown): Record type: stringOrNull(item.type), phase: stringOrNull(item.phase), status: stringOrNull(item.status), + resultOk: item.resultOk === true ? true : item.resultOk === false ? false : null, + failureKind: stringOrNull(item.failureKind)?.slice(0, 160) ?? null, + failedReason: stringOrNull(item.failedReason)?.slice(0, 240) ?? null, + message: stringOrNull(item.message)?.slice(0, 240) ?? null, exitCode: numberOrNull(item.exitCode), timedOut: item.timedOut === true ? true : item.timedOut === false ? false : null, stdoutPreview: stringOrNull(item.stdoutPreview)?.slice(0, 240) ?? null, stderrPreview: stringOrNull(item.stderrPreview)?.slice(0, 240) ?? null, + artifactBucket: stringOrNull(item.artifactBucket), artifactRef: stringOrNull(item.artifactRef), + failedAt: stringOrNull(item.failedAt), + completedAt: stringOrNull(item.completedAt), + abandonedAt: stringOrNull(item.abandonedAt), valuesRedacted: true, }; }