diff --git a/config/hwlab-web-probe-sentinel/check-catalog.yaml b/config/hwlab-web-probe-sentinel/check-catalog.yaml index 9962a219..75a82ef4 100644 --- a/config/hwlab-web-probe-sentinel/check-catalog.yaml +++ b/config/hwlab-web-probe-sentinel/check-catalog.yaml @@ -939,3 +939,67 @@ sentinel: actionZh: 先修复目标 node/lane 的 Web 访问或 Workbench 壳加载，再重新启动观察。 blocking: true order: 1150 + - code: WBC-116 + id: account-session-command-failed-before-observation + level: error + titleZh: 账号/会话准备命令失败 + summaryZh: 快速验证的账号、会话或 Workbench 准备命令在业务轮次观察前失败。 + actionZh: 查看首个失败命令的 type、phase、status、stdout/stderr 摘要和 artifact 引用。 + blocking: true + order: 1160 + - code: WBC-117 + id: observer-graceful-stop-timeout-force-stop-succeeded + level: warning + titleZh: 观察优雅停止超时但强停成功 + summaryZh: 观察程序优雅停止未及时完成，但强制停止已确认进程不再存活。 + actionZh: 作为清理耗时和遗留命令上下文记录；只有强停后仍存活才按阻塞处理。 + blocking: false + order: 1170 + - code: WBC-118 + id: observer-force-stop-left-process-alive + level: error + titleZh: 观察强停后进程仍存活 + summaryZh: 观察程序强制停止后仍有存活证据，可能留下浏览器或 runner 进程。 + actionZh: 检查 observe stop/force-stop 输出和目标节点进程树，清理后再恢复周期巡检。 + blocking: true + order: 1180 + - code: WBC-119 + id: observer-cleanup-evidence-missing + level: error + titleZh: 观察清理证据缺失 + summaryZh: 观察停止状态缺少 aliveAfter、term/kill 或命令结果证据，不能判定是否真正停止。 + actionZh: 先修复 observe stop 证据采集或 report 索引回填，再判断是否有进程泄漏。 + blocking: true + order: 1190 + - code: WBC-120 + id: observer-cleanup-index-mismatch + level: error + titleZh: 观察清理索引不一致 + summaryZh: 原始 cleanup payload 与 report/indexed finding 对同一次停止结果的分类不一致。 + actionZh: 对齐 cleanup payload、recordRunResult 和 report visible findings 的同一证据来源。 + blocking: true + order: 1200 + - code: WBC-121 + id: no-samples-after-observer-command-failed + level: error + titleZh: 命令失败后无采样 + summaryZh: 本次观察没有样本，且已有阻塞性的页面控制命令失败作为确定前置条件。 + actionZh: 优先处理 WBC-008 的首个失败命令，再重新启动采样。 + blocking: true + order: 1210 + - code: WBC-122 + id: no-samples-artifact-evidence-missing + level: error + titleZh: 无采样且证据不足 + summaryZh: 本次观察没有样本，也没有可归因的命令失败，属于采样或 artifact 证据缺口。 + actionZh: 检查 observer start、samples.jsonl、analysis/report.json 和 report fallback 读取路径。 + blocking: true + order: 1220 + - code: WBC-123 + id: observer-command-failure-evidence-missing + level: error + titleZh: 页面操作失败证据缺失 + summaryZh: 报告称页面控制命令失败，但缺少首个失败命令的 type、phase、status 或输出摘要。 + actionZh: 修复 analyzer/report 对 commandFailures 的 compact 输出，先补齐命令证据再判断页面业务状态。 + blocking: true + order: 1230 diff --git a/docs/reference/observability.md b/docs/reference/observability.md index 94839e6d..a29dfe72 100644 --- a/docs/reference/observability.md +++ b/docs/reference/observability.md @@ -37,7 +37,7 @@ Web 哨兵 dashboard/API 展示问题的第一事实源是 sentinel runner 的 ` Web 哨兵 findings 可见性要同时核对 runner API 和已有 observe artifact。若某个 run 的公开 `/api/report?view=findings&run=` 只显示 WBC-003，但 `web-probe sentinel report --run --view findings --raw` 能从 `analysis/report.json` 读出 red/amber analyzer findings，根因是索引或 artifact 可见性遮盖，不是业务没有产生 warning/error。此时应回填或重建这条既有 run 的 report index，并保留原有 report views；不要通过启动新的哨兵 run 来解释旧记录。 -Web 哨兵 check code 是排障和验收合同，必须保持语义单一且确定。一个 code/id 只能对应一种可执行处置路径；如果一个 finding 可能同时表示“没有采集到业务轮次”“目标轮次缺 traceId”“trace rows/projection 缺失”“Final Response 为空但仍在运行/取消”“Final Response 为空且已失败/终止”等多种状态，就必须拆成多个固定 code/id。不得用同一个 code 的动态标题、动态 summary 或 rootCause 文案来承载多种互斥语义；动态字段只能补充证据，不能改变 code 的语义。 +Web 哨兵 check code 是排障和验收合同，必须保持语义单一且确定。一个 code/id 只能对应一种可执行处置路径；如果一个 finding 可能同时表示“没有采集到业务轮次”“目标轮次缺 traceId”“trace rows/projection 缺失”“Final Response 为空但仍在运行/取消”“Final Response 为空且已失败/终止”等多种状态，就必须拆成多个固定 code/id。不得用同一个 code 的动态标题、动态 summary 或 rootCause 文案来承载多种互斥语义；动态字段只能补充证据，不能改变 code 的语义。`quick-verify` 只能作为运行阶段或证据上下文，不能作为 root-cause code 的宽泛语义；账号/会话命令失败、observer start/stop/cleanup、turn-summary、trace-frame、report analyze、无采样和 artifact/index 缺口必须各自用固定 code 表示。 Web 哨兵请求频率曲线的验收事实源是 runner `/api/runs/{id}.requestRate`、已有 observe artifact 中的 request-rate summary，以及 `web-probe sentinel dashboard screenshot` 的远程浏览器证据。阈值、采样间隔、bucket 大小和红黄线只从 YAML/source-of-truth 读取，长期文档只记录字段族与验证入口。验收时应核对 `bucketSeconds`、总请求曲线、页面曲线、API path 曲线、峰值每分钟计数、数据来源和 chart/DOM 是否显示在内存曲线上方并共享时间轴。若 quick-verify 的业务链路失败，但同一 run 的 `requestRate` API 和截图已经有曲线数据，应把请求频率能力验收与业务阻塞分开记录；反之，`requestRate.source=unavailable` 或曲线为空时要继续检查 analyzer compact 输出、artifact summary、索引回填和 report fallback。除非 `dashboard verify` 已显式输出 request-rate 专用字段，不得把旧的 `API_PAGES` / `API_SAMPLES` 列当作请求频率曲线验收结果。 diff --git a/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts index 35552e4a..851bdba5 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts @@ -5,7 +5,16 @@ export function nodeWebObserveAnalyzerFindingsSource(): string { return String.raw`function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, requestRate, pageProvenance, commandFailures = [], manifest = {}, apiDomLag = null, browserProcess = null, frontendPerformance = null) { const findings = []; const effectiveApiDomLag = apiDomLag || buildApiDomLagReport(samples, network); - if (commandFailures.length > 0) findings.push({ id: "observer-command-failed", severity: "red", summary: "observer control commands failed; analyze must surface command failure instead of hiding it in command artifacts", count: commandFailures.length, commands: commandFailures.slice(0, 20) }); + const compactCommandFailures = commandFailures.map(compactObserverCommandFailure); + const firstCommandFailure = compactCommandFailures[0] || null; + if (commandFailures.length > 0) findings.push({ + id: "observer-command-failed", + severity: "red", + summary: "observer control commands failed; analyze must surface the first failed command with phase/status/stdout/stderr evidence.", + count: commandFailures.length, + firstCommandFailure, + commands: compactCommandFailures.slice(0, 20) + }); findings.push(...buildFrontendFreezeFindings(errors, control)); findings.push(...buildBrowserProcessFindings(browserProcess, runtimeAlerts)); findings.push(...buildFrontendPerformanceFindings(frontendPerformance)); @@ -386,10 +395,56 @@ export function nodeWebObserveAnalyzerFindingsSource(): string { groups: Array.isArray(effectiveApiDomLag?.groups) ? effectiveApiDomLag.groups.slice(0, 12) : [], }); if (errors.length > 0) findings.push({ id: "browser-console-or-page-errors", severity: "amber", summary: "pageerror/runner errors were captured", count: errors.length, first: errors.slice(0, 5) }); - if (samples.length === 0) findings.push({ id: "no-samples", severity: "red", summary: "observer produced no samples" }); + if (samples.length === 0) { + if (commandFailures.length > 0) { + findings.push({ + id: "no-samples-after-observer-command-failed", + severity: "red", + summary: "observer produced zero samples after a blocking control command failed; the command failure is the deterministic precondition.", + count: 1, + firstCommandFailure, + rootCause: "observer-command-failed-before-sampling", + rootCauseStatus: "confirmed", + rootCauseConfidence: "high", + evidenceSummary: firstCommandFailure ? "command=" + (firstCommandFailure.commandId || "-") + " type=" + (firstCommandFailure.type || "-") + " phase=" + (firstCommandFailure.phase || "-") + " status=" + (firstCommandFailure.status || "-") + " exit=" + (firstCommandFailure.exitCode ?? "-") : "commandFailures>0", + valuesRedacted: true + }); + } else { + findings.push({ + id: "no-samples-artifact-evidence-missing", + severity: "red", + summary: "observer produced zero samples and analyzer has no failed command precondition; sampling/source evidence is missing.", + count: 1, + rootCause: "sample-artifact-evidence-missing", + rootCauseStatus: "evidence-gap", + rootCauseConfidence: "medium", + evidenceSummary: "samples=0 control=" + (Array.isArray(control) ? control.length : 0) + " errors=" + (Array.isArray(errors) ? errors.length : 0), + valuesRedacted: true + }); + } + } return findings; } +function compactObserverCommandFailure(item) { + const result = item && typeof item.result === "object" ? item.result : {}; + const output = item && typeof item.output === "object" ? item.output : {}; + const stdout = item?.stdoutPreview ?? item?.stdoutTail ?? result.stdoutPreview ?? result.stdoutTail ?? output.stdoutPreview ?? output.stdoutTail ?? null; + const stderr = item?.stderrPreview ?? item?.stderrTail ?? result.stderrPreview ?? result.stderrTail ?? output.stderrPreview ?? output.stderrTail ?? null; + return { + commandId: item?.commandId ?? item?.id ?? item?.command_id ?? null, + type: item?.type ?? item?.commandType ?? item?.command_type ?? null, + phase: item?.phase ?? item?.statusPhase ?? null, + status: item?.status ?? item?.resultStatus ?? null, + exitCode: item?.exitCode ?? result.exitCode ?? null, + timedOut: item?.timedOut === true || result.timedOut === true, + stdoutPreview: stdout == null ? null : String(stdout).slice(0, 240), + stderrPreview: stderr == null ? null : String(stderr).slice(0, 240), + artifactRef: item?.artifactRef ?? item?.artifactPath ?? item?.path ?? null, + valuesRedacted: true + }; +} + function buildCrossPageProjectionDrilldown(rows) { const sourceRows = Array.isArray(rows) ? rows.filter(Boolean) : []; const traceIds = uniqueSorted(sourceRows.flatMap((row) => collectIdsFromUnknown(row, "trace")).filter(Boolean)).slice(0, 12); diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts index 312458c9..cec190b4 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts @@ -37,10 +37,13 @@ test("quick verify cleanup downgrades abandoned graceful stop when force stop co assert.equal(cleanup.forceStopOk, true); assert.equal(cleanup.aliveAfter, false); assert.equal(cleanup.pendingAbandoned, 1); - assert.deepEqual(quickVerifyCleanupFindings(step), []); + const findings = quickVerifyCleanupFindings(step); + assert.equal(findings.length, 1); + assert.equal(findings[0]?.id, "observer-graceful-stop-timeout-force-stop-succeeded"); + assert.equal(findings[0]?.severity, "warning"); }); -test("quick verify cleanup emits WBC-096 when force stop leaves observer alive", () => { +test("quick verify cleanup emits process-alive code when force stop leaves observer alive", () => { const step = forceStopStep({ aliveBefore: true, aliveAfter: true, @@ -56,11 +59,11 @@ test("quick verify cleanup emits WBC-096 when force stop leaves observer alive", assert.equal(cleanup.forceStopOk, false); assert.equal(cleanup.aliveAfter, true); assert.equal(findings.length, 1); - assert.equal(findings[0]?.id, "quick-verify-observer-stop-failed"); + assert.equal(findings[0]?.id, "observer-force-stop-left-process-alive"); assert.match(String(findings[0]?.evidenceSummary), /aliveAfter=true/u); }); -test("quick verify cleanup emits WBC-096 when stopped state cannot be confirmed", () => { +test("quick verify cleanup emits evidence-gap code when stopped state cannot be confirmed", () => { const step = forceStopStep({ aliveBefore: true, termSent: true, @@ -74,5 +77,5 @@ test("quick verify cleanup emits WBC-096 when stopped state cannot be confirmed" assert.equal(cleanup.cleanupClass, "observer-stop-failed"); assert.equal(cleanup.aliveAfter, null); assert.equal(findings.length, 1); - assert.equal(findings[0]?.id, "quick-verify-observer-stop-failed"); + assert.equal(findings[0]?.id, "observer-cleanup-evidence-missing"); }); diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts index d8d0da5b..36553458 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts @@ -647,12 +647,30 @@ function stopQuickVerifyObserver(state: SentinelCicdState, observerId: string, p export function quickVerifyCleanupFindings(cleanupStep: Record): Record[] { const cleanup = classifyQuickVerifyCleanupStep(cleanupStep); - if (stringAtNullable(cleanup, "cleanupClass") !== "observer-stop-failed") return []; + const cleanupClass = stringAtNullable(cleanup, "cleanupClass"); + if (cleanupClass === "bounded-force-stop") { + return [{ + id: "observer-graceful-stop-timeout-force-stop-succeeded", + severity: "warning", + count: 1, + summary: "observer graceful stop did not drain before timeout, but force stop confirmed aliveAfter=false.", + cleanupPhase: cleanupStep.phase, + cleanup, + evidenceSummary: formatQuickVerifyCleanupLine(cleanup), + valuesRedacted: true, + }]; + } + if (cleanupClass !== "observer-stop-failed") return []; + const aliveAfter = booleanAtNullable(cleanup, "aliveAfter"); + const id = aliveAfter === true ? "observer-force-stop-left-process-alive" : "observer-cleanup-evidence-missing"; + const summary = aliveAfter === true + ? "observer force stop completed but process liveness evidence still reports aliveAfter=true." + : "observer cleanup failed because stop-state evidence is missing or incomplete; this is an evidence gap, not confirmed process leakage."; return [{ - id: "quick-verify-observer-stop-failed", + id, severity: "red", count: 1, - summary: "quick verify completed target analysis but failed to stop its observer runner; this can leak Chrome process trees on cadence runs.", + summary, cleanupPhase: cleanupStep.phase, cleanup, evidenceSummary: formatQuickVerifyCleanupLine(cleanup), @@ -1993,9 +2011,11 @@ function isQuickVerifyBlockingFinding(item: Record): boolean { "quick-verify-final-response-empty-status-unknown", "quick-verify-diagnostics-inconclusive", "quick-verify-analysis-summary-unreadable", - "quick-verify-command-sequence-failed", + "account-session-command-failed-before-observation", "quick-verify-observer-start-failed", - "quick-verify-observer-stop-failed", + "observer-force-stop-left-process-alive", + "observer-cleanup-evidence-missing", + "observer-cleanup-index-mismatch", "quick-verify-account-secret-missing", "prompt-chat-submit-failed", "workbench-turn-state-triad-inconsistent", @@ -2004,6 +2024,9 @@ function isQuickVerifyBlockingFinding(item: Record): boolean { "round-completion-final-response-missing", "turn-trace-id-missing", "no-samples", + "no-samples-after-observer-command-failed", + "no-samples-artifact-evidence-missing", + "observer-command-failure-evidence-missing", "jsonl-read-issues", ].includes(id); } @@ -2123,15 +2146,22 @@ function quickVerifyControlFindings(failure: string | null, promptIndex: number, const observerStartFailure = failure === "observe-start-failed"; const displayTitleZh = stringAtNullable(display ?? {}, "titleZh"); return [{ - id: observerStartFailure ? "quick-verify-observer-start-failed" : "quick-verify-command-sequence-failed", + id: observerStartFailure ? "quick-verify-observer-start-failed" : "account-session-command-failed-before-observation", severity: "red", count: 1, summary: displayTitleZh ?? (observerStartFailure ? "quick verify observer failed to start before the no-prompt scenario could run." - : "quick verify no-prompt command sequence failed before the account/session workflow completed."), + : "account/session setup command failed before any business turn could be observed."), displayTitleZh, errorTitleZh: displayTitleZh, timeoutDisplay: display, + rootCause: observerStartFailure ? "observer-start-failed" : `account-session-command-stage-failed:${failure}`, + rootCauseStatus: "confirmed", + rootCauseConfidence: "high", + evidenceSummary: observerStartFailure ? `failure=${failure}` : `stage=account-session-command failure=${failure}`, + nextAction: observerStartFailure + ? "Inspect observer start output and fix the reported degradedReason before interpreting samples." + : "Inspect the first failed account/session/workbench command and rerun quick verify after that command succeeds.", failure, promptIndex, valuesRedacted: true, diff --git a/scripts/src/hwlab-node-web-sentinel-p5.ts b/scripts/src/hwlab-node-web-sentinel-p5.ts index 235698d1..86419d4a 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5.ts @@ -264,7 +264,7 @@ function compactSentinelReportRawPayload( const artifact = record(artifactSummary); const findings = Array.isArray(body.findings) ? body.findings.map(record) : []; const artifactFindings = Array.isArray(artifact.findings) ? artifact.findings.map(record) : []; - const visibleFindings = mergeSentinelReportFindings(findings, artifactFindings); + const visibleFindings = reclassifySentinelReportFindings(state, run, mergeSentinelReportFindings(findings, artifactFindings)); const offlineReclassify = offlineQuickVerifyReclassify(state, run, visibleFindings); const storedFindingCount = numberAtNullable(run, "finding_count") ?? numberAtNullable(run, "findingCount") ?? findings.length; const artifactFindingCount = numberAtNullable(artifact, "findingCount") ?? artifactFindings.length; @@ -337,7 +337,14 @@ function compactSentinelReportRawPayload( } function offlineQuickVerifyReclassify(state: SentinelCicdState, run: Record, findings: readonly Record[]): Record | null { - const hasLegacyQuickVerifyControl = findings.some((item) => sentinelReportFindingIdentityCandidates(item).includes("quick-verify-no-business-turn")); + const hasLegacyQuickVerifyControl = findings.some((item) => sentinelReportFindingIdentityCandidates(item).some((id) => [ + "quick-verify-no-business-turn", + "quick-verify-command-sequence-failed", + "account-session-command-failed-before-observation", + "no-samples", + "no-samples-after-observer-command-failed", + "no-samples-artifact-evidence-missing", + ].includes(id))); if (!hasLegacyQuickVerifyControl) return null; const catalog = sentinelReportCheckCatalogById(state); const result = reclassifyQuickVerifyControlFindings(state, { @@ -357,6 +364,114 @@ function offlineQuickVerifyReclassify(state: SentinelCicdState, run: Record, findings: readonly Record[]): Record[] { + const hasObserverCommandFailure = findings.some((item) => sentinelReportFindingIdentityCandidates(item).includes("observer-command-failed")); + const catalog = sentinelReportCheckCatalogById(state); + return findings.map((item) => enrichSentinelReportFindingWithCatalog(reclassifySentinelReportFinding(rowFailure(run), item, hasObserverCommandFailure), catalog)); +} + +function rowFailure(row: Record): string | null { + return stringAtNullable(record(row.summary), "failure") + ?? stringAtNullable(row, "failure") + ?? stringAtNullable(record(row.businessStatus), "failure"); +} + +function reclassifySentinelReportFinding(failure: string | null, item: Record, hasObserverCommandFailure: boolean): Record { + const ids = sentinelReportFindingIdentityCandidates(item); + if (ids.includes("quick-verify-command-sequence-failed")) { + return retagSentinelReportFinding(item, "account-session-command-failed-before-observation", { + summary: "account/session setup command failed before any business turn could be observed.", + rootCause: `account-session-command-stage-failed:${failure ?? "unknown"}`, + rootCauseStatus: "confirmed", + rootCauseConfidence: "high", + evidenceSummary: `stage=account-session-command failure=${failure ?? "-"}`, + nextAction: "Inspect the first failed account/session/workbench command and rerun quick verify after that command succeeds.", + }); + } + if (ids.includes("observer-command-failed") && !hasCommandFailureEvidence(item)) { + return retagSentinelReportFinding(item, "observer-command-failure-evidence-missing", { + summary: "report says observer control commands failed, but the first failed command evidence is missing.", + rootCause: "observer-command-failure-evidence-missing", + rootCauseStatus: "evidence-gap", + rootCauseConfidence: "high", + evidenceSummary: "firstCommandFailure missing from indexed and artifact findings", + nextAction: "Fix analyzer/report commandFailures compact output before using this finding as a page root cause.", + }); + } + if (ids.includes("quick-verify-observer-stop-failed")) { + const evidence = `${stringAtNullable(item, "evidenceSummary") ?? ""} ${stringAtNullable(item, "summary") ?? ""}`; + if (/aliveAfter\s*=\s*true/u.test(evidence)) { + return retagSentinelReportFinding(item, "observer-force-stop-left-process-alive", { + summary: "observer force stop completed but process liveness evidence still reports aliveAfter=true.", + rootCause: "observer-process-still-alive-after-force-stop", + rootCauseStatus: "confirmed", + rootCauseConfidence: "high", + }); + } + if (/aliveAfter\s*=\s*false/u.test(evidence)) { + return retagSentinelReportFinding(item, "observer-graceful-stop-timeout-force-stop-succeeded", { + severity: "warning", + level: "warning", + summary: "observer graceful stop did not drain before timeout, but force stop confirmed aliveAfter=false.", + rootCause: "observer-graceful-stop-timeout-force-stop-succeeded", + rootCauseStatus: "confirmed", + rootCauseConfidence: "high", + }); + } + return retagSentinelReportFinding(item, "observer-cleanup-evidence-missing", { + summary: "observer cleanup stop-state evidence is missing or incomplete; this is an evidence gap, not confirmed process leakage.", + rootCause: "observer-cleanup-evidence-missing", + rootCauseStatus: "evidence-gap", + rootCauseConfidence: "medium", + evidenceSummary: stringAtNullable(item, "evidenceSummary") ?? "cleanup evidence missing", + }); + } + if (ids.includes("no-samples")) { + if (hasObserverCommandFailure) { + return retagSentinelReportFinding(item, "no-samples-after-observer-command-failed", { + summary: "observer produced zero samples after a blocking control command failed; the command failure is the deterministic precondition.", + rootCause: "observer-command-failed-before-sampling", + rootCauseStatus: "confirmed", + rootCauseConfidence: "high", + }); + } + return retagSentinelReportFinding(item, "no-samples-artifact-evidence-missing", { + summary: "observer produced zero samples and analyzer has no failed command precondition; sampling/source evidence is missing.", + rootCause: "sample-artifact-evidence-missing", + rootCauseStatus: "evidence-gap", + rootCauseConfidence: "medium", + }); + } + return item; +} + +function hasCommandFailureEvidence(item: Record): boolean { + const first = record(item.firstCommandFailure ?? firstArrayRecord(item.commands)); + return stringAtNullable(first, "type") !== null + || stringAtNullable(first, "phase") !== null + || stringAtNullable(first, "status") !== null + || stringAtNullable(first, "stdoutPreview") !== null + || stringAtNullable(first, "stderrPreview") !== null + || numberAtNullable(first, "exitCode") !== null; +} + +function retagSentinelReportFinding(item: Record, id: string, extra: Record): Record { + return { + ...item, + ...extra, + id, + kind: id, + code: id, + finding_id: id, + findingId: id, + check: null, + checkId: null, + checkCode: null, + checkTitleZh: null, + valuesRedacted: true, + }; +} + function sentinelReportCheckCatalogById(state: SentinelCicdState): Map> { try { const reportViews = record(readWebProbeSentinelConfigRefTarget(state.spec, state.configRefs.reportViews)); @@ -465,6 +580,8 @@ function compactSentinelReportFinding(value: Record): Record): Record | null { + const item = record(value); + if (Object.keys(item).length === 0) return null; + return { + commandId: stringAtNullable(item, "commandId"), + type: stringAtNullable(item, "type"), + phase: stringAtNullable(item, "phase"), + status: stringAtNullable(item, "status"), + exitCode: numberAtNullable(item, "exitCode"), + timedOut: item.timedOut === true ? true : item.timedOut === false ? false : null, + stdoutPreview: reportText(item.stdoutPreview, 160), + stderrPreview: reportText(item.stderrPreview, 160), + artifactRef: stringAtNullable(item, "artifactRef"), + valuesRedacted: true, + }; +} + +function firstArrayRecord(value: unknown): Record | null { + return Array.isArray(value) && value.length > 0 ? record(value[0]) : null; +} + function readSentinelReportArtifactSummary(state: SentinelCicdState, body: Record, timeoutSeconds: number): Record | null { const run = record(body.run); const stateDir = stringAtNullable(run, "state_dir") ?? stringAtNullable(run, "stateDir"); @@ -501,13 +639,14 @@ function readSentinelReportArtifactSummary(state: SentinelCicdState, body: Recor "const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;", "const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);", "const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};", + "const compactCommandFailure=(value)=>{const v=rec(value); if(Object.keys(v).length===0)return null; return {commandId:clip(v.commandId,80),type:clip(v.type,48),phase:clip(v.phase,80),status:clip(v.status,80),exitCode:Number.isFinite(Number(v.exitCode))?Number(v.exitCode):null,timedOut:v.timedOut===true?true:v.timedOut===false?false:null,stdoutPreview:clip(v.stdoutPreview,160),stderrPreview:clip(v.stderrPreview,160),artifactRef:clip(v.artifactRef,160),valuesRedacted:true};};", "const sleep=(ms)=>{try{Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms)}catch{}};", "let jsonBuf=read(reportPath); let report=null; let reportParseError=null; const deadline=Date.now()+waitMs;", "while(true){ if(jsonBuf){try{report=JSON.parse(jsonBuf.toString('utf8')); break}catch(err){reportParseError=String(err&&err.message?err.message:err)}} if(Date.now()>=deadline)break; sleep(Math.min(500, Math.max(50, deadline-Date.now()))); jsonBuf=read(reportPath); }", "let artifactCount=0; let screenshot=null;", "function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}", "walk(stateDir);", - "const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,rootCauseSignals:compactRootCauseSignals(v.rootCauseSignals),blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});", + "const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),firstCommandFailure:compactCommandFailure(v.firstCommandFailure || arr(v.commands)[0]),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,rootCauseSignals:compactRootCauseSignals(v.rootCauseSignals),blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});", "const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});", "console.log(JSON.stringify({ok:!!report,reason:report?null:(jsonBuf?'report-json-parse-failed':'report-json-missing'),reportReadWaitMs:waitMs,reportParseError:clip(reportParseError,220),reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));", "NODE", diff --git a/scripts/src/hwlab-node-web-sentinel-service.ts b/scripts/src/hwlab-node-web-sentinel-service.ts index 2cdb6471..ee97c408 100644 --- a/scripts/src/hwlab-node-web-sentinel-service.ts +++ b/scripts/src/hwlab-node-web-sentinel-service.ts @@ -1760,7 +1760,7 @@ function visibleFindingsForRun( const artifactFindings = arrayRecords(artifactSummary.findings) .slice(0, limit) .map((item) => enrichFindingWithCheck(config, compactStoredFinding(item))); - return mergeVisibleFindings(config, storedFindings, artifactFindings, limit); + return reclassifyVisibleFindings(config, row, mergeVisibleFindings(config, storedFindings, artifactFindings, limit), limit); } function mergeVisibleFindings(config: WebProbeSentinelServiceConfig, primary: readonly Record[], artifact: readonly Record[], limit: number): readonly Record[] { @@ -1793,6 +1793,126 @@ function canonicalFindingIdentity(config: WebProbeSentinelServiceConfig, item: R return stringOrNull(entry?.code) ?? stringOrNull(entry?.id) ?? findingId; } +function reclassifyVisibleFindings(config: WebProbeSentinelServiceConfig, row: Record, findings: readonly Record[], limit: number): readonly Record[] { + const hasObserverCommandFailure = findings.some((item) => findingIdentityCandidates(item).includes("observer-command-failed")); + const failure = stringOrNull(record(row.summary).failure) ?? stringOrNull(row.failure) ?? stringOrNull(record(row.businessStatus).failure); + return findings + .map((item) => enrichFindingWithCheck(config, reclassifyVisibleFinding(item, failure, hasObserverCommandFailure))) + .slice(0, limit); +} + +function findingIdentityCandidates(item: Record): readonly string[] { + const check = record(item.check); + const candidates = [ + stringOrNull(item.finding_id), + stringOrNull(item.findingId), + stringOrNull(item.id), + stringOrNull(item.kind), + stringOrNull(item.code), + stringOrNull(item.checkId), + stringOrNull(item.checkCode), + stringOrNull(check.id), + stringOrNull(check.code), + ]; + return [...new Set(candidates.filter((value): value is string => value !== null))]; +} + +function reclassifyVisibleFinding(item: Record, failure: string | null, hasObserverCommandFailure: boolean): Record { + const ids = findingIdentityCandidates(item); + if (ids.includes("quick-verify-command-sequence-failed")) { + return retagVisibleFinding(item, "account-session-command-failed-before-observation", { + summary: "account/session setup command failed before any business turn could be observed.", + rootCause: `account-session-command-stage-failed:${failure ?? "unknown"}`, + rootCauseStatus: "confirmed", + rootCauseConfidence: "high", + evidenceSummary: `stage=account-session-command failure=${failure ?? "-"}`, + nextAction: "Inspect the first failed account/session/workbench command and rerun quick verify after that command succeeds.", + }); + } + if (ids.includes("observer-command-failed") && !hasCommandFailureEvidence(item)) { + return retagVisibleFinding(item, "observer-command-failure-evidence-missing", { + summary: "report says observer control commands failed, but the first failed command evidence is missing.", + rootCause: "observer-command-failure-evidence-missing", + rootCauseStatus: "evidence-gap", + rootCauseConfidence: "high", + evidenceSummary: "firstCommandFailure missing from indexed and artifact findings", + nextAction: "Fix analyzer/report commandFailures compact output before using this finding as a page root cause.", + }); + } + if (ids.includes("quick-verify-observer-stop-failed")) { + const evidence = `${stringOrNull(item.evidenceSummary) ?? ""} ${stringOrNull(item.summary) ?? ""}`; + if (/aliveAfter\s*=\s*true/u.test(evidence)) { + return retagVisibleFinding(item, "observer-force-stop-left-process-alive", { + summary: "observer force stop completed but process liveness evidence still reports aliveAfter=true.", + rootCause: "observer-process-still-alive-after-force-stop", + rootCauseStatus: "confirmed", + rootCauseConfidence: "high", + }); + } + if (/aliveAfter\s*=\s*false/u.test(evidence)) { + return retagVisibleFinding(item, "observer-graceful-stop-timeout-force-stop-succeeded", { + severity: "warning", + level: "warning", + summary: "observer graceful stop did not drain before timeout, but force stop confirmed aliveAfter=false.", + rootCause: "observer-graceful-stop-timeout-force-stop-succeeded", + rootCauseStatus: "confirmed", + rootCauseConfidence: "high", + }); + } + return retagVisibleFinding(item, "observer-cleanup-evidence-missing", { + summary: "observer cleanup stop-state evidence is missing or incomplete; this is an evidence gap, not confirmed process leakage.", + rootCause: "observer-cleanup-evidence-missing", + rootCauseStatus: "evidence-gap", + rootCauseConfidence: "medium", + evidenceSummary: stringOrNull(item.evidenceSummary) ?? "cleanup evidence missing", + }); + } + if (ids.includes("no-samples")) { + if (hasObserverCommandFailure) { + return retagVisibleFinding(item, "no-samples-after-observer-command-failed", { + summary: "observer produced zero samples after a blocking control command failed; the command failure is the deterministic precondition.", + rootCause: "observer-command-failed-before-sampling", + rootCauseStatus: "confirmed", + rootCauseConfidence: "high", + }); + } + return retagVisibleFinding(item, "no-samples-artifact-evidence-missing", { + summary: "observer produced zero samples and analyzer has no failed command precondition; sampling/source evidence is missing.", + rootCause: "sample-artifact-evidence-missing", + rootCauseStatus: "evidence-gap", + rootCauseConfidence: "medium", + }); + } + return item; +} + +function hasCommandFailureEvidence(item: Record): boolean { + const first = record(item.firstCommandFailure ?? firstArrayRecord(item.commands)); + return stringOrNull(first.type) !== null + || stringOrNull(first.phase) !== null + || stringOrNull(first.status) !== null + || stringOrNull(first.stdoutPreview) !== null + || stringOrNull(first.stderrPreview) !== null + || numberOrNull(first.exitCode) !== null; +} + +function retagVisibleFinding(item: Record, id: string, extra: Record): Record { + return { + ...item, + ...extra, + id, + kind: id, + code: id, + finding_id: id, + findingId: id, + check: null, + checkId: null, + checkCode: null, + checkTitleZh: null, + valuesRedacted: true, + }; +} + function findingIdentity(item: Record): string | null { return stringOrNull(item.finding_id) ?? stringOrNull(item.findingId) @@ -1871,6 +1991,7 @@ function enrichFindingRowWithStoredDetail(config: WebProbeSentinelServiceConfig, rootCauseConfidence: stringOrNull(detail?.rootCauseConfidence), nextAction: stringOrNull(detail?.nextAction), evidenceSummary: stringOrNull(detail?.evidenceSummary), + firstCommandFailure: compactCommandFailureEvidence(detail?.firstCommandFailure ?? firstArrayRecord(detail?.commands)), rootCauseSignals: record(detail?.rootCauseSignals), timingSourceOfTruth: stringOrNull(detail?.timingSourceOfTruth), timingStatus: stringOrNull(detail?.timingStatus), @@ -1918,6 +2039,7 @@ function compactStoredFinding(value: unknown): Record { rootCauseConfidence: stringOrNull(item.rootCauseConfidence), nextAction: stringOrNull(item.nextAction), evidenceSummary: stringOrNull(item.evidenceSummary) ?? compactFindingEvidenceSummary(item.evidence), + firstCommandFailure: compactCommandFailureEvidence(item.firstCommandFailure ?? firstArrayRecord(item.commands)), rootCauseSignals: compactFindingRootCauseSignals(item.rootCauseSignals), timingSourceOfTruth: stringOrNull(item.timingSourceOfTruth) ?? stringOrNull(item.expectedElapsedSource) ?? stringOrNull(item.evidenceKind), timingStatus: stringOrNull(item.timingStatus), @@ -1926,6 +2048,27 @@ function compactStoredFinding(value: unknown): Record { }; } +function compactCommandFailureEvidence(value: unknown): Record | null { + const item = record(value); + if (Object.keys(item).length === 0) return null; + return { + commandId: stringOrNull(item.commandId), + type: stringOrNull(item.type), + phase: stringOrNull(item.phase), + status: stringOrNull(item.status), + exitCode: numberOrNull(item.exitCode), + timedOut: item.timedOut === true ? true : item.timedOut === false ? false : null, + stdoutPreview: stringOrNull(item.stdoutPreview)?.slice(0, 240) ?? null, + stderrPreview: stringOrNull(item.stderrPreview)?.slice(0, 240) ?? null, + artifactRef: stringOrNull(item.artifactRef), + valuesRedacted: true, + }; +} + +function firstArrayRecord(value: unknown): Record | null { + return Array.isArray(value) && value.length > 0 ? record(value[0]) : null; +} + function compactFindingRootCauseSignals(value: unknown): Record | null { const item = record(value); const keys = [