From d2ac68053dfe77c5a5833a4a17d18eea746f81da Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 1 Jul 2026 11:21:30 +0000 Subject: [PATCH] fix(web-sentinel): record analyze raw findings --- .../src/hwlab-node-web-sentinel-p5-observe.ts | 122 ++++++++++++- scripts/src/hwlab-node/web-probe-observe.ts | 162 +++++++++++++++++- 2 files changed, 277 insertions(+), 7 deletions(-) diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts index e3cb57cf..e5331173 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts @@ -310,11 +310,12 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, } } printQuickVerifyProgress(state, runId, "observe-analyze", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 120) }); - const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120))], remainingSeconds(deadline, 120)); + const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120)), "--raw", "--compact-raw"], remainingSeconds(deadline, 120)); steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result }); printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() }); const indexEntry = readLocalObserveIndex(observerId); - const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS); + const artifactSummary = analysisSummaryFromAnalyzeResult(analysis, indexEntry?.stateDir ?? null) + ?? (indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS)); const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30)); const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30)); const controlFindings = quickVerifyControlFindings(null, promptIndex, turnSummary, traceFrame); @@ -507,12 +508,14 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: { "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", "55", + "--raw", "--compact-raw", ], 60); cleanupSteps.push({ phase: "observe-analyze-after-failure", ok: analysis.ok, result: analysis.result }); const indexEntry = readLocalObserveIndex(input.observerId); - const artifactSummary = indexEntry === null + const artifactSummary = analysisSummaryFromAnalyzeResult(analysis, indexEntry?.stateDir ?? null) + ?? (indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId: input.observerId, valuesRedacted: true } - : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS); + : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS)); const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30); const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30); const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(input.promptIndex, turnSummary, traceFrame); @@ -780,13 +783,19 @@ function compactQuickVerifyRecordAnalysis(value: unknown): Record): Record | null { + if (Object.keys(value).length === 0) return null; + return { + exitCode: numberAtNullable(value, "exitCode"), + timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null, + stdoutBytes: numberAtNullable(value, "stdoutBytes"), + stderrBytes: numberAtNullable(value, "stderrBytes"), + stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 600), + stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 600), + valuesRedacted: true, + }; +} + function compactQuickVerifyRecordBrowserProcess(value: Record): Record | null { const pageSeries = Array.isArray(value.pageSeries) ? value.pageSeries.map(compactQuickVerifyRecordMemorySeries).filter((item) => item.points.length > 0) : []; if (pageSeries.length === 0) return null; @@ -965,6 +987,82 @@ function boundQuickVerifyRecordText(value: unknown, maxChars: number): string | return `${value.slice(0, maxChars)}\n[truncated ${value.length - maxChars} chars]`; } +function analysisSummaryFromAnalyzeResult(analysis: ChildCliResult, fallbackStateDir: string | null): Record | null { + const payload = cliDataPayload(analysis.parsed); + const source = record(payload.analysis); + const reportJsonSha256 = stringAtNullable(source, "reportJsonSha256"); + if (reportJsonSha256 === null) return null; + const counts = record(source.counts); + const archiveSummary = record(source.archiveSummary); + const findings = mergeFindingRecords( + compactAnalyzeFindings(source.findings), + compactAnalyzeFindings(source.archiveRedFindings ?? archiveSummary.redFindings), + ); + const findingCount = Math.max( + findings.length, + numberAtNullable(source, "findingCount") ?? 0, + numberAtNullable(archiveSummary, "findingCount") ?? 0, + numberAtNullable(archiveSummary, "redFindingCount") ?? 0, + ); + return { + ok: true, + source: "observe-analyze-raw", + reason: null, + reportOk: source.ok === true, + stateDir: stringAtNullable(source, "stateDir") ?? fallbackStateDir, + reportJsonPath: stringAtNullable(source, "reportJsonPath"), + reportJsonSha256, + reportMdPath: stringAtNullable(source, "reportMdPath"), + reportMdSha256: stringAtNullable(source, "reportMdSha256"), + findingCount, + artifactCount: numberAtNullable(source, "artifactCount") ?? numberAtNullable(counts, "artifacts") ?? 0, + findings, + counts, + analysisWindow: record(source.analysisWindow), + pagePerformanceSlowApi: Array.isArray(source.pagePerformanceSlowApi) ? source.pagePerformanceSlowApi.slice(0, 8).map(record) : [], + browserProcess: compactAnalyzeBrowserProcess(source.browserProcess), + result: record(payload.result), + valuesRedacted: true, + }; +} + +function compactAnalyzeFindings(value: unknown): Record[] { + if (!Array.isArray(value)) return []; + return value.slice(0, 20).map(record).map((item) => ({ + id: stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code"), + kind: stringAtNullable(item, "kind") ?? stringAtNullable(item, "id") ?? stringAtNullable(item, "code"), + code: stringAtNullable(item, "code") ?? stringAtNullable(item, "id") ?? stringAtNullable(item, "kind"), + severity: stringAtNullable(item, "severity") ?? stringAtNullable(item, "level"), + level: stringAtNullable(item, "level") ?? stringAtNullable(item, "severity"), + count: numberAtNullable(item, "count") ?? numberAtNullable(item, "sampleCount") ?? 1, + sampleCount: numberAtNullable(item, "sampleCount") ?? numberAtNullable(item, "count") ?? 1, + summary: stringAtNullable(item, "summary") ?? stringAtNullable(item, "message"), + message: stringAtNullable(item, "message") ?? stringAtNullable(item, "summary"), + rootCause: stringAtNullable(item, "rootCause"), + rootCauseStatus: stringAtNullable(item, "rootCauseStatus"), + rootCauseConfidence: stringAtNullable(item, "rootCauseConfidence"), + nextAction: stringAtNullable(item, "nextAction"), + evidenceSummary: stringAtNullable(item, "evidenceSummary"), + timingSourceOfTruth: stringAtNullable(item, "timingSourceOfTruth"), + timingStatus: stringAtNullable(item, "timingStatus"), + timingAlert: item.timingAlert === true, + valuesRedacted: true, + })).filter((item) => item.id !== null || item.kind !== null || item.code !== null); +} + +function compactAnalyzeBrowserProcess(value: unknown): Record | null { + const source = record(value); + if (Object.keys(source).length === 0) return null; + return { + source: stringAtNullable(source, "source") ?? "observe-analyze-raw", + unit: stringAtNullable(source, "unit"), + metric: stringAtNullable(source, "metric"), + pageCount: numberAtNullable(source, "pageCount"), + sampleCount: numberAtNullable(source, "sampleCount"), + valuesRedacted: true, + }; +} + function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: string, timeoutSeconds: number): Record { if (!isSafeRelativeStateDir(stateDir)) return { ok: false, reason: "unsafe-state-dir", stateDir, valuesRedacted: true }; const waitMs = Math.max(0, Math.min(50_000, (Math.max(5, timeoutSeconds) * 1000) - 5000)); @@ -996,7 +1094,21 @@ function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: st ].join("\n"); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const parsed = parseJsonObject(result.stdout); - return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; + const parsedRecord = record(parsed); + const reason = stringAtNullable(parsedRecord, "reason") + ?? (result.timedOut ? "workspace-artifact-read-timeout" + : result.exitCode !== 0 ? "workspace-artifact-read-command-failed" + : parsed === null ? "workspace-artifact-read-output-not-json" + : null); + return { + ok: result.exitCode === 0 && parsed?.ok === true, + ...parsedRecord, + source: "workspace-trans", + reason, + stateDir: stringAtNullable(parsedRecord, "stateDir") ?? stateDir, + result: compactCommand(result), + valuesRedacted: true, + }; } function waitForQuickVerifyObserverStartup(state: SentinelCicdState, observerId: string, deadline: number, pollIntervalMs: number, budgetSeconds: number): Record { diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index bbfd76d2..1acaebce 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -2545,7 +2545,7 @@ export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOption }, valuesRedacted: true, } : null); - return withWebObserveAnalyzeRendered({ + const payload = { ok: analysisOk, status: analysisOk ? "analyzed" : "blocked", command: webObserveCommandLabel("analyze", options), @@ -2560,7 +2560,165 @@ export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOption wrapper: buildWebObserveWrapperForObserveOptions("analyze", options, spec.workspace), result: analysis === null ? compactCommandResultWithStdoutTail(result) : compactCommandResult(result), valuesRedacted: true, - }); + }; + return options.raw ? compactWebObserveAnalyzePayloadForRaw(payload, options.compactRaw) : withWebObserveAnalyzeRendered(payload); +} + +function compactWebObserveAnalyzePayloadForRaw(payload: Record, compactRaw: boolean): Record { + if (!compactRaw) return payload; + const analysis = payload.analysis && typeof payload.analysis === "object" && !Array.isArray(payload.analysis) + ? payload.analysis as Record + : {}; + return { + ok: payload.ok, + status: payload.status, + command: payload.command, + id: payload.id, + node: payload.node, + lane: payload.lane, + workspace: payload.workspace, + analysis: compactWebObserveAnalyzeAnalysisForRaw(analysis), + failure: compactWebObserveAnalyzeFailureForRaw(payload.failure), + result: payload.result, + valuesRedacted: true, + }; +} + +function compactWebObserveAnalyzeAnalysisForRaw(analysis: Record): Record { + const counts = recordValue(analysis.counts); + const archiveSummary = recordValue(analysis.archiveSummary); + const findings = arrayRecordsValue(analysis.findings).slice(0, 16).map(compactWebObserveAnalyzeFindingForRaw); + const archiveRedFindings = arrayRecordsValue(analysis.archiveRedFindings ?? archiveSummary.redFindings).slice(0, 16).map(compactWebObserveAnalyzeFindingForRaw); + return { + ok: analysis.ok === true ? true : analysis.ok === false ? false : null, + command: stringOrNullValue(analysis.command), + stateDir: stringOrNullValue(analysis.stateDir), + counts, + analysisWindow: recordValue(analysis.analysisWindow), + archiveSummary: { + redFindingCount: numberOrNullValue(archiveSummary.redFindingCount), + findingCount: numberOrNullValue(archiveSummary.findingCount), + sampleCount: numberOrNullValue(recordValue(archiveSummary.sampleMetrics).sampleCount), + pagePerformance: recordValue(archiveSummary.pagePerformance), + runtimeAlerts: recordValue(archiveSummary.runtimeAlerts), + valuesRedacted: true, + }, + runtimeAlerts: compactWebObserveAnalyzeAlertSummaryForRaw(analysis.runtimeAlerts), + pagePerformanceSlowApi: arrayRecordsValue(analysis.pagePerformanceSlowApi ?? analysis.archivePagePerformanceSlowApi).slice(0, 8).map((item) => ({ + path: stringOrNullValue(item.path ?? item.route), + route: stringOrNullValue(item.route ?? item.path), + sampleCount: numberOrNullValue(item.sampleCount), + p95Ms: numberOrNullValue(item.p95Ms ?? item.p95), + maxMs: numberOrNullValue(item.maxMs ?? item.max), + budgetMs: numberOrNullValue(item.budgetMs), + overBudgetCount: numberOrNullValue(item.overBudgetCount), + overFiveSecondCount: numberOrNullValue(item.overFiveSecondCount), + valuesRedacted: true, + })), + findings, + archiveRedFindings, + reportJsonPath: stringOrNullValue(analysis.reportJsonPath), + reportJsonSha256: stringOrNullValue(analysis.reportJsonSha256), + reportMdPath: stringOrNullValue(analysis.reportMdPath), + reportMdSha256: stringOrNullValue(analysis.reportMdSha256), + analyzer: compactWebObserveAnalyzeAnalyzerForRaw(analysis.analyzer), + valuesRedacted: true, + }; +} + +function compactWebObserveAnalyzeFailureForRaw(value: unknown): Record | null { + const failure = recordValue(value); + if (Object.keys(failure).length === 0) return null; + return { + reason: stringOrNullValue(failure.reason), + exitCode: numberOrNullValue(failure.exitCode), + timedOut: failure.timedOut === true, + parsedJson: failure.parsedJson === true, + recoveredFromArtifacts: failure.recoveredFromArtifacts === true, + stdoutBytes: numberOrNullValue(failure.stdoutBytes), + stderrBytes: numberOrNullValue(failure.stderrBytes), + stdoutTail: stringOrNullValue(failure.stdoutTail)?.slice(-1200) ?? null, + stderrTail: stringOrNullValue(failure.stderrTail)?.slice(-1200) ?? null, + valuesRedacted: true, + }; +} + +function compactWebObserveAnalyzeAnalyzerForRaw(value: unknown): Record { + const analyzer = recordValue(value); + return { + exitCode: numberOrNullValue(analyzer.exitCode), + recoveredFrom: stringOrNullValue(analyzer.recoveredFrom), + stdoutBytes: numberOrNullValue(analyzer.stdoutBytes), + stderrBytes: numberOrNullValue(analyzer.stderrBytes), + reportJsonBytes: numberOrNullValue(analyzer.reportJsonBytes), + reportMdBytes: numberOrNullValue(analyzer.reportMdBytes), + transportExitCode: numberOrNullValue(analyzer.transportExitCode), + transportTimedOut: analyzer.transportTimedOut === true, + compactStdoutLimited: analyzer.compactStdoutLimited === true, + valuesRedacted: true, + }; +} + +function compactWebObserveAnalyzeAlertSummaryForRaw(value: unknown): Record { + const source = recordValue(value); + const summary = recordValue(source.summary); + const out: Record = {}; + for (const key of [ + "httpErrorCount", + "requestFailedCount", + "significantRequestFailedCount", + "workbenchSessionListReadCount", + "workbenchTraceEventsReadCount", + "webPerformanceBeaconFailureCount", + "workbenchEventSourceFailureCount", + "consoleAlertCount", + "significantConsoleAlertCount", + ]) { + const value = summary[key] ?? source[key]; + if (value !== undefined && value !== null) out[key] = value; + } + out.valuesRedacted = true; + return out; +} + +function compactWebObserveAnalyzeFindingForRaw(value: Record): Record { + return { + kind: stringOrNullValue(value.kind ?? value.id ?? value.code), + id: stringOrNullValue(value.id ?? value.kind ?? value.code), + code: stringOrNullValue(value.code ?? value.id ?? value.kind), + severity: stringOrNullValue(value.severity ?? value.level), + level: stringOrNullValue(value.level ?? value.severity), + count: numberOrNullValue(value.count ?? value.sampleCount), + sampleCount: numberOrNullValue(value.sampleCount ?? value.count), + timingSourceOfTruth: stringOrNullValue(value.timingSourceOfTruth ?? value.expectedElapsedSource ?? value.evidenceKind), + timingStatus: stringOrNullValue(value.timingStatus), + timingAlert: value.timingAlert === true, + summary: stringOrNullValue(value.summary ?? value.message)?.slice(0, 220) ?? null, + message: stringOrNullValue(value.message ?? value.summary)?.slice(0, 220) ?? null, + rootCause: stringOrNullValue(value.rootCause)?.slice(0, 140) ?? null, + rootCauseStatus: stringOrNullValue(value.rootCauseStatus)?.slice(0, 90) ?? null, + rootCauseConfidence: stringOrNullValue(value.rootCauseConfidence)?.slice(0, 40) ?? null, + nextAction: stringOrNullValue(value.nextAction)?.slice(0, 240) ?? null, + evidenceSummary: stringOrNullValue(value.evidenceSummary)?.slice(0, 220) ?? null, + valuesRedacted: true, + }; +} + +function arrayRecordsValue(value: unknown): Record[] { + return Array.isArray(value) ? value.map(recordValue).filter((item) => Object.keys(item).length > 0) : []; +} + +function recordValue(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +function stringOrNullValue(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function numberOrNullValue(value: unknown): number | null { + const number = Number(value); + return Number.isFinite(number) ? number : null; } export function recoverWebObserveAnalyzeFromArtifacts(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, result: { exitCode: number; timedOut: boolean }): Record | null {