// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel. // 职责:编排 Web Probe Sentinel P5 quick-verify,并投影有界终态证据。 import { createHash, randomUUID } from "node:crypto"; import type { CommandResult } from "./command"; import { runCommand } from "./command"; import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery"; import { repoRoot } from "./config"; import { runWebProbeMemoryGuard } from "./hwlab-node-web-probe-memory-guard"; import type { ChildCliResult, SentinelCicdState } from "./hwlab-node-web-sentinel-cicd"; import { arrayAt, compactCommand, mergeWarnings, numberAt, numberAtNullable, parseJsonObject, quickVerifyAccountEnv, record, nonEmptyString, shellQuote, short, stringAt, stringAtNullable, targetValidationElapsedWarnings, text, withWarnings, } from "./hwlab-node-web-sentinel-cicd"; import { analysisSummaryFromAnalyzeResult, cliDataPayload, collectObserveView, enrichObserveStartFailureFinding, findScenario, isQuickVerifyBlockingFinding, isRecoverableQuickVerifyWaitFailure, mergeFindingRecords, observerIdFromText, quickVerifyBusinessStatus, quickVerifyControlFindings, quickVerifyHasDurableBusinessTurn, quickVerifyPrimaryControlFailure, readAnalysisSummaryFromWorkspace, readLocalObserveIndex, readPromptSetForScenario, remainingSeconds, runChildCli, runSentinelExactProviderProfilePreflight, timeoutDisplayForQuickVerifyFailure, waitForQuickVerifyObserverStartup, waitForQuickVerifyPromptTurn, } from "./hwlab-node-web-sentinel-p5-observe-analysis"; import { emitWebProbeSentinelSpan } from "./hwlab-node-web-sentinel-otel"; import { buildMonitorTerminalIngest, MonitorIngestFailedError, resolveMonitorTerminalWriter } from "./monitor-terminal-ingest"; export { analysisSummaryFromAnalyzeResult, metricNames, readAnalysisSummaryFromWorkspace, remainingSeconds, runChildCli, runSentinelExactProviderProfilePreflight, sentinelP5Next, serviceUnavailableBlocker, validationBlocker, } from "./hwlab-node-web-sentinel-p5-observe-analysis"; const QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS = 55; export interface QuickVerifyObserverLifecycle { readonly observerId: string; cleanupStep: Record | null; stop(phase: string): Record; } type QuickVerifyObserverStop = (observerId: string, phase: string) => Record; export function createQuickVerifyObserverLifecycle(observerId: string, stopObserver: QuickVerifyObserverStop): QuickVerifyObserverLifecycle { const attempts: Record[] = []; const lifecycle: QuickVerifyObserverLifecycle = { observerId, cleanupStep: null, stop(phase) { if (lifecycle.cleanupStep?.ok === true) return lifecycle.cleanupStep; for (let callAttempt = 1; callAttempt <= 2; callAttempt += 1) { const attempt = attempts.length + 1; try { const result = stopObserver(observerId, phase); attempts.push({ attempt, ok: result.ok === true, valuesRedacted: true }); lifecycle.cleanupStep = { ...result, stopAttempts: attempts, valuesRedacted: true }; if (result.ok === true) return lifecycle.cleanupStep; } catch (error) { attempts.push({ attempt, ok: false, failure: "observer-stop-unhandled-exception", valuesRedacted: true }); lifecycle.cleanupStep = { phase, ok: false, failure: "observer-stop-unhandled-exception", observerId, exception: quickVerifyUnhandledExceptionSummary(error), stopAttempts: attempts, valuesRedacted: true, }; } } lifecycle.cleanupStep ??= { phase, ok: false, failure: "observer-stop-did-not-return", observerId, stopAttempts: attempts, valuesRedacted: true }; return lifecycle.cleanupStep; }, }; return lifecycle; } export function stopQuickVerifyObserverAfterPartialStart(observerId: string | null, stopObserver: QuickVerifyObserverStop): Record | null { if (observerId === null) return null; return createQuickVerifyObserverLifecycle(observerId, stopObserver).stop("observe-stop-after-start-failure"); } export function quickVerifyUnhandledExceptionSummary(error: unknown): Record { let errorName = typeof error; let errorMessage = typeof error === "string" ? error : ""; if (error instanceof Error) { errorName = error.name || "Error"; errorMessage = error.message; } const boundedMessage = errorMessage.slice(0, 4096); return { errorName: short(errorName), errorMessageSha256: boundedMessage.length === 0 ? null : createHash("sha256").update(boundedMessage).digest("hex"), errorMessageTruncated: errorMessage.length > boundedMessage.length, valuesRedacted: true, }; } export function sentinelChildStartMemorySkip(value: unknown): Record | null { const envelope = record(value); const payload = Object.keys(record(envelope.data)).length > 0 ? record(envelope.data) : envelope; if (payload.status !== "skipped-wait-next-round" || payload.observerCreated !== false) return null; return { status: "skipped-wait-next-round", decision: "skip-observer", observerCreated: false, memoryGuard: payload.memoryGuard ?? null, valuesRedacted: true, }; } function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record = {}): void { const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => { if (typeof value === "string") return [key, short(value)]; if (Array.isArray(value)) return [key, value.slice(0, 8)]; return [key, value]; })); process.stdout.write(`${JSON.stringify({ event: "sentinel.quick-verify.progress", at: new Date().toISOString(), node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, runId, ...compactExtra, phase, status, valuesRedacted: true, })}\n`); } export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record { const startedAt = Date.now(); const elapsedMs = () => Date.now() - startedAt; const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId"); const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds"); const scenario = findScenario(state, scenarioId); if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true }; const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; emitWebProbeSentinelSpan(sentinelOtelContext(state), "web_probe_sentinel.quick_verify.job_start", { scenarioId, runId, cadence: stringAtNullable(scenario, "cadence"), status: "running", valuesRedacted: true, }); const memoryGuard = runWebProbeMemoryGuard(state.spec, "sentinel-cadence"); if (memoryGuard.status === "skipped") { return recordSkippedQuickVerify(state, scenario, runId, scenarioId, reason, elapsedMs(), memoryGuard, [], "memavailable-below-threshold"); } if (memoryGuard.status !== "allowed") { return recordQuickVerify(state, { ok: false, runId, status: "blocked", reason: "memavailable-preflight-unavailable", scenarioId, observerId: null, observerCreated: false, runCreated: true, mutation: false, memoryGuard, elapsedMs: elapsedMs(), findingCount: 0, artifactCount: 0, findings: [], views: {}, updatedAt: new Date().toISOString(), valuesRedacted: true, }); } const providerProfilePreflight = runSentinelExactProviderProfilePreflight(state, timeoutSeconds, scenario); if (providerProfilePreflight.ok !== true) { return recordQuickVerify(state, { ok: false, runId, status: "blocked", reason: "exact-provider-profile-unavailable", scenarioId, providerProfile: stringAtNullable(scenario, "providerProfile"), providerProfileMode: stringAtNullable(scenario, "providerProfileMode"), providerProfilePreflight, observerId: null, observerCreated: false, runCreated: true, mutation: false, elapsedMs: elapsedMs(), findingCount: 0, artifactCount: 0, findings: [], views: {}, updatedAt: new Date().toISOString(), valuesRedacted: true, }); } const commandSequence = arrayAt(scenario, "commandSequence").map(record); const needsPromptSet = commandSequence.some((item) => stringAt(item, "type") === "sendPrompt" && inlinePromptText(item) === null); const prompts = needsPromptSet ? readPromptSetForScenario(state, scenario) : { ok: true as const, prompts: [], summary: { source: "not-required", promptCount: 0, valuesRedacted: true } }; if (!prompts.ok) { emitWebProbeSentinelSpan(sentinelOtelContext(state), "web_probe_sentinel.quick_verify.job_finish", { scenarioId, runId, status: "blocked", exitCode: 1, failureKind: "prompt-source-unavailable", valuesRedacted: true, }, false); return { ok: false, status: "blocked", reason: "prompt-source-unavailable", promptSource: prompts, valuesRedacted: true }; } const accountEnv = quickVerifyAccountEnv(state); if (!accountEnv.ok) { const findings = [{ id: "quick-verify-account-secret-missing", severity: "red", count: arrayAt(accountEnv.summary, "missing").length || 1, summary: "quick verify could not materialize YAML-declared web account credentials for the observer runner.", missing: arrayAt(accountEnv.summary, "missing"), valuesRedacted: true, }]; return recordQuickVerify(state, { ok: false, runId, scenarioId, reason, status: "blocked", observerId: null, elapsedMs: 0, businessStatus: quickVerifyBusinessStatus("quick-verify-account-secret-missing", 0, null, null, 0, maxSeconds), steps: [{ phase: "quick-verify-account-env", ok: false, result: accountEnv.summary }], failure: "quick-verify-account-secret-missing", findingCount: findings.length, findings, promptSource: prompts.summary, accountEnv: accountEnv.summary, views: { summary: { renderedText: renderQuickVerifySummary({ scenarioId, artifactSummary: { ok: false, findings, findingCount: findings.length }, steps: [], publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ scenarioId, steps: [], findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, }, warnings: [], valuesRedacted: true, }); } const sampleIntervalMs = numberAt(scenario, "sampleIntervalMs"); const warningBudgetSeconds = maxSeconds; const hardBudgetSeconds = Math.min(timeoutSeconds, Math.max(maxSeconds, numberAt(scenario, "maxRunSeconds"))); const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", warningBudgetSeconds); const deadline = Date.now() + hardBudgetSeconds * 1000; printQuickVerifyProgress(state, runId, "start", "running", { scenarioId, reason, warningBudgetSeconds, hardBudgetSeconds, timeoutSeconds }); const steps: Record[] = []; const startArgs = [ "web-probe", "observe", "start", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--target-path", stringAt(scenario, "observeTargetPath"), "--sample-interval-ms", String(sampleIntervalMs), "--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")), "--max-run-seconds", String(hardBudgetSeconds), "--command-timeout-seconds", "55", "--sentinel-cadence", ]; const viewport = stringAtNullable(scenario, "viewport"); if (viewport !== null) startArgs.push("--viewport", viewport); printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) }); const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env); steps.push({ phase: "observe-start", ok: started.ok, result: started.result }); const childMemorySkip = sentinelChildStartMemorySkip(started.parsed); if (childMemorySkip !== null) { return recordSkippedQuickVerify(state, scenario, runId, scenarioId, reason, elapsedMs(), childMemorySkip.memoryGuard, steps, "memavailable-below-threshold-after-cadence-preflight"); } const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? "")); printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() }); if (!started.ok || observerId === null) { const cleanupStep = stopQuickVerifyObserverAfterPartialStart( observerId, (partialObserverId, phase) => stopQuickVerifyObserver(state, partialObserverId, phase), ); const cleanup = cleanupStep === null ? null : classifyQuickVerifyCleanupStep(cleanupStep); const findings = mergeFindingRecords( quickVerifyControlFindings("observe-start-failed", 0, null, null) .map((finding) => enrichObserveStartFailureFinding(finding, record(started.result))), cleanupStep === null ? [] : quickVerifyCleanupFindings(cleanupStep), ); return recordQuickVerify(state, { ok: false, runId, scenarioId, reason, status: "blocked", observerId, elapsedMs: elapsedMs(), businessStatus: quickVerifyBusinessStatus("observe-start-failed", 0, null, null, elapsedMs(), maxSeconds), steps: cleanupStep === null ? steps : [...steps, cleanupStep], cleanup, failure: "observe-start-failed", findingCount: findings.length, findings, warnings: mergeWarnings(elapsedWarnings(), quickVerifyCleanupWarnings(cleanup)), valuesRedacted: true, }); } const observerLifecycle = createQuickVerifyObserverLifecycle( observerId, (activeObserverId, phase) => stopQuickVerifyObserver(state, activeObserverId, phase), ); let promptIndex = 0; try { printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) }); const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds); steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady }); printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", startupReady.ok === true ? "succeeded" : "failed", { observerId, failure: startupReady.failure ?? null, status: startupReady.status ?? null, heartbeatStatus: startupReady.heartbeatStatus ?? null, elapsedMs: elapsedMs() }); if (startupReady.ok !== true) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, observerLifecycle, promptIndex: 0, steps, failure: text(startupReady.failure ?? "observe-startup-ready-wait-failed"), elapsedMs: elapsedMs(), warnings: mergeWarnings(Array.isArray(startupReady.warnings) ? startupReady.warnings : [], elapsedWarnings()), promptSource: prompts.summary, })); } const sessionInvarianceChecks = sessionInvarianceChecksByRound(scenario); const nonBlockingCanaryWarnings: string[] = []; for (const item of commandSequence) { const type = stringAt(item, "type"); const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1); for (let index = 0; index < repeat; index += 1) { if (Date.now() >= deadline) { const timeoutDisplay = timeoutDisplayForQuickVerifyFailure("quick-verify-timeout-over-budget", promptIndex, warningBudgetSeconds, timeoutSeconds); printQuickVerifyProgress(state, runId, "timeout", "failed", { observerId, promptIndex, elapsedMs: elapsedMs(), hardBudgetSeconds }); return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, observerLifecycle, promptIndex, steps, failure: "quick-verify-timeout-over-budget", failureTitleZh: stringAtNullable(timeoutDisplay, "titleZh"), timeoutDisplay, elapsedMs: elapsedMs(), warnings: mergeWarnings(`quick verify exceeded the hard ${hardBudgetSeconds}s execution budget after the configured ${warningBudgetSeconds}s targetValidation warning budget.`, elapsedWarnings()), promptSource: prompts.summary, })); } const commandWaitMs = Math.max(1000, Math.trunc(numberAtNullable(item, "commandWaitMs") ?? numberAtNullable(scenario, "commandWaitMs") ?? 55_000)); const commandTimeoutSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "commandTimeoutSeconds") ?? numberAtNullable(scenario, "commandTimeoutSeconds") ?? 55)); const turnWaitChunkSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "turnWaitChunkSeconds") ?? numberAtNullable(scenario, "turnWaitChunkSeconds") ?? 55)); const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", String(commandWaitMs), "--command-timeout-seconds", String(remainingSeconds(deadline, commandTimeoutSeconds))]; if (type === "selectProvider") args.push("--provider", stringAt(item, "provider")); if (type === "loginAccount" || type === "listSessions" || type === "logout") { const accountId = stringAtNullable(item, "accountId"); if (accountId !== null) args.push("--account-id", accountId); } if (type === "switchSessions") { const fromAccountId = stringAtNullable(item, "fromAccountId"); const toAccountId = stringAtNullable(item, "toAccountId"); if (fromAccountId !== null) args.push("--from-account-id", fromAccountId); if (toAccountId !== null) args.push("--to-account-id", toAccountId); } if (type === "sendPrompt") { args.push("--text", inlinePromptText(item) ?? prompts.prompts[promptIndex % prompts.prompts.length] ?? ""); args.push("--expected-action-wait-ms", String(numberAtNullable(item, "expectedActionWaitMs") ?? 45000)); promptIndex += 1; } appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" }); printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat }); const commandResult = runChildCli(args, remainingSeconds(deadline, Math.max(60, commandTimeoutSeconds + 10))); steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result }); printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() }); if (!commandResult.ok) { if (type === "sendPrompt") { printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) }); const delayedWaitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds); steps.push({ phase: "observe-wait-turn-terminal-after-command-timeout", ok: delayedWaitResult.ok, promptIndex, result: delayedWaitResult }); printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", delayedWaitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: delayedWaitResult.failure ?? null, status: delayedWaitResult.status ?? null, traceId: delayedWaitResult.traceId ?? null, elapsedMs: elapsedMs() }); if (delayedWaitResult.ok === true) { const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps); nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings)); printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() }); if (invariantResult.ok !== true) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, observerLifecycle, promptIndex, steps, failure: text(record(invariantResult).failure ?? "observe-session-invariance-failed"), promptSource: prompts.summary, elapsedMs: elapsedMs(), warnings: mergeWarnings(record(invariantResult).warnings, elapsedWarnings()), })); } continue; } } return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, observerLifecycle, promptIndex, steps, failure: `observe-command-${type}-failed`, elapsedMs: elapsedMs(), warnings: elapsedWarnings(), promptSource: prompts.summary, })); } if (type === "sendPrompt") { printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) }); const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds); steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult }); printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() }); if (waitResult.ok !== true) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, observerLifecycle, promptIndex, steps, failure: text(waitResult.failure ?? "observe-turn-terminal-wait-failed"), promptSource: prompts.summary, elapsedMs: elapsedMs(), warnings: mergeWarnings(Array.isArray(waitResult.warnings) ? waitResult.warnings : [], elapsedWarnings()), })); } const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps); nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings)); printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() }); if (invariantResult.ok !== true) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, observerLifecycle, promptIndex, steps, failure: text(invariantResult.failure ?? "observe-session-invariance-check-failed"), promptSource: prompts.summary, elapsedMs: elapsedMs(), warnings: mergeWarnings(nonBlockingCanaryWarnings, elapsedWarnings()), })); } } } } printQuickVerifyProgress(state, runId, "observe-analyze", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 120) }); const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120)), "--raw", "--compact-raw"], remainingSeconds(deadline, 120)); steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result }); printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() }); const indexEntry = readLocalObserveIndex(observerId); const artifactSummary = analysisSummaryFromAnalyzeResult(analysis, indexEntry?.stateDir ?? null) ?? (indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS)); const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30)); const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30)); const controlFindings = quickVerifyControlFindings(null, promptIndex, turnSummary, traceFrame); const artifactSummaryRecord = record(artifactSummary); const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : []; const visibilityFindings = quickVerifyAnalysisVisibilityFindings(analysis, artifactSummary); const cleanupStep = observerLifecycle.stop("observe-stop-after-terminal"); const cleanup = classifyQuickVerifyCleanupStep(cleanupStep); const cleanupFindings = quickVerifyCleanupFindings(cleanupStep); const findings = mergeFindingRecords( mergeFindingRecords( mergeFindingRecords(artifactFindings, controlFindings), visibilityFindings, ), cleanupFindings, ); const blockingFindings = findings.filter(isQuickVerifyBlockingFinding); const analysisReadable = stringAtNullable(artifactSummary, "reportJsonSha256") !== null; const analysisWarnings = mergeWarnings( analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."], analysisReadable && artifactSummaryRecord.ok !== true ? ["quick verify analysis report contains non-blocking findings; targetValidation keeps them in report but does not fail the CronJob without red/blocking findings."] : [], ); const ok = analysisReadable && controlFindings.length === 0 && blockingFindings.length === 0; const businessStatus = quickVerifyBusinessStatus(null, promptIndex, turnSummary, traceFrame, elapsedMs(), maxSeconds); const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings); printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), analysisReadable, findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() }); return recordQuickVerify(state, { ok, runId, scenarioId, reason, status: ok ? "analyzed" : "blocked", observerId, elapsedMs: elapsedMs(), businessStatus, stateDir: indexEntry?.stateDir ?? null, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0, failure: !analysisReadable ? "quick-verify-analysis-missing" : primaryControlFailure ?? (blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null), promptSource: prompts.summary, accountEnv: accountEnv.summary, steps: [...steps, cleanupStep], cleanup, analysis: artifactSummary, views: { summary: { renderedText: renderQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, findings, findingCount: findings.length, steps, cleanup, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, cleanup, findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok }, "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok }, }, findings, screenshot: record(artifactSummary).screenshot, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), warnings: mergeWarnings(analysisWarnings, nonBlockingCanaryWarnings, quickVerifyCleanupWarnings(cleanup), elapsedWarnings()), valuesRedacted: true, }); } catch (error) { const exception = quickVerifyUnhandledExceptionSummary(error); steps.push({ phase: "quick-verify-unhandled-exception", ok: false, result: exception }); printQuickVerifyProgress(state, runId, "quick-verify-unhandled-exception", "failed", { observerId, promptIndex, errorName: exception.errorName, errorMessageSha256: exception.errorMessageSha256, elapsedMs: elapsedMs(), }); return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, observerLifecycle, promptIndex, steps, failure: "quick-verify-unhandled-exception", elapsedMs: elapsedMs(), warnings: mergeWarnings("quick verify caught an unexpected post-start exception; details are redacted and the observer cleanup fallback remained active.", elapsedWarnings()), promptSource: prompts.summary, })); } finally { observerLifecycle.stop("observe-stop-after-finally"); } } function sessionInvarianceChecksByRound(scenario: Record): Map[]> { const checks = new Map[]>(); const items = Array.isArray(scenario.sessionInvarianceChecks) ? scenario.sessionInvarianceChecks.map(record) : []; for (const item of items) { const afterRound = typeof item.afterRound === "number" && Number.isInteger(item.afterRound) ? item.afterRound : null; if (afterRound === null || afterRound < 0) continue; const list = checks.get(afterRound) ?? []; list.push(item); checks.set(afterRound, list); } return checks; } function runQuickVerifySessionInvarianceChecks( state: SentinelCicdState, observerId: string, checks: readonly Record[], deadline: number, promptIndex: number, steps: Record[], ): Record { const warnings: string[] = []; for (const check of checks) { const checkId = nonEmptyString(check.id) ?? `after-round-${promptIndex}`; const blocking = check.blocking === true; const commands: { readonly type: string; readonly enabled: boolean }[] = [ { type: "refreshCurrentSession", enabled: check.refreshCurrent === true }, { type: "switchAwayAndBack", enabled: check.switchAwayAndBack === true }, { type: "assertSessionInvariant", enabled: check.assertSessionInvariant !== false }, ]; for (const command of commands) { if (!command.enabled) continue; const args = [ "web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", command.type, "--after-round", String(promptIndex), "--wait-ms", "55000", "--command-timeout-seconds", String(remainingSeconds(deadline, 55)), ]; const severity = nonEmptyString(check.severity); const findingId = nonEmptyString(check.findingId); const expectedSentinelRange = nonEmptyString(check.expectedSentinelRange); const alternateSessionStrategy = nonEmptyString(check.alternateSessionStrategy); if (severity !== null) args.push("--severity", severity); if (findingId !== null) args.push("--finding-id", findingId); if (expectedSentinelRange !== null) args.push("--expected-sentinel-range", expectedSentinelRange); if (command.type === "switchAwayAndBack" && alternateSessionStrategy !== null) args.push("--alternate-session-strategy", alternateSessionStrategy); if (command.type === "assertSessionInvariant" && check.requireComposerReady === true) args.push("--require-composer-ready"); args.push(check.blocking === true ? "--blocking" : "--non-blocking"); const result = runChildCli(args, remainingSeconds(deadline, 60)); steps.push({ phase: `observe-session-invariance-${command.type}`, ok: result.ok, promptIndex, checkId, result: result.result }); if (!result.ok) { if (!blocking) { warnings.push(`non-blocking session invariance canary ${checkId}/${command.type} failed after round ${promptIndex}; continuing Code Agent multi-round quick verify because scenario marks this check blocking=false.`); continue; } return { ok: false, failure: `observe-session-invariance-${command.type}-failed`, checkId, promptIndex, valuesRedacted: true }; } } } return { ok: true, promptIndex, checkCount: checks.length, warnings, valuesRedacted: true }; } function appendScenarioObserveCommandArgs(args: string[], item: Record, options: { readonly skipText?: boolean } = {}): void { const mappings: readonly (readonly [string, string])[] = [ ["path", "--path"], ["label", "--label"], ["sessionId", "--session-id"], ["provider", "--provider"], ["accountId", "--account-id"], ["fromAccountId", "--from-account-id"], ["toAccountId", "--to-account-id"], ["sourceId", "--source-id"], ["fileRef", "--file-ref"], ["filename", "--filename"], ["taskRef", "--task-ref"], ["taskId", "--task-id"], ["task", "--task"], ["field", "--field"], ["link", "--link"], ["title", "--title"], ["body", "--body"], ["status", "--status"], ["hwpodId", "--hwpod-id"], ["nodeId", "--node-id"], ["workspaceRoot", "--workspace-root"], ["root", "--root"], ]; for (const [key, flag] of mappings) { if (args.includes(flag)) continue; const value = stringAtNullable(item, key); if (value !== null) args.push(flag, value); } if (options.skipText !== true && !args.includes("--text")) { const text = stringAtNullable(item, "text") ?? stringAtNullable(item, "value"); if (text !== null) args.push("--text", text); } if (item.waitProjectManagementReady === true && !args.includes("--wait-project-management-ready")) args.push("--wait-project-management-ready"); } function inlinePromptText(item: Record): string | null { return stringAtNullable(item, "text") ?? stringAtNullable(item, "prompt") ?? stringAtNullable(item, "value"); } export function reclassifyQuickVerifyControlFindings(state: SentinelCicdState, input: { readonly runId: string | null; readonly scenarioId: string | null; readonly observerId: string | null; readonly failure?: string | null; readonly timeoutSeconds?: number | null; }): Record { if (input.observerId === null || input.observerId.length === 0) { return { ok: false, reason: "observer-id-missing", runId: input.runId, valuesRedacted: true }; } const scenarioId = input.scenarioId ?? stringAt(state.cicd, "targetValidation.scenarioId"); const scenario = findScenario(state, scenarioId); if (scenario === null) { return { ok: false, reason: "scenario-not-found", runId: input.runId, scenarioId, observerId: input.observerId, valuesRedacted: true }; } const commandSequence = arrayAt(scenario, "commandSequence").map(record); const promptIndex = commandSequence.reduce((count, item) => { const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1); return stringAtNullable(item, "type") === "sendPrompt" ? count + repeat : count; }, 0); const timeoutSeconds = Math.max(5, Math.min(Math.trunc(input.timeoutSeconds ?? 55), 55)); const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, timeoutSeconds); const traceFrame = collectObserveView(state, input.observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, timeoutSeconds); const findings = quickVerifyControlFindings(input.failure ?? null, promptIndex, turnSummary, traceFrame); return { ok: true, runId: input.runId, scenarioId, observerId: input.observerId, promptIndex, findingCount: findings.length, findings, turnSummary: { ok: turnSummary.ok === true, collectShape: turnSummary.collectShape, valuesRedacted: true }, traceFrame: { ok: traceFrame.ok === true, collectShape: traceFrame.collectShape, valuesRedacted: true }, valuesRedacted: true, }; } function finalizeQuickVerifyFailure(state: SentinelCicdState, input: { readonly runId: string; readonly scenarioId: string; readonly reason: string; readonly observerId: string; readonly observerLifecycle: QuickVerifyObserverLifecycle; readonly promptIndex: number; readonly steps: readonly Record[]; readonly failure: string; readonly failureTitleZh?: string | null; readonly timeoutDisplay?: Record | null; readonly promptSource?: Record; readonly elapsedMs?: number; readonly warnings?: readonly unknown[]; }): Record { const targetValidationSeconds = numberAt(state.cicd, "targetValidation.maxSeconds"); const failureDisplay = input.timeoutDisplay ?? timeoutDisplayForQuickVerifyFailure(input.failure, input.promptIndex, targetValidationSeconds, null); const failureTitle = input.failureTitleZh ?? stringAtNullable(failureDisplay, "titleZh"); const cleanupSteps: Record[] = []; if (input.promptIndex > 0) { const cancel = runChildCli([ "web-probe", "observe", "command", input.observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", "cancel", "--wait-ms", "55000", "--command-timeout-seconds", "55", ], 60); cleanupSteps.push({ phase: "observe-cancel-after-failure", ok: cancel.ok, result: cancel.result }); } cleanupSteps.push(input.observerLifecycle.stop("observe-stop-after-failure")); const analysis = runChildCli([ "web-probe", "observe", "analyze", input.observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", "55", "--raw", "--compact-raw", ], 60); cleanupSteps.push({ phase: "observe-analyze-after-failure", ok: analysis.ok, result: analysis.result }); const indexEntry = readLocalObserveIndex(input.observerId); const artifactSummary = analysisSummaryFromAnalyzeResult(analysis, indexEntry?.stateDir ?? null) ?? (indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId: input.observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS)); const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30); const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30); const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(input.promptIndex, turnSummary, traceFrame); const controlFindings = quickVerifyControlFindings(input.failure, input.promptIndex, turnSummary, traceFrame, failureDisplay); const artifactSummaryRecord = record(artifactSummary); const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : []; const visibilityFindings = quickVerifyAnalysisVisibilityFindings(analysis, artifactSummary); const cleanup = quickVerifyCleanupFromSteps(cleanupSteps); const cleanupFindings = cleanupSteps.flatMap((step) => quickVerifyCleanupFindings(step)); const findings = mergeFindingRecords( mergeFindingRecords( mergeFindingRecords(artifactFindings, controlFindings), visibilityFindings, ), cleanupFindings, ); const blockingFindings = findings.filter(isQuickVerifyBlockingFinding); const recoveredWaitFailure = durableBusinessTurn && isRecoverableQuickVerifyWaitFailure(input.failure) && record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0; const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, targetValidationSeconds); const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings); return { ok: recoveredWaitFailure, runId: input.runId, scenarioId: input.scenarioId, reason: input.reason, status: recoveredWaitFailure ? "analyzed" : "blocked", observerId: input.observerId, elapsedMs: input.elapsedMs ?? null, businessStatus, failureTitleZh: recoveredWaitFailure ? null : failureTitle, errorTitleZh: recoveredWaitFailure ? null : failureTitle, timeoutDisplay: recoveredWaitFailure ? null : failureDisplay, stateDir: indexEntry?.stateDir ?? null, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0, failure: recoveredWaitFailure ? null : primaryControlFailure ?? input.failure, runnerFailure: recoveredWaitFailure ? null : input.failure, promptSource: input.promptSource, steps: [...input.steps, ...cleanupSteps], cleanup, analysis: artifactSummary, views: { summary: { renderedText: renderQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, findings, findingCount: findings.length, steps: input.steps, cleanup, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), failure: recoveredWaitFailure ? null : input.failure, failureTitleZh: recoveredWaitFailure ? null : failureTitle, timeoutDisplay: recoveredWaitFailure ? null : failureDisplay }) }, "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, cleanup, findings, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), failure: recoveredWaitFailure ? null : input.failure, failureTitleZh: recoveredWaitFailure ? null : failureTitle, timeoutDisplay: recoveredWaitFailure ? null : failureDisplay }) }, "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok }, "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok }, }, findings, screenshot: record(artifactSummary).screenshot, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), warnings: mergeWarnings( Array.isArray(input.warnings) ? input.warnings : [], recoveredWaitFailure ? ["quick verify wait command timed out, but collected turn-summary/trace-frame artifacts show a durable completed business turn; treating the wait timeout as a non-blocking tool finding."] : [], quickVerifyCleanupWarnings(cleanup), targetValidationElapsedWarnings(input.elapsedMs ?? null, "quick verify confirm-wait", targetValidationSeconds), ), valuesRedacted: true, }; } function stopQuickVerifyObserver(state: SentinelCicdState, observerId: string, phase: string): Record { const stop = runChildCli([ "web-probe", "observe", "stop", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--force", "--raw", "--wait-ms", "55000", "--command-timeout-seconds", "55", ], 60); const step = { phase, ok: stop.ok, payload: cliDataPayload(stop.parsed), result: stop.result }; return { ...step, cleanup: classifyQuickVerifyCleanupStep(step) }; } export function quickVerifyCleanupFindings(cleanupStep: Record): Record[] { const cleanup = classifyQuickVerifyCleanupStep(cleanupStep); const cleanupClass = stringAtNullable(cleanup, "cleanupClass"); if (cleanupClass === "bounded-force-stop") { return [{ id: "observer-graceful-stop-timeout-force-stop-succeeded", severity: "warning", count: 1, summary: "observer graceful stop did not drain before timeout, but force stop confirmed aliveAfter=false.", cleanupPhase: cleanupStep.phase, cleanup, evidenceSummary: formatQuickVerifyCleanupLine(cleanup), valuesRedacted: true, }]; } if (cleanupClass !== "observer-stop-failed") return []; const aliveAfter = booleanAtNullable(cleanup, "aliveAfter"); const id = aliveAfter === true ? "observer-force-stop-left-process-alive" : "observer-cleanup-evidence-missing"; const summary = aliveAfter === true ? "observer force stop completed but process liveness evidence still reports aliveAfter=true." : "observer cleanup failed because stop-state evidence is missing or incomplete; this is an evidence gap, not confirmed process leakage."; return [{ id, severity: "red", count: 1, summary, cleanupPhase: cleanupStep.phase, cleanup, evidenceSummary: formatQuickVerifyCleanupLine(cleanup), valuesRedacted: true, }]; } export function classifyQuickVerifyCleanupStep(cleanupStep: Record): Record { const explicit = record(cleanupStep.cleanup); if (stringAtNullable(explicit, "cleanupClass") !== null) return explicit; const payload = quickVerifyCleanupPayload(cleanupStep); const observer = record(payload.observer); const forceStop = record(payload.forceStop); const detail = record(payload.detail); const forceResult = record(payload.forceResult); const sources = cleanupEvidenceSources([ ["payload.observer", observer], ["payload.forceStop", forceStop], ["payload.detail", detail], ["payload", payload], ["payload.forceResult", forceResult], ]); const forceReason = firstStringFromCleanupSources(sources, ["forceReason", "reason"]); const status = firstStringFromCleanupSources(sources, ["status"]); const command = firstStringFromCleanupSources(sources, ["command"]); const evidenceSourceNames = cleanupEvidenceSourceNames(sources, [ "aliveBefore", "aliveAfter", "termSent", "killSent", "pendingAbandoned", "processingAbandoned", ]); const forceStopAttempted = forceReason !== null || status === "forced-stopped" || status === "force-stopped" || command === "web-probe-observe force-stop" || payload.forced === true || numberAtNullable(forceResult, "exitCode") !== null || evidenceSourceNames.length > 0; const aliveBefore = firstBooleanFromCleanupSources(sources, "aliveBefore"); const aliveAfter = firstBooleanFromCleanupSources(sources, "aliveAfter"); const termSent = firstBooleanFromCleanupSources(sources, "termSent"); const killSent = firstBooleanFromCleanupSources(sources, "killSent"); const pendingAbandoned = firstNumberFromCleanupSources(sources, "pendingAbandoned"); const processingAbandoned = firstNumberFromCleanupSources(sources, "processingAbandoned"); const forceStopOk = forceStopAttempted && aliveAfter === false; const gracefulStopOk = !forceStopAttempted && (cleanupStep.ok === true || payload.ok === true); const cleanupClass = gracefulStopOk ? "stopped" : forceStopOk ? "bounded-force-stop" : "observer-stop-failed"; const missingEvidenceFields = forceStopAttempted && cleanupClass === "observer-stop-failed" ? [ aliveAfter === null ? "aliveAfter" : null, termSent === null ? "termSent" : null, killSent === null ? "killSent" : null, pendingAbandoned === null ? "pendingAbandoned" : null, processingAbandoned === null ? "processingAbandoned" : null, ].filter((value): value is string => value !== null) : []; return { cleanupClass, phase: stringAtNullable(cleanupStep, "phase"), commandOk: cleanupStep.ok === true ? true : cleanupStep.ok === false ? false : null, status, forceReason, gracefulStopOk, forceStopOk, aliveBefore, aliveAfter, termSent, killSent, pendingAbandoned, processingAbandoned, evidenceSources: evidenceSourceNames, missingEvidenceFields, valuesRedacted: true, }; } function quickVerifyCleanupPayload(cleanupStep: Record): Record { const payload = record(cleanupStep.payload); if (Object.keys(payload).length > 0) return payload; const result = record(cleanupStep.result); for (const key of ["stdoutPreview", "stdoutTail"]) { const parsed = parseJsonObject(stringAtNullable(result, key) ?? ""); const data = cliDataPayload(parsed); if (Object.keys(data).length > 0) return data; } return {}; } function cleanupEvidenceSources(entries: readonly (readonly [string, Record])[]): readonly { name: string; value: Record }[] { return entries .filter((entry) => Object.keys(entry[1]).length > 0) .map(([name, value]) => ({ name, value })); } function cleanupEvidenceSourceNames(sources: readonly { name: string; value: Record }[], keys: readonly string[]): string[] { const names: string[] = []; for (const source of sources) { if (keys.some((key) => source.value[key] !== undefined && source.value[key] !== null)) names.push(source.name); } return [...new Set(names)]; } function firstStringFromCleanupSources(sources: readonly { value: Record }[], keys: readonly string[]): string | null { for (const source of sources) { for (const key of keys) { const value = stringAtNullable(source.value, key); if (value !== null) return value; } } return null; } function firstBooleanFromCleanupSources(sources: readonly { value: Record }[], key: string): boolean | null { for (const source of sources) { const value = booleanAtNullable(source.value, key); if (value !== null) return value; } return null; } function firstNumberFromCleanupSources(sources: readonly { value: Record }[], key: string): number | null { for (const source of sources) { const value = numberAtNullable(source.value, key); if (value !== null) return value; } return null; } function quickVerifyCleanupFromSteps(steps: readonly Record[]): Record | null { for (const step of steps.slice().reverse()) { const phase = stringAtNullable(step, "phase"); if (phase !== null && phase.includes("observe-stop")) return classifyQuickVerifyCleanupStep(step); } return null; } function quickVerifyCleanupWarnings(cleanup: Record | null): string[] { if (cleanup === null) return []; const cleanupClass = stringAtNullable(cleanup, "cleanupClass"); if (cleanupClass === "bounded-force-stop") { return [`quick verify observer graceful stop did not drain, but force stop completed with aliveAfter=false; recorded bounded cleanup evidence (${formatQuickVerifyCleanupLine(cleanup)}).`]; } if (cleanupClass === "observer-stop-failed") { return ["quick verify observer stop failed; runner lifecycle cleanup is a blocking finding."]; } return []; } function formatQuickVerifyCleanupLine(cleanup: Record | null): string { if (cleanup === null) return "cleanup=-"; const fields = [ ["cleanup", stringAtNullable(cleanup, "cleanupClass")], ["aliveBefore", cleanup.aliveBefore], ["aliveAfter", cleanup.aliveAfter], ["termSent", cleanup.termSent], ["killSent", cleanup.killSent], ["pendingAbandoned", cleanup.pendingAbandoned], ["processingAbandoned", cleanup.processingAbandoned], ["gracefulStopOk", cleanup.gracefulStopOk], ["forceStopOk", cleanup.forceStopOk], ["missing", Array.isArray(cleanup.missingEvidenceFields) && cleanup.missingEvidenceFields.length > 0 ? cleanup.missingEvidenceFields.join(",") : null], ]; return fields.map(([key, value]) => `${key}=${value ?? "-"}`).join(" "); } function booleanAtNullable(value: Record, key: string): boolean | null { const raw = value[key]; return raw === true ? true : raw === false ? false : null; } function quickVerifyAnalysisVisibilityFindings(analysis: ChildCliResult, artifactSummary: Record): Record[] { if (analysis.ok !== true) return []; if (stringAtNullable(artifactSummary, "reportJsonSha256") !== null) return []; const result = record(analysis.result); const stdout = stringAtNullable(result, "stdoutPreview") ?? stringAtNullable(result, "stdoutTail"); const reportPathMatch = stdout === null ? null : stdout.match(/analysis\/report\.(?:json|md)/u); return [{ id: "quick-verify-analysis-summary-unreadable", severity: "red", count: 1, summary: "quick verify analyze exited successfully, but sentinel could not read analysis/report.json before recording the run.", rootCause: "The report index would otherwise contain only control blockers such as WBC-003 and hide analyzer red/amber findings.", rootCauseStatus: "confirmed", rootCauseConfidence: "high", evidenceSummary: `analyze ok=true reportJsonSha256=missing stdoutMentionsReport=${reportPathMatch === null ? "false" : "true"}`, nextAction: "Read stateDir/analysis/report.json again or rehydrate the report from stateDir before trusting WBC-003 as the only visible finding.", valuesRedacted: true, }]; } function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number): Record { const namespace = stringAt(state.runtime, "namespace"); const serviceName = stringAt(state.runtime, "serviceName"); const servicePort = numberAt(state.runtime, "servicePort"); const deploymentName = stringAt(state.runtime, "deploymentName"); const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`; const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64"); const postScript = [ "const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');", "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');", `const url = 'http://127.0.0.1:${servicePort}' + path;`, "fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {", " const text = await response.text();", " process.stdout.write(text);", " if (!response.ok) process.exit(22);", "}).catch((error) => {", " console.error(error && error.stack ? error.stack : String(error));", " process.exit(23);", "});", ].join(" "); const script = method === "GET" ? `kubectl get --raw ${shellQuote(proxyPath)}` : [ "set -eu", `kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`, ].join("\n"); const maxAttempts = method === "GET" ? 3 : 1; const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60)); const attempts: Record[] = []; let result: CommandResult | null = null; let parsed: Record | null = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const invocation = sentinelServiceProxyInvocation(stringAt(state.controlPlaneNode, "kubeRoute"), script); result = runCommand(invocation.argv, repoRoot, { input: invocation.input, timeoutMs: attemptTimeoutSeconds * 1000 }); const parsedResolution = resolveCliChildJsonCommandResult({ result, requestedStdoutType: "web-probe sentinel service proxy response JSON", acceptParsed: isSentinelServiceResponseContract, }); parsed = parsedResolution.parsed; attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, stdoutRecovery: parsedResolution.diagnostics, valuesRedacted: true }); if (result.exitCode === 0) break; } return { ok: result?.exitCode === 0, method, path: pathWithQuery, internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`, httpStatus: result?.exitCode === 0 ? 200 : null, bodyJson: record(compactSentinelServiceBodyJson(parsed)), bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "", bodyBytes: Buffer.byteLength(result?.stdout ?? ""), error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000), proxyPath, result: result === null ? null : compactCommand(result), attempts, valuesRedacted: true, }; } function isSentinelServiceResponseContract(value: Record): boolean { return value.ok === false || typeof value.view === "string" || typeof value.renderedText === "string" || typeof value.error === "string" || Array.isArray(value.availableViews) || Object.keys(record(value.run)).length > 0 || Object.keys(record(value.summary)).length > 0 || Array.isArray(value.findings); } export function sentinelServiceProxyInvocation(kubeRoute: string, script: string): { readonly argv: readonly string[]; readonly input: string } { return { argv: ["trans", kubeRoute, "sh"], input: script }; } function compactSentinelServiceBodyJson(value: Record | null): unknown { if (value === null || typeof value.renderedText !== "string") return value; return { ok: value.ok, view: value.view, error: value.error, availableViews: value.availableViews, valuesRedacted: true, }; } function clipTail(value: string, maxLength: number): string { return value.length <= maxLength ? value : value.slice(value.length - maxLength); } function recordSkippedQuickVerify( state: SentinelCicdState, scenario: Record, runId: string, scenarioId: string, triggerReason: string, elapsedMs: number, memoryGuard: unknown, steps: readonly Record[], skipReason: string, ): Record { const cadence = stringAtNullable(scenario, "cadence"); const renderedText = [ `# Web 哨兵巡检 ${runId}`, "", "本轮因主机物理可用内存低于 WebProbe 启动门槛而跳过,未创建 observer 或 browser。", `- 状态:skipped-wait-next-round`, `- 场景:${scenarioId}`, `- 下一步:等待下一次计划 cadence${cadence === null ? "" : `(${cadence})`}`, ].join("\n"); const recorded = recordQuickVerify(state, { ok: true, runId, scenarioId, reason: skipReason, triggerReason, status: "skipped-wait-next-round", decision: "skip-observer", observerId: null, observerCreated: false, browserCreated: false, runCreated: true, mutation: false, memoryGuard, elapsedMs, steps: [...steps], findingCount: 0, artifactCount: 0, findings: [], warnings: [], updatedAt: new Date().toISOString(), nextCadence: { action: "wait-for-next-scheduled-cadence", cadence, automaticRetry: true, valuesRedacted: true, }, views: { summary: { renderedText, valuesRedacted: true }, "auth-session-switch-summary": { renderedText, valuesRedacted: true }, }, valuesRedacted: true, }); return { ...recorded, runCreated: record(recorded.recordResult).ok === true, valuesRedacted: true, }; } function recordQuickVerify(state: SentinelCicdState, payload: Record): Record { const writer = resolveMonitorTerminalWriter(state.runtime); const views = compactQuickVerifyRecordViews(record(payload.views)); const summary = { reason: payload.reason, status: payload.status, businessStatus: payload.businessStatus ?? null, elapsedMs: payload.elapsedMs, failure: payload.failure, failureTitleZh: payload.failureTitleZh ?? payload.errorTitleZh ?? null, errorTitleZh: payload.errorTitleZh ?? payload.failureTitleZh ?? null, timeoutDisplay: payload.timeoutDisplay ?? null, warnings: Array.isArray(payload.warnings) ? payload.warnings : [], analysis: compactQuickVerifyRecordAnalysis(payload.analysis), cleanup: compactQuickVerifyCleanupEvidence(payload.cleanup), promptSource: payload.promptSource, steps: Array.isArray(payload.steps) ? payload.steps.map(compactQuickVerifyRecordStep) : [], valuesRedacted: true, }; const legacyRecord = { runId: payload.runId, scenarioId: payload.scenarioId, status: payload.status, observerId: payload.observerId, stateDir: payload.stateDir, reportJsonSha256: payload.reportJsonSha256, findingCount: payload.findingCount, artifactCount: payload.artifactCount, summary, businessStatus: payload.businessStatus ?? null, findings: payload.findings, views, screenshot: payload.screenshot, publicOrigin: payload.publicOrigin ?? stringAt(state.publicExposure, "publicBaseUrl"), maintenance: payload.reason === "maintenance-stop", valuesRedacted: true, }; const recordResult = writer.mode === "legacy-sqlite" ? callSentinelService(state, "POST", "/api/runs/record", legacyRecord, 60) : submitCentralTerminalIngest(state, payload, writer.ingest!, views, summary); emitWebProbeSentinelSpan(sentinelOtelContext(state), "web_probe_sentinel.quick_verify.job_finish", { scenarioId: payload.scenarioId, runId: payload.runId, observerId: payload.observerId, status: payload.status, exitCode: payload.ok === true && recordResult.ok === true ? 0 : 1, failureKind: payload.failure ?? (recordResult.ok === true ? null : writer.mode === "central" ? "monitor-ingest-failed" : "record-run-failed"), valuesRedacted: true, }, payload.ok === true && recordResult.ok === true); return withWarnings({ ...payload, views, terminalWriter: writer.mode, recordResult, valuesRedacted: true }, recordResult.ok === true ? [] : [writer.mode === "central" ? "monitor-ingest-failed: retry the same immutable analysis/report artifact after the central ingest endpoint recovers." : "quick verify completed but sentinel report index record failed; report/dashboard may lag until record payload is reduced or retried."]); } function submitCentralTerminalIngest(state: SentinelCicdState, payload: Record, config: NonNullable["ingest"]>, views: Record, summary: Record): Record { const analysis = record(payload.analysis); try { const stateDir = stringAtNullable(payload, "stateDir") ?? stringAtNullable(analysis, "stateDir"); const report = { ...analysis, updatedAt: stringAtNullable(analysis, "reportUpdatedAt") ?? stringAtNullable(payload, "updatedAt"), findings: Array.isArray(payload.findings) ? payload.findings.map(record) : [], summary, views, valuesRedacted: true }; const artifacts = payload.status === "skipped-wait-next-round" || (payload.observerId === null && payload.artifactCount === 0) ? [] : [{ relativePath: "analysis/report.json", kind: "report-json", sha256: stringAtNullable(payload, "reportJsonSha256") ?? stringAtNullable(analysis, "reportJsonSha256") ?? "", sizeBytes: numberAtNullable(analysis, "reportJsonBytes") ?? -1 }]; const ingest = buildMonitorTerminalIngest({ sentinelId: state.sentinelId, node: state.spec.nodeId, lane: state.spec.lane, runId: stringAt(payload, "runId") }, report, artifacts, { ownerNode: state.spec.nodeId, ownerLane: state.spec.lane, ownerPvc: stringAt(state.runtime, "pvcName"), stateDir: stateDir ?? "" }); const result = runCommand(["bun", "scripts/monitor-terminal-ingest-submit.ts"], repoRoot, { timeoutMs: config.retry.maxAttempts * config.timeoutMs + (config.retry.maxAttempts - 1) * config.retry.delayMs + 1_000, input: JSON.stringify({ config, ingest }) }); const response = parseJsonObject(result.stdout); if (result.exitCode === 0 && response?.ok === true) return { ...response, writer: "central", valuesRedacted: true }; return { ok: false, writer: "central", error: "monitor-ingest-failed", response, stderr: short(result.stderr), payloadHash: ingest.payloadHash, valuesRedacted: true }; } catch (error) { return { ok: false, writer: "central", error: "monitor-ingest-failed", details: error instanceof MonitorIngestFailedError ? error.details : { reason: error instanceof Error ? error.message : String(error) }, valuesRedacted: true }; } } function sentinelOtelContext(state: SentinelCicdState): { readonly node: string; readonly lane: string; readonly sentinelId: string; readonly namespace: string | null; readonly runtime: Record; readonly cicd: Record } { return { node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, namespace: stringAtNullable(state.runtime, "namespace"), runtime: state.runtime, cicd: state.cicd, }; } function compactQuickVerifyRecordViews(views: Record): Record { const compacted: Record = {}; for (const [key, value] of Object.entries(views)) { const item = record(value); const limit = key === "summary" || key === "auth-session-switch-summary" ? 8_000 : 6_000; compacted[key] = { ...item, renderedText: boundQuickVerifyRecordText(item.renderedText, limit), valuesRedacted: true, }; } return compacted; } function compactQuickVerifyRecordAnalysis(value: unknown): Record | null { const item = record(value); if (Object.keys(item).length === 0) return null; return { ok: item.ok === true ? true : item.ok === false ? false : null, reportOk: item.reportOk === true ? true : item.reportOk === false ? false : null, source: stringAtNullable(item, "source"), reason: stringAtNullable(item, "reason"), stateDir: stringAtNullable(item, "stateDir"), reportJsonPath: stringAtNullable(item, "reportJsonPath"), reportJsonSha256: stringAtNullable(item, "reportJsonSha256"), reportMdPath: stringAtNullable(item, "reportMdPath"), reportMdSha256: stringAtNullable(item, "reportMdSha256"), reportReadWaitMs: numberAtNullable(item, "reportReadWaitMs"), reportParseError: stringAtNullable(item, "reportParseError"), findingCount: numberAtNullable(item, "findingCount"), artifactCount: numberAtNullable(item, "artifactCount"), counts: compactQuickVerifyRecordCounts(record(item.counts)), readResult: compactQuickVerifyRecordArtifactReadResult(record(item.result)), screenshot: compactQuickVerifyRecordScreenshot(record(item.screenshot)), findings: Array.isArray(item.findings) ? item.findings.slice(0, 16).map(compactQuickVerifyRecordFinding) : [], pagePerformanceSlowApi: Array.isArray(item.pagePerformanceSlowApi) ? item.pagePerformanceSlowApi.slice(0, 6).map(record) : [], browserProcess: compactQuickVerifyRecordBrowserProcess(record(item.browserProcess)), requestRate: compactQuickVerifyRecordRequestRate(record(item.requestRate), record(item.requestRateCurve)), valuesRedacted: true, }; } function compactQuickVerifyRecordArtifactReadResult(value: Record): Record | null { if (Object.keys(value).length === 0) return null; return { exitCode: numberAtNullable(value, "exitCode"), timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null, stdoutBytes: numberAtNullable(value, "stdoutBytes"), stderrBytes: numberAtNullable(value, "stderrBytes"), stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 600), stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 600), valuesRedacted: true, }; } function compactQuickVerifyRecordBrowserProcess(value: Record): Record | null { const pageSeries = Array.isArray(value.pageSeries) ? value.pageSeries.map(compactQuickVerifyRecordMemorySeries).filter((item) => item.points.length > 0) : []; if (pageSeries.length === 0) return null; return { source: stringAtNullable(value, "source") ?? "analysis-browser-process", unit: stringAtNullable(value, "unit") ?? "MB", metric: stringAtNullable(value, "metric") ?? "page-heap-used", pageCount: pageSeries.length, sampleCount: pageSeries.reduce((sum, item) => sum + item.points.length, 0), pageSeries, valuesRedacted: true, }; } function compactQuickVerifyRecordRequestRate(summaryValue: Record, curveValue: Record): Record | null { const source = Object.keys(curveValue).length > 0 ? curveValue : summaryValue; const summary = record(source.summary); const effectiveSummary = Object.keys(summary).length > 0 ? summary : summaryValue; const pageCurves = Array.isArray(source.pageCurves) ? source.pageCurves.map((item) => compactQuickVerifyRecordRequestRateCurve(item, "page")).filter((item) => item.buckets.length > 0).slice(0, 12) : []; const apiPathCurves = Array.isArray(source.apiPathCurves) ? source.apiPathCurves.map((item) => compactQuickVerifyRecordRequestRateCurve(item, "apiPath")).filter((item) => item.buckets.length > 0).slice(0, 16) : []; const buckets = Array.isArray(source.buckets) ? source.buckets.map(compactQuickVerifyRecordRequestRateBucket).filter(hasCompactRequestRateBucketPoint).slice(-120) : []; if (Object.keys(source).length === 0 && pageCurves.length === 0 && apiPathCurves.length === 0 && buckets.length === 0) return null; return { summary: compactQuickVerifyRecordRequestRateSummary(effectiveSummary), buckets, pageCurves, apiPathCurves, peaks: Array.isArray(source.peaks) ? source.peaks.slice(0, 12).map(compactQuickVerifyRecordRequestRatePeak) : [], valuesRedacted: true, }; } function compactQuickVerifyRecordRequestRateSummary(value: Record): Record { return { bucketMs: numberAtNullable(value, "bucketMs"), bucketSeconds: numberAtNullable(value, "bucketSeconds"), requestCount: numberAtNullable(value, "requestCount"), bucketCount: numberAtNullable(value, "bucketCount"), pageCount: numberAtNullable(value, "pageCount"), apiPathCount: numberAtNullable(value, "apiPathCount"), firstAt: stringAtNullable(value, "firstAt"), lastAt: stringAtNullable(value, "lastAt"), totalRedPerMinute: numberAtNullable(value, "totalRedPerMinute"), pageRedPerMinute: numberAtNullable(value, "pageRedPerMinute"), apiPathRedPerMinute: numberAtNullable(value, "apiPathRedPerMinute"), totalPeakPerMinute: numberAtNullable(value, "totalPeakPerMinute"), totalPeakCount: numberAtNullable(value, "totalPeakCount"), totalPeakAt: stringAtNullable(value, "totalPeakAt"), pagePeakPerMinute: numberAtNullable(value, "pagePeakPerMinute"), pagePeakKey: stringAtNullable(value, "pagePeakKey"), pagePeakPath: stringAtNullable(value, "pagePeakPath"), apiPathPeakPerMinute: numberAtNullable(value, "apiPathPeakPerMinute"), apiPathPeakKey: stringAtNullable(value, "apiPathPeakKey"), overThresholdPeakCount: numberAtNullable(value, "overThresholdPeakCount"), valuesRedacted: true, }; } function compactQuickVerifyRecordRequestRateCurve(value: unknown, scope: string): Record & { buckets: Record[] } { const item = record(value); const buckets = Array.isArray(item.buckets) ? item.buckets.map(compactQuickVerifyRecordRequestRateBucket).filter(hasCompactRequestRateBucketPoint).slice(-120) : []; return { pageKey: stringAtNullable(item, "pageKey"), pageRole: stringAtNullable(item, "pageRole"), pageId: stringAtNullable(item, "pageId"), pageEpoch: numberAtNullable(item, "pageEpoch"), apiKey: stringAtNullable(item, "apiKey"), method: stringAtNullable(item, "method"), path: stringAtNullable(item, "path"), count: numberAtNullable(item, "count"), bucketCount: numberAtNullable(item, "bucketCount") ?? buckets.length, peakRequestPerMinute: numberAtNullable(item, "peakRequestPerMinute"), peakBucket: compactQuickVerifyRecordRequestRateBucket(record(item.peakBucket)), buckets, scope, valuesRedacted: true, }; } function compactQuickVerifyRecordRequestRateBucket(value: unknown): Record { const item = record(value); return { bucketStartMs: numberAtNullable(item, "bucketStartMs"), bucketEndMs: numberAtNullable(item, "bucketEndMs"), startAt: stringAtNullable(item, "startAt") ?? stringAtNullable(item, "t"), endAt: stringAtNullable(item, "endAt") ?? stringAtNullable(item, "e"), count: numberAtNullable(item, "count") ?? numberAtNullable(item, "c"), requestPerMinute: numberAtNullable(item, "requestPerMinute") ?? numberAtNullable(item, "rpm"), valuesRedacted: true, }; } function hasCompactRequestRateBucketPoint(value: Record): boolean { return stringAtNullable(value, "startAt") !== null && numberAtNullable(value, "requestPerMinute") !== null; } function compactQuickVerifyRecordRequestRatePeak(value: unknown): Record { const item = record(value); return { scope: stringAtNullable(item, "scope"), thresholdPerMinute: numberAtNullable(item, "thresholdPerMinute"), overThreshold: item.overThreshold === true, bucketMs: numberAtNullable(item, "bucketMs"), startAt: stringAtNullable(item, "startAt"), endAt: stringAtNullable(item, "endAt"), count: numberAtNullable(item, "count"), requestPerMinute: numberAtNullable(item, "requestPerMinute"), method: stringAtNullable(item, "method"), path: stringAtNullable(item, "path"), apiKey: stringAtNullable(item, "apiKey"), pageKey: stringAtNullable(item, "pageKey"), pageRole: stringAtNullable(item, "pageRole"), pageId: stringAtNullable(item, "pageId"), pageEpoch: numberAtNullable(item, "pageEpoch"), valuesRedacted: true, }; } function compactQuickVerifyRecordMemorySeries(value: unknown): Record & { points: Record[] } { const item = record(value); const points = Array.isArray(item.points) ? item.points.map(record).map((point) => ({ ts: stringAtNullable(point, "ts"), elapsedSeconds: numberAtNullable(point, "elapsedSeconds"), elapsedMinutes: numberAtNullable(point, "elapsedMinutes"), memoryMb: numberAtNullable(point, "memoryMb"), heapUsedMb: numberAtNullable(point, "heapUsedMb"), jsHeapUsedMb: numberAtNullable(point, "jsHeapUsedMb"), effectiveHeapUsedMb: numberAtNullable(point, "effectiveHeapUsedMb"), effectiveJsHeapUsedMb: numberAtNullable(point, "effectiveJsHeapUsedMb"), domNodes: numberAtNullable(point, "domNodes"), valuesRedacted: true, })).filter((point) => point.ts !== null && point.memoryMb !== null) : []; return { key: stringAtNullable(item, "key"), pageRole: stringAtNullable(item, "pageRole"), pageId: stringAtNullable(item, "pageId"), label: stringAtNullable(item, "label"), url: stringAtNullable(item, "url"), points, valuesRedacted: true, }; } function compactQuickVerifyRecordCounts(value: Record): Record { return { samples: numberAtNullable(value, "samples"), control: numberAtNullable(value, "control"), network: numberAtNullable(value, "network"), console: numberAtNullable(value, "console"), errors: numberAtNullable(value, "errors"), artifacts: numberAtNullable(value, "artifacts"), valuesRedacted: true, }; } function compactQuickVerifyRecordScreenshot(value: Record): Record | null { if (Object.keys(value).length === 0) return null; return { path: stringAtNullable(value, "path"), sha256: stringAtNullable(value, "sha256"), bytes: numberAtNullable(value, "bytes"), valuesRedacted: true, }; } function compactQuickVerifyRecordFinding(value: unknown): Record { const item = record(value); return { id: stringAtNullable(item, "id"), kind: stringAtNullable(item, "kind"), code: stringAtNullable(item, "code"), severity: stringAtNullable(item, "severity"), level: stringAtNullable(item, "level"), count: numberAtNullable(item, "count"), summary: boundQuickVerifyRecordText(item.summary ?? item.message, 220), rootCause: boundQuickVerifyRecordText(item.rootCause, 140), rootCauseStatus: boundQuickVerifyRecordText(item.rootCauseStatus, 90), rootCauseConfidence: boundQuickVerifyRecordText(item.rootCauseConfidence, 40), nextAction: boundQuickVerifyRecordText(item.nextAction, 240), evidenceSummary: stringAtNullable(item, "evidenceSummary") ?? compactQuickVerifyFindingEvidence(item.evidence), rootCauseSignals: compactQuickVerifyRootCauseSignals(item.rootCauseSignals), timingSourceOfTruth: boundQuickVerifyRecordText(item.timingSourceOfTruth ?? item.expectedElapsedSource ?? item.evidenceKind, 100), timingStatus: boundQuickVerifyRecordText(item.timingStatus, 60), timingAlert: item.timingAlert === true, blocking: item.blocking === true, valuesRedacted: true, }; } function compactQuickVerifyRootCauseSignals(value: unknown): Record | null { const item = record(value); const keys = [ "sessionListReadCount", "traceEventsReadCount", "webPerformanceBeaconFailureCount", "eventSourceFailureCount", "requestFailedCount", "httpErrorCount", "consoleAlertCount", "requestfailedTop", "httpStatusTop", ]; const compact: Record = {}; for (const key of keys) { const raw = item[key]; if (raw === null || raw === undefined) continue; compact[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw; } return Object.keys(compact).length === 0 ? null : { ...compact, valuesRedacted: true }; } function compactQuickVerifyFindingEvidence(value: unknown): string | null { const item = record(value); if (Object.keys(item).length === 0) return null; const keys = [ "http404Count", "responseErrorCount", "requestFailedCount", "statuses", "afterProjectedSeqs", "sinceSeqs", "traceIds", "maxFallbackRatio", "maxFallbackTitleCount", "overThresholdSampleCount", "majorityFallbackSampleCount", ]; const compact: Record = {}; for (const key of keys) { const raw = item[key]; if (raw === null || raw === undefined) continue; compact[key] = Array.isArray(raw) ? raw.slice(0, 6) : raw; } return Object.keys(compact).length === 0 ? null : boundQuickVerifyRecordText(JSON.stringify(compact), 240); } function compactQuickVerifyRecordStep(value: unknown): Record { const item = record(value); return { phase: stringAtNullable(item, "phase"), ok: item.ok === true ? true : item.ok === false ? false : null, promptIndex: numberAtNullable(item, "promptIndex"), checkId: stringAtNullable(item, "checkId"), failure: stringAtNullable(item, "failure"), cleanup: compactQuickVerifyCleanupEvidence(item.cleanup), result: compactQuickVerifyRecordStepResult(record(item.result)), valuesRedacted: true, }; } function compactQuickVerifyCleanupEvidence(value: unknown): Record | null { const item = record(value); if (stringAtNullable(item, "cleanupClass") === null) return null; return { cleanupClass: stringAtNullable(item, "cleanupClass"), phase: stringAtNullable(item, "phase"), commandOk: booleanAtNullable(item, "commandOk"), status: stringAtNullable(item, "status"), forceReason: stringAtNullable(item, "forceReason"), gracefulStopOk: booleanAtNullable(item, "gracefulStopOk"), forceStopOk: booleanAtNullable(item, "forceStopOk"), aliveBefore: booleanAtNullable(item, "aliveBefore"), aliveAfter: booleanAtNullable(item, "aliveAfter"), termSent: booleanAtNullable(item, "termSent"), killSent: booleanAtNullable(item, "killSent"), pendingAbandoned: numberAtNullable(item, "pendingAbandoned"), processingAbandoned: numberAtNullable(item, "processingAbandoned"), evidenceSources: Array.isArray(item.evidenceSources) ? item.evidenceSources.filter((value): value is string => typeof value === "string").slice(0, 6) : [], missingEvidenceFields: Array.isArray(item.missingEvidenceFields) ? item.missingEvidenceFields.filter((value): value is string => typeof value === "string").slice(0, 8) : [], valuesRedacted: true, }; } function compactQuickVerifyRecordStepResult(value: Record): Record { return { ok: value.ok === true ? true : value.ok === false ? false : null, status: stringAtNullable(value, "status"), view: stringAtNullable(value, "view"), exitCode: numberAtNullable(value, "exitCode"), timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null, stdoutBytes: numberAtNullable(value, "stdoutBytes"), stderrBytes: numberAtNullable(value, "stderrBytes"), stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 240), stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 240), stdoutTail: boundQuickVerifyRecordText(value.stdoutTail, 1200), stderrTail: boundQuickVerifyRecordText(value.stderrTail, 1200), valuesRedacted: true, }; } function boundQuickVerifyRecordText(value: unknown, maxChars: number): string | null { if (typeof value !== "string") return null; if (value.length <= maxChars) return value; return `${value.slice(0, maxChars)}\n[truncated ${value.length - maxChars} chars]`; } function renderQuickVerifySummary(input: Record): string { const artifact = record(input.artifactSummary); const findings = Array.isArray(input.findings) ? input.findings.map(record).slice(0, 8) : Array.isArray(artifact.findings) ? artifact.findings.map(record).slice(0, 8) : []; const findingCount = numberAtNullable(input, "findingCount") ?? numberAtNullable(artifact, "findingCount") ?? findings.length; const failureTitle = stringAtNullable(input, "failureTitleZh") ?? stringAtNullable(input, "errorTitleZh") ?? stringAtNullable(record(input.timeoutDisplay), "titleZh"); const cleanup = compactQuickVerifyCleanupEvidence(input.cleanup); return [ "Web Probe Sentinel Quick Verify", "=======================================================", `run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`, failureTitle === null ? "" : `errorTitle=${failureTitle} failure=${input.failure ?? "-"}`, `report=${artifact.reportJsonSha256 ?? "-"} artifacts=${artifact.artifactCount ?? "-"} findings=${findingCount}`, cleanup === null ? "" : formatQuickVerifyCleanupLine(cleanup), `publicOrigin=${input.publicOrigin ?? "-"}`, "", "Findings", findings.length === 0 ? "-" : findings.map(formatQuickVerifyFindingLine).join("\n"), ].filter((line) => line !== "").join("\n"); } function renderAuthSessionSwitchQuickVerifySummary(input: Record): string { const artifact = record(input.artifactSummary); const accountEnv = record(input.accountEnv); const cleanup = compactQuickVerifyCleanupEvidence(input.cleanup); const findingRows = Array.isArray(input.findings) ? input.findings.map(record).slice(0, 8) : Array.isArray(artifact.findings) ? artifact.findings.map(record).slice(0, 8) : []; const steps = Array.isArray(input.steps) ? input.steps.map(record) : []; const commandSteps = steps .filter((step) => { const phase = stringAtNullable(step, "phase") ?? ""; return phase.startsWith("observe-command-") || phase.startsWith("observe-session-invariance-") || phase === "quick-verify-account-env"; }) .map((step) => { const phase = stringAtNullable(step, "phase") ?? "-"; const ok = step.ok === true ? "ok" : step.ok === false ? "failed" : "-"; const result = record(step.result); const status = stringAtNullable(result, "status") ?? stringAtNullable(result, "failure") ?? "-"; return `${phase} ${ok} ${status}`; }); return [ "Auth Session Switch Quick Verify", "=======================================================", `run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`, stringAtNullable(input, "failureTitleZh") === null && stringAtNullable(input, "errorTitleZh") === null ? "" : `errorTitle=${stringAtNullable(input, "failureTitleZh") ?? stringAtNullable(input, "errorTitleZh") ?? "-"}`, `status=${artifact.ok === true ? "ok" : "blocked"} report=${artifact.reportJsonSha256 ?? "-"} publicOrigin=${input.publicOrigin ?? "-"}`, cleanup === null ? "" : formatQuickVerifyCleanupLine(cleanup), `accountEnv=${accountEnv.envCount ?? "-"} valuesRedacted=true`, "", "Command Sequence", commandSteps.length === 0 ? "-" : commandSteps.join("\n"), "", "Findings", findingRows.length === 0 ? "-" : findingRows.map(formatQuickVerifyFindingLine).join("\n"), ].filter((line) => line !== "").join("\n"); } function formatQuickVerifyFindingLine(item: Record): string { const title = stringAtNullable(item, "displayTitleZh") ?? stringAtNullable(item, "errorTitleZh") ?? stringAtNullable(record(item.timeoutDisplay), "titleZh"); const summary = title ?? item.summary ?? item.message ?? ""; return `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${summary}`; } function formatQuickVerifyTimingSuffix(item: Record): string { const status = stringAtNullable(item, "timingStatus"); const source = stringAtNullable(item, "timingSourceOfTruth") ?? stringAtNullable(item, "expectedElapsedSource") ?? stringAtNullable(item, "evidenceKind"); if (status === null && source === null) return ""; return ` timing=${[status === null ? null : `status=${status}`, source === null ? null : `source=${source}`].filter((part) => part !== null).join(" ")}`; }