// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard. // Responsibility: Quick-verify observe orchestration and artifact interpretation for web-probe sentinel P5 validation. import { createHash, randomUUID } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import type { CommandResult } from "./command"; import { runCommand } from "./command"; import { repoRoot, rootPath } from "./config"; import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref"; import type { ChildCliResult, CompactCommandResult, SentinelCicdState } from "./hwlab-node-web-sentinel-cicd"; import { arrayAt, compactCommand, displayPath, isRecord, mergeWarnings, numberAt, numberAtNullable, parseEnvFile, parseJsonObject, quickVerifyAccountEnv, record, recordTarget, nonEmptyString, secretSourcePaths, sentinelCliSuffix, shellQuote, short, stringAt, stringAtNullable, targetValidationElapsedWarnings, text, withWarnings, } from "./hwlab-node-web-sentinel-cicd"; function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record = {}): void { const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => { if (typeof value === "string") return [key, short(value)]; if (Array.isArray(value)) return [key, value.slice(0, 8)]; return [key, value]; })); process.stdout.write(`${JSON.stringify({ event: "sentinel.quick-verify.progress", at: new Date().toISOString(), node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, runId, ...compactExtra, phase, status, valuesRedacted: true, })}\n`); } export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record { const startedAt = Date.now(); const elapsedMs = () => Date.now() - startedAt; const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId"); const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds"); const scenario = findScenario(state, scenarioId); if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true }; const commandSequence = arrayAt(scenario, "commandSequence").map(record); const needsPromptSet = commandSequence.some((item) => stringAt(item, "type") === "sendPrompt" && inlinePromptText(item) === null); const prompts = needsPromptSet ? readPromptSetForScenario(state, scenario) : { ok: true as const, prompts: [], summary: { source: "not-required", promptCount: 0, valuesRedacted: true } }; if (!prompts.ok) return { ok: false, status: "blocked", reason: "prompt-source-unavailable", promptSource: prompts, valuesRedacted: true }; const accountEnv = quickVerifyAccountEnv(state); if (!accountEnv.ok) { const findings = [{ id: "quick-verify-account-secret-missing", severity: "red", count: arrayAt(accountEnv.summary, "missing").length || 1, summary: "quick verify could not materialize YAML-declared web account credentials for the observer runner.", missing: arrayAt(accountEnv.summary, "missing"), valuesRedacted: true, }]; return recordQuickVerify(state, { ok: false, runId: `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`, scenarioId, reason, status: "blocked", observerId: null, elapsedMs: 0, businessStatus: quickVerifyBusinessStatus("quick-verify-account-secret-missing", 0, null, null, 0, maxSeconds), steps: [{ phase: "quick-verify-account-env", ok: false, result: accountEnv.summary }], failure: "quick-verify-account-secret-missing", findingCount: findings.length, findings, promptSource: prompts.summary, accountEnv: accountEnv.summary, views: { summary: { renderedText: renderQuickVerifySummary({ scenarioId, artifactSummary: { ok: false, findings, findingCount: findings.length }, steps: [], publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ scenarioId, steps: [], findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, }, warnings: [], valuesRedacted: true, }); } const sampleIntervalMs = numberAt(scenario, "sampleIntervalMs"); const warningBudgetSeconds = maxSeconds; const hardBudgetSeconds = Math.min(timeoutSeconds, Math.max(maxSeconds, numberAt(scenario, "maxRunSeconds"))); const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", warningBudgetSeconds); const deadline = Date.now() + hardBudgetSeconds * 1000; const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; printQuickVerifyProgress(state, runId, "start", "running", { scenarioId, reason, warningBudgetSeconds, hardBudgetSeconds, timeoutSeconds }); const steps: Record[] = []; const startArgs = [ "web-probe", "observe", "start", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--target-path", stringAt(scenario, "observeTargetPath"), "--sample-interval-ms", String(sampleIntervalMs), "--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")), "--command-timeout-seconds", "55", ]; const viewport = stringAtNullable(scenario, "viewport"); if (viewport !== null) startArgs.push("--viewport", viewport); printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) }); const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env); steps.push({ phase: "observe-start", ok: started.ok, result: started.result }); const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? "")); printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() }); if (!started.ok || observerId === null) { const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null); return recordQuickVerify(state, { ok: false, runId, scenarioId, reason, status: "blocked", observerId, elapsedMs: elapsedMs(), businessStatus: quickVerifyBusinessStatus("observe-start-failed", 0, null, null, elapsedMs(), maxSeconds), steps, failure: "observe-start-failed", findingCount: findings.length, findings, warnings: elapsedWarnings(), valuesRedacted: true, }); } printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) }); const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds); steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady }); printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", startupReady.ok === true ? "succeeded" : "failed", { observerId, failure: startupReady.failure ?? null, status: startupReady.status ?? null, heartbeatStatus: startupReady.heartbeatStatus ?? null, elapsedMs: elapsedMs() }); if (startupReady.ok !== true) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, promptIndex: 0, steps, failure: text(startupReady.failure ?? "observe-startup-ready-wait-failed"), elapsedMs: elapsedMs(), warnings: mergeWarnings(Array.isArray(startupReady.warnings) ? startupReady.warnings : [], elapsedWarnings()), promptSource: prompts.summary, })); } let promptIndex = 0; const sessionInvarianceChecks = sessionInvarianceChecksByRound(scenario); const nonBlockingCanaryWarnings: string[] = []; for (const item of commandSequence) { const type = stringAt(item, "type"); const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1); for (let index = 0; index < repeat; index += 1) { if (Date.now() >= deadline) { printQuickVerifyProgress(state, runId, "timeout", "failed", { observerId, promptIndex, elapsedMs: elapsedMs(), hardBudgetSeconds }); return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, promptIndex, steps, failure: "quick-verify-timeout-over-budget", elapsedMs: elapsedMs(), warnings: mergeWarnings(`quick verify exceeded the hard ${hardBudgetSeconds}s execution budget after the configured ${warningBudgetSeconds}s targetValidation warning budget.`, elapsedWarnings()), promptSource: prompts.summary, })); } const commandWaitMs = Math.max(1000, Math.trunc(numberAtNullable(item, "commandWaitMs") ?? numberAtNullable(scenario, "commandWaitMs") ?? 55_000)); const commandTimeoutSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "commandTimeoutSeconds") ?? numberAtNullable(scenario, "commandTimeoutSeconds") ?? 55)); const turnWaitChunkSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "turnWaitChunkSeconds") ?? numberAtNullable(scenario, "turnWaitChunkSeconds") ?? 55)); const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", String(commandWaitMs), "--command-timeout-seconds", String(remainingSeconds(deadline, commandTimeoutSeconds))]; if (type === "selectProvider") args.push("--provider", stringAt(item, "provider")); if (type === "loginAccount" || type === "listSessions" || type === "logout") { const accountId = stringAtNullable(item, "accountId"); if (accountId !== null) args.push("--account-id", accountId); } if (type === "switchSessions") { const fromAccountId = stringAtNullable(item, "fromAccountId"); const toAccountId = stringAtNullable(item, "toAccountId"); if (fromAccountId !== null) args.push("--from-account-id", fromAccountId); if (toAccountId !== null) args.push("--to-account-id", toAccountId); } if (type === "sendPrompt") { args.push("--text", inlinePromptText(item) ?? prompts.prompts[promptIndex % prompts.prompts.length] ?? ""); args.push("--expected-action-wait-ms", String(numberAtNullable(item, "expectedActionWaitMs") ?? 45000)); promptIndex += 1; } appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" }); printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat }); const commandResult = runChildCli(args, remainingSeconds(deadline, Math.max(60, commandTimeoutSeconds + 10))); steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result }); printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() }); if (!commandResult.ok) { if (type === "sendPrompt") { printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) }); const delayedWaitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds); steps.push({ phase: "observe-wait-turn-terminal-after-command-timeout", ok: delayedWaitResult.ok, promptIndex, result: delayedWaitResult }); printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", delayedWaitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: delayedWaitResult.failure ?? null, status: delayedWaitResult.status ?? null, traceId: delayedWaitResult.traceId ?? null, elapsedMs: elapsedMs() }); if (delayedWaitResult.ok === true) { const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps); nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings)); printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() }); if (invariantResult.ok !== true) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, promptIndex, steps, failure: text(record(invariantResult).failure ?? "observe-session-invariance-failed"), promptSource: prompts.summary, elapsedMs: elapsedMs(), warnings: mergeWarnings(record(invariantResult).warnings, elapsedWarnings()), })); } continue; } } return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, promptIndex, steps, failure: `observe-command-${type}-failed`, elapsedMs: elapsedMs(), warnings: elapsedWarnings(), promptSource: prompts.summary, })); } if (type === "sendPrompt") { printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) }); const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds); steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult }); printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() }); if (waitResult.ok !== true) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, promptIndex, steps, failure: text(waitResult.failure ?? "observe-turn-terminal-wait-failed"), promptSource: prompts.summary, elapsedMs: elapsedMs(), warnings: mergeWarnings(Array.isArray(waitResult.warnings) ? waitResult.warnings : [], elapsedWarnings()), })); } const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps); nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings)); printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() }); if (invariantResult.ok !== true) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, reason, observerId, promptIndex, steps, failure: text(invariantResult.failure ?? "observe-session-invariance-check-failed"), promptSource: prompts.summary, elapsedMs: elapsedMs(), warnings: mergeWarnings(nonBlockingCanaryWarnings, elapsedWarnings()), })); } } } } printQuickVerifyProgress(state, runId, "observe-analyze", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 120) }); const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120))], remainingSeconds(deadline, 120)); steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result }); printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() }); const indexEntry = readLocalObserveIndex(observerId); const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, remainingSeconds(deadline, 30)); const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30)); const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30)); const controlFindings = quickVerifyControlFindings(null, promptIndex, turnSummary, traceFrame); const artifactSummaryRecord = record(artifactSummary); const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : []; const findings = mergeFindingRecords(artifactFindings, controlFindings); const blockingFindings = findings.filter(isQuickVerifyBlockingFinding); const analysisWarnings = analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."]; const ok = record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0; const businessStatus = quickVerifyBusinessStatus(null, promptIndex, turnSummary, traceFrame, elapsedMs(), maxSeconds); printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() }); return recordQuickVerify(state, { ok, runId, scenarioId, reason, status: ok ? "analyzed" : "blocked", observerId, elapsedMs: elapsedMs(), businessStatus, stateDir: indexEntry?.stateDir ?? null, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0, failure: controlFindings.length > 0 ? "quick-verify-no-business-turn" : blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null, promptSource: prompts.summary, accountEnv: accountEnv.summary, steps, analysis: artifactSummary, views: { summary: { renderedText: renderQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok }, "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok }, }, findings, screenshot: record(artifactSummary).screenshot, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), warnings: mergeWarnings(analysisWarnings, nonBlockingCanaryWarnings, elapsedWarnings()), valuesRedacted: true, }); } function sessionInvarianceChecksByRound(scenario: Record): Map[]> { const checks = new Map[]>(); const items = Array.isArray(scenario.sessionInvarianceChecks) ? scenario.sessionInvarianceChecks.map(record) : []; for (const item of items) { const afterRound = typeof item.afterRound === "number" && Number.isInteger(item.afterRound) ? item.afterRound : null; if (afterRound === null || afterRound < 0) continue; const list = checks.get(afterRound) ?? []; list.push(item); checks.set(afterRound, list); } return checks; } function runQuickVerifySessionInvarianceChecks( state: SentinelCicdState, observerId: string, checks: readonly Record[], deadline: number, promptIndex: number, steps: Record[], ): Record { const warnings: string[] = []; for (const check of checks) { const checkId = nonEmptyString(check.id) ?? `after-round-${promptIndex}`; const blocking = check.blocking === true; const commands: { readonly type: string; readonly enabled: boolean }[] = [ { type: "refreshCurrentSession", enabled: check.refreshCurrent === true }, { type: "switchAwayAndBack", enabled: check.switchAwayAndBack === true }, { type: "assertSessionInvariant", enabled: check.assertSessionInvariant !== false }, ]; for (const command of commands) { if (!command.enabled) continue; const args = [ "web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", command.type, "--after-round", String(promptIndex), "--wait-ms", "55000", "--command-timeout-seconds", String(remainingSeconds(deadline, 55)), ]; const severity = nonEmptyString(check.severity); const findingId = nonEmptyString(check.findingId); const expectedSentinelRange = nonEmptyString(check.expectedSentinelRange); const alternateSessionStrategy = nonEmptyString(check.alternateSessionStrategy); if (severity !== null) args.push("--severity", severity); if (findingId !== null) args.push("--finding-id", findingId); if (expectedSentinelRange !== null) args.push("--expected-sentinel-range", expectedSentinelRange); if (command.type === "switchAwayAndBack" && alternateSessionStrategy !== null) args.push("--alternate-session-strategy", alternateSessionStrategy); if (command.type === "assertSessionInvariant" && check.requireComposerReady === true) args.push("--require-composer-ready"); args.push(check.blocking === true ? "--blocking" : "--non-blocking"); const result = runChildCli(args, remainingSeconds(deadline, 60)); steps.push({ phase: `observe-session-invariance-${command.type}`, ok: result.ok, promptIndex, checkId, result: result.result }); if (!result.ok) { if (!blocking) { warnings.push(`non-blocking session invariance canary ${checkId}/${command.type} failed after round ${promptIndex}; continuing Code Agent multi-round quick verify because scenario marks this check blocking=false.`); continue; } return { ok: false, failure: `observe-session-invariance-${command.type}-failed`, checkId, promptIndex, valuesRedacted: true }; } } } return { ok: true, promptIndex, checkCount: checks.length, warnings, valuesRedacted: true }; } function appendScenarioObserveCommandArgs(args: string[], item: Record, options: { readonly skipText?: boolean } = {}): void { const mappings: readonly (readonly [string, string])[] = [ ["path", "--path"], ["label", "--label"], ["sessionId", "--session-id"], ["provider", "--provider"], ["accountId", "--account-id"], ["fromAccountId", "--from-account-id"], ["toAccountId", "--to-account-id"], ["sourceId", "--source-id"], ["fileRef", "--file-ref"], ["filename", "--filename"], ["taskRef", "--task-ref"], ["taskId", "--task-id"], ["task", "--task"], ["field", "--field"], ["link", "--link"], ["title", "--title"], ["body", "--body"], ["status", "--status"], ["hwpodId", "--hwpod-id"], ["nodeId", "--node-id"], ["workspaceRoot", "--workspace-root"], ["root", "--root"], ]; for (const [key, flag] of mappings) { if (args.includes(flag)) continue; const value = stringAtNullable(item, key); if (value !== null) args.push(flag, value); } if (options.skipText !== true && !args.includes("--text")) { const text = stringAtNullable(item, "text") ?? stringAtNullable(item, "value"); if (text !== null) args.push("--text", text); } if (item.waitProjectManagementReady === true && !args.includes("--wait-project-management-ready")) args.push("--wait-project-management-ready"); } function inlinePromptText(item: Record): string | null { return stringAtNullable(item, "text") ?? stringAtNullable(item, "prompt") ?? stringAtNullable(item, "value"); } function finalizeQuickVerifyFailure(state: SentinelCicdState, input: { readonly runId: string; readonly scenarioId: string; readonly reason: string; readonly observerId: string; readonly promptIndex: number; readonly steps: readonly Record[]; readonly failure: string; readonly promptSource?: Record; readonly elapsedMs?: number; readonly warnings?: readonly unknown[]; }): Record { const cleanupSteps: Record[] = []; if (input.promptIndex > 0) { const cancel = runChildCli([ "web-probe", "observe", "command", input.observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", "cancel", "--wait-ms", "55000", "--command-timeout-seconds", "55", ], 60); cleanupSteps.push({ phase: "observe-cancel-after-failure", ok: cancel.ok, result: cancel.result }); } const stop = runChildCli([ "web-probe", "observe", "stop", input.observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--force", "--command-timeout-seconds", "55", ], 30); cleanupSteps.push({ phase: "observe-stop-after-failure", ok: stop.ok, result: stop.result }); const analysis = runChildCli([ "web-probe", "observe", "analyze", input.observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", "55", ], 60); cleanupSteps.push({ phase: "observe-analyze-after-failure", ok: analysis.ok, result: analysis.result }); const indexEntry = readLocalObserveIndex(input.observerId); const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId: input.observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, 30); const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30); const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30); const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(input.promptIndex, turnSummary, traceFrame); const controlFindings = quickVerifyControlFindings(input.failure, input.promptIndex, turnSummary, traceFrame); const artifactSummaryRecord = record(artifactSummary); const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : []; const findings = mergeFindingRecords(artifactFindings, controlFindings); const blockingFindings = findings.filter(isQuickVerifyBlockingFinding); const recoveredWaitFailure = durableBusinessTurn && isRecoverableQuickVerifyWaitFailure(input.failure) && record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0; const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, numberAt(state.cicd, "targetValidation.maxSeconds")); return { ok: recoveredWaitFailure, runId: input.runId, scenarioId: input.scenarioId, reason: input.reason, status: recoveredWaitFailure ? "analyzed" : "blocked", observerId: input.observerId, elapsedMs: input.elapsedMs ?? null, businessStatus, stateDir: indexEntry?.stateDir ?? null, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0, failure: recoveredWaitFailure ? null : input.failure, promptSource: input.promptSource, steps: [...input.steps, ...cleanupSteps], analysis: artifactSummary, views: { summary: { renderedText: renderQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, findings, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok }, "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok }, }, findings, screenshot: record(artifactSummary).screenshot, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), warnings: mergeWarnings( Array.isArray(input.warnings) ? input.warnings : [], recoveredWaitFailure ? ["quick verify wait command timed out, but collected turn-summary/trace-frame artifacts show a durable completed business turn; treating the wait timeout as a non-blocking tool finding."] : [], targetValidationElapsedWarnings(input.elapsedMs ?? null, "quick verify confirm-wait", numberAt(state.cicd, "targetValidation.maxSeconds")), ), valuesRedacted: true, }; } function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number): Record { const namespace = stringAt(state.runtime, "namespace"); const serviceName = stringAt(state.runtime, "serviceName"); const servicePort = numberAt(state.runtime, "servicePort"); const deploymentName = stringAt(state.runtime, "deploymentName"); const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`; const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64"); const postScript = [ "const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');", "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');", `const url = 'http://127.0.0.1:${servicePort}' + path;`, "fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {", " const text = await response.text();", " process.stdout.write(text);", " if (!response.ok) process.exit(22);", "}).catch((error) => {", " console.error(error && error.stack ? error.stack : String(error));", " process.exit(23);", "});", ].join(" "); const script = method === "GET" ? `kubectl get --raw ${shellQuote(proxyPath)}` : [ "set -eu", `kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`, ].join("\n"); const maxAttempts = method === "GET" ? 3 : 1; const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60)); const attempts: Record[] = []; let result: CommandResult | null = null; let parsed: Record | null = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 }); parsed = parseJsonObject(result.stdout); attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, valuesRedacted: true }); if (result.exitCode === 0) break; } return { ok: result?.exitCode === 0, method, path: pathWithQuery, internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`, httpStatus: result?.exitCode === 0 ? 200 : null, bodyJson: record(compactSentinelServiceBodyJson(parsed)), bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "", bodyBytes: Buffer.byteLength(result?.stdout ?? ""), error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000), proxyPath, result: result === null ? null : compactCommand(result), attempts, valuesRedacted: true, }; } function compactSentinelServiceBodyJson(value: Record | null): unknown { if (value === null || typeof value.renderedText !== "string") return value; return { ok: value.ok, view: value.view, error: value.error, availableViews: value.availableViews, valuesRedacted: true, }; } function clipTail(value: string, maxLength: number): string { return value.length <= maxLength ? value : value.slice(value.length - maxLength); } function recordQuickVerify(state: SentinelCicdState, payload: Record): Record { const views = compactQuickVerifyRecordViews(record(payload.views)); const summary = { reason: payload.reason, status: payload.status, businessStatus: payload.businessStatus ?? null, elapsedMs: payload.elapsedMs, failure: payload.failure, warnings: Array.isArray(payload.warnings) ? payload.warnings : [], analysis: compactQuickVerifyRecordAnalysis(payload.analysis), promptSource: payload.promptSource, steps: Array.isArray(payload.steps) ? payload.steps.map(compactQuickVerifyRecordStep) : [], valuesRedacted: true, }; const recordResult = callSentinelService(state, "POST", "/api/runs/record", { runId: payload.runId, scenarioId: payload.scenarioId, status: payload.status, observerId: payload.observerId, stateDir: payload.stateDir, reportJsonSha256: payload.reportJsonSha256, findingCount: payload.findingCount, artifactCount: payload.artifactCount, summary, businessStatus: payload.businessStatus ?? null, findings: payload.findings, views, screenshot: payload.screenshot, publicOrigin: payload.publicOrigin ?? stringAt(state.publicExposure, "publicBaseUrl"), maintenance: payload.reason === "maintenance-stop", valuesRedacted: true, }, 60); return withWarnings({ ...payload, views, recordResult, valuesRedacted: true }, recordResult.ok === true ? [] : ["quick verify completed but sentinel report index record failed; report/dashboard may lag until record payload is reduced or retried."]); } function compactQuickVerifyRecordViews(views: Record): Record { const compacted: Record = {}; for (const [key, value] of Object.entries(views)) { const item = record(value); const limit = key === "summary" || key === "auth-session-switch-summary" ? 8_000 : 6_000; compacted[key] = { ...item, renderedText: boundQuickVerifyRecordText(item.renderedText, limit), valuesRedacted: true, }; } return compacted; } function compactQuickVerifyRecordAnalysis(value: unknown): Record | null { const item = record(value); if (Object.keys(item).length === 0) return null; return { ok: item.ok === true ? true : item.ok === false ? false : null, reportOk: item.reportOk === true ? true : item.reportOk === false ? false : null, reason: stringAtNullable(item, "reason"), stateDir: stringAtNullable(item, "stateDir"), reportJsonSha256: stringAtNullable(item, "reportJsonSha256"), reportMdSha256: stringAtNullable(item, "reportMdSha256"), findingCount: numberAtNullable(item, "findingCount"), artifactCount: numberAtNullable(item, "artifactCount"), counts: compactQuickVerifyRecordCounts(record(item.counts)), screenshot: compactQuickVerifyRecordScreenshot(record(item.screenshot)), findings: Array.isArray(item.findings) ? item.findings.slice(0, 16).map(compactQuickVerifyRecordFinding) : [], pagePerformanceSlowApi: Array.isArray(item.pagePerformanceSlowApi) ? item.pagePerformanceSlowApi.slice(0, 6).map(record) : [], valuesRedacted: true, }; } function compactQuickVerifyRecordCounts(value: Record): Record { return { samples: numberAtNullable(value, "samples"), control: numberAtNullable(value, "control"), network: numberAtNullable(value, "network"), console: numberAtNullable(value, "console"), errors: numberAtNullable(value, "errors"), artifacts: numberAtNullable(value, "artifacts"), valuesRedacted: true, }; } function compactQuickVerifyRecordScreenshot(value: Record): Record | null { if (Object.keys(value).length === 0) return null; return { path: stringAtNullable(value, "path"), sha256: stringAtNullable(value, "sha256"), bytes: numberAtNullable(value, "bytes"), valuesRedacted: true, }; } function compactQuickVerifyRecordFinding(value: unknown): Record { const item = record(value); return { id: stringAtNullable(item, "id"), kind: stringAtNullable(item, "kind"), code: stringAtNullable(item, "code"), severity: stringAtNullable(item, "severity"), level: stringAtNullable(item, "level"), count: numberAtNullable(item, "count"), summary: boundQuickVerifyRecordText(item.summary ?? item.message, 220), rootCause: boundQuickVerifyRecordText(item.rootCause, 140), rootCauseStatus: boundQuickVerifyRecordText(item.rootCauseStatus, 90), rootCauseConfidence: boundQuickVerifyRecordText(item.rootCauseConfidence, 40), nextAction: boundQuickVerifyRecordText(item.nextAction, 240), evidenceSummary: stringAtNullable(item, "evidenceSummary") ?? compactQuickVerifyFindingEvidence(item.evidence), rootCauseSignals: compactQuickVerifyRootCauseSignals(item.rootCauseSignals), timingSourceOfTruth: boundQuickVerifyRecordText(item.timingSourceOfTruth ?? item.expectedElapsedSource ?? item.evidenceKind, 100), timingStatus: boundQuickVerifyRecordText(item.timingStatus, 60), timingAlert: item.timingAlert === true, blocking: item.blocking === true, valuesRedacted: true, }; } function compactQuickVerifyRootCauseSignals(value: unknown): Record | null { const item = record(value); const keys = [ "sessionListReadCount", "traceEventsReadCount", "webPerformanceBeaconFailureCount", "eventSourceFailureCount", "requestFailedCount", "httpErrorCount", "consoleAlertCount", "requestfailedTop", "httpStatusTop", ]; const compact: Record = {}; for (const key of keys) { const raw = item[key]; if (raw === null || raw === undefined) continue; compact[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw; } return Object.keys(compact).length === 0 ? null : { ...compact, valuesRedacted: true }; } function compactQuickVerifyFindingEvidence(value: unknown): string | null { const item = record(value); if (Object.keys(item).length === 0) return null; const keys = [ "http404Count", "responseErrorCount", "requestFailedCount", "statuses", "afterProjectedSeqs", "sinceSeqs", "traceIds", "maxFallbackRatio", "maxFallbackTitleCount", "overThresholdSampleCount", "majorityFallbackSampleCount", ]; const compact: Record = {}; for (const key of keys) { const raw = item[key]; if (raw === null || raw === undefined) continue; compact[key] = Array.isArray(raw) ? raw.slice(0, 6) : raw; } return Object.keys(compact).length === 0 ? null : boundQuickVerifyRecordText(JSON.stringify(compact), 240); } function compactQuickVerifyRecordStep(value: unknown): Record { const item = record(value); return { phase: stringAtNullable(item, "phase"), ok: item.ok === true ? true : item.ok === false ? false : null, promptIndex: numberAtNullable(item, "promptIndex"), checkId: stringAtNullable(item, "checkId"), failure: stringAtNullable(item, "failure"), result: compactQuickVerifyRecordStepResult(record(item.result)), valuesRedacted: true, }; } function compactQuickVerifyRecordStepResult(value: Record): Record { return { ok: value.ok === true ? true : value.ok === false ? false : null, status: stringAtNullable(value, "status"), view: stringAtNullable(value, "view"), exitCode: numberAtNullable(value, "exitCode"), timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null, stdoutBytes: numberAtNullable(value, "stdoutBytes"), stderrBytes: numberAtNullable(value, "stderrBytes"), stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 240), stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 240), valuesRedacted: true, }; } function boundQuickVerifyRecordText(value: unknown, maxChars: number): string | null { if (typeof value !== "string") return null; if (value.length <= maxChars) return value; return `${value.slice(0, maxChars)}\n[truncated ${value.length - maxChars} chars]`; } function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: string, timeoutSeconds: number): Record { if (!isSafeRelativeStateDir(stateDir)) return { ok: false, reason: "unsafe-state-dir", stateDir, valuesRedacted: true }; const script = [ "set -eu", `state_dir=${shellQuote(stateDir)}`, "node - \"$state_dir\" <<'NODE'", "const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');", "const stateDir=process.argv[2]; const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');", "const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}}; const jsonBuf=read(reportPath);", "const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;", "const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);", "const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};", "let report=null; try{report=jsonBuf?JSON.parse(jsonBuf.toString('utf8')):null}catch{}", "let artifactCount=0; let screenshot=null;", "function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}", "walk(stateDir);", "const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,rootCauseSignals:compactRootCauseSignals(v.rootCauseSignals),blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});", "const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});", "console.log(JSON.stringify({ok:!!report,reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));", "NODE", ].join("\n"); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; } function waitForQuickVerifyObserverStartup(state: SentinelCicdState, observerId: string, deadline: number, pollIntervalMs: number, budgetSeconds: number): Record { const observations: Record[] = []; const indexEntry = readLocalObserveIndex(observerId); if (indexEntry === null) { return { ok: false, failure: "observe-index-entry-missing", observerId, valuesRedacted: true, }; } const pollSleepMs = Math.max(250, Math.min(500, Math.trunc(pollIntervalMs / 2) || 250)); while (Date.now() < deadline) { const waitMs = Math.max(1000, Math.min(55_000, deadline - Date.now())); const script = quickVerifyObserverStartupWaitScript(indexEntry.stateDir, waitMs, pollSleepMs); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 5000 }); const payload = parseJsonObject(result.stdout); if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record)); const terminalPayload = { observerId, stateDir: indexEntry.stateDir, status: typeof payload?.status === "string" ? payload.status : null, heartbeatStatus: typeof payload?.heartbeatStatus === "string" ? payload.heartbeatStatus : null, startup: record(payload?.startup), observations: observations.slice(-6), waitResult: compactCommand(result), valuesRedacted: true, }; if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-startup-wait-chunk-timeout") { return { ok: false, failure: text(payload?.failure ?? "quick-verify-startup-artifact-wait-failed"), ...terminalPayload, }; } if (payload.ok === true) return { ok: true, ...terminalPayload }; } return { ok: false, failure: "quick-verify-timeout-over-budget", observerId, stateDir: indexEntry.stateDir, observations: observations.slice(-6), warnings: [`quick verify exceeded the configured ${budgetSeconds}s targetValidation budget while waiting for the observe runner startup to finish before sending the first command.`], valuesRedacted: true, }; } function quickVerifyObserverStartupWaitScript(stateDir: string, timeoutMs: number, pollSleepMs: number): string { return [ "set -eu", `state_dir=${shellQuote(stateDir)}`, `timeout_ms=${shellQuote(String(Math.max(1, Math.trunc(timeoutMs))))}`, `poll_ms=${shellQuote(String(Math.max(250, Math.trunc(pollSleepMs))))}`, "test -d \"$state_dir\" || { printf '{\"ok\":false,\"failure\":\"state-dir-missing\",\"stateDir\":\"%s\",\"valuesRedacted\":true}\\n' \"$state_dir\"; exit 0; }", "node - \"$state_dir\" \"$timeout_ms\" \"$poll_ms\" <<'NODE'", "const fs = require('node:fs');", "const path = require('node:path');", "const dir = process.argv[2];", "const timeoutMs = Number(process.argv[3]);", "const pollMs = Number(process.argv[4]);", "const startedAt = Date.now();", "const startupIds = ['startup-login', 'startup-goto', 'startup-observer-goto'];", "const readJson = (rel) => { try { return JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); } catch { return null; } };", "const readJsonl = (rel) => { try { return fs.readFileSync(path.join(dir, rel), 'utf8').split(/\\r?\\n/u).filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean); } catch { return []; } };", "const clip = (value, limit = 160) => value == null ? null : String(value).replace(/\\s+/gu, ' ').trim().slice(0, limit);", "const norm = (value) => String(value || '').trim().toLowerCase().replace(/_/gu, '-');", "const terminal = new Set(['failed', 'force-stopped', 'stopped', 'abandoned', 'completed']);", "function commandEvents(control, id) { return control.filter((item) => item && item.commandId === id); }", "function lastPhase(control, id) { return commandEvents(control, id).filter((item) => typeof item.phase === 'string').slice(-1)[0]?.phase || null; }", "function firstFailedStartup(control) { return control.filter((item) => item && startupIds.includes(item.commandId) && item.phase === 'failed').slice(-1)[0] || null; }", "function rowFor() {", " const heartbeat = readJson('heartbeat.json') || {};", " const manifest = readJson('manifest.json') || {};", " const control = readJsonl('control.jsonl');", " const phases = Object.fromEntries(startupIds.map((id) => [id, lastPhase(control, id)]));", " const failed = firstFailedStartup(control);", " const heartbeatStatus = norm(heartbeat.status || manifest.status);", " const ready = startupIds.every((id) => phases[id] === 'completed') && heartbeatStatus === 'running';", " const terminalBeforeReady = !ready && terminal.has(heartbeatStatus);", " const degraded = control.filter((item) => item && item.type === 'observer-startup-degraded').slice(-1)[0] || null;", " return {", " ok: ready,", " status: ready ? 'startup-ready' : terminalBeforeReady ? 'startup-terminal' : 'startup-waiting',", " heartbeatStatus,", " startup: { phases, failedCommandId: failed?.commandId || null, failedType: failed?.type || null, failedMessage: clip(failed?.detail?.error?.message || failed?.detail?.error || failed?.error?.message), observerStartupDegraded: !!degraded, degradedReason: clip(degraded?.reason || degraded?.result?.failureKind || degraded?.result?.reason), sampleSeq: heartbeat.sampleSeq ?? null, commandSeq: heartbeat.commandSeq ?? null, currentUrl: clip(heartbeat.currentUrl, 180), observerUrl: clip(heartbeat.observerUrl, 180), valuesRedacted: true },", " valuesRedacted: true", " };", "}", "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", "(async () => {", " const observations = [];", " while (Date.now() - startedAt <= timeoutMs) {", " const row = rowFor();", " observations.push(row);", " if (row.ok === true) { console.log(JSON.stringify({ ok: true, ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", " if (row.startup.failedCommandId) { console.log(JSON.stringify({ ok: false, failure: 'observer-startup-command-failed', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", " if (row.status === 'startup-terminal') { console.log(JSON.stringify({ ok: false, failure: 'observer-startup-terminal', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", " await sleep(Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - startedAt))));", " }", " const row = rowFor();", " observations.push(row);", " console.log(JSON.stringify({ ok: false, failure: 'quick-verify-startup-wait-chunk-timeout', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true }));", "})().catch((error) => { console.log(JSON.stringify({ ok: false, failure: 'quick-verify-startup-wait-script-error', error: error instanceof Error ? error.message : String(error), valuesRedacted: true })); });", "NODE", ].join("\n"); } function collectObserveView(state: SentinelCicdState, observerId: string, view: "turn-summary" | "trace-frame", turn: number | null, timeoutSeconds: number): Record { const args = ["web-probe", "observe", "collect", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--view", view, "--command-timeout-seconds", String(Math.max(5, Math.min(timeoutSeconds, 55))), "--raw", "--compact-raw"]; if (turn !== null) args.push("--turn", String(turn)); const result = runChildCli(args, timeoutSeconds); const payload = cliDataPayload(result.parsed); const collect = record(payload.collect); return { ok: result.ok && result.parsed !== null && payload.ok !== false && collect.ok !== false, view, renderedText: typeof collect.renderedText === "string" ? collect.renderedText : typeof payload.renderedText === "string" ? payload.renderedText : String(record(result.result).stdoutTail ?? record(result.result).stdoutPreview ?? ""), collect, payload, result: result.result, valuesRedacted: true, }; } export function runChildCli(args: string[], timeoutSeconds: number, input?: string, env?: NodeJS.ProcessEnv): ChildCliResult { const result = runCommand(["bun", "scripts/cli.ts", ...args], repoRoot, { input, env: env === undefined ? undefined : { ...process.env, ...env }, timeoutMs: Math.max(5, Math.min(timeoutSeconds, 120)) * 1000, }); return { ok: result.exitCode === 0 && !result.timedOut, parsed: parseJsonObject(result.stdout), result: compactCommandWithTail(result), }; } function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: string, promptIndex: number, deadline: number, pollIntervalMs: number, budgetSeconds: number, chunkSeconds = 45): Record { const observations: Record[] = []; const indexEntry = readLocalObserveIndex(observerId); if (indexEntry === null) { return { ok: false, failure: "observe-index-entry-missing", round: promptIndex, observerId, valuesRedacted: true, }; } const pollSleepMs = Math.max(1000, Math.min(3000, Math.trunc(pollIntervalMs * 2) || 1000)); while (Date.now() < deadline) { const waitMs = Math.max(1000, Math.min(Math.max(1000, Math.trunc(chunkSeconds * 1000)), deadline - Date.now())); const script = quickVerifyPromptWaitScript(indexEntry.stateDir, promptIndex, waitMs, pollSleepMs); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 8000 }); const payload = parseJsonObject(result.stdout); if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record)); const status = typeof payload?.status === "string" ? payload.status : null; const terminalPayload = { round: promptIndex, status, traceId: payload?.traceId ?? null, finalResponseEmpty: payload?.finalResponseEmpty === true, composerReadyForTurn: payload?.composerReadyForTurn === true, composerAction: typeof payload?.composerAction === "string" ? payload.composerAction : null, observations: observations.slice(-6), waitResult: compactCommand(result), valuesRedacted: true, }; if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-wait-chunk-timeout") { const fallback = quickVerifyTurnSummaryFallback(state, observerId, promptIndex); if (fallback.ok === true) { return { ok: true, ...terminalPayload, status: stringAtNullable(fallback, "status") ?? status, traceId: stringAtNullable(fallback, "traceId") ?? payload?.traceId ?? null, finalResponseEmpty: false, fallback, warnings: ["quick verify artifact wait command failed, but bounded turn-summary artifacts show this round completed; continuing validation."], }; } return { ok: false, failure: text(payload?.failure ?? "quick-verify-artifact-wait-failed"), ...terminalPayload, fallback, }; } if (payload.ok === false && payload.failure === "quick-verify-wait-chunk-timeout") { const fallback = quickVerifyTurnSummaryFallback(state, observerId, promptIndex); if (fallback.ok === true) { return { ok: true, ...terminalPayload, status: stringAtNullable(fallback, "status") ?? status, traceId: stringAtNullable(fallback, "traceId") ?? payload.traceId ?? null, finalResponseEmpty: false, fallback, warnings: ["quick verify wait chunk timed out, but bounded turn-summary artifacts show this round completed; continuing validation."], }; } } if (payload.ok === true) return { ok: true, ...terminalPayload }; if (isQuickVerifyTurnSuccessful(status)) { if (payload?.finalResponseEmpty !== true) return { ok: true, ...terminalPayload }; continue; } if (isQuickVerifyTurnTerminal(status)) { return { ok: false, failure: "observe-turn-terminal-non-success", ...terminalPayload, }; } } return { ok: false, failure: "quick-verify-timeout-over-budget", round: promptIndex, observations: observations.slice(-6), warnings: [`quick verify exceeded the configured ${budgetSeconds}s targetValidation budget while waiting for a submitted turn to become terminal; investigate Code Agent multi-round continuity before retrying.`], valuesRedacted: true, }; } function quickVerifyPromptWaitScript(stateDir: string, promptIndex: number, timeoutMs: number, pollSleepMs: number): string { return [ "set -eu", `state_dir=${shellQuote(stateDir)}`, `prompt_index=${shellQuote(String(promptIndex))}`, `timeout_ms=${shellQuote(String(Math.max(1, Math.trunc(timeoutMs))))}`, `poll_ms=${shellQuote(String(Math.max(250, Math.trunc(pollSleepMs))))}`, "test -d \"$state_dir\" || { printf '{\"ok\":false,\"failure\":\"state-dir-missing\",\"stateDir\":\"%s\",\"valuesRedacted\":true}\\n' \"$state_dir\"; exit 0; }", "node - \"$state_dir\" \"$prompt_index\" \"$timeout_ms\" \"$poll_ms\" <<'NODE'", "const fs = require('node:fs');", "const path = require('node:path');", "const dir = process.argv[2];", "const promptIndex = Number(process.argv[3]);", "const timeoutMs = Number(process.argv[4]);", "const pollMs = Number(process.argv[5]);", "const startedAt = Date.now();", "const short = (value, limit = 160) => String(value || '').replace(/\\s+/gu, ' ').trim().slice(0, limit);", "const textOf = (value) => String(value?.text || value?.textPreview || value?.preview || '');", "const arr = (value) => Array.isArray(value) ? value : [];", "const unique = (values) => Array.from(new Set(values.filter(Boolean)));", "const numOrNull = (value) => { const n = Number(value); return Number.isFinite(n) ? n : null; };", "const tsMs = (value) => { const ms = Date.parse(String(value || '')); return Number.isFinite(ms) ? ms : null; };", "const readJson = (rel) => { try { return JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); } catch { return null; } };", "const readJsonl = (rel) => { try { return fs.readFileSync(path.join(dir, rel), 'utf8').split(/\\r?\\n/u).filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean); } catch { return []; } };", "const readJsonlTail = (rel, maxBytes = 2000000) => {", " try {", " const file = path.join(dir, rel);", " const stat = fs.statSync(file);", " const start = Math.max(0, stat.size - maxBytes);", " const length = stat.size - start;", " const fd = fs.openSync(file, 'r');", " try {", " const buffer = Buffer.alloc(length);", " fs.readSync(fd, buffer, 0, length, start);", " const lines = buffer.toString('utf8').split(/\\r?\\n/u);", " if (start > 0) lines.shift();", " return lines.filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean);", " } finally {", " fs.closeSync(fd);", " }", " } catch {", " return [];", " }", "};", "const readDone = (id) => id ? readJson(path.join('commands', 'done', `${id}.json`)) : null;", "const readFailed = (id) => id ? readJson(path.join('commands', 'failed', `${id}.json`)) : null;", "function sessionIdFromUrl(value) { const match = String(value || '').match(/\\/workbench\\/sessions\\/(ses_[A-Za-z0-9_-]+)/u); return match ? match[1] : null; }", "function commandSessionId(item) { const done = readDone(item?.commandId); return item?.sessionId || item?.detail?.sessionId || item?.input?.sessionId || item?.result?.sessionId || done?.result?.sessionId || done?.result?.observer?.sessionId || sessionIdFromUrl(item?.afterUrl) || sessionIdFromUrl(item?.detail?.afterUrl) || sessionIdFromUrl(done?.result?.afterUrl) || null; }", "function commandTextHash(item) { const done = readDone(item?.commandId); return item?.detail?.textHash || item?.input?.textHash || item?.result?.textHash || done?.result?.textHash || null; }", "function firstTraceId(value) { const match = String(value || '').match(/\\btrc_[A-Za-z0-9_-]+\\b/u); return match ? match[0] : null; }", "function commandTraceId(item) { const done = readDone(item?.commandId); return item?.traceId || item?.detail?.chatSubmit?.traceId || item?.detail?.traceId || item?.input?.traceId || item?.result?.chatSubmit?.traceId || done?.result?.chatSubmit?.traceId || done?.result?.traceId || null; }", "function itemTraceId(item) { return item?.traceId || firstTraceId(textOf(item)) || null; }", "function completedNewSessionIdsBefore(control, ts) { const limit = tsMs(ts); return control.filter((item) => item.type === 'newSession' && item.phase === 'completed').filter((item) => limit === null || tsMs(item.ts) === null || tsMs(item.ts) <= limit).map(commandSessionId).filter(Boolean); }", "function authoritativeSessionIdForPrompts(control, prompts) { const ids = completedNewSessionIdsBefore(control, prompts[0]?.firstTs || null); return ids.slice(-1)[0] || unique(prompts.map((item) => item.sessionId))[0] || null; }", "function promptCommands(control) {", " const map = new Map();", " for (const item of control) {", " if (item.type !== 'sendPrompt' || !['started', 'completed', 'failed'].includes(item.phase)) continue;", " const id = item.commandId || item.seq || String(map.size + 1);", " const existing = map.get(id) || {};", " map.set(id, { ...existing, ...item, input: { ...(existing.input || {}), ...(item.input || {}) }, sessionId: existing.sessionId || commandSessionId(item), traceId: existing.traceId || commandTraceId(item), textHash: existing.textHash || commandTextHash(item), firstTs: existing.firstTs || item.ts, lastTs: item.ts });", " }", " const prompts = Array.from(map.values()).filter((item) => tsMs(item.firstTs) !== null).sort((a, b) => tsMs(a.firstTs) - tsMs(b.firstTs));", " const sessionId = authoritativeSessionIdForPrompts(control, prompts);", " if (!sessionId) return prompts;", " const scoped = prompts.filter((item) => item.sessionId === sessionId);", " return scoped.length > 0 ? scoped : prompts;", "}", "function segmentFor(samples, prompts, index) { const start = tsMs(prompts[index]?.firstTs); const end = index + 1 < prompts.length ? tsMs(prompts[index + 1].firstTs) : Infinity; return samples.filter((sample) => { const ms = tsMs(sample.ts); return ms !== null && ms >= start && ms < end; }); }", "function entryGroups(sample) { return [...arr(sample.turns).map((item) => ({ group: 'turn', item })), ...arr(sample.traceRows).map((item) => ({ group: 'traceRow', item })), ...arr(sample.messages).map((item) => ({ group: 'message', item }))]; }", "function traceIdsFromSamples(items) { const ids = []; for (const sample of items) for (const entry of entryGroups(sample)) { const id = itemTraceId(entry.item); if (id) ids.push(id); } return unique(ids); }", "function chooseTraceId(segment, prompt) { const promptTraceId = commandTraceId(prompt) || prompt?.traceId || null; const ids = traceIdsFromSamples(segment); if (promptTraceId && (ids.length === 0 || ids.includes(promptTraceId))) return promptTraceId; return ids.slice(-1)[0] || promptTraceId || null; }", "function traceIdForPromptUserMessage(items, prompt) {", " const hash = prompt?.textHash || commandTextHash(prompt);", " if (!hash) return null;", " for (const sample of items) {", " for (const message of arr(sample.messages)) {", " const role = String(message?.role || message?.dataRole || message?.messageRole || '').toLowerCase();", " if (role && !/user/u.test(role)) continue;", " if (message?.textHash === hash) {", " const id = itemTraceId(message);", " if (id) return id;", " }", " }", " }", " return null;", "}", "function traceEntries(items, traceId) { const entries = []; for (const sample of items) for (const entry of entryGroups(sample)) { const text = textOf(entry.item); const id = itemTraceId(entry.item); if (!traceId || id === traceId || text.includes(traceId)) entries.push({ ...entry, sample, text }); } return entries; }", "function normalizeLifecycleStatus(value) { const raw = String(value || '').trim().toLowerCase(); if (/^(canceled|cancelled)$/u.test(raw)) return 'canceled'; if (/^(failed|failure|error)$/u.test(raw)) return 'failed'; if (/^(completed|complete|succeeded|success|terminal)$/u.test(raw)) return 'completed'; if (/^(running|admitting|queued|pending|in_progress|in-progress)$/u.test(raw)) return 'running'; return null; }", "function statusFor(items, traceId) { const entries = traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) }))); const lastTurn = entries.filter((entry) => entry.group === 'turn').slice(-1)[0]?.item || null; const turnStatus = normalizeLifecycleStatus(lastTurn?.status); if (turnStatus) return turnStatus; const lastMessage = entries.filter((entry) => entry.group === 'message').slice(-1)[0]?.item || null; return normalizeLifecycleStatus(lastMessage?.status) || 'unknown'; }", "function cleanFinalResponseText(value) { const raw = String(value || '').trim(); if (!raw) return ''; if (/^(completed|failed|canceled|cancelled|轮次完成|轮次失败|轮次取消|已记录)$/iu.test(raw.replace(/\\s+/gu, ' '))) return ''; if (/^(admitted|run|ok|error)\\s+/iu.test(raw)) return ''; return raw; }", "function finalResponseTextFromEntry(entry) {", " const explicit = cleanFinalResponseText(entry.item?.finalResponse?.text || entry.item?.finalResponse?.preview || '');", " if (explicit && !/^Code Agent\\s*耗时/iu.test(explicit)) return explicit;", " if (entry.group !== 'message') return '';", " const role = String(entry.item?.role || entry.item?.dataRole || entry.item?.messageRole || '').toLowerCase();", " if (role && !/assistant|agent|system/u.test(role)) return '';", " const text = cleanFinalResponseText(entry.text);", " return text && !/^Code Agent\\s*耗时/iu.test(text) ? text : '';", "}", "function finalResponseEmpty(items, traceId) { if (!/^(completed|failed|canceled)$/u.test(statusFor(items, traceId))) return true; const entries = (traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) })))).slice().reverse(); for (const entry of entries) { if (finalResponseTextFromEntry(entry)) return false; } return true; }", "function rowFor() {", " const control = readJsonl('control.jsonl');", " const samples = readJsonlTail('samples.jsonl');", " const prompts = promptCommands(control);", " const prompt = prompts[promptIndex - 1] || null;", " if (!prompt) return { ok: true, round: promptIndex, status: 'command-pending', traceId: null, finalResponseEmpty: true, lastSeq: null, lastTs: null, promptMissing: true, valuesRedacted: true };", " const done = readDone(prompt.commandId);", " const failed = readFailed(prompt.commandId);", " if (failed) return { ok: false, failure: 'observe-command-sendPrompt-failed', round: promptIndex, status: 'command-failed', commandId: prompt.commandId || null, traceId: null, finalResponseEmpty: true, commandFailure: short(failed.error?.message || failed.failure || failed.status || 'command failed'), valuesRedacted: true };", " const promptTraceId = commandTraceId(prompt);", " if (!done) return { ok: true, round: promptIndex, status: 'command-pending', commandId: prompt.commandId || null, traceId: promptTraceId || null, finalResponseEmpty: true, commandPhase: prompt.phase || null, traceMissing: !promptTraceId, valuesRedacted: true };", " const segment = segmentFor(samples, prompts, promptIndex - 1);", " const controlSegment = segment.filter((sample) => sample.pageRole === 'control');", " const traceId = promptTraceId || traceIdForPromptUserMessage(segment, prompt) || chooseTraceId(controlSegment, prompt) || chooseTraceId(segment, prompt);", " if (!traceId) return { ok: true, round: promptIndex, status: 'command-pending', commandId: prompt.commandId || null, traceId: null, finalResponseEmpty: true, commandPhase: prompt.phase || null, traceMissing: true, segmentSampleCount: segment.length, valuesRedacted: true };", " const controlTraceSegment = traceId ? controlSegment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)) : [];", " const statusSegment = controlTraceSegment.length > 0 ? controlSegment : segment;", " const status = statusFor(statusSegment, traceId);", " const sampleForTrace = traceId ? statusSegment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || segment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || null : null;", " const lastSample = sampleForTrace || segment.slice(-1)[0] || null;", " const composerSample = segment.filter((sample) => sample.pageRole === 'control').slice(-1)[0] || lastSample;", " const composer = composerSample?.composer || {};", " const composerReadyForTurn = composer.inputPresent === true && composer.inputDisabled !== true && composer.submitPresent === true && composer.warningPresent !== true && composer.submitAction === 'turn';", " return { ok: true, round: promptIndex, status, traceId, finalResponseEmpty: finalResponseEmpty(statusSegment, traceId), lastSeq: numOrNull(lastSample?.seq), lastTs: lastSample?.ts || null, composerReadyForTurn, composerAction: composer.submitAction || null, sampleScope: controlSegment.length > 0 ? 'control' : 'all', segmentSampleCount: segment.length, statusSampleCount: statusSegment.length, source: 'observe-artifact-wait-script', valuesRedacted: true };", "}", "function norm(value) { return String(value || '').trim().toLowerCase().replace(/_/gu, '-'); }", "function successful(value) { return ['completed', 'succeeded', 'success'].includes(norm(value)); }", "function terminal(value) { return ['completed', 'succeeded', 'success', 'failed', 'error', 'blocked', 'timeout', 'canceled', 'cancelled', 'terminal'].includes(norm(value)); }", "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", "(async () => {", " const observations = [];", " while (Date.now() - startedAt <= timeoutMs) {", " const row = rowFor();", " observations.push(row);", " if (row.ok === false) { console.log(JSON.stringify({ ...row, ok: false, failure: row.failure || 'quick-verify-artifact-row-failed', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", " if (successful(row.status) && row.finalResponseEmpty !== true) { console.log(JSON.stringify({ ...row, ok: true, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", " if (!successful(row.status) && terminal(row.status)) { console.log(JSON.stringify({ ...row, ok: false, failure: 'observe-turn-terminal-non-success', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", " await sleep(Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - startedAt))));", " }", " const row = rowFor();", " observations.push(row);", " console.log(JSON.stringify({ ...row, ok: false, failure: 'quick-verify-wait-chunk-timeout', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true }));", "})().catch((error) => { console.log(JSON.stringify({ ok: false, failure: 'quick-verify-artifact-wait-script-error', error: error instanceof Error ? error.message : String(error), valuesRedacted: true })); });", "NODE", ].join("\n"); } function isQuickVerifyTurnSuccessful(value: string | null): boolean { const status = normalizeQuickVerifyStatus(value); return status === "completed" || status === "succeeded" || status === "success"; } function isQuickVerifyTurnTerminal(value: string | null): boolean { const status = normalizeQuickVerifyStatus(value); return status === "completed" || status === "succeeded" || status === "success" || status === "failed" || status === "error" || status === "blocked" || status === "timeout" || status === "canceled" || status === "cancelled" || status === "terminal"; } function normalizeQuickVerifyStatus(value: string | null): string { return String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); } function cliDataPayload(parsed: Record | null): Record { const root = record(parsed); const payload = isRecord(root.data) ? root.data : root; return cliDumpPayload(payload) ?? payload; } function cliDumpPayload(payload: Record): Record | null { if (payload.outputTruncated !== true) return null; const dumpPath = stringAtNullable(record(payload.dump), "path"); if (dumpPath === null || !existsSync(dumpPath)) return null; const dumped = parseJsonObject(readFileSync(dumpPath, "utf8")); if (dumped === null) return null; const dumpedRoot = record(dumped); return isRecord(dumpedRoot.data) ? dumpedRoot.data : dumpedRoot; } function findScenario(state: SentinelCicdState, scenarioId: string): Record | null { const scenarios = readWebProbeSentinelConfigRefTarget(state.spec, state.configRefs.scenarios); const items = Array.isArray(scenarios) ? scenarios : isRecord(scenarios) ? [scenarios] : []; return items.map(record).find((item) => item.id === scenarioId) ?? null; } function readPromptSetForScenario(state: SentinelCicdState, scenario: Record): { ok: true; prompts: string[]; summary: Record } | { ok: false; error: string; summary: Record } { const promptSetRef = stringAt(scenario, "promptSetRef"); const promptSet = recordTarget(readWebProbeSentinelConfigRefTarget(state.spec, promptSetRef), promptSetRef); const sourceRef = stringAt(promptSet, "promptSourceRef"); const key = stringAt(promptSet, "promptSourceKey"); const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); const summary = { sourceRef, sourceKey: key, sourcePath: displayPath(sourcePath), valuesRedacted: true }; const runtimeRaw = process.env[key]; const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {}; const raw = values[key] ?? runtimeRaw; const sourceMode = values[key] !== undefined ? "secret-source-file" : runtimeRaw !== undefined && runtimeRaw.length > 0 ? "runtime-env" : null; if (!existsSync(sourcePath) && sourceMode === null) return { ok: false, error: "prompt-source-missing", summary }; if (raw === undefined || raw.length === 0) return { ok: false, error: "prompt-key-missing", summary }; const parsed = parsePromptJson(raw); if (parsed.length === 0) return { ok: false, error: "prompt-json-empty", summary }; return { ok: true, prompts: parsed, summary: { ...summary, sourceMode, promptCount: parsed.length, promptMarkers: parsed.map((item) => Array.from(new Set(Array.from(item.matchAll(/\bsentinel-(?:0[1-9]|10)\b/giu)).map((match) => match[0].toLowerCase())))), promptTextHashes: parsed.map((item) => `sha256:${createHash("sha256").update(item).digest("hex").slice(0, 16)}`), promptTextBytes: parsed.map((item) => Buffer.byteLength(item)), valuesRedacted: true, }, }; } function parsePromptJson(raw: string): string[] { try { const parsed = JSON.parse(raw) as unknown; if (Array.isArray(parsed)) return parsed.filter((item): item is string => typeof item === "string" && item.length > 0); const recordValue = record(parsed); if (Array.isArray(recordValue.prompts)) return recordValue.prompts.filter((item): item is string => typeof item === "string" && item.length > 0); if (typeof recordValue.prompt === "string" && recordValue.prompt.length > 0) return [recordValue.prompt]; } catch { if (raw.trim().length > 0) return [raw]; } return []; } function readLocalObserveIndex(observerId: string): { stateDir: string } | null { const path = rootPath(".state/web-observe/index.json"); if (!existsSync(path)) return null; const parsed = parseJsonObject(readFileSync(path, "utf8")); const entry = record(parsed?.[observerId]); const stateDir = typeof entry.stateDir === "string" ? entry.stateDir : null; return stateDir === null ? null : { stateDir }; } function observerIdFromText(textValue: string): string | null { return /\bwebobs-[a-z0-9-]+\b/iu.exec(textValue)?.[0] ?? null; } export function remainingSeconds(deadline: number, cap: number): number { return Math.max(5, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000))); } export function metricNames(textValue: unknown): string[] { if (typeof textValue !== "string") return []; return textValue.split(/\r?\n/u).map((line) => /^([A-Za-z_:][A-Za-z0-9_:]*)/u.exec(line)?.[1]).filter((item): item is string => typeof item === "string"); } export function validationBlocker(health: Record, metrics: Record, report: Record, publicExposure: Record, publicDashboard: Record, quickVerify: Record | null): Record { const blockers = []; if (!health.ok || record(health.bodyJson).ok !== true) blockers.push("health"); if (!metrics.ok || !metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health")) blockers.push("metrics"); if (!report.ok) blockers.push("recent-report"); if (publicExposure.ok !== true) blockers.push("public-exposure"); if (publicDashboard.ok !== true) blockers.push("public-dashboard"); if (quickVerify !== null && quickVerify.ok !== true) blockers.push("quick-verify"); return { code: "sentinel-validation-failed", blockers, valuesRedacted: true }; } export function serviceUnavailableBlocker(state: SentinelCicdState): Record { return { code: "sentinel-service-unavailable", policy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"), reason: "sentinel service must be reachable through k3s internal Service DNS before quick verify can run; no public/fallback path is used.", retry: `bun scripts/cli.ts web-probe sentinel validate --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, valuesRedacted: true, }; } export function sentinelP5Next(state: SentinelCicdState): Record { const node = state.spec.nodeId; const lane = state.spec.lane; const suffix = sentinelCliSuffix(state); return { validate: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix}`, quickVerify: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix} --quick-verify --confirm --wait`, maintenanceStart: `bun scripts/cli.ts web-probe sentinel maintenance start --node ${node} --lane ${lane}${suffix} --confirm --wait`, maintenanceStop: `bun scripts/cli.ts web-probe sentinel maintenance stop --node ${node} --lane ${lane}${suffix} --confirm --wait`, report: `bun scripts/cli.ts web-probe sentinel report --node ${node} --lane ${lane}${suffix} --view summary`, }; } function isSafeRelativeStateDir(value: string): boolean { return value.startsWith(".state/web-observe/") && !value.includes("\0") && !value.includes(".."); } function mergeFindingRecords(primary: readonly Record[], extra: readonly Record[]): Record[] { const merged: Record[] = []; const seen = new Set(); for (const item of [...primary, ...extra]) { const id = stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? stringAtNullable(item, "finding_id") ?? "finding"; const severity = stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "unknown"; const key = `${id}\0${severity}`; if (seen.has(key)) continue; seen.add(key); merged.push(item); } return merged; } function isQuickVerifyBlockingFinding(item: Record): boolean { const severity = (stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "").toLowerCase(); if (!["critical", "red", "fatal", "error", "failed", "blocked"].includes(severity)) return false; const id = (stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? "").toLowerCase(); if (id === "observer-command-failed") return observerCommandFailureBlocks(item); return [ "quick-verify-no-business-turn", "quick-verify-command-sequence-failed", "quick-verify-observer-start-failed", "quick-verify-account-secret-missing", "prompt-chat-submit-failed", "route-active-session-mismatch", "final-response-flicker", "round-completion-final-response-missing", "turn-trace-id-missing", "no-samples", "jsonl-read-issues", ].includes(id); } function observerCommandFailureBlocks(item: Record): boolean { const commands = Array.isArray(item.commands) ? item.commands.map(record) : []; if (commands.length === 0) return true; return commands.some((command) => { const type = (stringAtNullable(command, "type") ?? "").toLowerCase(); if (["stop", "cancel", "mark", "screenshot"].includes(type)) return false; return true; }); } function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record | null, traceFrame: Record | null): Record[] { if (quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame)) return []; const rendered = [ typeof turnSummary?.renderedText === "string" ? turnSummary.renderedText : "", typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : "", ].join("\n"); const noPromptScenario = promptIndex <= 0; if (noPromptScenario && failure === null) return []; if (noPromptScenario && failure !== null) { const observerStartFailure = failure === "observe-start-failed"; return [{ id: observerStartFailure ? "quick-verify-observer-start-failed" : "quick-verify-command-sequence-failed", severity: "red", count: 1, summary: observerStartFailure ? "quick verify observer failed to start before the no-prompt scenario could run." : "quick verify no-prompt command sequence failed before the account/session workflow completed.", failure, promptIndex, valuesRedacted: true, }]; } const noTrace = /无\s*sendPrompt|no\s+sendPrompt|无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered); const emptyFinal = /Final Response[\s\S]*\(空内容\)/iu.test(rendered); if (!noTrace && !emptyFinal && failure !== "observe-start-failed") return []; return [{ id: "quick-verify-no-business-turn", severity: "red", count: 1, summary: "quick verify did not reach a durable business turn/session/trace rows/final response; public dashboard health cannot be treated as HWLAB recovery.", failure: failure ?? null, promptIndex, valuesRedacted: true, }]; } function quickVerifyCompletedTurnSummaryRow(promptIndex: number, turnSummary: Record | null): Record | null { const rows = Array.isArray(record(turnSummary?.collect).rows) ? record(turnSummary?.collect).rows.map(record) : []; const scopedRows = promptIndex > 0 ? rows.filter((row) => numberAtNullable(row, "round") === promptIndex) : rows; return scopedRows.find((row) => { const finalResponse = record(row.finalResponse); return isQuickVerifyTurnSuccessful(stringAtNullable(row, "status")) && stringAtNullable(row, "traceId") !== null && finalResponse.empty !== true; }) ?? null; } function quickVerifyTurnSummaryFallback(state: SentinelCicdState, observerId: string, promptIndex: number): Record { const turnSummary = collectObserveView(state, observerId, "turn-summary", null, 25); const row = quickVerifyCompletedTurnSummaryRow(promptIndex, turnSummary); const rows = Array.isArray(record(turnSummary.collect).rows) ? record(turnSummary.collect).rows.map(record) : []; if (row === null) { return { ok: false, source: "turn-summary-fallback", collectOk: turnSummary.ok === true, rowCount: rows.length, promptIndex, result: turnSummary.result ?? null, valuesRedacted: true, }; } const finalResponse = record(row.finalResponse); return { ok: true, source: "turn-summary-fallback", collectOk: turnSummary.ok === true, rowCount: rows.length, promptIndex, status: stringAtNullable(row, "status"), traceId: stringAtNullable(row, "traceId"), finalResponseEmpty: finalResponse.empty === true, finalResponseBytes: numberAtNullable(finalResponse, "textBytes"), result: turnSummary.result ?? null, valuesRedacted: true, }; } function quickVerifyHasDurableBusinessTurn(promptIndex: number, turnSummary: Record | null, traceFrame: Record | null): boolean { if (quickVerifyCompletedTurnSummaryRow(promptIndex, turnSummary) !== null) return true; const renderedTrace = typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : ""; if (!renderedTrace) return false; if (/Final Response\s*\n\s*\(空内容\)/iu.test(renderedTrace)) return false; return /Code Agent[^\n]*completed|轮次完成（总耗时/iu.test(renderedTrace) && /Final Response\s*\n\s*\S/iu.test(renderedTrace) && !/无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(renderedTrace); } function quickVerifyBusinessStatus( failure: string | null, promptIndex: number, turnSummary: Record | null, traceFrame: Record | null, elapsedMs: unknown, budgetSeconds: number, ): Record { const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame); const elapsed = typeof elapsedMs === "number" && Number.isFinite(elapsedMs) ? elapsedMs : null; const budgetMs = Math.max(0, budgetSeconds) * 1000; const budgetExceeded = elapsed !== null && budgetMs > 0 && elapsed > budgetMs; const observerTimeout = budgetExceeded || (failure !== null && isRecoverableQuickVerifyWaitFailure(failure)); const status = durableBusinessTurn ? "business-turn-completed" : observerTimeout ? "observer-timeout" : "scenario-incomplete"; return { status, durableBusinessTurn, observerTimeout, scenarioComplete: durableBusinessTurn, failure: failure ?? null, promptIndex, elapsedMs: elapsed, budgetSeconds, sourceOfTruth: durableBusinessTurn ? "turn-summary-or-trace-frame" : observerTimeout ? "runner-wait-budget" : "control-findings", valuesRedacted: true, }; } function isRecoverableQuickVerifyWaitFailure(failure: string): boolean { return failure === "quick-verify-wait-chunk-timeout" || failure === "quick-verify-timeout-over-budget" || failure === "observe-turn-terminal-wait-failed"; } function compactCommandWithTail(result: CommandResult): CompactCommandResult & { stdoutTail: string; stderrTail: string } { return { ...compactCommand(result), stdoutPreview: result.stdout.trim().slice(0, 1200), stderrPreview: result.stderr.trim().slice(0, 1200), stdoutTail: result.stdout.trim().slice(-4000), stderrTail: result.stderr.trim().slice(-4000), }; } function renderQuickVerifySummary(input: Record): string { const artifact = record(input.artifactSummary); const findings = Array.isArray(artifact.findings) ? artifact.findings.map(record).slice(0, 8) : []; return [ "Web Probe Sentinel Quick Verify", "=======================================================", `run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`, `report=${artifact.reportJsonSha256 ?? "-"} artifacts=${artifact.artifactCount ?? "-"} findings=${artifact.findingCount ?? findings.length}`, `publicOrigin=${input.publicOrigin ?? "-"}`, "", "Findings", findings.length === 0 ? "-" : findings.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${item.summary ?? item.message ?? ""}`).join("\n"), ].join("\n"); } function renderAuthSessionSwitchQuickVerifySummary(input: Record): string { const artifact = record(input.artifactSummary); const accountEnv = record(input.accountEnv); const findingRows = Array.isArray(input.findings) ? input.findings.map(record).slice(0, 8) : Array.isArray(artifact.findings) ? artifact.findings.map(record).slice(0, 8) : []; const steps = Array.isArray(input.steps) ? input.steps.map(record) : []; const commandSteps = steps .filter((step) => { const phase = stringAtNullable(step, "phase") ?? ""; return phase.startsWith("observe-command-") || phase.startsWith("observe-session-invariance-") || phase === "quick-verify-account-env"; }) .map((step) => { const phase = stringAtNullable(step, "phase") ?? "-"; const ok = step.ok === true ? "ok" : step.ok === false ? "failed" : "-"; const result = record(step.result); const status = stringAtNullable(result, "status") ?? stringAtNullable(result, "failure") ?? "-"; return `${phase} ${ok} ${status}`; }); return [ "Auth Session Switch Quick Verify", "=======================================================", `run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`, `status=${artifact.ok === true ? "ok" : "blocked"} report=${artifact.reportJsonSha256 ?? "-"} publicOrigin=${input.publicOrigin ?? "-"}`, `accountEnv=${accountEnv.envCount ?? "-"} valuesRedacted=true`, "", "Command Sequence", commandSteps.length === 0 ? "-" : commandSteps.join("\n"), "", "Findings", findingRows.length === 0 ? "-" : findingRows.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${item.summary ?? item.message ?? ""}`).join("\n"), ].join("\n"); } function formatQuickVerifyTimingSuffix(item: Record): string { const status = stringAtNullable(item, "timingStatus"); const source = stringAtNullable(item, "timingSourceOfTruth") ?? stringAtNullable(item, "expectedElapsedSource") ?? stringAtNullable(item, "evidenceKind"); if (status === null && source === null) return ""; return ` timing=${[status === null ? null : `status=${status}`, source === null ? null : `source=${source}`].filter((part) => part !== null).join(" ")}`; }