Files
pikasTech-unidesk/scripts/src/hwlab-node-web-sentinel-p5-observe.ts
T

1736 lines
86 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel.
// 职责:编排 Web Probe Sentinel P5 quick-verify,并投影有界终态证据。
import { createHash, randomUUID } from "node:crypto";
import type { CommandResult } from "./command";
import { runCommand } from "./command";
import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery";
import { repoRoot } from "./config";
import { runWebProbeMemoryGuard } from "./hwlab-node-web-probe-memory-guard";
import type { ChildCliResult, SentinelCicdState } from "./hwlab-node-web-sentinel-cicd";
import {
arrayAt,
compactCommand,
mergeWarnings,
numberAt,
numberAtNullable,
parseJsonObject,
quickVerifyAccountEnv,
record,
nonEmptyString,
shellQuote,
short,
stringAt,
stringAtNullable,
targetValidationElapsedWarnings,
text,
withWarnings,
} from "./hwlab-node-web-sentinel-cicd";
import {
analysisSummaryFromAnalyzeResult,
cliDataPayload,
collectObserveView,
enrichObserveStartFailureFinding,
findScenario,
isQuickVerifyBlockingFinding,
isRecoverableQuickVerifyWaitFailure,
mergeFindingRecords,
observerIdFromText,
quickVerifyBusinessStatus,
quickVerifyControlFindings,
quickVerifyHasDurableBusinessTurn,
quickVerifyPrimaryControlFailure,
readAnalysisSummaryFromWorkspace,
readLocalObserveIndex,
readPromptSetForScenario,
remainingSeconds,
runChildCli,
runSentinelExactProviderProfilePreflight,
timeoutDisplayForQuickVerifyFailure,
waitForQuickVerifyObserverStartup,
waitForQuickVerifyPromptTurn,
} from "./hwlab-node-web-sentinel-p5-observe-analysis";
import { emitWebProbeSentinelSpan } from "./hwlab-node-web-sentinel-otel";
import { buildMonitorTerminalIngest, MonitorIngestFailedError, resolveMonitorTerminalWriter } from "./monitor-terminal-ingest";
export {
analysisSummaryFromAnalyzeResult,
metricNames,
readAnalysisSummaryFromWorkspace,
remainingSeconds,
runChildCli,
runSentinelExactProviderProfilePreflight,
sentinelP5Next,
serviceUnavailableBlocker,
validationBlocker,
} from "./hwlab-node-web-sentinel-p5-observe-analysis";
const QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS = 55;
export interface QuickVerifyObserverLifecycle {
readonly observerId: string;
cleanupStep: Record<string, unknown> | null;
stop(phase: string): Record<string, unknown>;
}
type QuickVerifyObserverStop = (observerId: string, phase: string) => Record<string, unknown>;
export function createQuickVerifyObserverLifecycle(observerId: string, stopObserver: QuickVerifyObserverStop): QuickVerifyObserverLifecycle {
const attempts: Record<string, unknown>[] = [];
const lifecycle: QuickVerifyObserverLifecycle = {
observerId,
cleanupStep: null,
stop(phase) {
if (lifecycle.cleanupStep?.ok === true) return lifecycle.cleanupStep;
for (let callAttempt = 1; callAttempt <= 2; callAttempt += 1) {
const attempt = attempts.length + 1;
try {
const result = stopObserver(observerId, phase);
attempts.push({ attempt, ok: result.ok === true, valuesRedacted: true });
lifecycle.cleanupStep = { ...result, stopAttempts: attempts, valuesRedacted: true };
if (result.ok === true) return lifecycle.cleanupStep;
} catch (error) {
attempts.push({ attempt, ok: false, failure: "observer-stop-unhandled-exception", valuesRedacted: true });
lifecycle.cleanupStep = {
phase,
ok: false,
failure: "observer-stop-unhandled-exception",
observerId,
exception: quickVerifyUnhandledExceptionSummary(error),
stopAttempts: attempts,
valuesRedacted: true,
};
}
}
lifecycle.cleanupStep ??= { phase, ok: false, failure: "observer-stop-did-not-return", observerId, stopAttempts: attempts, valuesRedacted: true };
return lifecycle.cleanupStep;
},
};
return lifecycle;
}
export function stopQuickVerifyObserverAfterPartialStart(observerId: string | null, stopObserver: QuickVerifyObserverStop): Record<string, unknown> | null {
if (observerId === null) return null;
return createQuickVerifyObserverLifecycle(observerId, stopObserver).stop("observe-stop-after-start-failure");
}
export function quickVerifyUnhandledExceptionSummary(error: unknown): Record<string, unknown> {
let errorName = typeof error;
let errorMessage = typeof error === "string" ? error : "";
if (error instanceof Error) {
errorName = error.name || "Error";
errorMessage = error.message;
}
const boundedMessage = errorMessage.slice(0, 4096);
return {
errorName: short(errorName),
errorMessageSha256: boundedMessage.length === 0 ? null : createHash("sha256").update(boundedMessage).digest("hex"),
errorMessageTruncated: errorMessage.length > boundedMessage.length,
valuesRedacted: true,
};
}
export function sentinelChildStartMemorySkip(value: unknown): Record<string, unknown> | null {
const envelope = record(value);
const payload = Object.keys(record(envelope.data)).length > 0 ? record(envelope.data) : envelope;
if (payload.status !== "skipped-wait-next-round" || payload.observerCreated !== false) return null;
return {
status: "skipped-wait-next-round",
decision: "skip-observer",
observerCreated: false,
memoryGuard: payload.memoryGuard ?? null,
valuesRedacted: true,
};
}
function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record<string, unknown> = {}): void {
const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => {
if (typeof value === "string") return [key, short(value)];
if (Array.isArray(value)) return [key, value.slice(0, 8)];
return [key, value];
}));
process.stdout.write(`${JSON.stringify({
event: "sentinel.quick-verify.progress",
at: new Date().toISOString(),
node: state.spec.nodeId,
lane: state.spec.lane,
sentinelId: state.sentinelId,
runId,
...compactExtra,
phase,
status,
valuesRedacted: true,
})}\n`);
}
export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record<string, unknown> {
const startedAt = Date.now();
const elapsedMs = () => Date.now() - startedAt;
const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId");
const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds");
const scenario = findScenario(state, scenarioId);
if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true };
const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;
emitWebProbeSentinelSpan(sentinelOtelContext(state), "web_probe_sentinel.quick_verify.job_start", {
scenarioId,
runId,
cadence: stringAtNullable(scenario, "cadence"),
status: "running",
valuesRedacted: true,
});
const memoryGuard = runWebProbeMemoryGuard(state.spec, "sentinel-cadence");
if (memoryGuard.status === "skipped") {
return recordSkippedQuickVerify(state, scenario, runId, scenarioId, reason, elapsedMs(), memoryGuard, [], "memavailable-below-threshold");
}
if (memoryGuard.status !== "allowed") {
return recordQuickVerify(state, {
ok: false,
runId,
status: "blocked",
reason: "memavailable-preflight-unavailable",
scenarioId,
observerId: null,
observerCreated: false,
runCreated: true,
mutation: false,
memoryGuard,
elapsedMs: elapsedMs(),
findingCount: 0,
artifactCount: 0,
findings: [],
views: {},
updatedAt: new Date().toISOString(),
valuesRedacted: true,
});
}
const providerProfilePreflight = runSentinelExactProviderProfilePreflight(state, timeoutSeconds, scenario);
if (providerProfilePreflight.ok !== true) {
return recordQuickVerify(state, {
ok: false,
runId,
status: "blocked",
reason: "exact-provider-profile-unavailable",
scenarioId,
providerProfile: stringAtNullable(scenario, "providerProfile"),
providerProfileMode: stringAtNullable(scenario, "providerProfileMode"),
providerProfilePreflight,
observerId: null,
observerCreated: false,
runCreated: true,
mutation: false,
elapsedMs: elapsedMs(),
findingCount: 0,
artifactCount: 0,
findings: [],
views: {},
updatedAt: new Date().toISOString(),
valuesRedacted: true,
});
}
const commandSequence = arrayAt(scenario, "commandSequence").map(record);
const needsPromptSet = commandSequence.some((item) => stringAt(item, "type") === "sendPrompt" && inlinePromptText(item) === null);
const prompts = needsPromptSet
? readPromptSetForScenario(state, scenario)
: { ok: true as const, prompts: [], summary: { source: "not-required", promptCount: 0, valuesRedacted: true } };
if (!prompts.ok) {
emitWebProbeSentinelSpan(sentinelOtelContext(state), "web_probe_sentinel.quick_verify.job_finish", {
scenarioId,
runId,
status: "blocked",
exitCode: 1,
failureKind: "prompt-source-unavailable",
valuesRedacted: true,
}, false);
return { ok: false, status: "blocked", reason: "prompt-source-unavailable", promptSource: prompts, valuesRedacted: true };
}
const accountEnv = quickVerifyAccountEnv(state);
if (!accountEnv.ok) {
const findings = [{
id: "quick-verify-account-secret-missing",
severity: "red",
count: arrayAt(accountEnv.summary, "missing").length || 1,
summary: "quick verify could not materialize YAML-declared web account credentials for the observer runner.",
missing: arrayAt(accountEnv.summary, "missing"),
valuesRedacted: true,
}];
return recordQuickVerify(state, {
ok: false,
runId,
scenarioId,
reason,
status: "blocked",
observerId: null,
elapsedMs: 0,
businessStatus: quickVerifyBusinessStatus("quick-verify-account-secret-missing", 0, null, null, 0, maxSeconds),
steps: [{ phase: "quick-verify-account-env", ok: false, result: accountEnv.summary }],
failure: "quick-verify-account-secret-missing",
findingCount: findings.length,
findings,
promptSource: prompts.summary,
accountEnv: accountEnv.summary,
views: {
summary: { renderedText: renderQuickVerifySummary({ scenarioId, artifactSummary: { ok: false, findings, findingCount: findings.length }, steps: [], publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ scenarioId, steps: [], findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
},
warnings: [],
valuesRedacted: true,
});
}
const sampleIntervalMs = numberAt(scenario, "sampleIntervalMs");
const warningBudgetSeconds = maxSeconds;
const hardBudgetSeconds = Math.min(timeoutSeconds, Math.max(maxSeconds, numberAt(scenario, "maxRunSeconds")));
const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", warningBudgetSeconds);
const deadline = Date.now() + hardBudgetSeconds * 1000;
printQuickVerifyProgress(state, runId, "start", "running", { scenarioId, reason, warningBudgetSeconds, hardBudgetSeconds, timeoutSeconds });
const steps: Record<string, unknown>[] = [];
const startArgs = [
"web-probe", "observe", "start",
"--node", state.spec.nodeId,
"--lane", state.spec.lane,
"--target-path", stringAt(scenario, "observeTargetPath"),
"--sample-interval-ms", String(sampleIntervalMs),
"--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")),
"--max-run-seconds", String(hardBudgetSeconds),
"--command-timeout-seconds", "55",
"--sentinel-cadence",
];
const viewport = stringAtNullable(scenario, "viewport");
if (viewport !== null) startArgs.push("--viewport", viewport);
printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) });
const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env);
steps.push({ phase: "observe-start", ok: started.ok, result: started.result });
const childMemorySkip = sentinelChildStartMemorySkip(started.parsed);
if (childMemorySkip !== null) {
return recordSkippedQuickVerify(state, scenario, runId, scenarioId, reason, elapsedMs(), childMemorySkip.memoryGuard, steps, "memavailable-below-threshold-after-cadence-preflight");
}
const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? ""));
printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() });
if (!started.ok || observerId === null) {
const cleanupStep = stopQuickVerifyObserverAfterPartialStart(
observerId,
(partialObserverId, phase) => stopQuickVerifyObserver(state, partialObserverId, phase),
);
const cleanup = cleanupStep === null ? null : classifyQuickVerifyCleanupStep(cleanupStep);
const findings = mergeFindingRecords(
quickVerifyControlFindings("observe-start-failed", 0, null, null)
.map((finding) => enrichObserveStartFailureFinding(finding, record(started.result))),
cleanupStep === null ? [] : quickVerifyCleanupFindings(cleanupStep),
);
return recordQuickVerify(state, {
ok: false,
runId,
scenarioId,
reason,
status: "blocked",
observerId,
elapsedMs: elapsedMs(),
businessStatus: quickVerifyBusinessStatus("observe-start-failed", 0, null, null, elapsedMs(), maxSeconds),
steps: cleanupStep === null ? steps : [...steps, cleanupStep],
cleanup,
failure: "observe-start-failed",
findingCount: findings.length,
findings,
warnings: mergeWarnings(elapsedWarnings(), quickVerifyCleanupWarnings(cleanup)),
valuesRedacted: true,
});
}
const observerLifecycle = createQuickVerifyObserverLifecycle(
observerId,
(activeObserverId, phase) => stopQuickVerifyObserver(state, activeObserverId, phase),
);
let promptIndex = 0;
try {
printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) });
const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds);
steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady });
printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", startupReady.ok === true ? "succeeded" : "failed", { observerId, failure: startupReady.failure ?? null, status: startupReady.status ?? null, heartbeatStatus: startupReady.heartbeatStatus ?? null, elapsedMs: elapsedMs() });
if (startupReady.ok !== true) {
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
observerLifecycle,
promptIndex: 0,
steps,
failure: text(startupReady.failure ?? "observe-startup-ready-wait-failed"),
elapsedMs: elapsedMs(),
warnings: mergeWarnings(Array.isArray(startupReady.warnings) ? startupReady.warnings : [], elapsedWarnings()),
promptSource: prompts.summary,
}));
}
const sessionInvarianceChecks = sessionInvarianceChecksByRound(scenario);
const nonBlockingCanaryWarnings: string[] = [];
for (const item of commandSequence) {
const type = stringAt(item, "type");
const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
for (let index = 0; index < repeat; index += 1) {
if (Date.now() >= deadline) {
const timeoutDisplay = timeoutDisplayForQuickVerifyFailure("quick-verify-timeout-over-budget", promptIndex, warningBudgetSeconds, timeoutSeconds);
printQuickVerifyProgress(state, runId, "timeout", "failed", { observerId, promptIndex, elapsedMs: elapsedMs(), hardBudgetSeconds });
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
observerLifecycle,
promptIndex,
steps,
failure: "quick-verify-timeout-over-budget",
failureTitleZh: stringAtNullable(timeoutDisplay, "titleZh"),
timeoutDisplay,
elapsedMs: elapsedMs(),
warnings: mergeWarnings(`quick verify exceeded the hard ${hardBudgetSeconds}s execution budget after the configured ${warningBudgetSeconds}s targetValidation warning budget.`, elapsedWarnings()),
promptSource: prompts.summary,
}));
}
const commandWaitMs = Math.max(1000, Math.trunc(numberAtNullable(item, "commandWaitMs") ?? numberAtNullable(scenario, "commandWaitMs") ?? 55_000));
const commandTimeoutSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "commandTimeoutSeconds") ?? numberAtNullable(scenario, "commandTimeoutSeconds") ?? 55));
const turnWaitChunkSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "turnWaitChunkSeconds") ?? numberAtNullable(scenario, "turnWaitChunkSeconds") ?? 55));
const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", String(commandWaitMs), "--command-timeout-seconds", String(remainingSeconds(deadline, commandTimeoutSeconds))];
if (type === "selectProvider") args.push("--provider", stringAt(item, "provider"));
if (type === "loginAccount" || type === "listSessions" || type === "logout") {
const accountId = stringAtNullable(item, "accountId");
if (accountId !== null) args.push("--account-id", accountId);
}
if (type === "switchSessions") {
const fromAccountId = stringAtNullable(item, "fromAccountId");
const toAccountId = stringAtNullable(item, "toAccountId");
if (fromAccountId !== null) args.push("--from-account-id", fromAccountId);
if (toAccountId !== null) args.push("--to-account-id", toAccountId);
}
if (type === "sendPrompt") {
args.push("--text", inlinePromptText(item) ?? prompts.prompts[promptIndex % prompts.prompts.length] ?? "");
args.push("--expected-action-wait-ms", String(numberAtNullable(item, "expectedActionWaitMs") ?? 45000));
promptIndex += 1;
}
appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" });
printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat });
const commandResult = runChildCli(args, remainingSeconds(deadline, Math.max(60, commandTimeoutSeconds + 10)));
steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result });
printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() });
if (!commandResult.ok) {
if (type === "sendPrompt") {
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) });
const delayedWaitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds);
steps.push({ phase: "observe-wait-turn-terminal-after-command-timeout", ok: delayedWaitResult.ok, promptIndex, result: delayedWaitResult });
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", delayedWaitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: delayedWaitResult.failure ?? null, status: delayedWaitResult.status ?? null, traceId: delayedWaitResult.traceId ?? null, elapsedMs: elapsedMs() });
if (delayedWaitResult.ok === true) {
const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps);
nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings));
printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() });
if (invariantResult.ok !== true) {
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
observerLifecycle,
promptIndex,
steps,
failure: text(record(invariantResult).failure ?? "observe-session-invariance-failed"),
promptSource: prompts.summary,
elapsedMs: elapsedMs(),
warnings: mergeWarnings(record(invariantResult).warnings, elapsedWarnings()),
}));
}
continue;
}
}
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
observerLifecycle,
promptIndex,
steps,
failure: `observe-command-${type}-failed`,
elapsedMs: elapsedMs(),
warnings: elapsedWarnings(),
promptSource: prompts.summary,
}));
}
if (type === "sendPrompt") {
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) });
const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds);
steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult });
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() });
if (waitResult.ok !== true) {
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
observerLifecycle,
promptIndex,
steps,
failure: text(waitResult.failure ?? "observe-turn-terminal-wait-failed"),
promptSource: prompts.summary,
elapsedMs: elapsedMs(),
warnings: mergeWarnings(Array.isArray(waitResult.warnings) ? waitResult.warnings : [], elapsedWarnings()),
}));
}
const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps);
nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings));
printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() });
if (invariantResult.ok !== true) {
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
observerLifecycle,
promptIndex,
steps,
failure: text(invariantResult.failure ?? "observe-session-invariance-check-failed"),
promptSource: prompts.summary,
elapsedMs: elapsedMs(),
warnings: mergeWarnings(nonBlockingCanaryWarnings, elapsedWarnings()),
}));
}
}
}
}
printQuickVerifyProgress(state, runId, "observe-analyze", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 120) });
const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120)), "--raw", "--compact-raw"], remainingSeconds(deadline, 120));
steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result });
printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() });
const indexEntry = readLocalObserveIndex(observerId);
const artifactSummary = analysisSummaryFromAnalyzeResult(analysis, indexEntry?.stateDir ?? null)
?? (indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS));
const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30));
const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30));
const controlFindings = quickVerifyControlFindings(null, promptIndex, turnSummary, traceFrame);
const artifactSummaryRecord = record(artifactSummary);
const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
const visibilityFindings = quickVerifyAnalysisVisibilityFindings(analysis, artifactSummary);
const cleanupStep = observerLifecycle.stop("observe-stop-after-terminal");
const cleanup = classifyQuickVerifyCleanupStep(cleanupStep);
const cleanupFindings = quickVerifyCleanupFindings(cleanupStep);
const findings = mergeFindingRecords(
mergeFindingRecords(
mergeFindingRecords(artifactFindings, controlFindings),
visibilityFindings,
),
cleanupFindings,
);
const blockingFindings = findings.filter(isQuickVerifyBlockingFinding);
const analysisReadable = stringAtNullable(artifactSummary, "reportJsonSha256") !== null;
const analysisWarnings = mergeWarnings(
analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."],
analysisReadable && artifactSummaryRecord.ok !== true ? ["quick verify analysis report contains non-blocking findings; targetValidation keeps them in report but does not fail the CronJob without red/blocking findings."] : [],
);
const ok = analysisReadable && controlFindings.length === 0 && blockingFindings.length === 0;
const businessStatus = quickVerifyBusinessStatus(null, promptIndex, turnSummary, traceFrame, elapsedMs(), maxSeconds);
const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings);
printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), analysisReadable, findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() });
return recordQuickVerify(state, {
ok,
runId,
scenarioId,
reason,
status: ok ? "analyzed" : "blocked",
observerId,
elapsedMs: elapsedMs(),
businessStatus,
stateDir: indexEntry?.stateDir ?? null,
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
findingCount: findings.length,
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
failure: !analysisReadable ? "quick-verify-analysis-missing" : primaryControlFailure ?? (blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null),
promptSource: prompts.summary,
accountEnv: accountEnv.summary,
steps: [...steps, cleanupStep],
cleanup,
analysis: artifactSummary,
views: {
summary: { renderedText: renderQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, findings, findingCount: findings.length, steps, cleanup, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, cleanup, findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
"turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok },
"trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok },
},
findings,
screenshot: record(artifactSummary).screenshot,
publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"),
warnings: mergeWarnings(analysisWarnings, nonBlockingCanaryWarnings, quickVerifyCleanupWarnings(cleanup), elapsedWarnings()),
valuesRedacted: true,
});
} catch (error) {
const exception = quickVerifyUnhandledExceptionSummary(error);
steps.push({ phase: "quick-verify-unhandled-exception", ok: false, result: exception });
printQuickVerifyProgress(state, runId, "quick-verify-unhandled-exception", "failed", {
observerId,
promptIndex,
errorName: exception.errorName,
errorMessageSha256: exception.errorMessageSha256,
elapsedMs: elapsedMs(),
});
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
observerLifecycle,
promptIndex,
steps,
failure: "quick-verify-unhandled-exception",
elapsedMs: elapsedMs(),
warnings: mergeWarnings("quick verify caught an unexpected post-start exception; details are redacted and the observer cleanup fallback remained active.", elapsedWarnings()),
promptSource: prompts.summary,
}));
} finally {
observerLifecycle.stop("observe-stop-after-finally");
}
}
function sessionInvarianceChecksByRound(scenario: Record<string, unknown>): Map<number, Record<string, unknown>[]> {
const checks = new Map<number, Record<string, unknown>[]>();
const items = Array.isArray(scenario.sessionInvarianceChecks) ? scenario.sessionInvarianceChecks.map(record) : [];
for (const item of items) {
const afterRound = typeof item.afterRound === "number" && Number.isInteger(item.afterRound) ? item.afterRound : null;
if (afterRound === null || afterRound < 0) continue;
const list = checks.get(afterRound) ?? [];
list.push(item);
checks.set(afterRound, list);
}
return checks;
}
function runQuickVerifySessionInvarianceChecks(
state: SentinelCicdState,
observerId: string,
checks: readonly Record<string, unknown>[],
deadline: number,
promptIndex: number,
steps: Record<string, unknown>[],
): Record<string, unknown> {
const warnings: string[] = [];
for (const check of checks) {
const checkId = nonEmptyString(check.id) ?? `after-round-${promptIndex}`;
const blocking = check.blocking === true;
const commands: { readonly type: string; readonly enabled: boolean }[] = [
{ type: "refreshCurrentSession", enabled: check.refreshCurrent === true },
{ type: "switchAwayAndBack", enabled: check.switchAwayAndBack === true },
{ type: "assertSessionInvariant", enabled: check.assertSessionInvariant !== false },
];
for (const command of commands) {
if (!command.enabled) continue;
const args = [
"web-probe", "observe", "command", observerId,
"--node", state.spec.nodeId,
"--lane", state.spec.lane,
"--type", command.type,
"--after-round", String(promptIndex),
"--wait-ms", "55000",
"--command-timeout-seconds", String(remainingSeconds(deadline, 55)),
];
const severity = nonEmptyString(check.severity);
const findingId = nonEmptyString(check.findingId);
const expectedSentinelRange = nonEmptyString(check.expectedSentinelRange);
const alternateSessionStrategy = nonEmptyString(check.alternateSessionStrategy);
if (severity !== null) args.push("--severity", severity);
if (findingId !== null) args.push("--finding-id", findingId);
if (expectedSentinelRange !== null) args.push("--expected-sentinel-range", expectedSentinelRange);
if (command.type === "switchAwayAndBack" && alternateSessionStrategy !== null) args.push("--alternate-session-strategy", alternateSessionStrategy);
if (command.type === "assertSessionInvariant" && check.requireComposerReady === true) args.push("--require-composer-ready");
args.push(check.blocking === true ? "--blocking" : "--non-blocking");
const result = runChildCli(args, remainingSeconds(deadline, 60));
steps.push({ phase: `observe-session-invariance-${command.type}`, ok: result.ok, promptIndex, checkId, result: result.result });
if (!result.ok) {
if (!blocking) {
warnings.push(`non-blocking session invariance canary ${checkId}/${command.type} failed after round ${promptIndex}; continuing Code Agent multi-round quick verify because scenario marks this check blocking=false.`);
continue;
}
return { ok: false, failure: `observe-session-invariance-${command.type}-failed`, checkId, promptIndex, valuesRedacted: true };
}
}
}
return { ok: true, promptIndex, checkCount: checks.length, warnings, valuesRedacted: true };
}
function appendScenarioObserveCommandArgs(args: string[], item: Record<string, unknown>, options: { readonly skipText?: boolean } = {}): void {
const mappings: readonly (readonly [string, string])[] = [
["path", "--path"],
["label", "--label"],
["sessionId", "--session-id"],
["provider", "--provider"],
["accountId", "--account-id"],
["fromAccountId", "--from-account-id"],
["toAccountId", "--to-account-id"],
["sourceId", "--source-id"],
["fileRef", "--file-ref"],
["filename", "--filename"],
["taskRef", "--task-ref"],
["taskId", "--task-id"],
["task", "--task"],
["field", "--field"],
["link", "--link"],
["title", "--title"],
["body", "--body"],
["status", "--status"],
["hwpodId", "--hwpod-id"],
["nodeId", "--node-id"],
["workspaceRoot", "--workspace-root"],
["root", "--root"],
];
for (const [key, flag] of mappings) {
if (args.includes(flag)) continue;
const value = stringAtNullable(item, key);
if (value !== null) args.push(flag, value);
}
if (options.skipText !== true && !args.includes("--text")) {
const text = stringAtNullable(item, "text") ?? stringAtNullable(item, "value");
if (text !== null) args.push("--text", text);
}
if (item.waitProjectManagementReady === true && !args.includes("--wait-project-management-ready")) args.push("--wait-project-management-ready");
}
function inlinePromptText(item: Record<string, unknown>): string | null {
return stringAtNullable(item, "text") ?? stringAtNullable(item, "prompt") ?? stringAtNullable(item, "value");
}
export function reclassifyQuickVerifyControlFindings(state: SentinelCicdState, input: {
readonly runId: string | null;
readonly scenarioId: string | null;
readonly observerId: string | null;
readonly failure?: string | null;
readonly timeoutSeconds?: number | null;
}): Record<string, unknown> {
if (input.observerId === null || input.observerId.length === 0) {
return { ok: false, reason: "observer-id-missing", runId: input.runId, valuesRedacted: true };
}
const scenarioId = input.scenarioId ?? stringAt(state.cicd, "targetValidation.scenarioId");
const scenario = findScenario(state, scenarioId);
if (scenario === null) {
return { ok: false, reason: "scenario-not-found", runId: input.runId, scenarioId, observerId: input.observerId, valuesRedacted: true };
}
const commandSequence = arrayAt(scenario, "commandSequence").map(record);
const promptIndex = commandSequence.reduce((count, item) => {
const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
return stringAtNullable(item, "type") === "sendPrompt" ? count + repeat : count;
}, 0);
const timeoutSeconds = Math.max(5, Math.min(Math.trunc(input.timeoutSeconds ?? 55), 55));
const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, timeoutSeconds);
const traceFrame = collectObserveView(state, input.observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, timeoutSeconds);
const findings = quickVerifyControlFindings(input.failure ?? null, promptIndex, turnSummary, traceFrame);
return {
ok: true,
runId: input.runId,
scenarioId,
observerId: input.observerId,
promptIndex,
findingCount: findings.length,
findings,
turnSummary: { ok: turnSummary.ok === true, collectShape: turnSummary.collectShape, valuesRedacted: true },
traceFrame: { ok: traceFrame.ok === true, collectShape: traceFrame.collectShape, valuesRedacted: true },
valuesRedacted: true,
};
}
function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
readonly runId: string;
readonly scenarioId: string;
readonly reason: string;
readonly observerId: string;
readonly observerLifecycle: QuickVerifyObserverLifecycle;
readonly promptIndex: number;
readonly steps: readonly Record<string, unknown>[];
readonly failure: string;
readonly failureTitleZh?: string | null;
readonly timeoutDisplay?: Record<string, unknown> | null;
readonly promptSource?: Record<string, unknown>;
readonly elapsedMs?: number;
readonly warnings?: readonly unknown[];
}): Record<string, unknown> {
const targetValidationSeconds = numberAt(state.cicd, "targetValidation.maxSeconds");
const failureDisplay = input.timeoutDisplay ?? timeoutDisplayForQuickVerifyFailure(input.failure, input.promptIndex, targetValidationSeconds, null);
const failureTitle = input.failureTitleZh ?? stringAtNullable(failureDisplay, "titleZh");
const cleanupSteps: Record<string, unknown>[] = [];
if (input.promptIndex > 0) {
const cancel = runChildCli([
"web-probe", "observe", "command", input.observerId,
"--node", state.spec.nodeId,
"--lane", state.spec.lane,
"--type", "cancel",
"--wait-ms", "55000",
"--command-timeout-seconds", "55",
], 60);
cleanupSteps.push({ phase: "observe-cancel-after-failure", ok: cancel.ok, result: cancel.result });
}
cleanupSteps.push(input.observerLifecycle.stop("observe-stop-after-failure"));
const analysis = runChildCli([
"web-probe", "observe", "analyze", input.observerId,
"--node", state.spec.nodeId,
"--lane", state.spec.lane,
"--command-timeout-seconds", "55",
"--raw", "--compact-raw",
], 60);
cleanupSteps.push({ phase: "observe-analyze-after-failure", ok: analysis.ok, result: analysis.result });
const indexEntry = readLocalObserveIndex(input.observerId);
const artifactSummary = analysisSummaryFromAnalyzeResult(analysis, indexEntry?.stateDir ?? null)
?? (indexEntry === null
? { ok: false, reason: "observe-index-entry-missing", observerId: input.observerId, valuesRedacted: true }
: readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS));
const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30);
const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30);
const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(input.promptIndex, turnSummary, traceFrame);
const controlFindings = quickVerifyControlFindings(input.failure, input.promptIndex, turnSummary, traceFrame, failureDisplay);
const artifactSummaryRecord = record(artifactSummary);
const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
const visibilityFindings = quickVerifyAnalysisVisibilityFindings(analysis, artifactSummary);
const cleanup = quickVerifyCleanupFromSteps(cleanupSteps);
const cleanupFindings = cleanupSteps.flatMap((step) => quickVerifyCleanupFindings(step));
const findings = mergeFindingRecords(
mergeFindingRecords(
mergeFindingRecords(artifactFindings, controlFindings),
visibilityFindings,
),
cleanupFindings,
);
const blockingFindings = findings.filter(isQuickVerifyBlockingFinding);
const recoveredWaitFailure = durableBusinessTurn
&& isRecoverableQuickVerifyWaitFailure(input.failure)
&& record(artifactSummary).ok === true
&& controlFindings.length === 0
&& blockingFindings.length === 0;
const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, targetValidationSeconds);
const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings);
return {
ok: recoveredWaitFailure,
runId: input.runId,
scenarioId: input.scenarioId,
reason: input.reason,
status: recoveredWaitFailure ? "analyzed" : "blocked",
observerId: input.observerId,
elapsedMs: input.elapsedMs ?? null,
businessStatus,
failureTitleZh: recoveredWaitFailure ? null : failureTitle,
errorTitleZh: recoveredWaitFailure ? null : failureTitle,
timeoutDisplay: recoveredWaitFailure ? null : failureDisplay,
stateDir: indexEntry?.stateDir ?? null,
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
findingCount: findings.length,
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
failure: recoveredWaitFailure ? null : primaryControlFailure ?? input.failure,
runnerFailure: recoveredWaitFailure ? null : input.failure,
promptSource: input.promptSource,
steps: [...input.steps, ...cleanupSteps],
cleanup,
analysis: artifactSummary,
views: {
summary: { renderedText: renderQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, findings, findingCount: findings.length, steps: input.steps, cleanup, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), failure: recoveredWaitFailure ? null : input.failure, failureTitleZh: recoveredWaitFailure ? null : failureTitle, timeoutDisplay: recoveredWaitFailure ? null : failureDisplay }) },
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, cleanup, findings, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), failure: recoveredWaitFailure ? null : input.failure, failureTitleZh: recoveredWaitFailure ? null : failureTitle, timeoutDisplay: recoveredWaitFailure ? null : failureDisplay }) },
"turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok },
"trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok },
},
findings,
screenshot: record(artifactSummary).screenshot,
publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"),
warnings: mergeWarnings(
Array.isArray(input.warnings) ? input.warnings : [],
recoveredWaitFailure ? ["quick verify wait command timed out, but collected turn-summary/trace-frame artifacts show a durable completed business turn; treating the wait timeout as a non-blocking tool finding."] : [],
quickVerifyCleanupWarnings(cleanup),
targetValidationElapsedWarnings(input.elapsedMs ?? null, "quick verify confirm-wait", targetValidationSeconds),
),
valuesRedacted: true,
};
}
function stopQuickVerifyObserver(state: SentinelCicdState, observerId: string, phase: string): Record<string, unknown> {
const stop = runChildCli([
"web-probe", "observe", "stop", observerId,
"--node", state.spec.nodeId,
"--lane", state.spec.lane,
"--force",
"--raw",
"--wait-ms", "55000",
"--command-timeout-seconds", "55",
], 60);
const step = { phase, ok: stop.ok, payload: cliDataPayload(stop.parsed), result: stop.result };
return { ...step, cleanup: classifyQuickVerifyCleanupStep(step) };
}
export function quickVerifyCleanupFindings(cleanupStep: Record<string, unknown>): Record<string, unknown>[] {
const cleanup = classifyQuickVerifyCleanupStep(cleanupStep);
const cleanupClass = stringAtNullable(cleanup, "cleanupClass");
if (cleanupClass === "bounded-force-stop") {
return [{
id: "observer-graceful-stop-timeout-force-stop-succeeded",
severity: "warning",
count: 1,
summary: "observer graceful stop did not drain before timeout, but force stop confirmed aliveAfter=false.",
cleanupPhase: cleanupStep.phase,
cleanup,
evidenceSummary: formatQuickVerifyCleanupLine(cleanup),
valuesRedacted: true,
}];
}
if (cleanupClass !== "observer-stop-failed") return [];
const aliveAfter = booleanAtNullable(cleanup, "aliveAfter");
const id = aliveAfter === true ? "observer-force-stop-left-process-alive" : "observer-cleanup-evidence-missing";
const summary = aliveAfter === true
? "observer force stop completed but process liveness evidence still reports aliveAfter=true."
: "observer cleanup failed because stop-state evidence is missing or incomplete; this is an evidence gap, not confirmed process leakage.";
return [{
id,
severity: "red",
count: 1,
summary,
cleanupPhase: cleanupStep.phase,
cleanup,
evidenceSummary: formatQuickVerifyCleanupLine(cleanup),
valuesRedacted: true,
}];
}
export function classifyQuickVerifyCleanupStep(cleanupStep: Record<string, unknown>): Record<string, unknown> {
const explicit = record(cleanupStep.cleanup);
if (stringAtNullable(explicit, "cleanupClass") !== null) return explicit;
const payload = quickVerifyCleanupPayload(cleanupStep);
const observer = record(payload.observer);
const forceStop = record(payload.forceStop);
const detail = record(payload.detail);
const forceResult = record(payload.forceResult);
const sources = cleanupEvidenceSources([
["payload.observer", observer],
["payload.forceStop", forceStop],
["payload.detail", detail],
["payload", payload],
["payload.forceResult", forceResult],
]);
const forceReason = firstStringFromCleanupSources(sources, ["forceReason", "reason"]);
const status = firstStringFromCleanupSources(sources, ["status"]);
const command = firstStringFromCleanupSources(sources, ["command"]);
const evidenceSourceNames = cleanupEvidenceSourceNames(sources, [
"aliveBefore",
"aliveAfter",
"termSent",
"killSent",
"pendingAbandoned",
"processingAbandoned",
]);
const forceStopAttempted = forceReason !== null
|| status === "forced-stopped"
|| status === "force-stopped"
|| command === "web-probe-observe force-stop"
|| payload.forced === true
|| numberAtNullable(forceResult, "exitCode") !== null
|| evidenceSourceNames.length > 0;
const aliveBefore = firstBooleanFromCleanupSources(sources, "aliveBefore");
const aliveAfter = firstBooleanFromCleanupSources(sources, "aliveAfter");
const termSent = firstBooleanFromCleanupSources(sources, "termSent");
const killSent = firstBooleanFromCleanupSources(sources, "killSent");
const pendingAbandoned = firstNumberFromCleanupSources(sources, "pendingAbandoned");
const processingAbandoned = firstNumberFromCleanupSources(sources, "processingAbandoned");
const forceStopOk = forceStopAttempted && aliveAfter === false;
const gracefulStopOk = !forceStopAttempted && (cleanupStep.ok === true || payload.ok === true);
const cleanupClass = gracefulStopOk
? "stopped"
: forceStopOk
? "bounded-force-stop"
: "observer-stop-failed";
const missingEvidenceFields = forceStopAttempted && cleanupClass === "observer-stop-failed"
? [
aliveAfter === null ? "aliveAfter" : null,
termSent === null ? "termSent" : null,
killSent === null ? "killSent" : null,
pendingAbandoned === null ? "pendingAbandoned" : null,
processingAbandoned === null ? "processingAbandoned" : null,
].filter((value): value is string => value !== null)
: [];
return {
cleanupClass,
phase: stringAtNullable(cleanupStep, "phase"),
commandOk: cleanupStep.ok === true ? true : cleanupStep.ok === false ? false : null,
status,
forceReason,
gracefulStopOk,
forceStopOk,
aliveBefore,
aliveAfter,
termSent,
killSent,
pendingAbandoned,
processingAbandoned,
evidenceSources: evidenceSourceNames,
missingEvidenceFields,
valuesRedacted: true,
};
}
function quickVerifyCleanupPayload(cleanupStep: Record<string, unknown>): Record<string, unknown> {
const payload = record(cleanupStep.payload);
if (Object.keys(payload).length > 0) return payload;
const result = record(cleanupStep.result);
for (const key of ["stdoutPreview", "stdoutTail"]) {
const parsed = parseJsonObject(stringAtNullable(result, key) ?? "");
const data = cliDataPayload(parsed);
if (Object.keys(data).length > 0) return data;
}
return {};
}
function cleanupEvidenceSources(entries: readonly (readonly [string, Record<string, unknown>])[]): readonly { name: string; value: Record<string, unknown> }[] {
return entries
.filter((entry) => Object.keys(entry[1]).length > 0)
.map(([name, value]) => ({ name, value }));
}
function cleanupEvidenceSourceNames(sources: readonly { name: string; value: Record<string, unknown> }[], keys: readonly string[]): string[] {
const names: string[] = [];
for (const source of sources) {
if (keys.some((key) => source.value[key] !== undefined && source.value[key] !== null)) names.push(source.name);
}
return [...new Set(names)];
}
function firstStringFromCleanupSources(sources: readonly { value: Record<string, unknown> }[], keys: readonly string[]): string | null {
for (const source of sources) {
for (const key of keys) {
const value = stringAtNullable(source.value, key);
if (value !== null) return value;
}
}
return null;
}
function firstBooleanFromCleanupSources(sources: readonly { value: Record<string, unknown> }[], key: string): boolean | null {
for (const source of sources) {
const value = booleanAtNullable(source.value, key);
if (value !== null) return value;
}
return null;
}
function firstNumberFromCleanupSources(sources: readonly { value: Record<string, unknown> }[], key: string): number | null {
for (const source of sources) {
const value = numberAtNullable(source.value, key);
if (value !== null) return value;
}
return null;
}
function quickVerifyCleanupFromSteps(steps: readonly Record<string, unknown>[]): Record<string, unknown> | null {
for (const step of steps.slice().reverse()) {
const phase = stringAtNullable(step, "phase");
if (phase !== null && phase.includes("observe-stop")) return classifyQuickVerifyCleanupStep(step);
}
return null;
}
function quickVerifyCleanupWarnings(cleanup: Record<string, unknown> | null): string[] {
if (cleanup === null) return [];
const cleanupClass = stringAtNullable(cleanup, "cleanupClass");
if (cleanupClass === "bounded-force-stop") {
return [`quick verify observer graceful stop did not drain, but force stop completed with aliveAfter=false; recorded bounded cleanup evidence (${formatQuickVerifyCleanupLine(cleanup)}).`];
}
if (cleanupClass === "observer-stop-failed") {
return ["quick verify observer stop failed; runner lifecycle cleanup is a blocking finding."];
}
return [];
}
function formatQuickVerifyCleanupLine(cleanup: Record<string, unknown> | null): string {
if (cleanup === null) return "cleanup=-";
const fields = [
["cleanup", stringAtNullable(cleanup, "cleanupClass")],
["aliveBefore", cleanup.aliveBefore],
["aliveAfter", cleanup.aliveAfter],
["termSent", cleanup.termSent],
["killSent", cleanup.killSent],
["pendingAbandoned", cleanup.pendingAbandoned],
["processingAbandoned", cleanup.processingAbandoned],
["gracefulStopOk", cleanup.gracefulStopOk],
["forceStopOk", cleanup.forceStopOk],
["missing", Array.isArray(cleanup.missingEvidenceFields) && cleanup.missingEvidenceFields.length > 0 ? cleanup.missingEvidenceFields.join(",") : null],
];
return fields.map(([key, value]) => `${key}=${value ?? "-"}`).join(" ");
}
function booleanAtNullable(value: Record<string, unknown>, key: string): boolean | null {
const raw = value[key];
return raw === true ? true : raw === false ? false : null;
}
function quickVerifyAnalysisVisibilityFindings(analysis: ChildCliResult, artifactSummary: Record<string, unknown>): Record<string, unknown>[] {
if (analysis.ok !== true) return [];
if (stringAtNullable(artifactSummary, "reportJsonSha256") !== null) return [];
const result = record(analysis.result);
const stdout = stringAtNullable(result, "stdoutPreview") ?? stringAtNullable(result, "stdoutTail");
const reportPathMatch = stdout === null ? null : stdout.match(/analysis\/report\.(?:json|md)/u);
return [{
id: "quick-verify-analysis-summary-unreadable",
severity: "red",
count: 1,
summary: "quick verify analyze exited successfully, but sentinel could not read analysis/report.json before recording the run.",
rootCause: "The report index would otherwise contain only control blockers such as WBC-003 and hide analyzer red/amber findings.",
rootCauseStatus: "confirmed",
rootCauseConfidence: "high",
evidenceSummary: `analyze ok=true reportJsonSha256=missing stdoutMentionsReport=${reportPathMatch === null ? "false" : "true"}`,
nextAction: "Read stateDir/analysis/report.json again or rehydrate the report from stateDir before trusting WBC-003 as the only visible finding.",
valuesRedacted: true,
}];
}
function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number): Record<string, unknown> {
const namespace = stringAt(state.runtime, "namespace");
const serviceName = stringAt(state.runtime, "serviceName");
const servicePort = numberAt(state.runtime, "servicePort");
const deploymentName = stringAt(state.runtime, "deploymentName");
const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`;
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64");
const postScript = [
"const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');",
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
`const url = 'http://127.0.0.1:${servicePort}' + path;`,
"fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {",
" const text = await response.text();",
" process.stdout.write(text);",
" if (!response.ok) process.exit(22);",
"}).catch((error) => {",
" console.error(error && error.stack ? error.stack : String(error));",
" process.exit(23);",
"});",
].join(" ");
const script = method === "GET"
? `kubectl get --raw ${shellQuote(proxyPath)}`
: [
"set -eu",
`kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`,
].join("\n");
const maxAttempts = method === "GET" ? 3 : 1;
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60));
const attempts: Record<string, unknown>[] = [];
let result: CommandResult | null = null;
let parsed: Record<string, unknown> | null = null;
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
const invocation = sentinelServiceProxyInvocation(stringAt(state.controlPlaneNode, "kubeRoute"), script);
result = runCommand(invocation.argv, repoRoot, { input: invocation.input, timeoutMs: attemptTimeoutSeconds * 1000 });
const parsedResolution = resolveCliChildJsonCommandResult({
result,
requestedStdoutType: "web-probe sentinel service proxy response JSON",
acceptParsed: isSentinelServiceResponseContract,
});
parsed = parsedResolution.parsed;
attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, stdoutRecovery: parsedResolution.diagnostics, valuesRedacted: true });
if (result.exitCode === 0) break;
}
return {
ok: result?.exitCode === 0,
method,
path: pathWithQuery,
internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`,
httpStatus: result?.exitCode === 0 ? 200 : null,
bodyJson: record(compactSentinelServiceBodyJson(parsed)),
bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "",
bodyBytes: Buffer.byteLength(result?.stdout ?? ""),
error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000),
proxyPath,
result: result === null ? null : compactCommand(result),
attempts,
valuesRedacted: true,
};
}
function isSentinelServiceResponseContract(value: Record<string, unknown>): boolean {
return value.ok === false
|| typeof value.view === "string"
|| typeof value.renderedText === "string"
|| typeof value.error === "string"
|| Array.isArray(value.availableViews)
|| Object.keys(record(value.run)).length > 0
|| Object.keys(record(value.summary)).length > 0
|| Array.isArray(value.findings);
}
export function sentinelServiceProxyInvocation(kubeRoute: string, script: string): { readonly argv: readonly string[]; readonly input: string } {
return { argv: ["trans", kubeRoute, "sh"], input: script };
}
function compactSentinelServiceBodyJson(value: Record<string, unknown> | null): unknown {
if (value === null || typeof value.renderedText !== "string") return value;
return {
ok: value.ok,
view: value.view,
error: value.error,
availableViews: value.availableViews,
valuesRedacted: true,
};
}
function clipTail(value: string, maxLength: number): string {
return value.length <= maxLength ? value : value.slice(value.length - maxLength);
}
function recordSkippedQuickVerify(
state: SentinelCicdState,
scenario: Record<string, unknown>,
runId: string,
scenarioId: string,
triggerReason: string,
elapsedMs: number,
memoryGuard: unknown,
steps: readonly Record<string, unknown>[],
skipReason: string,
): Record<string, unknown> {
const cadence = stringAtNullable(scenario, "cadence");
const renderedText = [
`# Web 哨兵巡检 ${runId}`,
"",
"本轮因主机物理可用内存低于 WebProbe 启动门槛而跳过,未创建 observer 或 browser。",
`- 状态:skipped-wait-next-round`,
`- 场景:${scenarioId}`,
`- 下一步:等待下一次计划 cadence${cadence === null ? "" : `${cadence}`}`,
].join("\n");
const recorded = recordQuickVerify(state, {
ok: true,
runId,
scenarioId,
reason: skipReason,
triggerReason,
status: "skipped-wait-next-round",
decision: "skip-observer",
observerId: null,
observerCreated: false,
browserCreated: false,
runCreated: true,
mutation: false,
memoryGuard,
elapsedMs,
steps: [...steps],
findingCount: 0,
artifactCount: 0,
findings: [],
warnings: [],
updatedAt: new Date().toISOString(),
nextCadence: {
action: "wait-for-next-scheduled-cadence",
cadence,
automaticRetry: true,
valuesRedacted: true,
},
views: {
summary: { renderedText, valuesRedacted: true },
"auth-session-switch-summary": { renderedText, valuesRedacted: true },
},
valuesRedacted: true,
});
return {
...recorded,
runCreated: record(recorded.recordResult).ok === true,
valuesRedacted: true,
};
}
function recordQuickVerify(state: SentinelCicdState, payload: Record<string, unknown>): Record<string, unknown> {
const writer = resolveMonitorTerminalWriter(state.runtime);
const views = compactQuickVerifyRecordViews(record(payload.views));
const summary = {
reason: payload.reason,
status: payload.status,
businessStatus: payload.businessStatus ?? null,
elapsedMs: payload.elapsedMs,
failure: payload.failure,
failureTitleZh: payload.failureTitleZh ?? payload.errorTitleZh ?? null,
errorTitleZh: payload.errorTitleZh ?? payload.failureTitleZh ?? null,
timeoutDisplay: payload.timeoutDisplay ?? null,
warnings: Array.isArray(payload.warnings) ? payload.warnings : [],
analysis: compactQuickVerifyRecordAnalysis(payload.analysis),
cleanup: compactQuickVerifyCleanupEvidence(payload.cleanup),
promptSource: payload.promptSource,
steps: Array.isArray(payload.steps) ? payload.steps.map(compactQuickVerifyRecordStep) : [],
valuesRedacted: true,
};
const legacyRecord = {
runId: payload.runId,
scenarioId: payload.scenarioId,
status: payload.status,
observerId: payload.observerId,
stateDir: payload.stateDir,
reportJsonSha256: payload.reportJsonSha256,
findingCount: payload.findingCount,
artifactCount: payload.artifactCount,
summary,
businessStatus: payload.businessStatus ?? null,
findings: payload.findings,
views,
screenshot: payload.screenshot,
publicOrigin: payload.publicOrigin ?? stringAt(state.publicExposure, "publicBaseUrl"),
maintenance: payload.reason === "maintenance-stop",
valuesRedacted: true,
};
const recordResult = writer.mode === "legacy-sqlite"
? callSentinelService(state, "POST", "/api/runs/record", legacyRecord, 60)
: submitCentralTerminalIngest(state, payload, writer.ingest!, views, summary);
emitWebProbeSentinelSpan(sentinelOtelContext(state), "web_probe_sentinel.quick_verify.job_finish", {
scenarioId: payload.scenarioId,
runId: payload.runId,
observerId: payload.observerId,
status: payload.status,
exitCode: payload.ok === true && recordResult.ok === true ? 0 : 1,
failureKind: payload.failure ?? (recordResult.ok === true ? null : writer.mode === "central" ? "monitor-ingest-failed" : "record-run-failed"),
valuesRedacted: true,
}, payload.ok === true && recordResult.ok === true);
return withWarnings({ ...payload, views, terminalWriter: writer.mode, recordResult, valuesRedacted: true }, recordResult.ok === true ? [] : [writer.mode === "central" ? "monitor-ingest-failed: retry the same immutable analysis/report artifact after the central ingest endpoint recovers." : "quick verify completed but sentinel report index record failed; report/dashboard may lag until record payload is reduced or retried."]);
}
function submitCentralTerminalIngest(state: SentinelCicdState, payload: Record<string, unknown>, config: NonNullable<ReturnType<typeof resolveMonitorTerminalWriter>["ingest"]>, views: Record<string, unknown>, summary: Record<string, unknown>): Record<string, unknown> {
const analysis = record(payload.analysis);
try {
const stateDir = stringAtNullable(payload, "stateDir") ?? stringAtNullable(analysis, "stateDir");
const report = { ...analysis, updatedAt: stringAtNullable(analysis, "reportUpdatedAt") ?? stringAtNullable(payload, "updatedAt"), findings: Array.isArray(payload.findings) ? payload.findings.map(record) : [], summary, views, valuesRedacted: true };
const artifacts = payload.status === "skipped-wait-next-round" || (payload.observerId === null && payload.artifactCount === 0)
? []
: [{ relativePath: "analysis/report.json", kind: "report-json", sha256: stringAtNullable(payload, "reportJsonSha256") ?? stringAtNullable(analysis, "reportJsonSha256") ?? "", sizeBytes: numberAtNullable(analysis, "reportJsonBytes") ?? -1 }];
const ingest = buildMonitorTerminalIngest({ sentinelId: state.sentinelId, node: state.spec.nodeId, lane: state.spec.lane, runId: stringAt(payload, "runId") }, report, artifacts, { ownerNode: state.spec.nodeId, ownerLane: state.spec.lane, ownerPvc: stringAt(state.runtime, "pvcName"), stateDir: stateDir ?? "" });
const result = runCommand(["bun", "scripts/monitor-terminal-ingest-submit.ts"], repoRoot, { timeoutMs: config.retry.maxAttempts * config.timeoutMs + (config.retry.maxAttempts - 1) * config.retry.delayMs + 1_000, input: JSON.stringify({ config, ingest }) });
const response = parseJsonObject(result.stdout);
if (result.exitCode === 0 && response?.ok === true) return { ...response, writer: "central", valuesRedacted: true };
return { ok: false, writer: "central", error: "monitor-ingest-failed", response, stderr: short(result.stderr), payloadHash: ingest.payloadHash, valuesRedacted: true };
} catch (error) {
return { ok: false, writer: "central", error: "monitor-ingest-failed", details: error instanceof MonitorIngestFailedError ? error.details : { reason: error instanceof Error ? error.message : String(error) }, valuesRedacted: true };
}
}
function sentinelOtelContext(state: SentinelCicdState): { readonly node: string; readonly lane: string; readonly sentinelId: string; readonly namespace: string | null; readonly runtime: Record<string, unknown>; readonly cicd: Record<string, unknown> } {
return {
node: state.spec.nodeId,
lane: state.spec.lane,
sentinelId: state.sentinelId,
namespace: stringAtNullable(state.runtime, "namespace"),
runtime: state.runtime,
cicd: state.cicd,
};
}
function compactQuickVerifyRecordViews(views: Record<string, unknown>): Record<string, unknown> {
const compacted: Record<string, unknown> = {};
for (const [key, value] of Object.entries(views)) {
const item = record(value);
const limit = key === "summary" || key === "auth-session-switch-summary" ? 8_000 : 6_000;
compacted[key] = {
...item,
renderedText: boundQuickVerifyRecordText(item.renderedText, limit),
valuesRedacted: true,
};
}
return compacted;
}
function compactQuickVerifyRecordAnalysis(value: unknown): Record<string, unknown> | null {
const item = record(value);
if (Object.keys(item).length === 0) return null;
return {
ok: item.ok === true ? true : item.ok === false ? false : null,
reportOk: item.reportOk === true ? true : item.reportOk === false ? false : null,
source: stringAtNullable(item, "source"),
reason: stringAtNullable(item, "reason"),
stateDir: stringAtNullable(item, "stateDir"),
reportJsonPath: stringAtNullable(item, "reportJsonPath"),
reportJsonSha256: stringAtNullable(item, "reportJsonSha256"),
reportMdPath: stringAtNullable(item, "reportMdPath"),
reportMdSha256: stringAtNullable(item, "reportMdSha256"),
reportReadWaitMs: numberAtNullable(item, "reportReadWaitMs"),
reportParseError: stringAtNullable(item, "reportParseError"),
findingCount: numberAtNullable(item, "findingCount"),
artifactCount: numberAtNullable(item, "artifactCount"),
counts: compactQuickVerifyRecordCounts(record(item.counts)),
readResult: compactQuickVerifyRecordArtifactReadResult(record(item.result)),
screenshot: compactQuickVerifyRecordScreenshot(record(item.screenshot)),
findings: Array.isArray(item.findings) ? item.findings.slice(0, 16).map(compactQuickVerifyRecordFinding) : [],
pagePerformanceSlowApi: Array.isArray(item.pagePerformanceSlowApi) ? item.pagePerformanceSlowApi.slice(0, 6).map(record) : [],
browserProcess: compactQuickVerifyRecordBrowserProcess(record(item.browserProcess)),
requestRate: compactQuickVerifyRecordRequestRate(record(item.requestRate), record(item.requestRateCurve)),
valuesRedacted: true,
};
}
function compactQuickVerifyRecordArtifactReadResult(value: Record<string, unknown>): Record<string, unknown> | null {
if (Object.keys(value).length === 0) return null;
return {
exitCode: numberAtNullable(value, "exitCode"),
timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null,
stdoutBytes: numberAtNullable(value, "stdoutBytes"),
stderrBytes: numberAtNullable(value, "stderrBytes"),
stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 600),
stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 600),
valuesRedacted: true,
};
}
function compactQuickVerifyRecordBrowserProcess(value: Record<string, unknown>): Record<string, unknown> | null {
const pageSeries = Array.isArray(value.pageSeries) ? value.pageSeries.map(compactQuickVerifyRecordMemorySeries).filter((item) => item.points.length > 0) : [];
if (pageSeries.length === 0) return null;
return {
source: stringAtNullable(value, "source") ?? "analysis-browser-process",
unit: stringAtNullable(value, "unit") ?? "MB",
metric: stringAtNullable(value, "metric") ?? "page-heap-used",
pageCount: pageSeries.length,
sampleCount: pageSeries.reduce((sum, item) => sum + item.points.length, 0),
pageSeries,
valuesRedacted: true,
};
}
function compactQuickVerifyRecordRequestRate(summaryValue: Record<string, unknown>, curveValue: Record<string, unknown>): Record<string, unknown> | null {
const source = Object.keys(curveValue).length > 0 ? curveValue : summaryValue;
const summary = record(source.summary);
const effectiveSummary = Object.keys(summary).length > 0 ? summary : summaryValue;
const pageCurves = Array.isArray(source.pageCurves) ? source.pageCurves.map((item) => compactQuickVerifyRecordRequestRateCurve(item, "page")).filter((item) => item.buckets.length > 0).slice(0, 12) : [];
const apiPathCurves = Array.isArray(source.apiPathCurves) ? source.apiPathCurves.map((item) => compactQuickVerifyRecordRequestRateCurve(item, "apiPath")).filter((item) => item.buckets.length > 0).slice(0, 16) : [];
const buckets = Array.isArray(source.buckets) ? source.buckets.map(compactQuickVerifyRecordRequestRateBucket).filter(hasCompactRequestRateBucketPoint).slice(-120) : [];
if (Object.keys(source).length === 0 && pageCurves.length === 0 && apiPathCurves.length === 0 && buckets.length === 0) return null;
return {
summary: compactQuickVerifyRecordRequestRateSummary(effectiveSummary),
buckets,
pageCurves,
apiPathCurves,
peaks: Array.isArray(source.peaks) ? source.peaks.slice(0, 12).map(compactQuickVerifyRecordRequestRatePeak) : [],
valuesRedacted: true,
};
}
function compactQuickVerifyRecordRequestRateSummary(value: Record<string, unknown>): Record<string, unknown> {
return {
bucketMs: numberAtNullable(value, "bucketMs"),
bucketSeconds: numberAtNullable(value, "bucketSeconds"),
requestCount: numberAtNullable(value, "requestCount"),
bucketCount: numberAtNullable(value, "bucketCount"),
pageCount: numberAtNullable(value, "pageCount"),
apiPathCount: numberAtNullable(value, "apiPathCount"),
firstAt: stringAtNullable(value, "firstAt"),
lastAt: stringAtNullable(value, "lastAt"),
totalRedPerMinute: numberAtNullable(value, "totalRedPerMinute"),
pageRedPerMinute: numberAtNullable(value, "pageRedPerMinute"),
apiPathRedPerMinute: numberAtNullable(value, "apiPathRedPerMinute"),
totalPeakPerMinute: numberAtNullable(value, "totalPeakPerMinute"),
totalPeakCount: numberAtNullable(value, "totalPeakCount"),
totalPeakAt: stringAtNullable(value, "totalPeakAt"),
pagePeakPerMinute: numberAtNullable(value, "pagePeakPerMinute"),
pagePeakKey: stringAtNullable(value, "pagePeakKey"),
pagePeakPath: stringAtNullable(value, "pagePeakPath"),
apiPathPeakPerMinute: numberAtNullable(value, "apiPathPeakPerMinute"),
apiPathPeakKey: stringAtNullable(value, "apiPathPeakKey"),
overThresholdPeakCount: numberAtNullable(value, "overThresholdPeakCount"),
valuesRedacted: true,
};
}
function compactQuickVerifyRecordRequestRateCurve(value: unknown, scope: string): Record<string, unknown> & { buckets: Record<string, unknown>[] } {
const item = record(value);
const buckets = Array.isArray(item.buckets) ? item.buckets.map(compactQuickVerifyRecordRequestRateBucket).filter(hasCompactRequestRateBucketPoint).slice(-120) : [];
return {
pageKey: stringAtNullable(item, "pageKey"),
pageRole: stringAtNullable(item, "pageRole"),
pageId: stringAtNullable(item, "pageId"),
pageEpoch: numberAtNullable(item, "pageEpoch"),
apiKey: stringAtNullable(item, "apiKey"),
method: stringAtNullable(item, "method"),
path: stringAtNullable(item, "path"),
count: numberAtNullable(item, "count"),
bucketCount: numberAtNullable(item, "bucketCount") ?? buckets.length,
peakRequestPerMinute: numberAtNullable(item, "peakRequestPerMinute"),
peakBucket: compactQuickVerifyRecordRequestRateBucket(record(item.peakBucket)),
buckets,
scope,
valuesRedacted: true,
};
}
function compactQuickVerifyRecordRequestRateBucket(value: unknown): Record<string, unknown> {
const item = record(value);
return {
bucketStartMs: numberAtNullable(item, "bucketStartMs"),
bucketEndMs: numberAtNullable(item, "bucketEndMs"),
startAt: stringAtNullable(item, "startAt") ?? stringAtNullable(item, "t"),
endAt: stringAtNullable(item, "endAt") ?? stringAtNullable(item, "e"),
count: numberAtNullable(item, "count") ?? numberAtNullable(item, "c"),
requestPerMinute: numberAtNullable(item, "requestPerMinute") ?? numberAtNullable(item, "rpm"),
valuesRedacted: true,
};
}
function hasCompactRequestRateBucketPoint(value: Record<string, unknown>): boolean {
return stringAtNullable(value, "startAt") !== null && numberAtNullable(value, "requestPerMinute") !== null;
}
function compactQuickVerifyRecordRequestRatePeak(value: unknown): Record<string, unknown> {
const item = record(value);
return {
scope: stringAtNullable(item, "scope"),
thresholdPerMinute: numberAtNullable(item, "thresholdPerMinute"),
overThreshold: item.overThreshold === true,
bucketMs: numberAtNullable(item, "bucketMs"),
startAt: stringAtNullable(item, "startAt"),
endAt: stringAtNullable(item, "endAt"),
count: numberAtNullable(item, "count"),
requestPerMinute: numberAtNullable(item, "requestPerMinute"),
method: stringAtNullable(item, "method"),
path: stringAtNullable(item, "path"),
apiKey: stringAtNullable(item, "apiKey"),
pageKey: stringAtNullable(item, "pageKey"),
pageRole: stringAtNullable(item, "pageRole"),
pageId: stringAtNullable(item, "pageId"),
pageEpoch: numberAtNullable(item, "pageEpoch"),
valuesRedacted: true,
};
}
function compactQuickVerifyRecordMemorySeries(value: unknown): Record<string, unknown> & { points: Record<string, unknown>[] } {
const item = record(value);
const points = Array.isArray(item.points)
? item.points.map(record).map((point) => ({
ts: stringAtNullable(point, "ts"),
elapsedSeconds: numberAtNullable(point, "elapsedSeconds"),
elapsedMinutes: numberAtNullable(point, "elapsedMinutes"),
memoryMb: numberAtNullable(point, "memoryMb"),
heapUsedMb: numberAtNullable(point, "heapUsedMb"),
jsHeapUsedMb: numberAtNullable(point, "jsHeapUsedMb"),
effectiveHeapUsedMb: numberAtNullable(point, "effectiveHeapUsedMb"),
effectiveJsHeapUsedMb: numberAtNullable(point, "effectiveJsHeapUsedMb"),
domNodes: numberAtNullable(point, "domNodes"),
valuesRedacted: true,
})).filter((point) => point.ts !== null && point.memoryMb !== null)
: [];
return {
key: stringAtNullable(item, "key"),
pageRole: stringAtNullable(item, "pageRole"),
pageId: stringAtNullable(item, "pageId"),
label: stringAtNullable(item, "label"),
url: stringAtNullable(item, "url"),
points,
valuesRedacted: true,
};
}
function compactQuickVerifyRecordCounts(value: Record<string, unknown>): Record<string, unknown> {
return {
samples: numberAtNullable(value, "samples"),
control: numberAtNullable(value, "control"),
network: numberAtNullable(value, "network"),
console: numberAtNullable(value, "console"),
errors: numberAtNullable(value, "errors"),
artifacts: numberAtNullable(value, "artifacts"),
valuesRedacted: true,
};
}
function compactQuickVerifyRecordScreenshot(value: Record<string, unknown>): Record<string, unknown> | null {
if (Object.keys(value).length === 0) return null;
return {
path: stringAtNullable(value, "path"),
sha256: stringAtNullable(value, "sha256"),
bytes: numberAtNullable(value, "bytes"),
valuesRedacted: true,
};
}
function compactQuickVerifyRecordFinding(value: unknown): Record<string, unknown> {
const item = record(value);
return {
id: stringAtNullable(item, "id"),
kind: stringAtNullable(item, "kind"),
code: stringAtNullable(item, "code"),
severity: stringAtNullable(item, "severity"),
level: stringAtNullable(item, "level"),
count: numberAtNullable(item, "count"),
summary: boundQuickVerifyRecordText(item.summary ?? item.message, 220),
rootCause: boundQuickVerifyRecordText(item.rootCause, 140),
rootCauseStatus: boundQuickVerifyRecordText(item.rootCauseStatus, 90),
rootCauseConfidence: boundQuickVerifyRecordText(item.rootCauseConfidence, 40),
nextAction: boundQuickVerifyRecordText(item.nextAction, 240),
evidenceSummary: stringAtNullable(item, "evidenceSummary") ?? compactQuickVerifyFindingEvidence(item.evidence),
rootCauseSignals: compactQuickVerifyRootCauseSignals(item.rootCauseSignals),
timingSourceOfTruth: boundQuickVerifyRecordText(item.timingSourceOfTruth ?? item.expectedElapsedSource ?? item.evidenceKind, 100),
timingStatus: boundQuickVerifyRecordText(item.timingStatus, 60),
timingAlert: item.timingAlert === true,
blocking: item.blocking === true,
valuesRedacted: true,
};
}
function compactQuickVerifyRootCauseSignals(value: unknown): Record<string, unknown> | null {
const item = record(value);
const keys = [
"sessionListReadCount",
"traceEventsReadCount",
"webPerformanceBeaconFailureCount",
"eventSourceFailureCount",
"requestFailedCount",
"httpErrorCount",
"consoleAlertCount",
"requestfailedTop",
"httpStatusTop",
];
const compact: Record<string, unknown> = {};
for (const key of keys) {
const raw = item[key];
if (raw === null || raw === undefined) continue;
compact[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw;
}
return Object.keys(compact).length === 0 ? null : { ...compact, valuesRedacted: true };
}
function compactQuickVerifyFindingEvidence(value: unknown): string | null {
const item = record(value);
if (Object.keys(item).length === 0) return null;
const keys = [
"http404Count",
"responseErrorCount",
"requestFailedCount",
"statuses",
"afterProjectedSeqs",
"sinceSeqs",
"traceIds",
"maxFallbackRatio",
"maxFallbackTitleCount",
"overThresholdSampleCount",
"majorityFallbackSampleCount",
];
const compact: Record<string, unknown> = {};
for (const key of keys) {
const raw = item[key];
if (raw === null || raw === undefined) continue;
compact[key] = Array.isArray(raw) ? raw.slice(0, 6) : raw;
}
return Object.keys(compact).length === 0 ? null : boundQuickVerifyRecordText(JSON.stringify(compact), 240);
}
function compactQuickVerifyRecordStep(value: unknown): Record<string, unknown> {
const item = record(value);
return {
phase: stringAtNullable(item, "phase"),
ok: item.ok === true ? true : item.ok === false ? false : null,
promptIndex: numberAtNullable(item, "promptIndex"),
checkId: stringAtNullable(item, "checkId"),
failure: stringAtNullable(item, "failure"),
cleanup: compactQuickVerifyCleanupEvidence(item.cleanup),
result: compactQuickVerifyRecordStepResult(record(item.result)),
valuesRedacted: true,
};
}
function compactQuickVerifyCleanupEvidence(value: unknown): Record<string, unknown> | null {
const item = record(value);
if (stringAtNullable(item, "cleanupClass") === null) return null;
return {
cleanupClass: stringAtNullable(item, "cleanupClass"),
phase: stringAtNullable(item, "phase"),
commandOk: booleanAtNullable(item, "commandOk"),
status: stringAtNullable(item, "status"),
forceReason: stringAtNullable(item, "forceReason"),
gracefulStopOk: booleanAtNullable(item, "gracefulStopOk"),
forceStopOk: booleanAtNullable(item, "forceStopOk"),
aliveBefore: booleanAtNullable(item, "aliveBefore"),
aliveAfter: booleanAtNullable(item, "aliveAfter"),
termSent: booleanAtNullable(item, "termSent"),
killSent: booleanAtNullable(item, "killSent"),
pendingAbandoned: numberAtNullable(item, "pendingAbandoned"),
processingAbandoned: numberAtNullable(item, "processingAbandoned"),
evidenceSources: Array.isArray(item.evidenceSources) ? item.evidenceSources.filter((value): value is string => typeof value === "string").slice(0, 6) : [],
missingEvidenceFields: Array.isArray(item.missingEvidenceFields) ? item.missingEvidenceFields.filter((value): value is string => typeof value === "string").slice(0, 8) : [],
valuesRedacted: true,
};
}
function compactQuickVerifyRecordStepResult(value: Record<string, unknown>): Record<string, unknown> {
return {
ok: value.ok === true ? true : value.ok === false ? false : null,
status: stringAtNullable(value, "status"),
view: stringAtNullable(value, "view"),
exitCode: numberAtNullable(value, "exitCode"),
timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null,
stdoutBytes: numberAtNullable(value, "stdoutBytes"),
stderrBytes: numberAtNullable(value, "stderrBytes"),
stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 240),
stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 240),
stdoutTail: boundQuickVerifyRecordText(value.stdoutTail, 1200),
stderrTail: boundQuickVerifyRecordText(value.stderrTail, 1200),
valuesRedacted: true,
};
}
function boundQuickVerifyRecordText(value: unknown, maxChars: number): string | null {
if (typeof value !== "string") return null;
if (value.length <= maxChars) return value;
return `${value.slice(0, maxChars)}\n[truncated ${value.length - maxChars} chars]`;
}
function renderQuickVerifySummary(input: Record<string, unknown>): string {
const artifact = record(input.artifactSummary);
const findings = Array.isArray(input.findings)
? input.findings.map(record).slice(0, 8)
: Array.isArray(artifact.findings)
? artifact.findings.map(record).slice(0, 8)
: [];
const findingCount = numberAtNullable(input, "findingCount") ?? numberAtNullable(artifact, "findingCount") ?? findings.length;
const failureTitle = stringAtNullable(input, "failureTitleZh") ?? stringAtNullable(input, "errorTitleZh") ?? stringAtNullable(record(input.timeoutDisplay), "titleZh");
const cleanup = compactQuickVerifyCleanupEvidence(input.cleanup);
return [
"Web Probe Sentinel Quick Verify",
"=======================================================",
`run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`,
failureTitle === null ? "" : `errorTitle=${failureTitle} failure=${input.failure ?? "-"}`,
`report=${artifact.reportJsonSha256 ?? "-"} artifacts=${artifact.artifactCount ?? "-"} findings=${findingCount}`,
cleanup === null ? "" : formatQuickVerifyCleanupLine(cleanup),
`publicOrigin=${input.publicOrigin ?? "-"}`,
"",
"Findings",
findings.length === 0 ? "-" : findings.map(formatQuickVerifyFindingLine).join("\n"),
].filter((line) => line !== "").join("\n");
}
function renderAuthSessionSwitchQuickVerifySummary(input: Record<string, unknown>): string {
const artifact = record(input.artifactSummary);
const accountEnv = record(input.accountEnv);
const cleanup = compactQuickVerifyCleanupEvidence(input.cleanup);
const findingRows = Array.isArray(input.findings)
? input.findings.map(record).slice(0, 8)
: Array.isArray(artifact.findings)
? artifact.findings.map(record).slice(0, 8)
: [];
const steps = Array.isArray(input.steps) ? input.steps.map(record) : [];
const commandSteps = steps
.filter((step) => {
const phase = stringAtNullable(step, "phase") ?? "";
return phase.startsWith("observe-command-") || phase.startsWith("observe-session-invariance-") || phase === "quick-verify-account-env";
})
.map((step) => {
const phase = stringAtNullable(step, "phase") ?? "-";
const ok = step.ok === true ? "ok" : step.ok === false ? "failed" : "-";
const result = record(step.result);
const status = stringAtNullable(result, "status") ?? stringAtNullable(result, "failure") ?? "-";
return `${phase} ${ok} ${status}`;
});
return [
"Auth Session Switch Quick Verify",
"=======================================================",
`run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`,
stringAtNullable(input, "failureTitleZh") === null && stringAtNullable(input, "errorTitleZh") === null ? "" : `errorTitle=${stringAtNullable(input, "failureTitleZh") ?? stringAtNullable(input, "errorTitleZh") ?? "-"}`,
`status=${artifact.ok === true ? "ok" : "blocked"} report=${artifact.reportJsonSha256 ?? "-"} publicOrigin=${input.publicOrigin ?? "-"}`,
cleanup === null ? "" : formatQuickVerifyCleanupLine(cleanup),
`accountEnv=${accountEnv.envCount ?? "-"} valuesRedacted=true`,
"",
"Command Sequence",
commandSteps.length === 0 ? "-" : commandSteps.join("\n"),
"",
"Findings",
findingRows.length === 0 ? "-" : findingRows.map(formatQuickVerifyFindingLine).join("\n"),
].filter((line) => line !== "").join("\n");
}
function formatQuickVerifyFindingLine(item: Record<string, unknown>): string {
const title = stringAtNullable(item, "displayTitleZh") ?? stringAtNullable(item, "errorTitleZh") ?? stringAtNullable(record(item.timeoutDisplay), "titleZh");
const summary = title ?? item.summary ?? item.message ?? "";
return `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${summary}`;
}
function formatQuickVerifyTimingSuffix(item: Record<string, unknown>): string {
const status = stringAtNullable(item, "timingStatus");
const source = stringAtNullable(item, "timingSourceOfTruth") ?? stringAtNullable(item, "expectedElapsedSource") ?? stringAtNullable(item, "evidenceKind");
if (status === null && source === null) return "";
return ` timing=${[status === null ? null : `status=${status}`, source === null ? null : `source=${source}`].filter((part) => part !== null).join(" ")}`;
}