1736 lines
86 KiB
TypeScript
1736 lines
86 KiB
TypeScript
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
|
||
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel.
|
||
// 职责:编排 Web Probe Sentinel P5 quick-verify,并投影有界终态证据。
|
||
import { createHash, randomUUID } from "node:crypto";
|
||
import type { CommandResult } from "./command";
|
||
import { runCommand } from "./command";
|
||
import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery";
|
||
import { repoRoot } from "./config";
|
||
import { runWebProbeMemoryGuard } from "./hwlab-node-web-probe-memory-guard";
|
||
import type { ChildCliResult, SentinelCicdState } from "./hwlab-node-web-sentinel-cicd";
|
||
import {
|
||
arrayAt,
|
||
compactCommand,
|
||
mergeWarnings,
|
||
numberAt,
|
||
numberAtNullable,
|
||
parseJsonObject,
|
||
quickVerifyAccountEnv,
|
||
record,
|
||
nonEmptyString,
|
||
shellQuote,
|
||
short,
|
||
stringAt,
|
||
stringAtNullable,
|
||
targetValidationElapsedWarnings,
|
||
text,
|
||
withWarnings,
|
||
} from "./hwlab-node-web-sentinel-cicd";
|
||
import {
|
||
analysisSummaryFromAnalyzeResult,
|
||
cliDataPayload,
|
||
collectObserveView,
|
||
enrichObserveStartFailureFinding,
|
||
findScenario,
|
||
isQuickVerifyBlockingFinding,
|
||
isRecoverableQuickVerifyWaitFailure,
|
||
mergeFindingRecords,
|
||
observerIdFromText,
|
||
quickVerifyBusinessStatus,
|
||
quickVerifyControlFindings,
|
||
quickVerifyHasDurableBusinessTurn,
|
||
quickVerifyPrimaryControlFailure,
|
||
readAnalysisSummaryFromWorkspace,
|
||
readLocalObserveIndex,
|
||
readPromptSetForScenario,
|
||
remainingSeconds,
|
||
runChildCli,
|
||
runSentinelExactProviderProfilePreflight,
|
||
timeoutDisplayForQuickVerifyFailure,
|
||
waitForQuickVerifyObserverStartup,
|
||
waitForQuickVerifyPromptTurn,
|
||
} from "./hwlab-node-web-sentinel-p5-observe-analysis";
|
||
import { emitWebProbeSentinelSpan } from "./hwlab-node-web-sentinel-otel";
|
||
import { buildMonitorTerminalIngest, MonitorIngestFailedError, resolveMonitorTerminalWriter } from "./monitor-terminal-ingest";
|
||
|
||
export {
|
||
analysisSummaryFromAnalyzeResult,
|
||
metricNames,
|
||
readAnalysisSummaryFromWorkspace,
|
||
remainingSeconds,
|
||
runChildCli,
|
||
runSentinelExactProviderProfilePreflight,
|
||
sentinelP5Next,
|
||
serviceUnavailableBlocker,
|
||
validationBlocker,
|
||
} from "./hwlab-node-web-sentinel-p5-observe-analysis";
|
||
|
||
const QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS = 55;
|
||
|
||
export interface QuickVerifyObserverLifecycle {
|
||
readonly observerId: string;
|
||
cleanupStep: Record<string, unknown> | null;
|
||
stop(phase: string): Record<string, unknown>;
|
||
}
|
||
|
||
type QuickVerifyObserverStop = (observerId: string, phase: string) => Record<string, unknown>;
|
||
|
||
export function createQuickVerifyObserverLifecycle(observerId: string, stopObserver: QuickVerifyObserverStop): QuickVerifyObserverLifecycle {
|
||
const attempts: Record<string, unknown>[] = [];
|
||
const lifecycle: QuickVerifyObserverLifecycle = {
|
||
observerId,
|
||
cleanupStep: null,
|
||
stop(phase) {
|
||
if (lifecycle.cleanupStep?.ok === true) return lifecycle.cleanupStep;
|
||
for (let callAttempt = 1; callAttempt <= 2; callAttempt += 1) {
|
||
const attempt = attempts.length + 1;
|
||
try {
|
||
const result = stopObserver(observerId, phase);
|
||
attempts.push({ attempt, ok: result.ok === true, valuesRedacted: true });
|
||
lifecycle.cleanupStep = { ...result, stopAttempts: attempts, valuesRedacted: true };
|
||
if (result.ok === true) return lifecycle.cleanupStep;
|
||
} catch (error) {
|
||
attempts.push({ attempt, ok: false, failure: "observer-stop-unhandled-exception", valuesRedacted: true });
|
||
lifecycle.cleanupStep = {
|
||
phase,
|
||
ok: false,
|
||
failure: "observer-stop-unhandled-exception",
|
||
observerId,
|
||
exception: quickVerifyUnhandledExceptionSummary(error),
|
||
stopAttempts: attempts,
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
}
|
||
lifecycle.cleanupStep ??= { phase, ok: false, failure: "observer-stop-did-not-return", observerId, stopAttempts: attempts, valuesRedacted: true };
|
||
return lifecycle.cleanupStep;
|
||
},
|
||
};
|
||
return lifecycle;
|
||
}
|
||
|
||
export function stopQuickVerifyObserverAfterPartialStart(observerId: string | null, stopObserver: QuickVerifyObserverStop): Record<string, unknown> | null {
|
||
if (observerId === null) return null;
|
||
return createQuickVerifyObserverLifecycle(observerId, stopObserver).stop("observe-stop-after-start-failure");
|
||
}
|
||
|
||
export function quickVerifyUnhandledExceptionSummary(error: unknown): Record<string, unknown> {
|
||
let errorName = typeof error;
|
||
let errorMessage = typeof error === "string" ? error : "";
|
||
if (error instanceof Error) {
|
||
errorName = error.name || "Error";
|
||
errorMessage = error.message;
|
||
}
|
||
const boundedMessage = errorMessage.slice(0, 4096);
|
||
return {
|
||
errorName: short(errorName),
|
||
errorMessageSha256: boundedMessage.length === 0 ? null : createHash("sha256").update(boundedMessage).digest("hex"),
|
||
errorMessageTruncated: errorMessage.length > boundedMessage.length,
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
export function sentinelChildStartMemorySkip(value: unknown): Record<string, unknown> | null {
|
||
const envelope = record(value);
|
||
const payload = Object.keys(record(envelope.data)).length > 0 ? record(envelope.data) : envelope;
|
||
if (payload.status !== "skipped-wait-next-round" || payload.observerCreated !== false) return null;
|
||
return {
|
||
status: "skipped-wait-next-round",
|
||
decision: "skip-observer",
|
||
observerCreated: false,
|
||
memoryGuard: payload.memoryGuard ?? null,
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record<string, unknown> = {}): void {
|
||
const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => {
|
||
if (typeof value === "string") return [key, short(value)];
|
||
if (Array.isArray(value)) return [key, value.slice(0, 8)];
|
||
return [key, value];
|
||
}));
|
||
process.stdout.write(`${JSON.stringify({
|
||
event: "sentinel.quick-verify.progress",
|
||
at: new Date().toISOString(),
|
||
node: state.spec.nodeId,
|
||
lane: state.spec.lane,
|
||
sentinelId: state.sentinelId,
|
||
runId,
|
||
...compactExtra,
|
||
phase,
|
||
status,
|
||
valuesRedacted: true,
|
||
})}\n`);
|
||
}
|
||
|
||
export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record<string, unknown> {
|
||
const startedAt = Date.now();
|
||
const elapsedMs = () => Date.now() - startedAt;
|
||
const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId");
|
||
const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds");
|
||
const scenario = findScenario(state, scenarioId);
|
||
if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true };
|
||
const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;
|
||
emitWebProbeSentinelSpan(sentinelOtelContext(state), "web_probe_sentinel.quick_verify.job_start", {
|
||
scenarioId,
|
||
runId,
|
||
cadence: stringAtNullable(scenario, "cadence"),
|
||
status: "running",
|
||
valuesRedacted: true,
|
||
});
|
||
const memoryGuard = runWebProbeMemoryGuard(state.spec, "sentinel-cadence");
|
||
if (memoryGuard.status === "skipped") {
|
||
return recordSkippedQuickVerify(state, scenario, runId, scenarioId, reason, elapsedMs(), memoryGuard, [], "memavailable-below-threshold");
|
||
}
|
||
if (memoryGuard.status !== "allowed") {
|
||
return recordQuickVerify(state, {
|
||
ok: false,
|
||
runId,
|
||
status: "blocked",
|
||
reason: "memavailable-preflight-unavailable",
|
||
scenarioId,
|
||
observerId: null,
|
||
observerCreated: false,
|
||
runCreated: true,
|
||
mutation: false,
|
||
memoryGuard,
|
||
elapsedMs: elapsedMs(),
|
||
findingCount: 0,
|
||
artifactCount: 0,
|
||
findings: [],
|
||
views: {},
|
||
updatedAt: new Date().toISOString(),
|
||
valuesRedacted: true,
|
||
});
|
||
}
|
||
const providerProfilePreflight = runSentinelExactProviderProfilePreflight(state, timeoutSeconds, scenario);
|
||
if (providerProfilePreflight.ok !== true) {
|
||
return recordQuickVerify(state, {
|
||
ok: false,
|
||
runId,
|
||
status: "blocked",
|
||
reason: "exact-provider-profile-unavailable",
|
||
scenarioId,
|
||
providerProfile: stringAtNullable(scenario, "providerProfile"),
|
||
providerProfileMode: stringAtNullable(scenario, "providerProfileMode"),
|
||
providerProfilePreflight,
|
||
observerId: null,
|
||
observerCreated: false,
|
||
runCreated: true,
|
||
mutation: false,
|
||
elapsedMs: elapsedMs(),
|
||
findingCount: 0,
|
||
artifactCount: 0,
|
||
findings: [],
|
||
views: {},
|
||
updatedAt: new Date().toISOString(),
|
||
valuesRedacted: true,
|
||
});
|
||
}
|
||
const commandSequence = arrayAt(scenario, "commandSequence").map(record);
|
||
const needsPromptSet = commandSequence.some((item) => stringAt(item, "type") === "sendPrompt" && inlinePromptText(item) === null);
|
||
const prompts = needsPromptSet
|
||
? readPromptSetForScenario(state, scenario)
|
||
: { ok: true as const, prompts: [], summary: { source: "not-required", promptCount: 0, valuesRedacted: true } };
|
||
if (!prompts.ok) {
|
||
emitWebProbeSentinelSpan(sentinelOtelContext(state), "web_probe_sentinel.quick_verify.job_finish", {
|
||
scenarioId,
|
||
runId,
|
||
status: "blocked",
|
||
exitCode: 1,
|
||
failureKind: "prompt-source-unavailable",
|
||
valuesRedacted: true,
|
||
}, false);
|
||
return { ok: false, status: "blocked", reason: "prompt-source-unavailable", promptSource: prompts, valuesRedacted: true };
|
||
}
|
||
const accountEnv = quickVerifyAccountEnv(state);
|
||
if (!accountEnv.ok) {
|
||
const findings = [{
|
||
id: "quick-verify-account-secret-missing",
|
||
severity: "red",
|
||
count: arrayAt(accountEnv.summary, "missing").length || 1,
|
||
summary: "quick verify could not materialize YAML-declared web account credentials for the observer runner.",
|
||
missing: arrayAt(accountEnv.summary, "missing"),
|
||
valuesRedacted: true,
|
||
}];
|
||
return recordQuickVerify(state, {
|
||
ok: false,
|
||
runId,
|
||
scenarioId,
|
||
reason,
|
||
status: "blocked",
|
||
observerId: null,
|
||
elapsedMs: 0,
|
||
businessStatus: quickVerifyBusinessStatus("quick-verify-account-secret-missing", 0, null, null, 0, maxSeconds),
|
||
steps: [{ phase: "quick-verify-account-env", ok: false, result: accountEnv.summary }],
|
||
failure: "quick-verify-account-secret-missing",
|
||
findingCount: findings.length,
|
||
findings,
|
||
promptSource: prompts.summary,
|
||
accountEnv: accountEnv.summary,
|
||
views: {
|
||
summary: { renderedText: renderQuickVerifySummary({ scenarioId, artifactSummary: { ok: false, findings, findingCount: findings.length }, steps: [], publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
||
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ scenarioId, steps: [], findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
||
},
|
||
warnings: [],
|
||
valuesRedacted: true,
|
||
});
|
||
}
|
||
const sampleIntervalMs = numberAt(scenario, "sampleIntervalMs");
|
||
const warningBudgetSeconds = maxSeconds;
|
||
const hardBudgetSeconds = Math.min(timeoutSeconds, Math.max(maxSeconds, numberAt(scenario, "maxRunSeconds")));
|
||
const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", warningBudgetSeconds);
|
||
const deadline = Date.now() + hardBudgetSeconds * 1000;
|
||
printQuickVerifyProgress(state, runId, "start", "running", { scenarioId, reason, warningBudgetSeconds, hardBudgetSeconds, timeoutSeconds });
|
||
const steps: Record<string, unknown>[] = [];
|
||
const startArgs = [
|
||
"web-probe", "observe", "start",
|
||
"--node", state.spec.nodeId,
|
||
"--lane", state.spec.lane,
|
||
"--target-path", stringAt(scenario, "observeTargetPath"),
|
||
"--sample-interval-ms", String(sampleIntervalMs),
|
||
"--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")),
|
||
"--max-run-seconds", String(hardBudgetSeconds),
|
||
"--command-timeout-seconds", "55",
|
||
"--sentinel-cadence",
|
||
];
|
||
const viewport = stringAtNullable(scenario, "viewport");
|
||
if (viewport !== null) startArgs.push("--viewport", viewport);
|
||
printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) });
|
||
const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env);
|
||
steps.push({ phase: "observe-start", ok: started.ok, result: started.result });
|
||
const childMemorySkip = sentinelChildStartMemorySkip(started.parsed);
|
||
if (childMemorySkip !== null) {
|
||
return recordSkippedQuickVerify(state, scenario, runId, scenarioId, reason, elapsedMs(), childMemorySkip.memoryGuard, steps, "memavailable-below-threshold-after-cadence-preflight");
|
||
}
|
||
const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? ""));
|
||
printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() });
|
||
if (!started.ok || observerId === null) {
|
||
const cleanupStep = stopQuickVerifyObserverAfterPartialStart(
|
||
observerId,
|
||
(partialObserverId, phase) => stopQuickVerifyObserver(state, partialObserverId, phase),
|
||
);
|
||
const cleanup = cleanupStep === null ? null : classifyQuickVerifyCleanupStep(cleanupStep);
|
||
const findings = mergeFindingRecords(
|
||
quickVerifyControlFindings("observe-start-failed", 0, null, null)
|
||
.map((finding) => enrichObserveStartFailureFinding(finding, record(started.result))),
|
||
cleanupStep === null ? [] : quickVerifyCleanupFindings(cleanupStep),
|
||
);
|
||
return recordQuickVerify(state, {
|
||
ok: false,
|
||
runId,
|
||
scenarioId,
|
||
reason,
|
||
status: "blocked",
|
||
observerId,
|
||
elapsedMs: elapsedMs(),
|
||
businessStatus: quickVerifyBusinessStatus("observe-start-failed", 0, null, null, elapsedMs(), maxSeconds),
|
||
steps: cleanupStep === null ? steps : [...steps, cleanupStep],
|
||
cleanup,
|
||
failure: "observe-start-failed",
|
||
findingCount: findings.length,
|
||
findings,
|
||
warnings: mergeWarnings(elapsedWarnings(), quickVerifyCleanupWarnings(cleanup)),
|
||
valuesRedacted: true,
|
||
});
|
||
}
|
||
const observerLifecycle = createQuickVerifyObserverLifecycle(
|
||
observerId,
|
||
(activeObserverId, phase) => stopQuickVerifyObserver(state, activeObserverId, phase),
|
||
);
|
||
let promptIndex = 0;
|
||
try {
|
||
printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) });
|
||
const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds);
|
||
steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady });
|
||
printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", startupReady.ok === true ? "succeeded" : "failed", { observerId, failure: startupReady.failure ?? null, status: startupReady.status ?? null, heartbeatStatus: startupReady.heartbeatStatus ?? null, elapsedMs: elapsedMs() });
|
||
if (startupReady.ok !== true) {
|
||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||
runId,
|
||
scenarioId,
|
||
reason,
|
||
observerId,
|
||
observerLifecycle,
|
||
promptIndex: 0,
|
||
steps,
|
||
failure: text(startupReady.failure ?? "observe-startup-ready-wait-failed"),
|
||
elapsedMs: elapsedMs(),
|
||
warnings: mergeWarnings(Array.isArray(startupReady.warnings) ? startupReady.warnings : [], elapsedWarnings()),
|
||
promptSource: prompts.summary,
|
||
}));
|
||
}
|
||
const sessionInvarianceChecks = sessionInvarianceChecksByRound(scenario);
|
||
const nonBlockingCanaryWarnings: string[] = [];
|
||
for (const item of commandSequence) {
|
||
const type = stringAt(item, "type");
|
||
const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
|
||
for (let index = 0; index < repeat; index += 1) {
|
||
if (Date.now() >= deadline) {
|
||
const timeoutDisplay = timeoutDisplayForQuickVerifyFailure("quick-verify-timeout-over-budget", promptIndex, warningBudgetSeconds, timeoutSeconds);
|
||
printQuickVerifyProgress(state, runId, "timeout", "failed", { observerId, promptIndex, elapsedMs: elapsedMs(), hardBudgetSeconds });
|
||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||
runId,
|
||
scenarioId,
|
||
reason,
|
||
observerId,
|
||
observerLifecycle,
|
||
promptIndex,
|
||
steps,
|
||
failure: "quick-verify-timeout-over-budget",
|
||
failureTitleZh: stringAtNullable(timeoutDisplay, "titleZh"),
|
||
timeoutDisplay,
|
||
elapsedMs: elapsedMs(),
|
||
warnings: mergeWarnings(`quick verify exceeded the hard ${hardBudgetSeconds}s execution budget after the configured ${warningBudgetSeconds}s targetValidation warning budget.`, elapsedWarnings()),
|
||
promptSource: prompts.summary,
|
||
}));
|
||
}
|
||
const commandWaitMs = Math.max(1000, Math.trunc(numberAtNullable(item, "commandWaitMs") ?? numberAtNullable(scenario, "commandWaitMs") ?? 55_000));
|
||
const commandTimeoutSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "commandTimeoutSeconds") ?? numberAtNullable(scenario, "commandTimeoutSeconds") ?? 55));
|
||
const turnWaitChunkSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "turnWaitChunkSeconds") ?? numberAtNullable(scenario, "turnWaitChunkSeconds") ?? 55));
|
||
const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", String(commandWaitMs), "--command-timeout-seconds", String(remainingSeconds(deadline, commandTimeoutSeconds))];
|
||
if (type === "selectProvider") args.push("--provider", stringAt(item, "provider"));
|
||
if (type === "loginAccount" || type === "listSessions" || type === "logout") {
|
||
const accountId = stringAtNullable(item, "accountId");
|
||
if (accountId !== null) args.push("--account-id", accountId);
|
||
}
|
||
if (type === "switchSessions") {
|
||
const fromAccountId = stringAtNullable(item, "fromAccountId");
|
||
const toAccountId = stringAtNullable(item, "toAccountId");
|
||
if (fromAccountId !== null) args.push("--from-account-id", fromAccountId);
|
||
if (toAccountId !== null) args.push("--to-account-id", toAccountId);
|
||
}
|
||
if (type === "sendPrompt") {
|
||
args.push("--text", inlinePromptText(item) ?? prompts.prompts[promptIndex % prompts.prompts.length] ?? "");
|
||
args.push("--expected-action-wait-ms", String(numberAtNullable(item, "expectedActionWaitMs") ?? 45000));
|
||
promptIndex += 1;
|
||
}
|
||
appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" });
|
||
printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat });
|
||
const commandResult = runChildCli(args, remainingSeconds(deadline, Math.max(60, commandTimeoutSeconds + 10)));
|
||
steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result });
|
||
printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() });
|
||
if (!commandResult.ok) {
|
||
if (type === "sendPrompt") {
|
||
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) });
|
||
const delayedWaitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds);
|
||
steps.push({ phase: "observe-wait-turn-terminal-after-command-timeout", ok: delayedWaitResult.ok, promptIndex, result: delayedWaitResult });
|
||
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", delayedWaitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: delayedWaitResult.failure ?? null, status: delayedWaitResult.status ?? null, traceId: delayedWaitResult.traceId ?? null, elapsedMs: elapsedMs() });
|
||
if (delayedWaitResult.ok === true) {
|
||
const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps);
|
||
nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings));
|
||
printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() });
|
||
if (invariantResult.ok !== true) {
|
||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||
runId,
|
||
scenarioId,
|
||
reason,
|
||
observerId,
|
||
observerLifecycle,
|
||
promptIndex,
|
||
steps,
|
||
failure: text(record(invariantResult).failure ?? "observe-session-invariance-failed"),
|
||
promptSource: prompts.summary,
|
||
elapsedMs: elapsedMs(),
|
||
warnings: mergeWarnings(record(invariantResult).warnings, elapsedWarnings()),
|
||
}));
|
||
}
|
||
continue;
|
||
}
|
||
}
|
||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||
runId,
|
||
scenarioId,
|
||
reason,
|
||
observerId,
|
||
observerLifecycle,
|
||
promptIndex,
|
||
steps,
|
||
failure: `observe-command-${type}-failed`,
|
||
elapsedMs: elapsedMs(),
|
||
warnings: elapsedWarnings(),
|
||
promptSource: prompts.summary,
|
||
}));
|
||
}
|
||
if (type === "sendPrompt") {
|
||
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) });
|
||
const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds);
|
||
steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult });
|
||
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() });
|
||
if (waitResult.ok !== true) {
|
||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||
runId,
|
||
scenarioId,
|
||
reason,
|
||
observerId,
|
||
observerLifecycle,
|
||
promptIndex,
|
||
steps,
|
||
failure: text(waitResult.failure ?? "observe-turn-terminal-wait-failed"),
|
||
promptSource: prompts.summary,
|
||
elapsedMs: elapsedMs(),
|
||
warnings: mergeWarnings(Array.isArray(waitResult.warnings) ? waitResult.warnings : [], elapsedWarnings()),
|
||
}));
|
||
}
|
||
const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps);
|
||
nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings));
|
||
printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() });
|
||
if (invariantResult.ok !== true) {
|
||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||
runId,
|
||
scenarioId,
|
||
reason,
|
||
observerId,
|
||
observerLifecycle,
|
||
promptIndex,
|
||
steps,
|
||
failure: text(invariantResult.failure ?? "observe-session-invariance-check-failed"),
|
||
promptSource: prompts.summary,
|
||
elapsedMs: elapsedMs(),
|
||
warnings: mergeWarnings(nonBlockingCanaryWarnings, elapsedWarnings()),
|
||
}));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
printQuickVerifyProgress(state, runId, "observe-analyze", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 120) });
|
||
const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120)), "--raw", "--compact-raw"], remainingSeconds(deadline, 120));
|
||
steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result });
|
||
printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() });
|
||
const indexEntry = readLocalObserveIndex(observerId);
|
||
const artifactSummary = analysisSummaryFromAnalyzeResult(analysis, indexEntry?.stateDir ?? null)
|
||
?? (indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS));
|
||
const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30));
|
||
const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30));
|
||
const controlFindings = quickVerifyControlFindings(null, promptIndex, turnSummary, traceFrame);
|
||
const artifactSummaryRecord = record(artifactSummary);
|
||
const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
|
||
const visibilityFindings = quickVerifyAnalysisVisibilityFindings(analysis, artifactSummary);
|
||
const cleanupStep = observerLifecycle.stop("observe-stop-after-terminal");
|
||
const cleanup = classifyQuickVerifyCleanupStep(cleanupStep);
|
||
const cleanupFindings = quickVerifyCleanupFindings(cleanupStep);
|
||
const findings = mergeFindingRecords(
|
||
mergeFindingRecords(
|
||
mergeFindingRecords(artifactFindings, controlFindings),
|
||
visibilityFindings,
|
||
),
|
||
cleanupFindings,
|
||
);
|
||
const blockingFindings = findings.filter(isQuickVerifyBlockingFinding);
|
||
const analysisReadable = stringAtNullable(artifactSummary, "reportJsonSha256") !== null;
|
||
const analysisWarnings = mergeWarnings(
|
||
analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."],
|
||
analysisReadable && artifactSummaryRecord.ok !== true ? ["quick verify analysis report contains non-blocking findings; targetValidation keeps them in report but does not fail the CronJob without red/blocking findings."] : [],
|
||
);
|
||
const ok = analysisReadable && controlFindings.length === 0 && blockingFindings.length === 0;
|
||
const businessStatus = quickVerifyBusinessStatus(null, promptIndex, turnSummary, traceFrame, elapsedMs(), maxSeconds);
|
||
const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings);
|
||
printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), analysisReadable, findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() });
|
||
return recordQuickVerify(state, {
|
||
ok,
|
||
runId,
|
||
scenarioId,
|
||
reason,
|
||
status: ok ? "analyzed" : "blocked",
|
||
observerId,
|
||
elapsedMs: elapsedMs(),
|
||
businessStatus,
|
||
stateDir: indexEntry?.stateDir ?? null,
|
||
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
|
||
findingCount: findings.length,
|
||
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
|
||
failure: !analysisReadable ? "quick-verify-analysis-missing" : primaryControlFailure ?? (blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null),
|
||
promptSource: prompts.summary,
|
||
accountEnv: accountEnv.summary,
|
||
steps: [...steps, cleanupStep],
|
||
cleanup,
|
||
analysis: artifactSummary,
|
||
views: {
|
||
summary: { renderedText: renderQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, findings, findingCount: findings.length, steps, cleanup, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
||
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, cleanup, findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
||
"turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok },
|
||
"trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok },
|
||
},
|
||
findings,
|
||
screenshot: record(artifactSummary).screenshot,
|
||
publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"),
|
||
warnings: mergeWarnings(analysisWarnings, nonBlockingCanaryWarnings, quickVerifyCleanupWarnings(cleanup), elapsedWarnings()),
|
||
valuesRedacted: true,
|
||
});
|
||
} catch (error) {
|
||
const exception = quickVerifyUnhandledExceptionSummary(error);
|
||
steps.push({ phase: "quick-verify-unhandled-exception", ok: false, result: exception });
|
||
printQuickVerifyProgress(state, runId, "quick-verify-unhandled-exception", "failed", {
|
||
observerId,
|
||
promptIndex,
|
||
errorName: exception.errorName,
|
||
errorMessageSha256: exception.errorMessageSha256,
|
||
elapsedMs: elapsedMs(),
|
||
});
|
||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||
runId,
|
||
scenarioId,
|
||
reason,
|
||
observerId,
|
||
observerLifecycle,
|
||
promptIndex,
|
||
steps,
|
||
failure: "quick-verify-unhandled-exception",
|
||
elapsedMs: elapsedMs(),
|
||
warnings: mergeWarnings("quick verify caught an unexpected post-start exception; details are redacted and the observer cleanup fallback remained active.", elapsedWarnings()),
|
||
promptSource: prompts.summary,
|
||
}));
|
||
} finally {
|
||
observerLifecycle.stop("observe-stop-after-finally");
|
||
}
|
||
}
|
||
|
||
function sessionInvarianceChecksByRound(scenario: Record<string, unknown>): Map<number, Record<string, unknown>[]> {
|
||
const checks = new Map<number, Record<string, unknown>[]>();
|
||
const items = Array.isArray(scenario.sessionInvarianceChecks) ? scenario.sessionInvarianceChecks.map(record) : [];
|
||
for (const item of items) {
|
||
const afterRound = typeof item.afterRound === "number" && Number.isInteger(item.afterRound) ? item.afterRound : null;
|
||
if (afterRound === null || afterRound < 0) continue;
|
||
const list = checks.get(afterRound) ?? [];
|
||
list.push(item);
|
||
checks.set(afterRound, list);
|
||
}
|
||
return checks;
|
||
}
|
||
|
||
function runQuickVerifySessionInvarianceChecks(
|
||
state: SentinelCicdState,
|
||
observerId: string,
|
||
checks: readonly Record<string, unknown>[],
|
||
deadline: number,
|
||
promptIndex: number,
|
||
steps: Record<string, unknown>[],
|
||
): Record<string, unknown> {
|
||
const warnings: string[] = [];
|
||
for (const check of checks) {
|
||
const checkId = nonEmptyString(check.id) ?? `after-round-${promptIndex}`;
|
||
const blocking = check.blocking === true;
|
||
const commands: { readonly type: string; readonly enabled: boolean }[] = [
|
||
{ type: "refreshCurrentSession", enabled: check.refreshCurrent === true },
|
||
{ type: "switchAwayAndBack", enabled: check.switchAwayAndBack === true },
|
||
{ type: "assertSessionInvariant", enabled: check.assertSessionInvariant !== false },
|
||
];
|
||
for (const command of commands) {
|
||
if (!command.enabled) continue;
|
||
const args = [
|
||
"web-probe", "observe", "command", observerId,
|
||
"--node", state.spec.nodeId,
|
||
"--lane", state.spec.lane,
|
||
"--type", command.type,
|
||
"--after-round", String(promptIndex),
|
||
"--wait-ms", "55000",
|
||
"--command-timeout-seconds", String(remainingSeconds(deadline, 55)),
|
||
];
|
||
const severity = nonEmptyString(check.severity);
|
||
const findingId = nonEmptyString(check.findingId);
|
||
const expectedSentinelRange = nonEmptyString(check.expectedSentinelRange);
|
||
const alternateSessionStrategy = nonEmptyString(check.alternateSessionStrategy);
|
||
if (severity !== null) args.push("--severity", severity);
|
||
if (findingId !== null) args.push("--finding-id", findingId);
|
||
if (expectedSentinelRange !== null) args.push("--expected-sentinel-range", expectedSentinelRange);
|
||
if (command.type === "switchAwayAndBack" && alternateSessionStrategy !== null) args.push("--alternate-session-strategy", alternateSessionStrategy);
|
||
if (command.type === "assertSessionInvariant" && check.requireComposerReady === true) args.push("--require-composer-ready");
|
||
args.push(check.blocking === true ? "--blocking" : "--non-blocking");
|
||
const result = runChildCli(args, remainingSeconds(deadline, 60));
|
||
steps.push({ phase: `observe-session-invariance-${command.type}`, ok: result.ok, promptIndex, checkId, result: result.result });
|
||
if (!result.ok) {
|
||
if (!blocking) {
|
||
warnings.push(`non-blocking session invariance canary ${checkId}/${command.type} failed after round ${promptIndex}; continuing Code Agent multi-round quick verify because scenario marks this check blocking=false.`);
|
||
continue;
|
||
}
|
||
return { ok: false, failure: `observe-session-invariance-${command.type}-failed`, checkId, promptIndex, valuesRedacted: true };
|
||
}
|
||
}
|
||
}
|
||
return { ok: true, promptIndex, checkCount: checks.length, warnings, valuesRedacted: true };
|
||
}
|
||
|
||
function appendScenarioObserveCommandArgs(args: string[], item: Record<string, unknown>, options: { readonly skipText?: boolean } = {}): void {
|
||
const mappings: readonly (readonly [string, string])[] = [
|
||
["path", "--path"],
|
||
["label", "--label"],
|
||
["sessionId", "--session-id"],
|
||
["provider", "--provider"],
|
||
["accountId", "--account-id"],
|
||
["fromAccountId", "--from-account-id"],
|
||
["toAccountId", "--to-account-id"],
|
||
["sourceId", "--source-id"],
|
||
["fileRef", "--file-ref"],
|
||
["filename", "--filename"],
|
||
["taskRef", "--task-ref"],
|
||
["taskId", "--task-id"],
|
||
["task", "--task"],
|
||
["field", "--field"],
|
||
["link", "--link"],
|
||
["title", "--title"],
|
||
["body", "--body"],
|
||
["status", "--status"],
|
||
["hwpodId", "--hwpod-id"],
|
||
["nodeId", "--node-id"],
|
||
["workspaceRoot", "--workspace-root"],
|
||
["root", "--root"],
|
||
];
|
||
for (const [key, flag] of mappings) {
|
||
if (args.includes(flag)) continue;
|
||
const value = stringAtNullable(item, key);
|
||
if (value !== null) args.push(flag, value);
|
||
}
|
||
if (options.skipText !== true && !args.includes("--text")) {
|
||
const text = stringAtNullable(item, "text") ?? stringAtNullable(item, "value");
|
||
if (text !== null) args.push("--text", text);
|
||
}
|
||
if (item.waitProjectManagementReady === true && !args.includes("--wait-project-management-ready")) args.push("--wait-project-management-ready");
|
||
}
|
||
|
||
function inlinePromptText(item: Record<string, unknown>): string | null {
|
||
return stringAtNullable(item, "text") ?? stringAtNullable(item, "prompt") ?? stringAtNullable(item, "value");
|
||
}
|
||
|
||
export function reclassifyQuickVerifyControlFindings(state: SentinelCicdState, input: {
|
||
readonly runId: string | null;
|
||
readonly scenarioId: string | null;
|
||
readonly observerId: string | null;
|
||
readonly failure?: string | null;
|
||
readonly timeoutSeconds?: number | null;
|
||
}): Record<string, unknown> {
|
||
if (input.observerId === null || input.observerId.length === 0) {
|
||
return { ok: false, reason: "observer-id-missing", runId: input.runId, valuesRedacted: true };
|
||
}
|
||
const scenarioId = input.scenarioId ?? stringAt(state.cicd, "targetValidation.scenarioId");
|
||
const scenario = findScenario(state, scenarioId);
|
||
if (scenario === null) {
|
||
return { ok: false, reason: "scenario-not-found", runId: input.runId, scenarioId, observerId: input.observerId, valuesRedacted: true };
|
||
}
|
||
const commandSequence = arrayAt(scenario, "commandSequence").map(record);
|
||
const promptIndex = commandSequence.reduce((count, item) => {
|
||
const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
|
||
return stringAtNullable(item, "type") === "sendPrompt" ? count + repeat : count;
|
||
}, 0);
|
||
const timeoutSeconds = Math.max(5, Math.min(Math.trunc(input.timeoutSeconds ?? 55), 55));
|
||
const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, timeoutSeconds);
|
||
const traceFrame = collectObserveView(state, input.observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, timeoutSeconds);
|
||
const findings = quickVerifyControlFindings(input.failure ?? null, promptIndex, turnSummary, traceFrame);
|
||
return {
|
||
ok: true,
|
||
runId: input.runId,
|
||
scenarioId,
|
||
observerId: input.observerId,
|
||
promptIndex,
|
||
findingCount: findings.length,
|
||
findings,
|
||
turnSummary: { ok: turnSummary.ok === true, collectShape: turnSummary.collectShape, valuesRedacted: true },
|
||
traceFrame: { ok: traceFrame.ok === true, collectShape: traceFrame.collectShape, valuesRedacted: true },
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
|
||
readonly runId: string;
|
||
readonly scenarioId: string;
|
||
readonly reason: string;
|
||
readonly observerId: string;
|
||
readonly observerLifecycle: QuickVerifyObserverLifecycle;
|
||
readonly promptIndex: number;
|
||
readonly steps: readonly Record<string, unknown>[];
|
||
readonly failure: string;
|
||
readonly failureTitleZh?: string | null;
|
||
readonly timeoutDisplay?: Record<string, unknown> | null;
|
||
readonly promptSource?: Record<string, unknown>;
|
||
readonly elapsedMs?: number;
|
||
readonly warnings?: readonly unknown[];
|
||
}): Record<string, unknown> {
|
||
const targetValidationSeconds = numberAt(state.cicd, "targetValidation.maxSeconds");
|
||
const failureDisplay = input.timeoutDisplay ?? timeoutDisplayForQuickVerifyFailure(input.failure, input.promptIndex, targetValidationSeconds, null);
|
||
const failureTitle = input.failureTitleZh ?? stringAtNullable(failureDisplay, "titleZh");
|
||
const cleanupSteps: Record<string, unknown>[] = [];
|
||
if (input.promptIndex > 0) {
|
||
const cancel = runChildCli([
|
||
"web-probe", "observe", "command", input.observerId,
|
||
"--node", state.spec.nodeId,
|
||
"--lane", state.spec.lane,
|
||
"--type", "cancel",
|
||
"--wait-ms", "55000",
|
||
"--command-timeout-seconds", "55",
|
||
], 60);
|
||
cleanupSteps.push({ phase: "observe-cancel-after-failure", ok: cancel.ok, result: cancel.result });
|
||
}
|
||
cleanupSteps.push(input.observerLifecycle.stop("observe-stop-after-failure"));
|
||
const analysis = runChildCli([
|
||
"web-probe", "observe", "analyze", input.observerId,
|
||
"--node", state.spec.nodeId,
|
||
"--lane", state.spec.lane,
|
||
"--command-timeout-seconds", "55",
|
||
"--raw", "--compact-raw",
|
||
], 60);
|
||
cleanupSteps.push({ phase: "observe-analyze-after-failure", ok: analysis.ok, result: analysis.result });
|
||
const indexEntry = readLocalObserveIndex(input.observerId);
|
||
const artifactSummary = analysisSummaryFromAnalyzeResult(analysis, indexEntry?.stateDir ?? null)
|
||
?? (indexEntry === null
|
||
? { ok: false, reason: "observe-index-entry-missing", observerId: input.observerId, valuesRedacted: true }
|
||
: readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS));
|
||
const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30);
|
||
const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30);
|
||
const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(input.promptIndex, turnSummary, traceFrame);
|
||
const controlFindings = quickVerifyControlFindings(input.failure, input.promptIndex, turnSummary, traceFrame, failureDisplay);
|
||
const artifactSummaryRecord = record(artifactSummary);
|
||
const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
|
||
const visibilityFindings = quickVerifyAnalysisVisibilityFindings(analysis, artifactSummary);
|
||
const cleanup = quickVerifyCleanupFromSteps(cleanupSteps);
|
||
const cleanupFindings = cleanupSteps.flatMap((step) => quickVerifyCleanupFindings(step));
|
||
const findings = mergeFindingRecords(
|
||
mergeFindingRecords(
|
||
mergeFindingRecords(artifactFindings, controlFindings),
|
||
visibilityFindings,
|
||
),
|
||
cleanupFindings,
|
||
);
|
||
const blockingFindings = findings.filter(isQuickVerifyBlockingFinding);
|
||
const recoveredWaitFailure = durableBusinessTurn
|
||
&& isRecoverableQuickVerifyWaitFailure(input.failure)
|
||
&& record(artifactSummary).ok === true
|
||
&& controlFindings.length === 0
|
||
&& blockingFindings.length === 0;
|
||
const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, targetValidationSeconds);
|
||
const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings);
|
||
return {
|
||
ok: recoveredWaitFailure,
|
||
runId: input.runId,
|
||
scenarioId: input.scenarioId,
|
||
reason: input.reason,
|
||
status: recoveredWaitFailure ? "analyzed" : "blocked",
|
||
observerId: input.observerId,
|
||
elapsedMs: input.elapsedMs ?? null,
|
||
businessStatus,
|
||
failureTitleZh: recoveredWaitFailure ? null : failureTitle,
|
||
errorTitleZh: recoveredWaitFailure ? null : failureTitle,
|
||
timeoutDisplay: recoveredWaitFailure ? null : failureDisplay,
|
||
stateDir: indexEntry?.stateDir ?? null,
|
||
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
|
||
findingCount: findings.length,
|
||
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
|
||
failure: recoveredWaitFailure ? null : primaryControlFailure ?? input.failure,
|
||
runnerFailure: recoveredWaitFailure ? null : input.failure,
|
||
promptSource: input.promptSource,
|
||
steps: [...input.steps, ...cleanupSteps],
|
||
cleanup,
|
||
analysis: artifactSummary,
|
||
views: {
|
||
summary: { renderedText: renderQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, findings, findingCount: findings.length, steps: input.steps, cleanup, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), failure: recoveredWaitFailure ? null : input.failure, failureTitleZh: recoveredWaitFailure ? null : failureTitle, timeoutDisplay: recoveredWaitFailure ? null : failureDisplay }) },
|
||
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, cleanup, findings, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), failure: recoveredWaitFailure ? null : input.failure, failureTitleZh: recoveredWaitFailure ? null : failureTitle, timeoutDisplay: recoveredWaitFailure ? null : failureDisplay }) },
|
||
"turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok },
|
||
"trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok },
|
||
},
|
||
findings,
|
||
screenshot: record(artifactSummary).screenshot,
|
||
publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"),
|
||
warnings: mergeWarnings(
|
||
Array.isArray(input.warnings) ? input.warnings : [],
|
||
recoveredWaitFailure ? ["quick verify wait command timed out, but collected turn-summary/trace-frame artifacts show a durable completed business turn; treating the wait timeout as a non-blocking tool finding."] : [],
|
||
quickVerifyCleanupWarnings(cleanup),
|
||
targetValidationElapsedWarnings(input.elapsedMs ?? null, "quick verify confirm-wait", targetValidationSeconds),
|
||
),
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function stopQuickVerifyObserver(state: SentinelCicdState, observerId: string, phase: string): Record<string, unknown> {
|
||
const stop = runChildCli([
|
||
"web-probe", "observe", "stop", observerId,
|
||
"--node", state.spec.nodeId,
|
||
"--lane", state.spec.lane,
|
||
"--force",
|
||
"--raw",
|
||
"--wait-ms", "55000",
|
||
"--command-timeout-seconds", "55",
|
||
], 60);
|
||
const step = { phase, ok: stop.ok, payload: cliDataPayload(stop.parsed), result: stop.result };
|
||
return { ...step, cleanup: classifyQuickVerifyCleanupStep(step) };
|
||
}
|
||
|
||
export function quickVerifyCleanupFindings(cleanupStep: Record<string, unknown>): Record<string, unknown>[] {
|
||
const cleanup = classifyQuickVerifyCleanupStep(cleanupStep);
|
||
const cleanupClass = stringAtNullable(cleanup, "cleanupClass");
|
||
if (cleanupClass === "bounded-force-stop") {
|
||
return [{
|
||
id: "observer-graceful-stop-timeout-force-stop-succeeded",
|
||
severity: "warning",
|
||
count: 1,
|
||
summary: "observer graceful stop did not drain before timeout, but force stop confirmed aliveAfter=false.",
|
||
cleanupPhase: cleanupStep.phase,
|
||
cleanup,
|
||
evidenceSummary: formatQuickVerifyCleanupLine(cleanup),
|
||
valuesRedacted: true,
|
||
}];
|
||
}
|
||
if (cleanupClass !== "observer-stop-failed") return [];
|
||
const aliveAfter = booleanAtNullable(cleanup, "aliveAfter");
|
||
const id = aliveAfter === true ? "observer-force-stop-left-process-alive" : "observer-cleanup-evidence-missing";
|
||
const summary = aliveAfter === true
|
||
? "observer force stop completed but process liveness evidence still reports aliveAfter=true."
|
||
: "observer cleanup failed because stop-state evidence is missing or incomplete; this is an evidence gap, not confirmed process leakage.";
|
||
return [{
|
||
id,
|
||
severity: "red",
|
||
count: 1,
|
||
summary,
|
||
cleanupPhase: cleanupStep.phase,
|
||
cleanup,
|
||
evidenceSummary: formatQuickVerifyCleanupLine(cleanup),
|
||
valuesRedacted: true,
|
||
}];
|
||
}
|
||
|
||
export function classifyQuickVerifyCleanupStep(cleanupStep: Record<string, unknown>): Record<string, unknown> {
|
||
const explicit = record(cleanupStep.cleanup);
|
||
if (stringAtNullable(explicit, "cleanupClass") !== null) return explicit;
|
||
const payload = quickVerifyCleanupPayload(cleanupStep);
|
||
const observer = record(payload.observer);
|
||
const forceStop = record(payload.forceStop);
|
||
const detail = record(payload.detail);
|
||
const forceResult = record(payload.forceResult);
|
||
const sources = cleanupEvidenceSources([
|
||
["payload.observer", observer],
|
||
["payload.forceStop", forceStop],
|
||
["payload.detail", detail],
|
||
["payload", payload],
|
||
["payload.forceResult", forceResult],
|
||
]);
|
||
const forceReason = firstStringFromCleanupSources(sources, ["forceReason", "reason"]);
|
||
const status = firstStringFromCleanupSources(sources, ["status"]);
|
||
const command = firstStringFromCleanupSources(sources, ["command"]);
|
||
const evidenceSourceNames = cleanupEvidenceSourceNames(sources, [
|
||
"aliveBefore",
|
||
"aliveAfter",
|
||
"termSent",
|
||
"killSent",
|
||
"pendingAbandoned",
|
||
"processingAbandoned",
|
||
]);
|
||
const forceStopAttempted = forceReason !== null
|
||
|| status === "forced-stopped"
|
||
|| status === "force-stopped"
|
||
|| command === "web-probe-observe force-stop"
|
||
|| payload.forced === true
|
||
|| numberAtNullable(forceResult, "exitCode") !== null
|
||
|| evidenceSourceNames.length > 0;
|
||
const aliveBefore = firstBooleanFromCleanupSources(sources, "aliveBefore");
|
||
const aliveAfter = firstBooleanFromCleanupSources(sources, "aliveAfter");
|
||
const termSent = firstBooleanFromCleanupSources(sources, "termSent");
|
||
const killSent = firstBooleanFromCleanupSources(sources, "killSent");
|
||
const pendingAbandoned = firstNumberFromCleanupSources(sources, "pendingAbandoned");
|
||
const processingAbandoned = firstNumberFromCleanupSources(sources, "processingAbandoned");
|
||
const forceStopOk = forceStopAttempted && aliveAfter === false;
|
||
const gracefulStopOk = !forceStopAttempted && (cleanupStep.ok === true || payload.ok === true);
|
||
const cleanupClass = gracefulStopOk
|
||
? "stopped"
|
||
: forceStopOk
|
||
? "bounded-force-stop"
|
||
: "observer-stop-failed";
|
||
const missingEvidenceFields = forceStopAttempted && cleanupClass === "observer-stop-failed"
|
||
? [
|
||
aliveAfter === null ? "aliveAfter" : null,
|
||
termSent === null ? "termSent" : null,
|
||
killSent === null ? "killSent" : null,
|
||
pendingAbandoned === null ? "pendingAbandoned" : null,
|
||
processingAbandoned === null ? "processingAbandoned" : null,
|
||
].filter((value): value is string => value !== null)
|
||
: [];
|
||
return {
|
||
cleanupClass,
|
||
phase: stringAtNullable(cleanupStep, "phase"),
|
||
commandOk: cleanupStep.ok === true ? true : cleanupStep.ok === false ? false : null,
|
||
status,
|
||
forceReason,
|
||
gracefulStopOk,
|
||
forceStopOk,
|
||
aliveBefore,
|
||
aliveAfter,
|
||
termSent,
|
||
killSent,
|
||
pendingAbandoned,
|
||
processingAbandoned,
|
||
evidenceSources: evidenceSourceNames,
|
||
missingEvidenceFields,
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function quickVerifyCleanupPayload(cleanupStep: Record<string, unknown>): Record<string, unknown> {
|
||
const payload = record(cleanupStep.payload);
|
||
if (Object.keys(payload).length > 0) return payload;
|
||
const result = record(cleanupStep.result);
|
||
for (const key of ["stdoutPreview", "stdoutTail"]) {
|
||
const parsed = parseJsonObject(stringAtNullable(result, key) ?? "");
|
||
const data = cliDataPayload(parsed);
|
||
if (Object.keys(data).length > 0) return data;
|
||
}
|
||
return {};
|
||
}
|
||
|
||
function cleanupEvidenceSources(entries: readonly (readonly [string, Record<string, unknown>])[]): readonly { name: string; value: Record<string, unknown> }[] {
|
||
return entries
|
||
.filter((entry) => Object.keys(entry[1]).length > 0)
|
||
.map(([name, value]) => ({ name, value }));
|
||
}
|
||
|
||
function cleanupEvidenceSourceNames(sources: readonly { name: string; value: Record<string, unknown> }[], keys: readonly string[]): string[] {
|
||
const names: string[] = [];
|
||
for (const source of sources) {
|
||
if (keys.some((key) => source.value[key] !== undefined && source.value[key] !== null)) names.push(source.name);
|
||
}
|
||
return [...new Set(names)];
|
||
}
|
||
|
||
function firstStringFromCleanupSources(sources: readonly { value: Record<string, unknown> }[], keys: readonly string[]): string | null {
|
||
for (const source of sources) {
|
||
for (const key of keys) {
|
||
const value = stringAtNullable(source.value, key);
|
||
if (value !== null) return value;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function firstBooleanFromCleanupSources(sources: readonly { value: Record<string, unknown> }[], key: string): boolean | null {
|
||
for (const source of sources) {
|
||
const value = booleanAtNullable(source.value, key);
|
||
if (value !== null) return value;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function firstNumberFromCleanupSources(sources: readonly { value: Record<string, unknown> }[], key: string): number | null {
|
||
for (const source of sources) {
|
||
const value = numberAtNullable(source.value, key);
|
||
if (value !== null) return value;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function quickVerifyCleanupFromSteps(steps: readonly Record<string, unknown>[]): Record<string, unknown> | null {
|
||
for (const step of steps.slice().reverse()) {
|
||
const phase = stringAtNullable(step, "phase");
|
||
if (phase !== null && phase.includes("observe-stop")) return classifyQuickVerifyCleanupStep(step);
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function quickVerifyCleanupWarnings(cleanup: Record<string, unknown> | null): string[] {
|
||
if (cleanup === null) return [];
|
||
const cleanupClass = stringAtNullable(cleanup, "cleanupClass");
|
||
if (cleanupClass === "bounded-force-stop") {
|
||
return [`quick verify observer graceful stop did not drain, but force stop completed with aliveAfter=false; recorded bounded cleanup evidence (${formatQuickVerifyCleanupLine(cleanup)}).`];
|
||
}
|
||
if (cleanupClass === "observer-stop-failed") {
|
||
return ["quick verify observer stop failed; runner lifecycle cleanup is a blocking finding."];
|
||
}
|
||
return [];
|
||
}
|
||
|
||
function formatQuickVerifyCleanupLine(cleanup: Record<string, unknown> | null): string {
|
||
if (cleanup === null) return "cleanup=-";
|
||
const fields = [
|
||
["cleanup", stringAtNullable(cleanup, "cleanupClass")],
|
||
["aliveBefore", cleanup.aliveBefore],
|
||
["aliveAfter", cleanup.aliveAfter],
|
||
["termSent", cleanup.termSent],
|
||
["killSent", cleanup.killSent],
|
||
["pendingAbandoned", cleanup.pendingAbandoned],
|
||
["processingAbandoned", cleanup.processingAbandoned],
|
||
["gracefulStopOk", cleanup.gracefulStopOk],
|
||
["forceStopOk", cleanup.forceStopOk],
|
||
["missing", Array.isArray(cleanup.missingEvidenceFields) && cleanup.missingEvidenceFields.length > 0 ? cleanup.missingEvidenceFields.join(",") : null],
|
||
];
|
||
return fields.map(([key, value]) => `${key}=${value ?? "-"}`).join(" ");
|
||
}
|
||
|
||
function booleanAtNullable(value: Record<string, unknown>, key: string): boolean | null {
|
||
const raw = value[key];
|
||
return raw === true ? true : raw === false ? false : null;
|
||
}
|
||
|
||
function quickVerifyAnalysisVisibilityFindings(analysis: ChildCliResult, artifactSummary: Record<string, unknown>): Record<string, unknown>[] {
|
||
if (analysis.ok !== true) return [];
|
||
if (stringAtNullable(artifactSummary, "reportJsonSha256") !== null) return [];
|
||
const result = record(analysis.result);
|
||
const stdout = stringAtNullable(result, "stdoutPreview") ?? stringAtNullable(result, "stdoutTail");
|
||
const reportPathMatch = stdout === null ? null : stdout.match(/analysis\/report\.(?:json|md)/u);
|
||
return [{
|
||
id: "quick-verify-analysis-summary-unreadable",
|
||
severity: "red",
|
||
count: 1,
|
||
summary: "quick verify analyze exited successfully, but sentinel could not read analysis/report.json before recording the run.",
|
||
rootCause: "The report index would otherwise contain only control blockers such as WBC-003 and hide analyzer red/amber findings.",
|
||
rootCauseStatus: "confirmed",
|
||
rootCauseConfidence: "high",
|
||
evidenceSummary: `analyze ok=true reportJsonSha256=missing stdoutMentionsReport=${reportPathMatch === null ? "false" : "true"}`,
|
||
nextAction: "Read stateDir/analysis/report.json again or rehydrate the report from stateDir before trusting WBC-003 as the only visible finding.",
|
||
valuesRedacted: true,
|
||
}];
|
||
}
|
||
|
||
function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number): Record<string, unknown> {
|
||
const namespace = stringAt(state.runtime, "namespace");
|
||
const serviceName = stringAt(state.runtime, "serviceName");
|
||
const servicePort = numberAt(state.runtime, "servicePort");
|
||
const deploymentName = stringAt(state.runtime, "deploymentName");
|
||
const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`;
|
||
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
|
||
const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64");
|
||
const postScript = [
|
||
"const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');",
|
||
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
|
||
`const url = 'http://127.0.0.1:${servicePort}' + path;`,
|
||
"fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {",
|
||
" const text = await response.text();",
|
||
" process.stdout.write(text);",
|
||
" if (!response.ok) process.exit(22);",
|
||
"}).catch((error) => {",
|
||
" console.error(error && error.stack ? error.stack : String(error));",
|
||
" process.exit(23);",
|
||
"});",
|
||
].join(" ");
|
||
const script = method === "GET"
|
||
? `kubectl get --raw ${shellQuote(proxyPath)}`
|
||
: [
|
||
"set -eu",
|
||
`kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`,
|
||
].join("\n");
|
||
const maxAttempts = method === "GET" ? 3 : 1;
|
||
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60));
|
||
const attempts: Record<string, unknown>[] = [];
|
||
let result: CommandResult | null = null;
|
||
let parsed: Record<string, unknown> | null = null;
|
||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||
const invocation = sentinelServiceProxyInvocation(stringAt(state.controlPlaneNode, "kubeRoute"), script);
|
||
result = runCommand(invocation.argv, repoRoot, { input: invocation.input, timeoutMs: attemptTimeoutSeconds * 1000 });
|
||
const parsedResolution = resolveCliChildJsonCommandResult({
|
||
result,
|
||
requestedStdoutType: "web-probe sentinel service proxy response JSON",
|
||
acceptParsed: isSentinelServiceResponseContract,
|
||
});
|
||
parsed = parsedResolution.parsed;
|
||
attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, stdoutRecovery: parsedResolution.diagnostics, valuesRedacted: true });
|
||
if (result.exitCode === 0) break;
|
||
}
|
||
return {
|
||
ok: result?.exitCode === 0,
|
||
method,
|
||
path: pathWithQuery,
|
||
internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`,
|
||
httpStatus: result?.exitCode === 0 ? 200 : null,
|
||
bodyJson: record(compactSentinelServiceBodyJson(parsed)),
|
||
bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "",
|
||
bodyBytes: Buffer.byteLength(result?.stdout ?? ""),
|
||
error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000),
|
||
proxyPath,
|
||
result: result === null ? null : compactCommand(result),
|
||
attempts,
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function isSentinelServiceResponseContract(value: Record<string, unknown>): boolean {
|
||
return value.ok === false
|
||
|| typeof value.view === "string"
|
||
|| typeof value.renderedText === "string"
|
||
|| typeof value.error === "string"
|
||
|| Array.isArray(value.availableViews)
|
||
|| Object.keys(record(value.run)).length > 0
|
||
|| Object.keys(record(value.summary)).length > 0
|
||
|| Array.isArray(value.findings);
|
||
}
|
||
|
||
export function sentinelServiceProxyInvocation(kubeRoute: string, script: string): { readonly argv: readonly string[]; readonly input: string } {
|
||
return { argv: ["trans", kubeRoute, "sh"], input: script };
|
||
}
|
||
|
||
function compactSentinelServiceBodyJson(value: Record<string, unknown> | null): unknown {
|
||
if (value === null || typeof value.renderedText !== "string") return value;
|
||
return {
|
||
ok: value.ok,
|
||
view: value.view,
|
||
error: value.error,
|
||
availableViews: value.availableViews,
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function clipTail(value: string, maxLength: number): string {
|
||
return value.length <= maxLength ? value : value.slice(value.length - maxLength);
|
||
}
|
||
|
||
function recordSkippedQuickVerify(
|
||
state: SentinelCicdState,
|
||
scenario: Record<string, unknown>,
|
||
runId: string,
|
||
scenarioId: string,
|
||
triggerReason: string,
|
||
elapsedMs: number,
|
||
memoryGuard: unknown,
|
||
steps: readonly Record<string, unknown>[],
|
||
skipReason: string,
|
||
): Record<string, unknown> {
|
||
const cadence = stringAtNullable(scenario, "cadence");
|
||
const renderedText = [
|
||
`# Web 哨兵巡检 ${runId}`,
|
||
"",
|
||
"本轮因主机物理可用内存低于 WebProbe 启动门槛而跳过,未创建 observer 或 browser。",
|
||
`- 状态:skipped-wait-next-round`,
|
||
`- 场景:${scenarioId}`,
|
||
`- 下一步:等待下一次计划 cadence${cadence === null ? "" : `(${cadence})`}`,
|
||
].join("\n");
|
||
const recorded = recordQuickVerify(state, {
|
||
ok: true,
|
||
runId,
|
||
scenarioId,
|
||
reason: skipReason,
|
||
triggerReason,
|
||
status: "skipped-wait-next-round",
|
||
decision: "skip-observer",
|
||
observerId: null,
|
||
observerCreated: false,
|
||
browserCreated: false,
|
||
runCreated: true,
|
||
mutation: false,
|
||
memoryGuard,
|
||
elapsedMs,
|
||
steps: [...steps],
|
||
findingCount: 0,
|
||
artifactCount: 0,
|
||
findings: [],
|
||
warnings: [],
|
||
updatedAt: new Date().toISOString(),
|
||
nextCadence: {
|
||
action: "wait-for-next-scheduled-cadence",
|
||
cadence,
|
||
automaticRetry: true,
|
||
valuesRedacted: true,
|
||
},
|
||
views: {
|
||
summary: { renderedText, valuesRedacted: true },
|
||
"auth-session-switch-summary": { renderedText, valuesRedacted: true },
|
||
},
|
||
valuesRedacted: true,
|
||
});
|
||
return {
|
||
...recorded,
|
||
runCreated: record(recorded.recordResult).ok === true,
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function recordQuickVerify(state: SentinelCicdState, payload: Record<string, unknown>): Record<string, unknown> {
|
||
const writer = resolveMonitorTerminalWriter(state.runtime);
|
||
const views = compactQuickVerifyRecordViews(record(payload.views));
|
||
const summary = {
|
||
reason: payload.reason,
|
||
status: payload.status,
|
||
businessStatus: payload.businessStatus ?? null,
|
||
elapsedMs: payload.elapsedMs,
|
||
failure: payload.failure,
|
||
failureTitleZh: payload.failureTitleZh ?? payload.errorTitleZh ?? null,
|
||
errorTitleZh: payload.errorTitleZh ?? payload.failureTitleZh ?? null,
|
||
timeoutDisplay: payload.timeoutDisplay ?? null,
|
||
warnings: Array.isArray(payload.warnings) ? payload.warnings : [],
|
||
analysis: compactQuickVerifyRecordAnalysis(payload.analysis),
|
||
cleanup: compactQuickVerifyCleanupEvidence(payload.cleanup),
|
||
promptSource: payload.promptSource,
|
||
steps: Array.isArray(payload.steps) ? payload.steps.map(compactQuickVerifyRecordStep) : [],
|
||
valuesRedacted: true,
|
||
};
|
||
const legacyRecord = {
|
||
runId: payload.runId,
|
||
scenarioId: payload.scenarioId,
|
||
status: payload.status,
|
||
observerId: payload.observerId,
|
||
stateDir: payload.stateDir,
|
||
reportJsonSha256: payload.reportJsonSha256,
|
||
findingCount: payload.findingCount,
|
||
artifactCount: payload.artifactCount,
|
||
summary,
|
||
businessStatus: payload.businessStatus ?? null,
|
||
findings: payload.findings,
|
||
views,
|
||
screenshot: payload.screenshot,
|
||
publicOrigin: payload.publicOrigin ?? stringAt(state.publicExposure, "publicBaseUrl"),
|
||
maintenance: payload.reason === "maintenance-stop",
|
||
valuesRedacted: true,
|
||
};
|
||
const recordResult = writer.mode === "legacy-sqlite"
|
||
? callSentinelService(state, "POST", "/api/runs/record", legacyRecord, 60)
|
||
: submitCentralTerminalIngest(state, payload, writer.ingest!, views, summary);
|
||
emitWebProbeSentinelSpan(sentinelOtelContext(state), "web_probe_sentinel.quick_verify.job_finish", {
|
||
scenarioId: payload.scenarioId,
|
||
runId: payload.runId,
|
||
observerId: payload.observerId,
|
||
status: payload.status,
|
||
exitCode: payload.ok === true && recordResult.ok === true ? 0 : 1,
|
||
failureKind: payload.failure ?? (recordResult.ok === true ? null : writer.mode === "central" ? "monitor-ingest-failed" : "record-run-failed"),
|
||
valuesRedacted: true,
|
||
}, payload.ok === true && recordResult.ok === true);
|
||
return withWarnings({ ...payload, views, terminalWriter: writer.mode, recordResult, valuesRedacted: true }, recordResult.ok === true ? [] : [writer.mode === "central" ? "monitor-ingest-failed: retry the same immutable analysis/report artifact after the central ingest endpoint recovers." : "quick verify completed but sentinel report index record failed; report/dashboard may lag until record payload is reduced or retried."]);
|
||
}
|
||
|
||
function submitCentralTerminalIngest(state: SentinelCicdState, payload: Record<string, unknown>, config: NonNullable<ReturnType<typeof resolveMonitorTerminalWriter>["ingest"]>, views: Record<string, unknown>, summary: Record<string, unknown>): Record<string, unknown> {
|
||
const analysis = record(payload.analysis);
|
||
try {
|
||
const stateDir = stringAtNullable(payload, "stateDir") ?? stringAtNullable(analysis, "stateDir");
|
||
const report = { ...analysis, updatedAt: stringAtNullable(analysis, "reportUpdatedAt") ?? stringAtNullable(payload, "updatedAt"), findings: Array.isArray(payload.findings) ? payload.findings.map(record) : [], summary, views, valuesRedacted: true };
|
||
const artifacts = payload.status === "skipped-wait-next-round" || (payload.observerId === null && payload.artifactCount === 0)
|
||
? []
|
||
: [{ relativePath: "analysis/report.json", kind: "report-json", sha256: stringAtNullable(payload, "reportJsonSha256") ?? stringAtNullable(analysis, "reportJsonSha256") ?? "", sizeBytes: numberAtNullable(analysis, "reportJsonBytes") ?? -1 }];
|
||
const ingest = buildMonitorTerminalIngest({ sentinelId: state.sentinelId, node: state.spec.nodeId, lane: state.spec.lane, runId: stringAt(payload, "runId") }, report, artifacts, { ownerNode: state.spec.nodeId, ownerLane: state.spec.lane, ownerPvc: stringAt(state.runtime, "pvcName"), stateDir: stateDir ?? "" });
|
||
const result = runCommand(["bun", "scripts/monitor-terminal-ingest-submit.ts"], repoRoot, { timeoutMs: config.retry.maxAttempts * config.timeoutMs + (config.retry.maxAttempts - 1) * config.retry.delayMs + 1_000, input: JSON.stringify({ config, ingest }) });
|
||
const response = parseJsonObject(result.stdout);
|
||
if (result.exitCode === 0 && response?.ok === true) return { ...response, writer: "central", valuesRedacted: true };
|
||
return { ok: false, writer: "central", error: "monitor-ingest-failed", response, stderr: short(result.stderr), payloadHash: ingest.payloadHash, valuesRedacted: true };
|
||
} catch (error) {
|
||
return { ok: false, writer: "central", error: "monitor-ingest-failed", details: error instanceof MonitorIngestFailedError ? error.details : { reason: error instanceof Error ? error.message : String(error) }, valuesRedacted: true };
|
||
}
|
||
}
|
||
|
||
function sentinelOtelContext(state: SentinelCicdState): { readonly node: string; readonly lane: string; readonly sentinelId: string; readonly namespace: string | null; readonly runtime: Record<string, unknown>; readonly cicd: Record<string, unknown> } {
|
||
return {
|
||
node: state.spec.nodeId,
|
||
lane: state.spec.lane,
|
||
sentinelId: state.sentinelId,
|
||
namespace: stringAtNullable(state.runtime, "namespace"),
|
||
runtime: state.runtime,
|
||
cicd: state.cicd,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordViews(views: Record<string, unknown>): Record<string, unknown> {
|
||
const compacted: Record<string, unknown> = {};
|
||
for (const [key, value] of Object.entries(views)) {
|
||
const item = record(value);
|
||
const limit = key === "summary" || key === "auth-session-switch-summary" ? 8_000 : 6_000;
|
||
compacted[key] = {
|
||
...item,
|
||
renderedText: boundQuickVerifyRecordText(item.renderedText, limit),
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
return compacted;
|
||
}
|
||
|
||
function compactQuickVerifyRecordAnalysis(value: unknown): Record<string, unknown> | null {
|
||
const item = record(value);
|
||
if (Object.keys(item).length === 0) return null;
|
||
return {
|
||
ok: item.ok === true ? true : item.ok === false ? false : null,
|
||
reportOk: item.reportOk === true ? true : item.reportOk === false ? false : null,
|
||
source: stringAtNullable(item, "source"),
|
||
reason: stringAtNullable(item, "reason"),
|
||
stateDir: stringAtNullable(item, "stateDir"),
|
||
reportJsonPath: stringAtNullable(item, "reportJsonPath"),
|
||
reportJsonSha256: stringAtNullable(item, "reportJsonSha256"),
|
||
reportMdPath: stringAtNullable(item, "reportMdPath"),
|
||
reportMdSha256: stringAtNullable(item, "reportMdSha256"),
|
||
reportReadWaitMs: numberAtNullable(item, "reportReadWaitMs"),
|
||
reportParseError: stringAtNullable(item, "reportParseError"),
|
||
findingCount: numberAtNullable(item, "findingCount"),
|
||
artifactCount: numberAtNullable(item, "artifactCount"),
|
||
counts: compactQuickVerifyRecordCounts(record(item.counts)),
|
||
readResult: compactQuickVerifyRecordArtifactReadResult(record(item.result)),
|
||
screenshot: compactQuickVerifyRecordScreenshot(record(item.screenshot)),
|
||
findings: Array.isArray(item.findings) ? item.findings.slice(0, 16).map(compactQuickVerifyRecordFinding) : [],
|
||
pagePerformanceSlowApi: Array.isArray(item.pagePerformanceSlowApi) ? item.pagePerformanceSlowApi.slice(0, 6).map(record) : [],
|
||
browserProcess: compactQuickVerifyRecordBrowserProcess(record(item.browserProcess)),
|
||
requestRate: compactQuickVerifyRecordRequestRate(record(item.requestRate), record(item.requestRateCurve)),
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordArtifactReadResult(value: Record<string, unknown>): Record<string, unknown> | null {
|
||
if (Object.keys(value).length === 0) return null;
|
||
return {
|
||
exitCode: numberAtNullable(value, "exitCode"),
|
||
timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null,
|
||
stdoutBytes: numberAtNullable(value, "stdoutBytes"),
|
||
stderrBytes: numberAtNullable(value, "stderrBytes"),
|
||
stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 600),
|
||
stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 600),
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordBrowserProcess(value: Record<string, unknown>): Record<string, unknown> | null {
|
||
const pageSeries = Array.isArray(value.pageSeries) ? value.pageSeries.map(compactQuickVerifyRecordMemorySeries).filter((item) => item.points.length > 0) : [];
|
||
if (pageSeries.length === 0) return null;
|
||
return {
|
||
source: stringAtNullable(value, "source") ?? "analysis-browser-process",
|
||
unit: stringAtNullable(value, "unit") ?? "MB",
|
||
metric: stringAtNullable(value, "metric") ?? "page-heap-used",
|
||
pageCount: pageSeries.length,
|
||
sampleCount: pageSeries.reduce((sum, item) => sum + item.points.length, 0),
|
||
pageSeries,
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordRequestRate(summaryValue: Record<string, unknown>, curveValue: Record<string, unknown>): Record<string, unknown> | null {
|
||
const source = Object.keys(curveValue).length > 0 ? curveValue : summaryValue;
|
||
const summary = record(source.summary);
|
||
const effectiveSummary = Object.keys(summary).length > 0 ? summary : summaryValue;
|
||
const pageCurves = Array.isArray(source.pageCurves) ? source.pageCurves.map((item) => compactQuickVerifyRecordRequestRateCurve(item, "page")).filter((item) => item.buckets.length > 0).slice(0, 12) : [];
|
||
const apiPathCurves = Array.isArray(source.apiPathCurves) ? source.apiPathCurves.map((item) => compactQuickVerifyRecordRequestRateCurve(item, "apiPath")).filter((item) => item.buckets.length > 0).slice(0, 16) : [];
|
||
const buckets = Array.isArray(source.buckets) ? source.buckets.map(compactQuickVerifyRecordRequestRateBucket).filter(hasCompactRequestRateBucketPoint).slice(-120) : [];
|
||
if (Object.keys(source).length === 0 && pageCurves.length === 0 && apiPathCurves.length === 0 && buckets.length === 0) return null;
|
||
return {
|
||
summary: compactQuickVerifyRecordRequestRateSummary(effectiveSummary),
|
||
buckets,
|
||
pageCurves,
|
||
apiPathCurves,
|
||
peaks: Array.isArray(source.peaks) ? source.peaks.slice(0, 12).map(compactQuickVerifyRecordRequestRatePeak) : [],
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordRequestRateSummary(value: Record<string, unknown>): Record<string, unknown> {
|
||
return {
|
||
bucketMs: numberAtNullable(value, "bucketMs"),
|
||
bucketSeconds: numberAtNullable(value, "bucketSeconds"),
|
||
requestCount: numberAtNullable(value, "requestCount"),
|
||
bucketCount: numberAtNullable(value, "bucketCount"),
|
||
pageCount: numberAtNullable(value, "pageCount"),
|
||
apiPathCount: numberAtNullable(value, "apiPathCount"),
|
||
firstAt: stringAtNullable(value, "firstAt"),
|
||
lastAt: stringAtNullable(value, "lastAt"),
|
||
totalRedPerMinute: numberAtNullable(value, "totalRedPerMinute"),
|
||
pageRedPerMinute: numberAtNullable(value, "pageRedPerMinute"),
|
||
apiPathRedPerMinute: numberAtNullable(value, "apiPathRedPerMinute"),
|
||
totalPeakPerMinute: numberAtNullable(value, "totalPeakPerMinute"),
|
||
totalPeakCount: numberAtNullable(value, "totalPeakCount"),
|
||
totalPeakAt: stringAtNullable(value, "totalPeakAt"),
|
||
pagePeakPerMinute: numberAtNullable(value, "pagePeakPerMinute"),
|
||
pagePeakKey: stringAtNullable(value, "pagePeakKey"),
|
||
pagePeakPath: stringAtNullable(value, "pagePeakPath"),
|
||
apiPathPeakPerMinute: numberAtNullable(value, "apiPathPeakPerMinute"),
|
||
apiPathPeakKey: stringAtNullable(value, "apiPathPeakKey"),
|
||
overThresholdPeakCount: numberAtNullable(value, "overThresholdPeakCount"),
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordRequestRateCurve(value: unknown, scope: string): Record<string, unknown> & { buckets: Record<string, unknown>[] } {
|
||
const item = record(value);
|
||
const buckets = Array.isArray(item.buckets) ? item.buckets.map(compactQuickVerifyRecordRequestRateBucket).filter(hasCompactRequestRateBucketPoint).slice(-120) : [];
|
||
return {
|
||
pageKey: stringAtNullable(item, "pageKey"),
|
||
pageRole: stringAtNullable(item, "pageRole"),
|
||
pageId: stringAtNullable(item, "pageId"),
|
||
pageEpoch: numberAtNullable(item, "pageEpoch"),
|
||
apiKey: stringAtNullable(item, "apiKey"),
|
||
method: stringAtNullable(item, "method"),
|
||
path: stringAtNullable(item, "path"),
|
||
count: numberAtNullable(item, "count"),
|
||
bucketCount: numberAtNullable(item, "bucketCount") ?? buckets.length,
|
||
peakRequestPerMinute: numberAtNullable(item, "peakRequestPerMinute"),
|
||
peakBucket: compactQuickVerifyRecordRequestRateBucket(record(item.peakBucket)),
|
||
buckets,
|
||
scope,
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordRequestRateBucket(value: unknown): Record<string, unknown> {
|
||
const item = record(value);
|
||
return {
|
||
bucketStartMs: numberAtNullable(item, "bucketStartMs"),
|
||
bucketEndMs: numberAtNullable(item, "bucketEndMs"),
|
||
startAt: stringAtNullable(item, "startAt") ?? stringAtNullable(item, "t"),
|
||
endAt: stringAtNullable(item, "endAt") ?? stringAtNullable(item, "e"),
|
||
count: numberAtNullable(item, "count") ?? numberAtNullable(item, "c"),
|
||
requestPerMinute: numberAtNullable(item, "requestPerMinute") ?? numberAtNullable(item, "rpm"),
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function hasCompactRequestRateBucketPoint(value: Record<string, unknown>): boolean {
|
||
return stringAtNullable(value, "startAt") !== null && numberAtNullable(value, "requestPerMinute") !== null;
|
||
}
|
||
|
||
function compactQuickVerifyRecordRequestRatePeak(value: unknown): Record<string, unknown> {
|
||
const item = record(value);
|
||
return {
|
||
scope: stringAtNullable(item, "scope"),
|
||
thresholdPerMinute: numberAtNullable(item, "thresholdPerMinute"),
|
||
overThreshold: item.overThreshold === true,
|
||
bucketMs: numberAtNullable(item, "bucketMs"),
|
||
startAt: stringAtNullable(item, "startAt"),
|
||
endAt: stringAtNullable(item, "endAt"),
|
||
count: numberAtNullable(item, "count"),
|
||
requestPerMinute: numberAtNullable(item, "requestPerMinute"),
|
||
method: stringAtNullable(item, "method"),
|
||
path: stringAtNullable(item, "path"),
|
||
apiKey: stringAtNullable(item, "apiKey"),
|
||
pageKey: stringAtNullable(item, "pageKey"),
|
||
pageRole: stringAtNullable(item, "pageRole"),
|
||
pageId: stringAtNullable(item, "pageId"),
|
||
pageEpoch: numberAtNullable(item, "pageEpoch"),
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordMemorySeries(value: unknown): Record<string, unknown> & { points: Record<string, unknown>[] } {
|
||
const item = record(value);
|
||
const points = Array.isArray(item.points)
|
||
? item.points.map(record).map((point) => ({
|
||
ts: stringAtNullable(point, "ts"),
|
||
elapsedSeconds: numberAtNullable(point, "elapsedSeconds"),
|
||
elapsedMinutes: numberAtNullable(point, "elapsedMinutes"),
|
||
memoryMb: numberAtNullable(point, "memoryMb"),
|
||
heapUsedMb: numberAtNullable(point, "heapUsedMb"),
|
||
jsHeapUsedMb: numberAtNullable(point, "jsHeapUsedMb"),
|
||
effectiveHeapUsedMb: numberAtNullable(point, "effectiveHeapUsedMb"),
|
||
effectiveJsHeapUsedMb: numberAtNullable(point, "effectiveJsHeapUsedMb"),
|
||
domNodes: numberAtNullable(point, "domNodes"),
|
||
valuesRedacted: true,
|
||
})).filter((point) => point.ts !== null && point.memoryMb !== null)
|
||
: [];
|
||
return {
|
||
key: stringAtNullable(item, "key"),
|
||
pageRole: stringAtNullable(item, "pageRole"),
|
||
pageId: stringAtNullable(item, "pageId"),
|
||
label: stringAtNullable(item, "label"),
|
||
url: stringAtNullable(item, "url"),
|
||
points,
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordCounts(value: Record<string, unknown>): Record<string, unknown> {
|
||
return {
|
||
samples: numberAtNullable(value, "samples"),
|
||
control: numberAtNullable(value, "control"),
|
||
network: numberAtNullable(value, "network"),
|
||
console: numberAtNullable(value, "console"),
|
||
errors: numberAtNullable(value, "errors"),
|
||
artifacts: numberAtNullable(value, "artifacts"),
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordScreenshot(value: Record<string, unknown>): Record<string, unknown> | null {
|
||
if (Object.keys(value).length === 0) return null;
|
||
return {
|
||
path: stringAtNullable(value, "path"),
|
||
sha256: stringAtNullable(value, "sha256"),
|
||
bytes: numberAtNullable(value, "bytes"),
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordFinding(value: unknown): Record<string, unknown> {
|
||
const item = record(value);
|
||
return {
|
||
id: stringAtNullable(item, "id"),
|
||
kind: stringAtNullable(item, "kind"),
|
||
code: stringAtNullable(item, "code"),
|
||
severity: stringAtNullable(item, "severity"),
|
||
level: stringAtNullable(item, "level"),
|
||
count: numberAtNullable(item, "count"),
|
||
summary: boundQuickVerifyRecordText(item.summary ?? item.message, 220),
|
||
rootCause: boundQuickVerifyRecordText(item.rootCause, 140),
|
||
rootCauseStatus: boundQuickVerifyRecordText(item.rootCauseStatus, 90),
|
||
rootCauseConfidence: boundQuickVerifyRecordText(item.rootCauseConfidence, 40),
|
||
nextAction: boundQuickVerifyRecordText(item.nextAction, 240),
|
||
evidenceSummary: stringAtNullable(item, "evidenceSummary") ?? compactQuickVerifyFindingEvidence(item.evidence),
|
||
rootCauseSignals: compactQuickVerifyRootCauseSignals(item.rootCauseSignals),
|
||
timingSourceOfTruth: boundQuickVerifyRecordText(item.timingSourceOfTruth ?? item.expectedElapsedSource ?? item.evidenceKind, 100),
|
||
timingStatus: boundQuickVerifyRecordText(item.timingStatus, 60),
|
||
timingAlert: item.timingAlert === true,
|
||
blocking: item.blocking === true,
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRootCauseSignals(value: unknown): Record<string, unknown> | null {
|
||
const item = record(value);
|
||
const keys = [
|
||
"sessionListReadCount",
|
||
"traceEventsReadCount",
|
||
"webPerformanceBeaconFailureCount",
|
||
"eventSourceFailureCount",
|
||
"requestFailedCount",
|
||
"httpErrorCount",
|
||
"consoleAlertCount",
|
||
"requestfailedTop",
|
||
"httpStatusTop",
|
||
];
|
||
const compact: Record<string, unknown> = {};
|
||
for (const key of keys) {
|
||
const raw = item[key];
|
||
if (raw === null || raw === undefined) continue;
|
||
compact[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw;
|
||
}
|
||
return Object.keys(compact).length === 0 ? null : { ...compact, valuesRedacted: true };
|
||
}
|
||
|
||
function compactQuickVerifyFindingEvidence(value: unknown): string | null {
|
||
const item = record(value);
|
||
if (Object.keys(item).length === 0) return null;
|
||
const keys = [
|
||
"http404Count",
|
||
"responseErrorCount",
|
||
"requestFailedCount",
|
||
"statuses",
|
||
"afterProjectedSeqs",
|
||
"sinceSeqs",
|
||
"traceIds",
|
||
"maxFallbackRatio",
|
||
"maxFallbackTitleCount",
|
||
"overThresholdSampleCount",
|
||
"majorityFallbackSampleCount",
|
||
];
|
||
const compact: Record<string, unknown> = {};
|
||
for (const key of keys) {
|
||
const raw = item[key];
|
||
if (raw === null || raw === undefined) continue;
|
||
compact[key] = Array.isArray(raw) ? raw.slice(0, 6) : raw;
|
||
}
|
||
return Object.keys(compact).length === 0 ? null : boundQuickVerifyRecordText(JSON.stringify(compact), 240);
|
||
}
|
||
|
||
function compactQuickVerifyRecordStep(value: unknown): Record<string, unknown> {
|
||
const item = record(value);
|
||
return {
|
||
phase: stringAtNullable(item, "phase"),
|
||
ok: item.ok === true ? true : item.ok === false ? false : null,
|
||
promptIndex: numberAtNullable(item, "promptIndex"),
|
||
checkId: stringAtNullable(item, "checkId"),
|
||
failure: stringAtNullable(item, "failure"),
|
||
cleanup: compactQuickVerifyCleanupEvidence(item.cleanup),
|
||
result: compactQuickVerifyRecordStepResult(record(item.result)),
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyCleanupEvidence(value: unknown): Record<string, unknown> | null {
|
||
const item = record(value);
|
||
if (stringAtNullable(item, "cleanupClass") === null) return null;
|
||
return {
|
||
cleanupClass: stringAtNullable(item, "cleanupClass"),
|
||
phase: stringAtNullable(item, "phase"),
|
||
commandOk: booleanAtNullable(item, "commandOk"),
|
||
status: stringAtNullable(item, "status"),
|
||
forceReason: stringAtNullable(item, "forceReason"),
|
||
gracefulStopOk: booleanAtNullable(item, "gracefulStopOk"),
|
||
forceStopOk: booleanAtNullable(item, "forceStopOk"),
|
||
aliveBefore: booleanAtNullable(item, "aliveBefore"),
|
||
aliveAfter: booleanAtNullable(item, "aliveAfter"),
|
||
termSent: booleanAtNullable(item, "termSent"),
|
||
killSent: booleanAtNullable(item, "killSent"),
|
||
pendingAbandoned: numberAtNullable(item, "pendingAbandoned"),
|
||
processingAbandoned: numberAtNullable(item, "processingAbandoned"),
|
||
evidenceSources: Array.isArray(item.evidenceSources) ? item.evidenceSources.filter((value): value is string => typeof value === "string").slice(0, 6) : [],
|
||
missingEvidenceFields: Array.isArray(item.missingEvidenceFields) ? item.missingEvidenceFields.filter((value): value is string => typeof value === "string").slice(0, 8) : [],
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function compactQuickVerifyRecordStepResult(value: Record<string, unknown>): Record<string, unknown> {
|
||
return {
|
||
ok: value.ok === true ? true : value.ok === false ? false : null,
|
||
status: stringAtNullable(value, "status"),
|
||
view: stringAtNullable(value, "view"),
|
||
exitCode: numberAtNullable(value, "exitCode"),
|
||
timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null,
|
||
stdoutBytes: numberAtNullable(value, "stdoutBytes"),
|
||
stderrBytes: numberAtNullable(value, "stderrBytes"),
|
||
stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 240),
|
||
stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 240),
|
||
stdoutTail: boundQuickVerifyRecordText(value.stdoutTail, 1200),
|
||
stderrTail: boundQuickVerifyRecordText(value.stderrTail, 1200),
|
||
valuesRedacted: true,
|
||
};
|
||
}
|
||
|
||
function boundQuickVerifyRecordText(value: unknown, maxChars: number): string | null {
|
||
if (typeof value !== "string") return null;
|
||
if (value.length <= maxChars) return value;
|
||
return `${value.slice(0, maxChars)}\n[truncated ${value.length - maxChars} chars]`;
|
||
}
|
||
|
||
function renderQuickVerifySummary(input: Record<string, unknown>): string {
|
||
const artifact = record(input.artifactSummary);
|
||
const findings = Array.isArray(input.findings)
|
||
? input.findings.map(record).slice(0, 8)
|
||
: Array.isArray(artifact.findings)
|
||
? artifact.findings.map(record).slice(0, 8)
|
||
: [];
|
||
const findingCount = numberAtNullable(input, "findingCount") ?? numberAtNullable(artifact, "findingCount") ?? findings.length;
|
||
const failureTitle = stringAtNullable(input, "failureTitleZh") ?? stringAtNullable(input, "errorTitleZh") ?? stringAtNullable(record(input.timeoutDisplay), "titleZh");
|
||
const cleanup = compactQuickVerifyCleanupEvidence(input.cleanup);
|
||
return [
|
||
"Web Probe Sentinel Quick Verify",
|
||
"=======================================================",
|
||
`run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`,
|
||
failureTitle === null ? "" : `errorTitle=${failureTitle} failure=${input.failure ?? "-"}`,
|
||
`report=${artifact.reportJsonSha256 ?? "-"} artifacts=${artifact.artifactCount ?? "-"} findings=${findingCount}`,
|
||
cleanup === null ? "" : formatQuickVerifyCleanupLine(cleanup),
|
||
`publicOrigin=${input.publicOrigin ?? "-"}`,
|
||
"",
|
||
"Findings",
|
||
findings.length === 0 ? "-" : findings.map(formatQuickVerifyFindingLine).join("\n"),
|
||
].filter((line) => line !== "").join("\n");
|
||
}
|
||
|
||
function renderAuthSessionSwitchQuickVerifySummary(input: Record<string, unknown>): string {
|
||
const artifact = record(input.artifactSummary);
|
||
const accountEnv = record(input.accountEnv);
|
||
const cleanup = compactQuickVerifyCleanupEvidence(input.cleanup);
|
||
const findingRows = Array.isArray(input.findings)
|
||
? input.findings.map(record).slice(0, 8)
|
||
: Array.isArray(artifact.findings)
|
||
? artifact.findings.map(record).slice(0, 8)
|
||
: [];
|
||
const steps = Array.isArray(input.steps) ? input.steps.map(record) : [];
|
||
const commandSteps = steps
|
||
.filter((step) => {
|
||
const phase = stringAtNullable(step, "phase") ?? "";
|
||
return phase.startsWith("observe-command-") || phase.startsWith("observe-session-invariance-") || phase === "quick-verify-account-env";
|
||
})
|
||
.map((step) => {
|
||
const phase = stringAtNullable(step, "phase") ?? "-";
|
||
const ok = step.ok === true ? "ok" : step.ok === false ? "failed" : "-";
|
||
const result = record(step.result);
|
||
const status = stringAtNullable(result, "status") ?? stringAtNullable(result, "failure") ?? "-";
|
||
return `${phase} ${ok} ${status}`;
|
||
});
|
||
return [
|
||
"Auth Session Switch Quick Verify",
|
||
"=======================================================",
|
||
`run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`,
|
||
stringAtNullable(input, "failureTitleZh") === null && stringAtNullable(input, "errorTitleZh") === null ? "" : `errorTitle=${stringAtNullable(input, "failureTitleZh") ?? stringAtNullable(input, "errorTitleZh") ?? "-"}`,
|
||
`status=${artifact.ok === true ? "ok" : "blocked"} report=${artifact.reportJsonSha256 ?? "-"} publicOrigin=${input.publicOrigin ?? "-"}`,
|
||
cleanup === null ? "" : formatQuickVerifyCleanupLine(cleanup),
|
||
`accountEnv=${accountEnv.envCount ?? "-"} valuesRedacted=true`,
|
||
"",
|
||
"Command Sequence",
|
||
commandSteps.length === 0 ? "-" : commandSteps.join("\n"),
|
||
"",
|
||
"Findings",
|
||
findingRows.length === 0 ? "-" : findingRows.map(formatQuickVerifyFindingLine).join("\n"),
|
||
].filter((line) => line !== "").join("\n");
|
||
}
|
||
|
||
function formatQuickVerifyFindingLine(item: Record<string, unknown>): string {
|
||
const title = stringAtNullable(item, "displayTitleZh") ?? stringAtNullable(item, "errorTitleZh") ?? stringAtNullable(record(item.timeoutDisplay), "titleZh");
|
||
const summary = title ?? item.summary ?? item.message ?? "";
|
||
return `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${summary}`;
|
||
}
|
||
|
||
function formatQuickVerifyTimingSuffix(item: Record<string, unknown>): string {
|
||
const status = stringAtNullable(item, "timingStatus");
|
||
const source = stringAtNullable(item, "timingSourceOfTruth") ?? stringAtNullable(item, "expectedElapsedSource") ?? stringAtNullable(item, "evidenceKind");
|
||
if (status === null && source === null) return "";
|
||
return ` timing=${[status === null ? null : `status=${status}`, source === null ? null : `source=${source}`].filter((part) => part !== null).join(" ")}`;
|
||
}
|