|
|
|
@@ -191,6 +191,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
|
|
|
|
|
const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
|
|
|
|
|
for (let index = 0; index < repeat; index += 1) {
|
|
|
|
|
if (Date.now() >= deadline) {
|
|
|
|
|
const timeoutDisplay = timeoutDisplayForQuickVerifyFailure("quick-verify-timeout-over-budget", promptIndex, warningBudgetSeconds, timeoutSeconds);
|
|
|
|
|
printQuickVerifyProgress(state, runId, "timeout", "failed", { observerId, promptIndex, elapsedMs: elapsedMs(), hardBudgetSeconds });
|
|
|
|
|
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
|
|
|
|
runId,
|
|
|
|
@@ -200,6 +201,8 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
|
|
|
|
|
promptIndex,
|
|
|
|
|
steps,
|
|
|
|
|
failure: "quick-verify-timeout-over-budget",
|
|
|
|
|
failureTitleZh: stringAtNullable(timeoutDisplay, "titleZh"),
|
|
|
|
|
timeoutDisplay,
|
|
|
|
|
elapsedMs: elapsedMs(),
|
|
|
|
|
warnings: mergeWarnings(`quick verify exceeded the hard ${hardBudgetSeconds}s execution budget after the configured ${warningBudgetSeconds}s targetValidation warning budget.`, elapsedWarnings()),
|
|
|
|
|
promptSource: prompts.summary,
|
|
|
|
@@ -486,10 +489,15 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
|
|
|
|
|
readonly promptIndex: number;
|
|
|
|
|
readonly steps: readonly Record<string, unknown>[];
|
|
|
|
|
readonly failure: string;
|
|
|
|
|
readonly failureTitleZh?: string | null;
|
|
|
|
|
readonly timeoutDisplay?: Record<string, unknown> | null;
|
|
|
|
|
readonly promptSource?: Record<string, unknown>;
|
|
|
|
|
readonly elapsedMs?: number;
|
|
|
|
|
readonly warnings?: readonly unknown[];
|
|
|
|
|
}): Record<string, unknown> {
|
|
|
|
|
const targetValidationSeconds = numberAt(state.cicd, "targetValidation.maxSeconds");
|
|
|
|
|
const failureDisplay = input.timeoutDisplay ?? timeoutDisplayForQuickVerifyFailure(input.failure, input.promptIndex, targetValidationSeconds, null);
|
|
|
|
|
const failureTitle = input.failureTitleZh ?? stringAtNullable(failureDisplay, "titleZh");
|
|
|
|
|
const cleanupSteps: Record<string, unknown>[] = [];
|
|
|
|
|
if (input.promptIndex > 0) {
|
|
|
|
|
const cancel = runChildCli([
|
|
|
|
@@ -519,7 +527,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
|
|
|
|
|
const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30);
|
|
|
|
|
const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30);
|
|
|
|
|
const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(input.promptIndex, turnSummary, traceFrame);
|
|
|
|
|
const controlFindings = quickVerifyControlFindings(input.failure, input.promptIndex, turnSummary, traceFrame);
|
|
|
|
|
const controlFindings = quickVerifyControlFindings(input.failure, input.promptIndex, turnSummary, traceFrame, failureDisplay);
|
|
|
|
|
const artifactSummaryRecord = record(artifactSummary);
|
|
|
|
|
const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
|
|
|
|
|
const visibilityFindings = quickVerifyAnalysisVisibilityFindings(analysis, artifactSummary);
|
|
|
|
@@ -533,7 +541,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
|
|
|
|
|
&& record(artifactSummary).ok === true
|
|
|
|
|
&& controlFindings.length === 0
|
|
|
|
|
&& blockingFindings.length === 0;
|
|
|
|
|
const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, numberAt(state.cicd, "targetValidation.maxSeconds"));
|
|
|
|
|
const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, targetValidationSeconds);
|
|
|
|
|
return {
|
|
|
|
|
ok: recoveredWaitFailure,
|
|
|
|
|
runId: input.runId,
|
|
|
|
@@ -543,6 +551,9 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
|
|
|
|
|
observerId: input.observerId,
|
|
|
|
|
elapsedMs: input.elapsedMs ?? null,
|
|
|
|
|
businessStatus,
|
|
|
|
|
failureTitleZh: recoveredWaitFailure ? null : failureTitle,
|
|
|
|
|
errorTitleZh: recoveredWaitFailure ? null : failureTitle,
|
|
|
|
|
timeoutDisplay: recoveredWaitFailure ? null : failureDisplay,
|
|
|
|
|
stateDir: indexEntry?.stateDir ?? null,
|
|
|
|
|
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
|
|
|
|
|
findingCount: findings.length,
|
|
|
|
@@ -552,8 +563,8 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
|
|
|
|
|
steps: [...input.steps, ...cleanupSteps],
|
|
|
|
|
analysis: artifactSummary,
|
|
|
|
|
views: {
|
|
|
|
|
summary: { renderedText: renderQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, findings, findingCount: findings.length, steps: input.steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
|
|
|
|
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, findings, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
|
|
|
|
|
summary: { renderedText: renderQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, findings, findingCount: findings.length, steps: input.steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), failure: recoveredWaitFailure ? null : input.failure, failureTitleZh: recoveredWaitFailure ? null : failureTitle, timeoutDisplay: recoveredWaitFailure ? null : failureDisplay }) },
|
|
|
|
|
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, findings, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), failure: recoveredWaitFailure ? null : input.failure, failureTitleZh: recoveredWaitFailure ? null : failureTitle, timeoutDisplay: recoveredWaitFailure ? null : failureDisplay }) },
|
|
|
|
|
"turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok },
|
|
|
|
|
"trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok },
|
|
|
|
|
},
|
|
|
|
@@ -563,7 +574,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
|
|
|
|
|
warnings: mergeWarnings(
|
|
|
|
|
Array.isArray(input.warnings) ? input.warnings : [],
|
|
|
|
|
recoveredWaitFailure ? ["quick verify wait command timed out, but collected turn-summary/trace-frame artifacts show a durable completed business turn; treating the wait timeout as a non-blocking tool finding."] : [],
|
|
|
|
|
targetValidationElapsedWarnings(input.elapsedMs ?? null, "quick verify confirm-wait", numberAt(state.cicd, "targetValidation.maxSeconds")),
|
|
|
|
|
targetValidationElapsedWarnings(input.elapsedMs ?? null, "quick verify confirm-wait", targetValidationSeconds),
|
|
|
|
|
),
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
@@ -716,6 +727,9 @@ function recordQuickVerify(state: SentinelCicdState, payload: Record<string, unk
|
|
|
|
|
businessStatus: payload.businessStatus ?? null,
|
|
|
|
|
elapsedMs: payload.elapsedMs,
|
|
|
|
|
failure: payload.failure,
|
|
|
|
|
failureTitleZh: payload.failureTitleZh ?? payload.errorTitleZh ?? null,
|
|
|
|
|
errorTitleZh: payload.errorTitleZh ?? payload.failureTitleZh ?? null,
|
|
|
|
|
timeoutDisplay: payload.timeoutDisplay ?? null,
|
|
|
|
|
warnings: Array.isArray(payload.warnings) ? payload.warnings : [],
|
|
|
|
|
analysis: compactQuickVerifyRecordAnalysis(payload.analysis),
|
|
|
|
|
promptSource: payload.promptSource,
|
|
|
|
@@ -1822,7 +1836,7 @@ function observerCommandFailureBlocks(item: Record<string, unknown>): boolean {
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record<string, unknown> | null, traceFrame: Record<string, unknown> | null): Record<string, unknown>[] {
|
|
|
|
|
function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record<string, unknown> | null, traceFrame: Record<string, unknown> | null, display: Record<string, unknown> | null = null): Record<string, unknown>[] {
|
|
|
|
|
if (quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame)) return [];
|
|
|
|
|
const turnDiagnostics = quickVerifyTurnSummaryDiagnostics(promptIndex, turnSummary);
|
|
|
|
|
const traceDiagnostics = quickVerifyTraceFrameDiagnostics(traceFrame);
|
|
|
|
@@ -1834,13 +1848,17 @@ function quickVerifyControlFindings(failure: string | null, promptIndex: number,
|
|
|
|
|
if (noPromptScenario && failure === null) return [];
|
|
|
|
|
if (noPromptScenario && failure !== null) {
|
|
|
|
|
const observerStartFailure = failure === "observe-start-failed";
|
|
|
|
|
const displayTitleZh = stringAtNullable(display ?? {}, "titleZh");
|
|
|
|
|
return [{
|
|
|
|
|
id: observerStartFailure ? "quick-verify-observer-start-failed" : "quick-verify-command-sequence-failed",
|
|
|
|
|
severity: "red",
|
|
|
|
|
count: 1,
|
|
|
|
|
summary: observerStartFailure
|
|
|
|
|
summary: displayTitleZh ?? (observerStartFailure
|
|
|
|
|
? "quick verify observer failed to start before the no-prompt scenario could run."
|
|
|
|
|
: "quick verify no-prompt command sequence failed before the account/session workflow completed.",
|
|
|
|
|
: "quick verify no-prompt command sequence failed before the account/session workflow completed."),
|
|
|
|
|
displayTitleZh,
|
|
|
|
|
errorTitleZh: displayTitleZh,
|
|
|
|
|
timeoutDisplay: display,
|
|
|
|
|
failure,
|
|
|
|
|
promptIndex,
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
@@ -1849,11 +1867,15 @@ function quickVerifyControlFindings(failure: string | null, promptIndex: number,
|
|
|
|
|
const noTrace = /无\s*sendPrompt|no\s+sendPrompt|无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered);
|
|
|
|
|
const emptyFinal = /Final Response[\s\S]*\(空内容\)/iu.test(rendered);
|
|
|
|
|
if (!noTrace && !emptyFinal && failure !== "observe-start-failed") return [];
|
|
|
|
|
const displayTitleZh = stringAtNullable(display ?? {}, "titleZh");
|
|
|
|
|
return [{
|
|
|
|
|
id: "quick-verify-no-business-turn",
|
|
|
|
|
severity: "red",
|
|
|
|
|
count: 1,
|
|
|
|
|
summary: "quick verify did not reach a durable business turn/session/trace rows/final response; public dashboard health cannot be treated as HWLAB recovery.",
|
|
|
|
|
summary: displayTitleZh ?? "quick verify did not reach a durable business turn/session/trace rows/final response; public dashboard health cannot be treated as HWLAB recovery.",
|
|
|
|
|
displayTitleZh,
|
|
|
|
|
errorTitleZh: displayTitleZh,
|
|
|
|
|
timeoutDisplay: display,
|
|
|
|
|
rootCause: `quick verify could not confirm a durable completed turn: turn-summary scopedRows=${String(turnDiagnostics.scopedRowCount ?? 0)} rowCount=${String(turnDiagnostics.rowCount ?? 0)}, traceFrame traceIdPresent=${traceDiagnostics.traceIdPresent === true} finalResponseEmpty=${traceDiagnostics.finalResponseEmpty === true}.`,
|
|
|
|
|
rootCauseStatus: "confirmed",
|
|
|
|
|
rootCauseConfidence: "high",
|
|
|
|
@@ -2030,6 +2052,8 @@ function quickVerifyBusinessStatus(
|
|
|
|
|
observerTimeout,
|
|
|
|
|
scenarioComplete: durableBusinessTurn,
|
|
|
|
|
failure: failure ?? null,
|
|
|
|
|
errorTitleZh: observerTimeout ? stringAtNullable(timeoutDisplayForQuickVerifyFailure(failure, promptIndex, budgetSeconds, null), "titleZh") : null,
|
|
|
|
|
timeoutDisplay: observerTimeout ? timeoutDisplayForQuickVerifyFailure(failure, promptIndex, budgetSeconds, null) : null,
|
|
|
|
|
promptIndex,
|
|
|
|
|
elapsedMs: elapsed,
|
|
|
|
|
budgetSeconds,
|
|
|
|
@@ -2044,6 +2068,42 @@ function isRecoverableQuickVerifyWaitFailure(failure: string): boolean {
|
|
|
|
|
|| failure === "observe-turn-terminal-wait-failed";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function timeoutDisplayForFailure(failure: string | null, phase: string, targetValidationSeconds: number | null, runnerTimeoutSeconds: number | null): Record<string, unknown> {
|
|
|
|
|
const targetSeconds = typeof targetValidationSeconds === "number" && Number.isFinite(targetValidationSeconds) ? Math.max(1, Math.trunc(targetValidationSeconds)) : null;
|
|
|
|
|
const runnerSeconds = typeof runnerTimeoutSeconds === "number" && Number.isFinite(runnerTimeoutSeconds) ? Math.max(1, Math.trunc(runnerTimeoutSeconds)) : null;
|
|
|
|
|
const effectiveSeconds = targetSeconds ?? runnerSeconds;
|
|
|
|
|
const timeout = isTimeoutFailure(failure);
|
|
|
|
|
const phaseTitle = phase === "observe-wait-turn-terminal" ? "Workbench 等待终态" : phase === "observe-wait-startup-ready" ? "Workbench 启动等待" : "quick-verify";
|
|
|
|
|
const titleZh = timeout && effectiveSeconds !== null ? `${phaseTitle} 超时（${effectiveSeconds} 秒）` : failure === null ? "" : `${phaseTitle} 失败`;
|
|
|
|
|
return {
|
|
|
|
|
titleZh,
|
|
|
|
|
budgetKind: targetSeconds !== null ? "targetValidation" : runnerSeconds !== null ? "runnerTimeoutSeconds" : "unknown",
|
|
|
|
|
budgetSeconds: effectiveSeconds,
|
|
|
|
|
targetValidationSeconds: targetSeconds,
|
|
|
|
|
runnerTimeoutSeconds: runnerSeconds,
|
|
|
|
|
phase,
|
|
|
|
|
failure,
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function timeoutDisplayForQuickVerifyFailure(failure: string | null, promptIndex: number, targetValidationSeconds: number | null, runnerTimeoutSeconds: number | null): Record<string, unknown> {
|
|
|
|
|
if (!isTimeoutFailure(failure) || promptIndex <= 0) return timeoutDisplayForFailure(failure, "quick-verify", targetValidationSeconds, runnerTimeoutSeconds);
|
|
|
|
|
const base = timeoutDisplayForFailure(failure, "observe-wait-turn-terminal", targetValidationSeconds, runnerTimeoutSeconds);
|
|
|
|
|
const budgetSeconds = typeof base.budgetSeconds === "number" && Number.isFinite(base.budgetSeconds) ? Math.trunc(base.budgetSeconds) : null;
|
|
|
|
|
return {
|
|
|
|
|
...base,
|
|
|
|
|
titleZh: budgetSeconds === null ? `Workbench 第 ${promptIndex} 轮等待终态超时` : `Workbench 第 ${promptIndex} 轮等待终态超时（${budgetSeconds} 秒）`,
|
|
|
|
|
round: promptIndex,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function isTimeoutFailure(failure: string | null): boolean {
|
|
|
|
|
return failure === "quick-verify-timeout-over-budget"
|
|
|
|
|
|| failure === "quick-verify-wait-chunk-timeout"
|
|
|
|
|
|| failure === "observe-turn-terminal-wait-failed";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function compactCommandWithTail(result: CommandResult): CompactCommandResult & { stdoutTail: string; stderrTail: string } {
|
|
|
|
|
return {
|
|
|
|
|
...compactCommand(result),
|
|
|
|
@@ -2062,16 +2122,18 @@ function renderQuickVerifySummary(input: Record<string, unknown>): string {
|
|
|
|
|
? artifact.findings.map(record).slice(0, 8)
|
|
|
|
|
: [];
|
|
|
|
|
const findingCount = numberAtNullable(input, "findingCount") ?? numberAtNullable(artifact, "findingCount") ?? findings.length;
|
|
|
|
|
const failureTitle = stringAtNullable(input, "failureTitleZh") ?? stringAtNullable(input, "errorTitleZh") ?? stringAtNullable(record(input.timeoutDisplay), "titleZh");
|
|
|
|
|
return [
|
|
|
|
|
"Web Probe Sentinel Quick Verify",
|
|
|
|
|
"=======================================================",
|
|
|
|
|
`run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`,
|
|
|
|
|
failureTitle === null ? "" : `errorTitle=${failureTitle} failure=${input.failure ?? "-"}`,
|
|
|
|
|
`report=${artifact.reportJsonSha256 ?? "-"} artifacts=${artifact.artifactCount ?? "-"} findings=${findingCount}`,
|
|
|
|
|
`publicOrigin=${input.publicOrigin ?? "-"}`,
|
|
|
|
|
"",
|
|
|
|
|
"Findings",
|
|
|
|
|
findings.length === 0 ? "-" : findings.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${item.summary ?? item.message ?? ""}`).join("\n"),
|
|
|
|
|
].join("\n");
|
|
|
|
|
findings.length === 0 ? "-" : findings.map(formatQuickVerifyFindingLine).join("\n"),
|
|
|
|
|
].filter((line) => line !== "").join("\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function renderAuthSessionSwitchQuickVerifySummary(input: Record<string, unknown>): string {
|
|
|
|
@@ -2099,6 +2161,7 @@ function renderAuthSessionSwitchQuickVerifySummary(input: Record<string, unknown
|
|
|
|
|
"Auth Session Switch Quick Verify",
|
|
|
|
|
"=======================================================",
|
|
|
|
|
`run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`,
|
|
|
|
|
stringAtNullable(input, "failureTitleZh") === null && stringAtNullable(input, "errorTitleZh") === null ? "" : `errorTitle=${stringAtNullable(input, "failureTitleZh") ?? stringAtNullable(input, "errorTitleZh") ?? "-"}`,
|
|
|
|
|
`status=${artifact.ok === true ? "ok" : "blocked"} report=${artifact.reportJsonSha256 ?? "-"} publicOrigin=${input.publicOrigin ?? "-"}`,
|
|
|
|
|
`accountEnv=${accountEnv.envCount ?? "-"} valuesRedacted=true`,
|
|
|
|
|
"",
|
|
|
|
@@ -2106,8 +2169,14 @@ function renderAuthSessionSwitchQuickVerifySummary(input: Record<string, unknown
|
|
|
|
|
commandSteps.length === 0 ? "-" : commandSteps.join("\n"),
|
|
|
|
|
"",
|
|
|
|
|
"Findings",
|
|
|
|
|
findingRows.length === 0 ? "-" : findingRows.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${item.summary ?? item.message ?? ""}`).join("\n"),
|
|
|
|
|
].join("\n");
|
|
|
|
|
findingRows.length === 0 ? "-" : findingRows.map(formatQuickVerifyFindingLine).join("\n"),
|
|
|
|
|
].filter((line) => line !== "").join("\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function formatQuickVerifyFindingLine(item: Record<string, unknown>): string {
|
|
|
|
|
const title = stringAtNullable(item, "displayTitleZh") ?? stringAtNullable(item, "errorTitleZh") ?? stringAtNullable(record(item.timeoutDisplay), "titleZh");
|
|
|
|
|
const summary = title ?? item.summary ?? item.message ?? "";
|
|
|
|
|
return `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${summary}`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function formatQuickVerifyTimingSuffix(item: Record<string, unknown>): string {
|
|
|
|
|
Codex