fix: show sentinel quick verify progress
This commit is contained in:
@@ -2156,6 +2156,26 @@ function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record<string, unknown> = {}): void {
|
||||||
|
const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => {
|
||||||
|
if (typeof value === "string") return [key, short(value)];
|
||||||
|
if (Array.isArray(value)) return [key, value.slice(0, 8)];
|
||||||
|
return [key, value];
|
||||||
|
}));
|
||||||
|
process.stdout.write(`${JSON.stringify({
|
||||||
|
event: "sentinel.quick-verify.progress",
|
||||||
|
at: new Date().toISOString(),
|
||||||
|
node: state.spec.nodeId,
|
||||||
|
lane: state.spec.lane,
|
||||||
|
sentinelId: state.sentinelId,
|
||||||
|
runId,
|
||||||
|
...compactExtra,
|
||||||
|
phase,
|
||||||
|
status,
|
||||||
|
valuesRedacted: true,
|
||||||
|
})}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record<string, unknown> {
|
function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record<string, unknown> {
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
const elapsedMs = () => Date.now() - startedAt;
|
const elapsedMs = () => Date.now() - startedAt;
|
||||||
@@ -2207,6 +2227,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
|
|||||||
const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", warningBudgetSeconds);
|
const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", warningBudgetSeconds);
|
||||||
const deadline = Date.now() + hardBudgetSeconds * 1000;
|
const deadline = Date.now() + hardBudgetSeconds * 1000;
|
||||||
const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;
|
const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;
|
||||||
|
printQuickVerifyProgress(state, runId, "start", "running", { scenarioId, reason, warningBudgetSeconds, hardBudgetSeconds, timeoutSeconds });
|
||||||
const steps: Record<string, unknown>[] = [];
|
const steps: Record<string, unknown>[] = [];
|
||||||
const startArgs = [
|
const startArgs = [
|
||||||
"web-probe", "observe", "start",
|
"web-probe", "observe", "start",
|
||||||
@@ -2219,9 +2240,11 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
|
|||||||
];
|
];
|
||||||
const viewport = stringAtNullable(scenario, "viewport");
|
const viewport = stringAtNullable(scenario, "viewport");
|
||||||
if (viewport !== null) startArgs.push("--viewport", viewport);
|
if (viewport !== null) startArgs.push("--viewport", viewport);
|
||||||
|
printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) });
|
||||||
const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env);
|
const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env);
|
||||||
steps.push({ phase: "observe-start", ok: started.ok, result: started.result });
|
steps.push({ phase: "observe-start", ok: started.ok, result: started.result });
|
||||||
const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? ""));
|
const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? ""));
|
||||||
|
printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() });
|
||||||
if (!started.ok || observerId === null) {
|
if (!started.ok || observerId === null) {
|
||||||
const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null);
|
const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null);
|
||||||
return recordQuickVerify(state, {
|
return recordQuickVerify(state, {
|
||||||
@@ -2240,8 +2263,10 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
|
|||||||
valuesRedacted: true,
|
valuesRedacted: true,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) });
|
||||||
const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds);
|
const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds);
|
||||||
steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady });
|
steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady });
|
||||||
|
printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", startupReady.ok === true ? "succeeded" : "failed", { observerId, failure: startupReady.failure ?? null, status: startupReady.status ?? null, heartbeatStatus: startupReady.heartbeatStatus ?? null, elapsedMs: elapsedMs() });
|
||||||
if (startupReady.ok !== true) {
|
if (startupReady.ok !== true) {
|
||||||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||||||
runId,
|
runId,
|
||||||
@@ -2264,6 +2289,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
|
|||||||
const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
|
const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
|
||||||
for (let index = 0; index < repeat; index += 1) {
|
for (let index = 0; index < repeat; index += 1) {
|
||||||
if (Date.now() >= deadline) {
|
if (Date.now() >= deadline) {
|
||||||
|
printQuickVerifyProgress(state, runId, "timeout", "failed", { observerId, promptIndex, elapsedMs: elapsedMs(), hardBudgetSeconds });
|
||||||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||||||
runId,
|
runId,
|
||||||
scenarioId,
|
scenarioId,
|
||||||
@@ -2294,8 +2320,10 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
|
|||||||
promptIndex += 1;
|
promptIndex += 1;
|
||||||
}
|
}
|
||||||
appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" });
|
appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" });
|
||||||
|
printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat });
|
||||||
const commandResult = runChildCli(args, remainingSeconds(deadline, 60));
|
const commandResult = runChildCli(args, remainingSeconds(deadline, 60));
|
||||||
steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result });
|
steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result });
|
||||||
|
printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() });
|
||||||
if (!commandResult.ok) {
|
if (!commandResult.ok) {
|
||||||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||||||
runId,
|
runId,
|
||||||
@@ -2311,8 +2339,10 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
if (type === "sendPrompt") {
|
if (type === "sendPrompt") {
|
||||||
|
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, 55) });
|
||||||
const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds);
|
const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds);
|
||||||
steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult });
|
steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult });
|
||||||
|
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() });
|
||||||
if (waitResult.ok !== true) {
|
if (waitResult.ok !== true) {
|
||||||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||||||
runId,
|
runId,
|
||||||
@@ -2329,6 +2359,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
|
|||||||
}
|
}
|
||||||
const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps);
|
const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps);
|
||||||
nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings));
|
nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings));
|
||||||
|
printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() });
|
||||||
if (invariantResult.ok !== true) {
|
if (invariantResult.ok !== true) {
|
||||||
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
|
||||||
runId,
|
runId,
|
||||||
@@ -2346,8 +2377,10 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
printQuickVerifyProgress(state, runId, "observe-analyze", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 120) });
|
||||||
const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120))], remainingSeconds(deadline, 120));
|
const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120))], remainingSeconds(deadline, 120));
|
||||||
steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result });
|
steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result });
|
||||||
|
printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() });
|
||||||
const indexEntry = readLocalObserveIndex(observerId);
|
const indexEntry = readLocalObserveIndex(observerId);
|
||||||
const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, remainingSeconds(deadline, 30));
|
const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, remainingSeconds(deadline, 30));
|
||||||
const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30));
|
const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30));
|
||||||
@@ -2359,6 +2392,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
|
|||||||
const blockingFindings = findings.filter(isQuickVerifyBlockingFinding);
|
const blockingFindings = findings.filter(isQuickVerifyBlockingFinding);
|
||||||
const analysisWarnings = analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."];
|
const analysisWarnings = analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."];
|
||||||
const ok = record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0;
|
const ok = record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0;
|
||||||
|
printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() });
|
||||||
return recordQuickVerify(state, {
|
return recordQuickVerify(state, {
|
||||||
ok,
|
ok,
|
||||||
runId,
|
runId,
|
||||||
|
|||||||
Reference in New Issue
Block a user