diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index a7df8861..7d036e9b 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -2156,6 +2156,26 @@ function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[ })); } +function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record = {}): void { + const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => { + if (typeof value === "string") return [key, short(value)]; + if (Array.isArray(value)) return [key, value.slice(0, 8)]; + return [key, value]; + })); + process.stdout.write(`${JSON.stringify({ + event: "sentinel.quick-verify.progress", + at: new Date().toISOString(), + node: state.spec.nodeId, + lane: state.spec.lane, + sentinelId: state.sentinelId, + runId, + ...compactExtra, + phase, + status, + valuesRedacted: true, + })}\n`); +} + function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record { const startedAt = Date.now(); const elapsedMs = () => Date.now() - startedAt; @@ -2207,6 +2227,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", warningBudgetSeconds); const deadline = Date.now() + hardBudgetSeconds * 1000; const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; + printQuickVerifyProgress(state, runId, "start", "running", { scenarioId, reason, warningBudgetSeconds, hardBudgetSeconds, timeoutSeconds }); const steps: Record[] = []; const startArgs = [ "web-probe", "observe", "start", @@ -2219,9 +2240,11 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou ]; const viewport = stringAtNullable(scenario, "viewport"); if (viewport !== null) startArgs.push("--viewport", viewport); + printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) }); const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env); steps.push({ phase: "observe-start", ok: started.ok, result: started.result }); const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? "")); + printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() }); if (!started.ok || observerId === null) { const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null); return recordQuickVerify(state, { @@ -2240,8 +2263,10 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou valuesRedacted: true, }); } + printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) }); const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds); steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady }); + printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", startupReady.ok === true ? "succeeded" : "failed", { observerId, failure: startupReady.failure ?? null, status: startupReady.status ?? null, heartbeatStatus: startupReady.heartbeatStatus ?? null, elapsedMs: elapsedMs() }); if (startupReady.ok !== true) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, @@ -2264,6 +2289,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1); for (let index = 0; index < repeat; index += 1) { if (Date.now() >= deadline) { + printQuickVerifyProgress(state, runId, "timeout", "failed", { observerId, promptIndex, elapsedMs: elapsedMs(), hardBudgetSeconds }); return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, @@ -2294,8 +2320,10 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou promptIndex += 1; } appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" }); + printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat }); const commandResult = runChildCli(args, remainingSeconds(deadline, 60)); steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result }); + printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() }); if (!commandResult.ok) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, @@ -2311,8 +2339,10 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou })); } if (type === "sendPrompt") { + printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, 55) }); const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds); steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult }); + printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() }); if (waitResult.ok !== true) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, @@ -2329,6 +2359,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou } const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps); nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings)); + printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() }); if (invariantResult.ok !== true) { return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, @@ -2346,8 +2377,10 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou } } } + printQuickVerifyProgress(state, runId, "observe-analyze", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 120) }); const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120))], remainingSeconds(deadline, 120)); steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result }); + printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() }); const indexEntry = readLocalObserveIndex(observerId); const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, remainingSeconds(deadline, 30)); const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30)); @@ -2359,6 +2392,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou const blockingFindings = findings.filter(isQuickVerifyBlockingFinding); const analysisWarnings = analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."]; const ok = record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0; + printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() }); return recordQuickVerify(state, { ok, runId,