diff --git a/scripts/src/hwlab-node-web-observe-runner-runtime-source.ts b/scripts/src/hwlab-node-web-observe-runner-runtime-source.ts index d6671143..c066f3f9 100644 --- a/scripts/src/hwlab-node-web-observe-runner-runtime-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-runtime-source.ts @@ -57,7 +57,17 @@ async function writeManifest(extra = {}) { navigation: { maxAttempts: navigationMaxAttempts, valuesRedacted: true }, pageAuthority: { browser: "chromium", context: "shared-auth", pageMode: "dual-control-observer", controlPageId: pageId, observerPageId, continuityBreaksRecorded: true }, pageProvenance: compactPageProvenance(currentPageProvenance), - sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerRefreshIntervalMs, observerInitiatedDefault: false, responseBodyReadDefault: false }, + sampling: { + mode: "passive", + sampleIntervalMs, + screenshotIntervalMs, + maxSamples, + maxRunMs, + maxRunSeconds: maxRunMs > 0 ? Math.ceil(maxRunMs / 1000) : 0, + observerRefreshIntervalMs, + observerInitiatedDefault: false, + responseBodyReadDefault: false, + }, alertThresholds, browserFreezePolicy, projectManagement, @@ -84,6 +94,12 @@ async function writeHeartbeat(extra = {}) { currentUrl: currentPageUrl(), observerUrl: pageUrl(observerPage), observerRefreshIntervalMs, + observerLifecycle: { + maxSamples, + maxRunMs, + maxRunSeconds: maxRunMs > 0 ? Math.ceil(maxRunMs / 1000) : 0, + valuesRedacted: true, + }, lastObserverRefreshAt: Number.isFinite(lastObserverRefreshAtMs) ? new Date(lastObserverRefreshAtMs).toISOString() : null, pageProvenance: compactPageProvenance(currentPageProvenance), sampleSeq, diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts index f06c27fa..dd574a2f 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts @@ -6,7 +6,16 @@ import { join } from "node:path"; import { test } from "bun:test"; import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; -import { classifyQuickVerifyCleanupStep, quickVerifyCleanupFindings, readAnalysisSummaryFromWorkspace, sentinelServiceProxyInvocation } from "./hwlab-node-web-sentinel-p5-observe"; +import { nodeWebObserveRunnerSource } from "./hwlab-node-web-observe-runner-source"; +import { + classifyQuickVerifyCleanupStep, + createQuickVerifyObserverLifecycle, + quickVerifyCleanupFindings, + quickVerifyUnhandledExceptionSummary, + readAnalysisSummaryFromWorkspace, + sentinelServiceProxyInvocation, + stopQuickVerifyObserverAfterPartialStart, +} from "./hwlab-node-web-sentinel-p5-observe"; import { createWebProbeSentinelService } from "./hwlab-node-web-sentinel-service"; test("sentinel service proxy executes its script through trans standard input", () => { @@ -116,7 +125,6 @@ test("latest report selects the newest stored run without a report SHA", () => { rmSync(stateRoot, { recursive: true, force: true }); } }); - function forceStopStep(observer: Record): Record { return { phase: "observe-stop-after-terminal", @@ -224,3 +232,75 @@ test("quick verify cleanup emits evidence-gap code when stopped state cannot be assert.equal(findings[0]?.id, "observer-cleanup-evidence-missing"); assert.match(String(findings[0]?.evidenceSummary), /missing=aliveAfter/u); }); + +test("quick verify observer lifecycle stops once on success, failure, and exception fallback", () => { + const cases = [ + { name: "success", initialPhase: "observe-stop-after-terminal", expectedPhase: "observe-stop-after-terminal" }, + { name: "failure", initialPhase: "observe-stop-after-failure", expectedPhase: "observe-stop-after-failure" }, + { name: "exception-fallback", initialPhase: null, expectedPhase: "observe-stop-after-finally" }, + ] as const; + + for (const scenario of cases) { + const calls: { observerId: string; phase: string }[] = []; + const lifecycle = createQuickVerifyObserverLifecycle("webobs-fixture", (observerId, phase) => { + calls.push({ observerId, phase }); + return { phase, ok: true, observerId }; + }); + if (scenario.initialPhase !== null) lifecycle.stop(scenario.initialPhase); + const cleanupStep = lifecycle.stop("observe-stop-after-finally"); + + assert.deepEqual(calls, [{ observerId: "webobs-fixture", phase: scenario.expectedPhase }], scenario.name); + assert.equal(cleanupStep.phase, scenario.expectedPhase, scenario.name); + assert.strictEqual(cleanupStep, lifecycle.cleanupStep, scenario.name); + } +}); + +test("quick verify observer lifecycle records a thrown stop without retrying it", () => { + let calls = 0; + const lifecycle = createQuickVerifyObserverLifecycle("webobs-stop-throws", () => { + calls += 1; + throw new Error("sensitive-stop-error"); + }); + + const first = lifecycle.stop("observe-stop-after-failure"); + const fallback = lifecycle.stop("observe-stop-after-finally"); + + assert.equal(calls, 1); + assert.strictEqual(first, fallback); + assert.equal(first.failure, "observer-stop-unhandled-exception"); + assert.equal(JSON.stringify(first).includes("sensitive-stop-error"), false); +}); + +test("quick verify cleans a partial start only when start output contains an observer id", () => { + const calls: { observerId: string; phase: string }[] = []; + const stopObserver = (observerId: string, phase: string) => { + calls.push({ observerId, phase }); + return { phase, ok: true, observerId }; + }; + + assert.equal(stopQuickVerifyObserverAfterPartialStart(null, stopObserver), null); + const cleanupStep = stopQuickVerifyObserverAfterPartialStart("webobs-partial", stopObserver); + + assert.deepEqual(calls, [{ observerId: "webobs-partial", phase: "observe-stop-after-start-failure" }]); + assert.equal(cleanupStep?.ok, true); +}); + +test("quick verify unexpected exception evidence is bounded and redacted", () => { + const secret = "sensitive-upstream-message"; + const summary = quickVerifyUnhandledExceptionSummary(new Error(secret)); + const serialized = JSON.stringify(summary); + + assert.equal(summary.errorName, "Error"); + assert.match(String(summary.errorMessageSha256), /^[a-f0-9]{64}$/u); + assert.equal(summary.errorMessageTruncated, false); + assert.equal(serialized.includes(secret), false); +}); + +test("web observe runner records finite lifecycle policy and closes the browser", () => { + const source = nodeWebObserveRunnerSource(); + + assert.match(source, /sampling:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u); + assert.match(source, /observerLifecycle:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u); + assert.match(source, /if \(maxRunMs > 0 && Date\.now\(\) - startedAtMs >= maxRunMs\)/u); + assert.match(source, /finally \{[\s\S]*?if \(browser\) await browser\.close\(\)\.catch/u); +}); diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts index 5ffdfdc5..a434966f 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts @@ -38,6 +38,66 @@ import { buildMonitorTerminalIngest, MonitorIngestFailedError, resolveMonitorTer const QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS = 55; +export interface QuickVerifyObserverLifecycle { + readonly observerId: string; + cleanupStep: Record | null; + stop(phase: string): Record; +} + +type QuickVerifyObserverStop = (observerId: string, phase: string) => Record; + +export function createQuickVerifyObserverLifecycle(observerId: string, stopObserver: QuickVerifyObserverStop): QuickVerifyObserverLifecycle { + const lifecycle: QuickVerifyObserverLifecycle = { + observerId, + cleanupStep: null, + stop(phase) { + if (lifecycle.cleanupStep !== null) return lifecycle.cleanupStep; + lifecycle.cleanupStep = { + phase, + ok: false, + failure: "observer-stop-did-not-return", + observerId, + valuesRedacted: true, + }; + try { + lifecycle.cleanupStep = stopObserver(observerId, phase); + } catch (error) { + lifecycle.cleanupStep = { + phase, + ok: false, + failure: "observer-stop-unhandled-exception", + observerId, + exception: quickVerifyUnhandledExceptionSummary(error), + valuesRedacted: true, + }; + } + return lifecycle.cleanupStep; + }, + }; + return lifecycle; +} + +export function stopQuickVerifyObserverAfterPartialStart(observerId: string | null, stopObserver: QuickVerifyObserverStop): Record | null { + if (observerId === null) return null; + return createQuickVerifyObserverLifecycle(observerId, stopObserver).stop("observe-stop-after-start-failure"); +} + +export function quickVerifyUnhandledExceptionSummary(error: unknown): Record { + let errorName = typeof error; + let errorMessage = typeof error === "string" ? error : ""; + if (error instanceof Error) { + errorName = error.name || "Error"; + errorMessage = error.message; + } + const boundedMessage = errorMessage.slice(0, 4096); + return { + errorName: short(errorName), + errorMessageSha256: boundedMessage.length === 0 ? null : createHash("sha256").update(boundedMessage).digest("hex"), + errorMessageTruncated: errorMessage.length > boundedMessage.length, + valuesRedacted: true, + }; +} + function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record = {}): void { const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => { if (typeof value === "string") return [key, short(value)]; @@ -163,8 +223,16 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? "")); printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() }); if (!started.ok || observerId === null) { - const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null) - .map((finding) => enrichObserveStartFailureFinding(finding, record(started.result))); + const cleanupStep = stopQuickVerifyObserverAfterPartialStart( + observerId, + (partialObserverId, phase) => stopQuickVerifyObserver(state, partialObserverId, phase), + ); + const cleanup = cleanupStep === null ? null : classifyQuickVerifyCleanupStep(cleanupStep); + const findings = mergeFindingRecords( + quickVerifyControlFindings("observe-start-failed", 0, null, null) + .map((finding) => enrichObserveStartFailureFinding(finding, record(started.result))), + cleanupStep === null ? [] : quickVerifyCleanupFindings(cleanupStep), + ); return recordQuickVerify(state, { ok: false, runId, @@ -174,14 +242,21 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, observerId, elapsedMs: elapsedMs(), businessStatus: quickVerifyBusinessStatus("observe-start-failed", 0, null, null, elapsedMs(), maxSeconds), - steps, + steps: cleanupStep === null ? steps : [...steps, cleanupStep], + cleanup, failure: "observe-start-failed", findingCount: findings.length, findings, - warnings: elapsedWarnings(), + warnings: mergeWarnings(elapsedWarnings(), quickVerifyCleanupWarnings(cleanup)), valuesRedacted: true, }); } + const observerLifecycle = createQuickVerifyObserverLifecycle( + observerId, + (activeObserverId, phase) => stopQuickVerifyObserver(state, activeObserverId, phase), + ); + let promptIndex = 0; + try { printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) }); const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds); steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady }); @@ -192,6 +267,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, scenarioId, reason, observerId, + observerLifecycle, promptIndex: 0, steps, failure: text(startupReady.failure ?? "observe-startup-ready-wait-failed"), @@ -200,7 +276,6 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, promptSource: prompts.summary, })); } - let promptIndex = 0; const sessionInvarianceChecks = sessionInvarianceChecksByRound(scenario); const nonBlockingCanaryWarnings: string[] = []; for (const item of commandSequence) { @@ -215,6 +290,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, scenarioId, reason, observerId, + observerLifecycle, promptIndex, steps, failure: "quick-verify-timeout-over-budget", @@ -266,6 +342,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, scenarioId, reason, observerId, + observerLifecycle, promptIndex, steps, failure: text(record(invariantResult).failure ?? "observe-session-invariance-failed"), @@ -282,6 +359,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, scenarioId, reason, observerId, + observerLifecycle, promptIndex, steps, failure: `observe-command-${type}-failed`, @@ -301,6 +379,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, scenarioId, reason, observerId, + observerLifecycle, promptIndex, steps, failure: text(waitResult.failure ?? "observe-turn-terminal-wait-failed"), @@ -318,6 +397,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, scenarioId, reason, observerId, + observerLifecycle, promptIndex, steps, failure: text(invariantResult.failure ?? "observe-session-invariance-check-failed"), @@ -342,7 +422,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, const artifactSummaryRecord = record(artifactSummary); const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : []; const visibilityFindings = quickVerifyAnalysisVisibilityFindings(analysis, artifactSummary); - const cleanupStep = stopQuickVerifyObserver(state, observerId, "observe-stop-after-terminal"); + const cleanupStep = observerLifecycle.stop("observe-stop-after-terminal"); const cleanup = classifyQuickVerifyCleanupStep(cleanupStep); const cleanupFindings = quickVerifyCleanupFindings(cleanupStep); const findings = mergeFindingRecords( @@ -393,6 +473,32 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, warnings: mergeWarnings(analysisWarnings, nonBlockingCanaryWarnings, quickVerifyCleanupWarnings(cleanup), elapsedWarnings()), valuesRedacted: true, }); + } catch (error) { + const exception = quickVerifyUnhandledExceptionSummary(error); + steps.push({ phase: "quick-verify-unhandled-exception", ok: false, result: exception }); + printQuickVerifyProgress(state, runId, "quick-verify-unhandled-exception", "failed", { + observerId, + promptIndex, + errorName: exception.errorName, + errorMessageSha256: exception.errorMessageSha256, + elapsedMs: elapsedMs(), + }); + return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { + runId, + scenarioId, + reason, + observerId, + observerLifecycle, + promptIndex, + steps, + failure: "quick-verify-unhandled-exception", + elapsedMs: elapsedMs(), + warnings: mergeWarnings("quick verify caught an unexpected post-start exception; details are redacted and the observer cleanup fallback remained active.", elapsedWarnings()), + promptSource: prompts.summary, + })); + } finally { + observerLifecycle.stop("observe-stop-after-finally"); + } } function sessionInvarianceChecksByRound(scenario: Record): Map[]> { @@ -544,6 +650,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: { readonly scenarioId: string; readonly reason: string; readonly observerId: string; + readonly observerLifecycle: QuickVerifyObserverLifecycle; readonly promptIndex: number; readonly steps: readonly Record[]; readonly failure: string; @@ -568,7 +675,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: { ], 60); cleanupSteps.push({ phase: "observe-cancel-after-failure", ok: cancel.ok, result: cancel.result }); } - cleanupSteps.push(stopQuickVerifyObserver(state, input.observerId, "observe-stop-after-failure")); + cleanupSteps.push(input.observerLifecycle.stop("observe-stop-after-failure")); const analysis = runChildCli([ "web-probe", "observe", "analyze", input.observerId, "--node", state.spec.nodeId, diff --git a/scripts/src/hwlab-node/web-probe-observe-options.test.ts b/scripts/src/hwlab-node/web-probe-observe-options.test.ts index d281c524..f5f2bbea 100644 --- a/scripts/src/hwlab-node/web-probe-observe-options.test.ts +++ b/scripts/src/hwlab-node/web-probe-observe-options.test.ts @@ -171,6 +171,22 @@ test("observe start selects the YAML public origin semantically", () => { assert.equal(options.browserProxyModeSource, "yaml-origin"); }); +test("observe start inherits finite observer lifecycle limits from the owning YAML", () => { + const options = parseNodeWebProbeObserveOptions( + ["start"], + "NC01", + "v03", + spec, + null, + null, + ); + + assert.equal(options.maxRunSeconds, 3600); + assert.equal(options.maxSamples, 720); + assert.ok(options.maxRunSeconds > 0); + assert.ok(options.maxSamples > 0); +}); + test("observe start refuses URL-based internal/public selection ambiguity", () => { assert.throws( () => parseNodeWebProbeObserveOptions(