fix: 保证 Web observer 有界收尾

复用 stop-once 生命周期覆盖成功、失败、部分启动与异常兜底路径。\n\n将 YAML 派生的 maxRun/maxSamples 投影到 manifest 和 heartbeat，并补齐聚焦回归测试。
2026-07-12 12:59:30 +02:00
parent 3056a2b343
commit 95bf0b7de6
4 changed files with 229 additions and 10 deletions
@@ -57,7 +57,17 @@ async function writeManifest(extra = {}) {
    navigation: { maxAttempts: navigationMaxAttempts, valuesRedacted: true },
    pageAuthority: { browser: "chromium", context: "shared-auth", pageMode: "dual-control-observer", controlPageId: pageId, observerPageId, continuityBreaksRecorded: true },
    pageProvenance: compactPageProvenance(currentPageProvenance),
-    sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerRefreshIntervalMs, observerInitiatedDefault: false, responseBodyReadDefault: false },
+    sampling: {
+      mode: "passive",
+      sampleIntervalMs,
+      screenshotIntervalMs,
+      maxSamples,
+      maxRunMs,
+      maxRunSeconds: maxRunMs > 0 ? Math.ceil(maxRunMs / 1000) : 0,
+      observerRefreshIntervalMs,
+      observerInitiatedDefault: false,
+      responseBodyReadDefault: false,
+    },
    alertThresholds,
    browserFreezePolicy,
    projectManagement,
@@ -84,6 +94,12 @@ async function writeHeartbeat(extra = {}) {
    currentUrl: currentPageUrl(),
    observerUrl: pageUrl(observerPage),
    observerRefreshIntervalMs,
+    observerLifecycle: {
+      maxSamples,
+      maxRunMs,
+      maxRunSeconds: maxRunMs > 0 ? Math.ceil(maxRunMs / 1000) : 0,
+      valuesRedacted: true,
+    },
    lastObserverRefreshAt: Number.isFinite(lastObserverRefreshAtMs) ? new Date(lastObserverRefreshAtMs).toISOString() : null,
    pageProvenance: compactPageProvenance(currentPageProvenance),
    sampleSeq,
@@ -6,7 +6,16 @@ import { join } from "node:path";
 import { test } from "bun:test";

 import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
-import { classifyQuickVerifyCleanupStep, quickVerifyCleanupFindings, readAnalysisSummaryFromWorkspace, sentinelServiceProxyInvocation } from "./hwlab-node-web-sentinel-p5-observe";
+import { nodeWebObserveRunnerSource } from "./hwlab-node-web-observe-runner-source";
+import {
+  classifyQuickVerifyCleanupStep,
+  createQuickVerifyObserverLifecycle,
+  quickVerifyCleanupFindings,
+  quickVerifyUnhandledExceptionSummary,
+  readAnalysisSummaryFromWorkspace,
+  sentinelServiceProxyInvocation,
+  stopQuickVerifyObserverAfterPartialStart,
+} from "./hwlab-node-web-sentinel-p5-observe";
 import { createWebProbeSentinelService } from "./hwlab-node-web-sentinel-service";

 test("sentinel service proxy executes its script through trans standard input", () => {
@@ -116,7 +125,6 @@ test("latest report selects the newest stored run without a report SHA", () => {
    rmSync(stateRoot, { recursive: true, force: true });
  }
 });
-
 function forceStopStep(observer: Record<string, unknown>): Record<string, unknown> {
  return {
    phase: "observe-stop-after-terminal",
@@ -224,3 +232,75 @@ test("quick verify cleanup emits evidence-gap code when stopped state cannot be
  assert.equal(findings[0]?.id, "observer-cleanup-evidence-missing");
  assert.match(String(findings[0]?.evidenceSummary), /missing=aliveAfter/u);
 });
+
+test("quick verify observer lifecycle stops once on success, failure, and exception fallback", () => {
+  const cases = [
+    { name: "success", initialPhase: "observe-stop-after-terminal", expectedPhase: "observe-stop-after-terminal" },
+    { name: "failure", initialPhase: "observe-stop-after-failure", expectedPhase: "observe-stop-after-failure" },
+    { name: "exception-fallback", initialPhase: null, expectedPhase: "observe-stop-after-finally" },
+  ] as const;
+
+  for (const scenario of cases) {
+    const calls: { observerId: string; phase: string }[] = [];
+    const lifecycle = createQuickVerifyObserverLifecycle("webobs-fixture", (observerId, phase) => {
+      calls.push({ observerId, phase });
+      return { phase, ok: true, observerId };
+    });
+    if (scenario.initialPhase !== null) lifecycle.stop(scenario.initialPhase);
+    const cleanupStep = lifecycle.stop("observe-stop-after-finally");
+
+    assert.deepEqual(calls, [{ observerId: "webobs-fixture", phase: scenario.expectedPhase }], scenario.name);
+    assert.equal(cleanupStep.phase, scenario.expectedPhase, scenario.name);
+    assert.strictEqual(cleanupStep, lifecycle.cleanupStep, scenario.name);
+  }
+});
+
+test("quick verify observer lifecycle records a thrown stop without retrying it", () => {
+  let calls = 0;
+  const lifecycle = createQuickVerifyObserverLifecycle("webobs-stop-throws", () => {
+    calls += 1;
+    throw new Error("sensitive-stop-error");
+  });
+
+  const first = lifecycle.stop("observe-stop-after-failure");
+  const fallback = lifecycle.stop("observe-stop-after-finally");
+
+  assert.equal(calls, 1);
+  assert.strictEqual(first, fallback);
+  assert.equal(first.failure, "observer-stop-unhandled-exception");
+  assert.equal(JSON.stringify(first).includes("sensitive-stop-error"), false);
+});
+
+test("quick verify cleans a partial start only when start output contains an observer id", () => {
+  const calls: { observerId: string; phase: string }[] = [];
+  const stopObserver = (observerId: string, phase: string) => {
+    calls.push({ observerId, phase });
+    return { phase, ok: true, observerId };
+  };
+
+  assert.equal(stopQuickVerifyObserverAfterPartialStart(null, stopObserver), null);
+  const cleanupStep = stopQuickVerifyObserverAfterPartialStart("webobs-partial", stopObserver);
+
+  assert.deepEqual(calls, [{ observerId: "webobs-partial", phase: "observe-stop-after-start-failure" }]);
+  assert.equal(cleanupStep?.ok, true);
+});
+
+test("quick verify unexpected exception evidence is bounded and redacted", () => {
+  const secret = "sensitive-upstream-message";
+  const summary = quickVerifyUnhandledExceptionSummary(new Error(secret));
+  const serialized = JSON.stringify(summary);
+
+  assert.equal(summary.errorName, "Error");
+  assert.match(String(summary.errorMessageSha256), /^[a-f0-9]{64}$/u);
+  assert.equal(summary.errorMessageTruncated, false);
+  assert.equal(serialized.includes(secret), false);
+});
+
+test("web observe runner records finite lifecycle policy and closes the browser", () => {
+  const source = nodeWebObserveRunnerSource();
+
+  assert.match(source, /sampling:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u);
+  assert.match(source, /observerLifecycle:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u);
+  assert.match(source, /if \(maxRunMs > 0 && Date\.now\(\) - startedAtMs >= maxRunMs\)/u);
+  assert.match(source, /finally \{[\s\S]*?if \(browser\) await browser\.close\(\)\.catch/u);
+});
@@ -38,6 +38,66 @@ import { buildMonitorTerminalIngest, MonitorIngestFailedError, resolveMonitorTer

 const QUICK_VERIFY_ANALYSIS_SUMMARY_TIMEOUT_SECONDS = 55;

+export interface QuickVerifyObserverLifecycle {
+  readonly observerId: string;
+  cleanupStep: Record<string, unknown> | null;
+  stop(phase: string): Record<string, unknown>;
+}
+
+type QuickVerifyObserverStop = (observerId: string, phase: string) => Record<string, unknown>;
+
+export function createQuickVerifyObserverLifecycle(observerId: string, stopObserver: QuickVerifyObserverStop): QuickVerifyObserverLifecycle {
+  const lifecycle: QuickVerifyObserverLifecycle = {
+    observerId,
+    cleanupStep: null,
+    stop(phase) {
+      if (lifecycle.cleanupStep !== null) return lifecycle.cleanupStep;
+      lifecycle.cleanupStep = {
+        phase,
+        ok: false,
+        failure: "observer-stop-did-not-return",
+        observerId,
+        valuesRedacted: true,
+      };
+      try {
+        lifecycle.cleanupStep = stopObserver(observerId, phase);
+      } catch (error) {
+        lifecycle.cleanupStep = {
+          phase,
+          ok: false,
+          failure: "observer-stop-unhandled-exception",
+          observerId,
+          exception: quickVerifyUnhandledExceptionSummary(error),
+          valuesRedacted: true,
+        };
+      }
+      return lifecycle.cleanupStep;
+    },
+  };
+  return lifecycle;
+}
+
+export function stopQuickVerifyObserverAfterPartialStart(observerId: string | null, stopObserver: QuickVerifyObserverStop): Record<string, unknown> | null {
+  if (observerId === null) return null;
+  return createQuickVerifyObserverLifecycle(observerId, stopObserver).stop("observe-stop-after-start-failure");
+}
+
+export function quickVerifyUnhandledExceptionSummary(error: unknown): Record<string, unknown> {
+  let errorName = typeof error;
+  let errorMessage = typeof error === "string" ? error : "";
+  if (error instanceof Error) {
+    errorName = error.name || "Error";
+    errorMessage = error.message;
+  }
+  const boundedMessage = errorMessage.slice(0, 4096);
+  return {
+    errorName: short(errorName),
+    errorMessageSha256: boundedMessage.length === 0 ? null : createHash("sha256").update(boundedMessage).digest("hex"),
+    errorMessageTruncated: errorMessage.length > boundedMessage.length,
+    valuesRedacted: true,
+  };
+}
+
 function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record<string, unknown> = {}): void {
  const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => {
    if (typeof value === "string") return [key, short(value)];
@@ -163,8 +223,16 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
  const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? ""));
  printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() });
  if (!started.ok || observerId === null) {
-    const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null)
-      .map((finding) => enrichObserveStartFailureFinding(finding, record(started.result)));
+    const cleanupStep = stopQuickVerifyObserverAfterPartialStart(
+      observerId,
+      (partialObserverId, phase) => stopQuickVerifyObserver(state, partialObserverId, phase),
+    );
+    const cleanup = cleanupStep === null ? null : classifyQuickVerifyCleanupStep(cleanupStep);
+    const findings = mergeFindingRecords(
+      quickVerifyControlFindings("observe-start-failed", 0, null, null)
+        .map((finding) => enrichObserveStartFailureFinding(finding, record(started.result))),
+      cleanupStep === null ? [] : quickVerifyCleanupFindings(cleanupStep),
+    );
    return recordQuickVerify(state, {
      ok: false,
      runId,
@@ -174,14 +242,21 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
      observerId,
      elapsedMs: elapsedMs(),
      businessStatus: quickVerifyBusinessStatus("observe-start-failed", 0, null, null, elapsedMs(), maxSeconds),
-      steps,
+      steps: cleanupStep === null ? steps : [...steps, cleanupStep],
+      cleanup,
      failure: "observe-start-failed",
      findingCount: findings.length,
      findings,
-      warnings: elapsedWarnings(),
+      warnings: mergeWarnings(elapsedWarnings(), quickVerifyCleanupWarnings(cleanup)),
      valuesRedacted: true,
    });
  }
+  const observerLifecycle = createQuickVerifyObserverLifecycle(
+    observerId,
+    (activeObserverId, phase) => stopQuickVerifyObserver(state, activeObserverId, phase),
+  );
+  let promptIndex = 0;
+  try {
  printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) });
  const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds);
  steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady });
@@ -192,6 +267,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
      scenarioId,
      reason,
      observerId,
+      observerLifecycle,
      promptIndex: 0,
      steps,
      failure: text(startupReady.failure ?? "observe-startup-ready-wait-failed"),
@@ -200,7 +276,6 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
      promptSource: prompts.summary,
    }));
  }
-  let promptIndex = 0;
  const sessionInvarianceChecks = sessionInvarianceChecksByRound(scenario);
  const nonBlockingCanaryWarnings: string[] = [];
  for (const item of commandSequence) {
@@ -215,6 +290,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
          scenarioId,
          reason,
          observerId,
+          observerLifecycle,
          promptIndex,
          steps,
          failure: "quick-verify-timeout-over-budget",
@@ -266,6 +342,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
                scenarioId,
                reason,
                observerId,
+                observerLifecycle,
                promptIndex,
                steps,
                failure: text(record(invariantResult).failure ?? "observe-session-invariance-failed"),
@@ -282,6 +359,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
          scenarioId,
          reason,
          observerId,
+          observerLifecycle,
          promptIndex,
          steps,
          failure: `observe-command-${type}-failed`,
@@ -301,6 +379,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
            scenarioId,
            reason,
            observerId,
+            observerLifecycle,
            promptIndex,
            steps,
            failure: text(waitResult.failure ?? "observe-turn-terminal-wait-failed"),
@@ -318,6 +397,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
            scenarioId,
            reason,
            observerId,
+            observerLifecycle,
            promptIndex,
            steps,
            failure: text(invariantResult.failure ?? "observe-session-invariance-check-failed"),
@@ -342,7 +422,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
  const artifactSummaryRecord = record(artifactSummary);
  const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
  const visibilityFindings = quickVerifyAnalysisVisibilityFindings(analysis, artifactSummary);
-  const cleanupStep = stopQuickVerifyObserver(state, observerId, "observe-stop-after-terminal");
+  const cleanupStep = observerLifecycle.stop("observe-stop-after-terminal");
  const cleanup = classifyQuickVerifyCleanupStep(cleanupStep);
  const cleanupFindings = quickVerifyCleanupFindings(cleanupStep);
  const findings = mergeFindingRecords(
@@ -393,6 +473,32 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
    warnings: mergeWarnings(analysisWarnings, nonBlockingCanaryWarnings, quickVerifyCleanupWarnings(cleanup), elapsedWarnings()),
    valuesRedacted: true,
  });
+  } catch (error) {
+    const exception = quickVerifyUnhandledExceptionSummary(error);
+    steps.push({ phase: "quick-verify-unhandled-exception", ok: false, result: exception });
+    printQuickVerifyProgress(state, runId, "quick-verify-unhandled-exception", "failed", {
+      observerId,
+      promptIndex,
+      errorName: exception.errorName,
+      errorMessageSha256: exception.errorMessageSha256,
+      elapsedMs: elapsedMs(),
+    });
+    return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
+      runId,
+      scenarioId,
+      reason,
+      observerId,
+      observerLifecycle,
+      promptIndex,
+      steps,
+      failure: "quick-verify-unhandled-exception",
+      elapsedMs: elapsedMs(),
+      warnings: mergeWarnings("quick verify caught an unexpected post-start exception; details are redacted and the observer cleanup fallback remained active.", elapsedWarnings()),
+      promptSource: prompts.summary,
+    }));
+  } finally {
+    observerLifecycle.stop("observe-stop-after-finally");
+  }
 }

 function sessionInvarianceChecksByRound(scenario: Record<string, unknown>): Map<number, Record<string, unknown>[]> {
@@ -544,6 +650,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
  readonly scenarioId: string;
  readonly reason: string;
  readonly observerId: string;
+  readonly observerLifecycle: QuickVerifyObserverLifecycle;
  readonly promptIndex: number;
  readonly steps: readonly Record<string, unknown>[];
  readonly failure: string;
@@ -568,7 +675,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
    ], 60);
    cleanupSteps.push({ phase: "observe-cancel-after-failure", ok: cancel.ok, result: cancel.result });
  }
-  cleanupSteps.push(stopQuickVerifyObserver(state, input.observerId, "observe-stop-after-failure"));
+  cleanupSteps.push(input.observerLifecycle.stop("observe-stop-after-failure"));
  const analysis = runChildCli([
    "web-probe", "observe", "analyze", input.observerId,
    "--node", state.spec.nodeId,
@@ -171,6 +171,22 @@ test("observe start selects the YAML public origin semantically", () => {
  assert.equal(options.browserProxyModeSource, "yaml-origin");
 });

+test("observe start inherits finite observer lifecycle limits from the owning YAML", () => {
+  const options = parseNodeWebProbeObserveOptions(
+    ["start"],
+    "NC01",
+    "v03",
+    spec,
+    null,
+    null,
+  );
+
+  assert.equal(options.maxRunSeconds, 3600);
+  assert.equal(options.maxSamples, 720);
+  assert.ok(options.maxRunSeconds > 0);
+  assert.ok(options.maxSamples > 0);
+});
+
 test("observe start refuses URL-based internal/public selection ambiguity", () => {
  assert.throws(
    () => parseNodeWebProbeObserveOptions(