fix: 收紧 WebProbe 浏览器启动与清理

2026-07-13 10:26:48 +02:00
parent 11b1f3d647
commit cd2863e030
5 changed files with 79 additions and 35 deletions
@@ -5,7 +5,7 @@ import { test } from "bun:test";
 import type { CommandResult } from "./command";
 import { rootPath } from "./config";
 import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
-import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeMemoryStartGuardConfigRef } from "./hwlab-node-web-probe-memory-guard";
+import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeMemoryStartGuardConfigRef, webProbeScreenshotFullPage } from "./hwlab-node-web-probe-memory-guard";
 import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source";

 const spec = hwlabRuntimeLaneSpecForNode("v03", "NC01");
@@ -226,11 +226,16 @@ test("generated runner bounds network evidence and defaults screenshots to viewp
  const parserSource = exportedFunctionSource(readFileSync(rootPath("scripts/src/hwlab-node/web-probe.ts"), "utf8"), "parseNodeWebProbeOptions");
  assert.match(parserSource, /fullPage: args\.includes\("--full-page"\) && !args\.includes\("--no-full-page"\)/u);
  const screenshotSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
+  assert.match(screenshotSource, /webProbeScreenshotFullPage\(spec, options\.fullPage\)/u);
  assert.match(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE === "1"/u);
  assert.doesNotMatch(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE !== "0"/u);
  const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5.ts"), "utf8");
+  assert.match(sentinelSource, /webProbeScreenshotFullPage\(state\.spec, options\.fullPage\)/u);
  assert.match(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE === "1"/u);
  assert.doesNotMatch(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0"/u);
+  assert.equal(webProbeScreenshotFullPage(spec, false), false);
+  assert.equal(webProbeScreenshotFullPage({ ...spec, webProbe: { ...spec.webProbe!, resourcePolicy: { ...spec.webProbe!.resourcePolicy!, browserEvidence: { ...spec.webProbe!.resourcePolicy!.browserEvidence!, screenshotMode: "full-page" } } } }, false), true);
+  assert.equal(webProbeScreenshotFullPage(spec, true), true);
 });

 test("final launch guard rechecks MemAvailable under the configured lock", () => {
@@ -240,9 +245,17 @@ test("final launch guard rechecks MemAvailable under the configured lock", () =>
  assert.match(sync, /MemAvailable:/u);
  assert.match(sync, /mem_available_bytes.*-le '4294967296'/u);
  assert.match(sync, /setsid sh -c/u);
-  assert.match(sync, /kill -TERM -- "-\$browser_job_pid"/u);
-  assert.match(sync, /kill -KILL -- "-\$browser_job_pid"/u);
-  assert.match(async, /browserCreated === true/u);
-  assert.match(async, /data\.workerPid, data\.jobPid, data\.pid/u);
-  assert.match(async, /\/proc\/.*\/cmdline/u);
+  assert.match(sync, /exec 9>&-/u);
+  assert.equal(sync.includes("playwright"), false);
+  assert.equal(sync.includes("fs.readlinkSync('/proc/' + name + '/exe')"), true);
+  assert.equal(sync.includes("trap 'browser_exit=$?; trap - EXIT TERM INT HUP; cleanup_browser_group"), true);
+  assert.equal(sync.includes('kill -TERM -- "-$browser_job_pid"'), true);
+  assert.equal(sync.includes('kill -0 -- "-$browser_job_pid"'), true);
+  assert.equal(sync.includes('kill -KILL -- "-$browser_job_pid"'), true);
+  assert.equal(async.includes("browserCreated === true"), false);
+  assert.equal(async.includes("data.workerPid, data.jobPid, data.pid"), true);
+  assert.equal(async.includes("[data.browserPid, data.workerPid"), false);
+  assert.equal(async.includes("reportedBrowserPid === null || row.pid === reportedBrowserPid"), true);
+  assert.equal(async.includes("then flock -u 9; exit 43; fi"), true);
+  assert.equal(async.includes("fs.readlinkSync('/proc/' + name + '/exe')"), true);
 });
@@ -40,6 +40,12 @@ export function webProbeBrowserEvidenceEnvAssignments(spec: HwlabRuntimeLaneSpec
  ];
 }

+export function webProbeScreenshotFullPage(spec: HwlabRuntimeLaneSpec, explicitFullPage: boolean): boolean {
+  const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
+  if (policy === undefined) throw new Error("web-probe browser evidence policy is missing from the owning YAML");
+  return explicitFullPage || policy.screenshotMode !== "viewport";
+}
+
 export function webProbeGuardedLaunchShell(
  spec: HwlabRuntimeLaneSpec,
  mode: WebProbeMemoryGuardMode,
@@ -48,6 +54,11 @@ export function webProbeGuardedLaunchShell(
 ): string {
  const policy = memoryPolicy(spec);
  if (policy === null) throw new Error("web-probe memory start policy is missing from the owning YAML");
+  const closeStepTimeoutMs = spec.webProbe?.resourcePolicy?.observerLifecycle.closeStepTimeoutMs;
+  if (closeStepTimeoutMs === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML");
+  const cleanupPollMilliseconds = policy.launchReadyPollMilliseconds;
+  const cleanupPollSeconds = Math.max(0.05, cleanupPollMilliseconds / 1000);
+  const cleanupPollAttempts = Math.max(1, Math.ceil(closeStepTimeoutMs / cleanupPollMilliseconds));
  const comparator = mode === "manual-start" ? "-le" : "-lt";
  const common = [
    `exec 9>${shellQuote(policy.launchLockPath)}`,
@@ -62,10 +73,10 @@ export function webProbeGuardedLaunchShell(
    "launch_output=$(mktemp)",
    "launch_error=$(mktemp)",
    "trap 'rm -f \"$launch_output\" \"$launch_error\"' EXIT",
-    `if ! ${command} >"$launch_output" 2>"$launch_error"; then cat "$launch_output"; cat "$launch_error" >&2; exit 43; fi`,
+    `if ! (exec 9>&-; ${command}) >"$launch_output" 2>"$launch_error"; then cat "$launch_output"; cat "$launch_error" >&2; exit 43; fi`,
    "cat \"$launch_output\"",
    "cat \"$launch_error\" >&2",
-    `node - "$launch_output" ${shellQuote(String(policy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(policy.launchReadyPollMilliseconds))} <<'UNIDESK_WEB_PROBE_STRUCTURED_READINESS'`,
+    `if ! node - "$launch_output" ${shellQuote(String(policy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(policy.launchReadyPollMilliseconds))} <<'UNIDESK_WEB_PROBE_STRUCTURED_READINESS'`,
    "const fs = require('fs');",
    "const text = fs.readFileSync(process.argv[2], 'utf8').trim();",
    "const timeoutMs = Number(process.argv[3]);",
@@ -73,9 +84,9 @@ export function webProbeGuardedLaunchShell(
    "let value = null; try { value = JSON.parse(text); } catch {}",
    "const data = value && typeof value === 'object' && value.data && typeof value.data === 'object' ? value.data : value;",
    "if (!data || typeof data !== 'object' || !data.jobId) process.exit(43);",
-    "if (data.browserCreated === true && Number.isInteger(data.browserPid) && data.browserPid > 0) process.exit(0);",
-    "const rootPid = [data.browserPid, data.workerPid, data.jobPid, data.pid].find((item) => Number.isInteger(item) && item > 0);",
+    "const rootPid = [data.workerPid, data.jobPid, data.pid].find((item) => Number.isInteger(item) && item > 0);",
    "if (!rootPid) process.exit(43);",
+    "const reportedBrowserPid = Number.isInteger(data.browserPid) && data.browserPid > 0 ? data.browserPid : null;",
    "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
    "const browserDescendant = () => {",
    "  const rows = [];",
@@ -86,14 +97,14 @@ export function webProbeGuardedLaunchShell(
    "      const close = stat.lastIndexOf(')');",
    "      const fields = stat.slice(close + 2).split(' ');",
    "      const ppid = Number(fields[1]);",
-    "      const command = fs.readFileSync('/proc/' + name + '/cmdline', 'utf8').replace(/\\0/g, ' ');",
-    "      rows.push({ pid: Number(name), ppid, command });",
+    "      const executable = fs.readlinkSync('/proc/' + name + '/exe');",
+    "      rows.push({ pid: Number(name), ppid, executable });",
    "    } catch {}",
    "  }",
    "  const descendants = new Set([rootPid]);",
    "  let changed = true;",
    "  while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }",
-    "  return rows.some((row) => descendants.has(row.pid) && /(chromium|chrome|playwright)/i.test(row.command));",
+    "  return rows.some((row) => descendants.has(row.pid) && (reportedBrowserPid === null || row.pid === reportedBrowserPid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable));",
    "};",
    "(async () => {",
    "  const deadline = Date.now() + timeoutMs;",
@@ -101,12 +112,24 @@ export function webProbeGuardedLaunchShell(
    "  process.exit(43);",
    "})().catch(() => process.exit(43));",
    "UNIDESK_WEB_PROBE_STRUCTURED_READINESS",
+    "then flock -u 9; exit 43; fi",
    "flock -u 9",
  ].join("\n");
  return [
    ...common,
-    `setsid sh -c ${shellQuote(command)} &`,
+    `setsid sh -c ${shellQuote(`exec 9>&-\n${command}`)} &`,
    "browser_job_pid=$!",
+    "cleanup_browser_group() {",
+    "  kill -TERM -- \"-$browser_job_pid\" 2>/dev/null || true",
+    "  cleanup_attempt=0",
+    `  while kill -0 -- "-$browser_job_pid" 2>/dev/null && [ "$cleanup_attempt" -lt ${cleanupPollAttempts} ]; do cleanup_attempt=$((cleanup_attempt + 1)); sleep ${cleanupPollSeconds}; done`,
+    "  if kill -0 -- \"-$browser_job_pid\" 2>/dev/null; then kill -KILL -- \"-$browser_job_pid\" 2>/dev/null || true; fi",
+    "  wait \"$browser_job_pid\" 2>/dev/null || true",
+    "}",
+    "trap 'browser_exit=$?; trap - EXIT TERM INT HUP; cleanup_browser_group; exit \"$browser_exit\"' EXIT",
+    "trap 'exit 143' TERM",
+    "trap 'exit 130' INT",
+    "trap 'exit 129' HUP",
    "browser_ready=false",
    `browser_ready_deadline=$(( $(date +%s) + ${policy.launchReadyTimeoutSeconds} ))`,
    "while kill -0 \"$browser_job_pid\" 2>/dev/null; do",
@@ -120,13 +143,13 @@ export function webProbeGuardedLaunchShell(
    "    const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');",
    "    const close = stat.lastIndexOf(')');",
    "    const fields = stat.slice(close + 2).split(' ');",
-    "    rows.push({ pid: Number(name), ppid: Number(fields[1]), command: fs.readFileSync('/proc/' + name + '/cmdline', 'utf8').replace(/\\0/g, ' ') });",
+    "    rows.push({ pid: Number(name), ppid: Number(fields[1]), executable: fs.readlinkSync('/proc/' + name + '/exe') });",
    "  } catch {}",
    "}",
    "const descendants = new Set([rootPid]);",
    "let changed = true;",
    "while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }",
-    "process.exit(rows.some((row) => descendants.has(row.pid) && /(chromium|chrome|playwright)/i.test(row.command)) ? 0 : 1);",
+    "process.exit(rows.some((row) => descendants.has(row.pid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable)) ? 0 : 1);",
    "UNIDESK_WEB_PROBE_CHILD_READINESS",
    "  then browser_ready=true; break; fi",
    "  if [ \"$(date +%s)\" -ge \"$browser_ready_deadline\" ]; then break; fi",
@@ -134,14 +157,14 @@ export function webProbeGuardedLaunchShell(
    "done",
    "flock -u 9",
    "if [ \"$browser_ready\" != true ]; then",
-    "  kill -TERM -- \"-$browser_job_pid\" 2>/dev/null || true",
-    "  sleep 1",
-    "  if kill -0 \"$browser_job_pid\" 2>/dev/null; then kill -KILL -- \"-$browser_job_pid\" 2>/dev/null || true; fi",
-    "  wait \"$browser_job_pid\" 2>/dev/null || true",
    "  printf '%s\\n' 'web-probe browser launch readiness was not confirmed' >&2",
    "  exit 43",
    "fi",
+    "set +e",
    "wait \"$browser_job_pid\"",
+    "browser_exit=$?",
+    "set -e",
+    "exit \"$browser_exit\"",
  ].join("\n");
 }

@@ -111,6 +111,7 @@ try {
  });
  process.exitCode = 2;
 } finally {
+  if (context) await context.close().catch(() => {});
  if (browser) await browser.close().catch(() => {});
 }

@@ -9,7 +9,7 @@ import { startJob } from "./jobs";
 import type { RenderedCliResult } from "./output";
 import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact";
 import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
-import { runWebProbeMemoryGuard, webProbeGuardedLaunchShell } from "./hwlab-node-web-probe-memory-guard";
+import { runWebProbeMemoryGuard, webProbeGuardedLaunchShell, webProbeScreenshotFullPage } from "./hwlab-node-web-probe-memory-guard";
 import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd";
 import {
  clipTail,
@@ -1160,7 +1160,7 @@ export function probeSentinelDashboardBrowser(state: SentinelCicdState, options:
    `export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`,
    `export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`,
    `export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
-    `export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`,
+    `export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(webProbeScreenshotFullPage(state.spec, options.fullPage) ? "1" : "0")}`,
    `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID=${shellQuote(state.sentinelId)}`,
    `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX=${shellQuote(stringAtNullable(state.publicExposure, "routePrefix") ?? "/")}`,
    `export UNIDESK_SENTINEL_DASHBOARD_RUN_ID=${shellQuote(options.runId ?? "")}`,
@@ -1279,12 +1279,15 @@ if (!url) throw new Error("missing dashboard URL");
 const consoleMessages = [];
 const pageErrors = [];
 const requestFailures = [];
-const browser = await chromium.launch({
+let browser = null;
+let context = null;
+try {
+browser = await chromium.launch({
  headless: true,
  args: ["--disable-gpu", "--no-sandbox"],
  ...(executablePath ? { executablePath } : {}),
 });
-const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
+context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
 const page = await context.newPage();
 page.on("console", (message) => {
  if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) });
@@ -1681,8 +1684,10 @@ console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({
  valuesRedacted: true,
 }));

-await context.close().catch(() => {});
-await browser.close().catch(() => {});
+} finally {
+  if (context) await context.close().catch(() => {});
+  if (browser) await browser.close().catch(() => {});
+}
 `;
 }

@@ -26,7 +26,7 @@ import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapp
 import { runWebProbeSentinelCommand, type WebProbeSentinelDashboardAction, type WebProbeSentinelOptions, type WebProbeSentinelReportView, type WebProbeSentinelSourceAuthority, type WebProbeSentinelSourceOverrideOptions } from "../hwlab-node-web-sentinel-cicd";
 import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
 import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
-import { runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell } from "../hwlab-node-web-probe-memory-guard";
+import { runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeScreenshotFullPage } from "../hwlab-node-web-probe-memory-guard";
 import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
 import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
 import { runWebProbeRemoteArtifactJob } from "../web-probe-remote-artifact";
@@ -938,7 +938,7 @@ export function runNodeWebProbeScreenshot(options: NodeWebProbeScreenshotOptions
  const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
  if (memoryGuard.status !== "allowed") return memoryGuard;
  const route = `${options.node}:${spec.workspace}`;
-  const script = webProbeGuardedLaunchShell(spec, "manual-start", webProbeScreenshotRemoteScript(options));
+  const script = webProbeGuardedLaunchShell(spec, "manual-start", webProbeScreenshotRemoteScript(options, spec));
  const job = runWebProbeRemoteArtifactJob({
    route,
    localDir: options.localDir,
@@ -1063,7 +1063,7 @@ function compactWebProbeScreenshotCleanup(value: unknown): Record<string, unknow
  };
 }

-function webProbeScreenshotRemoteScript(options: NodeWebProbeScreenshotOptions): string {
+function webProbeScreenshotRemoteScript(options: NodeWebProbeScreenshotOptions, spec: HwlabRuntimeLaneSpec): string {
  const [widthRaw, heightRaw] = options.viewport.split("x");
  return [
    "set -eu",
@@ -1073,7 +1073,7 @@ function webProbeScreenshotRemoteScript(options: NodeWebProbeScreenshotOptions):
    `export UNIDESK_WEB_PROBE_SCREENSHOT_HEIGHT=${shellQuote(heightRaw ?? "900")}`,
    `export UNIDESK_WEB_PROBE_SCREENSHOT_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
    `export UNIDESK_WEB_PROBE_SCREENSHOT_WAIT_UNTIL=${shellQuote(options.waitUntil)}`,
-    `export UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`,
+    `export UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE=${shellQuote(webProbeScreenshotFullPage(spec, options.fullPage) ? "1" : "0")}`,
    `export UNIDESK_WEB_PROBE_SCREENSHOT_SELECTOR=${shellQuote(options.selector ?? "")}`,
    "if command -v chromium >/dev/null 2>&1; then",
    "  export UNIDESK_WEB_PROBE_SCREENSHOT_EXECUTABLE_PATH=$(command -v chromium)",
@@ -1113,8 +1113,11 @@ const launchOptions = {
  args: ["--disable-gpu", "--no-sandbox"],
  ...(executablePath ? { executablePath } : {}),
 };
-const browser = await chromium.launch(launchOptions);
-const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
+let browser = null;
+let context = null;
+try {
+browser = await chromium.launch(launchOptions);
+context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
 const page = await context.newPage();
 page.on("console", (message) => {
  if (consoleMessages.length < 20) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 240) });
@@ -1124,7 +1127,6 @@ page.on("requestfailed", (request) => {
 });

 let status = null;
-try {
  const response = await page.goto(url, { timeout, waitUntil });
  status = response?.status() ?? null;
  await page.waitForTimeout(350);
@@ -1193,8 +1195,8 @@ try {
    valuesRedacted: true,
  }));
 } finally {
-  await context.close().catch(() => {});
-  await browser.close().catch(() => {});
+  if (context) await context.close().catch(() => {});
+  if (browser) await browser.close().catch(() => {});
 }
 `;
 }