diff --git a/scripts/src/hwlab-node-web-probe-memory-guard.test.ts b/scripts/src/hwlab-node-web-probe-memory-guard.test.ts index 8a98cbc3..095061c6 100644 --- a/scripts/src/hwlab-node-web-probe-memory-guard.test.ts +++ b/scripts/src/hwlab-node-web-probe-memory-guard.test.ts @@ -5,7 +5,7 @@ import { test } from "bun:test"; import type { CommandResult } from "./command"; import { rootPath } from "./config"; import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; -import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeMemoryStartGuardConfigRef } from "./hwlab-node-web-probe-memory-guard"; +import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeMemoryStartGuardConfigRef, webProbeScreenshotFullPage } from "./hwlab-node-web-probe-memory-guard"; import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source"; const spec = hwlabRuntimeLaneSpecForNode("v03", "NC01"); @@ -226,11 +226,16 @@ test("generated runner bounds network evidence and defaults screenshots to viewp const parserSource = exportedFunctionSource(readFileSync(rootPath("scripts/src/hwlab-node/web-probe.ts"), "utf8"), "parseNodeWebProbeOptions"); assert.match(parserSource, /fullPage: args\.includes\("--full-page"\) && !args\.includes\("--no-full-page"\)/u); const screenshotSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8"); + assert.match(screenshotSource, /webProbeScreenshotFullPage\(spec, options\.fullPage\)/u); assert.match(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE === "1"/u); assert.doesNotMatch(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE !== "0"/u); const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5.ts"), "utf8"); + assert.match(sentinelSource, /webProbeScreenshotFullPage\(state\.spec, options\.fullPage\)/u); assert.match(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE === "1"/u); assert.doesNotMatch(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0"/u); + assert.equal(webProbeScreenshotFullPage(spec, false), false); + assert.equal(webProbeScreenshotFullPage({ ...spec, webProbe: { ...spec.webProbe!, resourcePolicy: { ...spec.webProbe!.resourcePolicy!, browserEvidence: { ...spec.webProbe!.resourcePolicy!.browserEvidence!, screenshotMode: "full-page" } } } }, false), true); + assert.equal(webProbeScreenshotFullPage(spec, true), true); }); test("final launch guard rechecks MemAvailable under the configured lock", () => { @@ -240,9 +245,17 @@ test("final launch guard rechecks MemAvailable under the configured lock", () => assert.match(sync, /MemAvailable:/u); assert.match(sync, /mem_available_bytes.*-le '4294967296'/u); assert.match(sync, /setsid sh -c/u); - assert.match(sync, /kill -TERM -- "-\$browser_job_pid"/u); - assert.match(sync, /kill -KILL -- "-\$browser_job_pid"/u); - assert.match(async, /browserCreated === true/u); - assert.match(async, /data\.workerPid, data\.jobPid, data\.pid/u); - assert.match(async, /\/proc\/.*\/cmdline/u); + assert.match(sync, /exec 9>&-/u); + assert.equal(sync.includes("playwright"), false); + assert.equal(sync.includes("fs.readlinkSync('/proc/' + name + '/exe')"), true); + assert.equal(sync.includes("trap 'browser_exit=$?; trap - EXIT TERM INT HUP; cleanup_browser_group"), true); + assert.equal(sync.includes('kill -TERM -- "-$browser_job_pid"'), true); + assert.equal(sync.includes('kill -0 -- "-$browser_job_pid"'), true); + assert.equal(sync.includes('kill -KILL -- "-$browser_job_pid"'), true); + assert.equal(async.includes("browserCreated === true"), false); + assert.equal(async.includes("data.workerPid, data.jobPid, data.pid"), true); + assert.equal(async.includes("[data.browserPid, data.workerPid"), false); + assert.equal(async.includes("reportedBrowserPid === null || row.pid === reportedBrowserPid"), true); + assert.equal(async.includes("then flock -u 9; exit 43; fi"), true); + assert.equal(async.includes("fs.readlinkSync('/proc/' + name + '/exe')"), true); }); diff --git a/scripts/src/hwlab-node-web-probe-memory-guard.ts b/scripts/src/hwlab-node-web-probe-memory-guard.ts index dc658055..2db3ae3a 100644 --- a/scripts/src/hwlab-node-web-probe-memory-guard.ts +++ b/scripts/src/hwlab-node-web-probe-memory-guard.ts @@ -40,6 +40,12 @@ export function webProbeBrowserEvidenceEnvAssignments(spec: HwlabRuntimeLaneSpec ]; } +export function webProbeScreenshotFullPage(spec: HwlabRuntimeLaneSpec, explicitFullPage: boolean): boolean { + const policy = spec.webProbe?.resourcePolicy?.browserEvidence; + if (policy === undefined) throw new Error("web-probe browser evidence policy is missing from the owning YAML"); + return explicitFullPage || policy.screenshotMode !== "viewport"; +} + export function webProbeGuardedLaunchShell( spec: HwlabRuntimeLaneSpec, mode: WebProbeMemoryGuardMode, @@ -48,6 +54,11 @@ export function webProbeGuardedLaunchShell( ): string { const policy = memoryPolicy(spec); if (policy === null) throw new Error("web-probe memory start policy is missing from the owning YAML"); + const closeStepTimeoutMs = spec.webProbe?.resourcePolicy?.observerLifecycle.closeStepTimeoutMs; + if (closeStepTimeoutMs === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML"); + const cleanupPollMilliseconds = policy.launchReadyPollMilliseconds; + const cleanupPollSeconds = Math.max(0.05, cleanupPollMilliseconds / 1000); + const cleanupPollAttempts = Math.max(1, Math.ceil(closeStepTimeoutMs / cleanupPollMilliseconds)); const comparator = mode === "manual-start" ? "-le" : "-lt"; const common = [ `exec 9>${shellQuote(policy.launchLockPath)}`, @@ -62,10 +73,10 @@ export function webProbeGuardedLaunchShell( "launch_output=$(mktemp)", "launch_error=$(mktemp)", "trap 'rm -f \"$launch_output\" \"$launch_error\"' EXIT", - `if ! ${command} >"$launch_output" 2>"$launch_error"; then cat "$launch_output"; cat "$launch_error" >&2; exit 43; fi`, + `if ! (exec 9>&-; ${command}) >"$launch_output" 2>"$launch_error"; then cat "$launch_output"; cat "$launch_error" >&2; exit 43; fi`, "cat \"$launch_output\"", "cat \"$launch_error\" >&2", - `node - "$launch_output" ${shellQuote(String(policy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(policy.launchReadyPollMilliseconds))} <<'UNIDESK_WEB_PROBE_STRUCTURED_READINESS'`, + `if ! node - "$launch_output" ${shellQuote(String(policy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(policy.launchReadyPollMilliseconds))} <<'UNIDESK_WEB_PROBE_STRUCTURED_READINESS'`, "const fs = require('fs');", "const text = fs.readFileSync(process.argv[2], 'utf8').trim();", "const timeoutMs = Number(process.argv[3]);", @@ -73,9 +84,9 @@ export function webProbeGuardedLaunchShell( "let value = null; try { value = JSON.parse(text); } catch {}", "const data = value && typeof value === 'object' && value.data && typeof value.data === 'object' ? value.data : value;", "if (!data || typeof data !== 'object' || !data.jobId) process.exit(43);", - "if (data.browserCreated === true && Number.isInteger(data.browserPid) && data.browserPid > 0) process.exit(0);", - "const rootPid = [data.browserPid, data.workerPid, data.jobPid, data.pid].find((item) => Number.isInteger(item) && item > 0);", + "const rootPid = [data.workerPid, data.jobPid, data.pid].find((item) => Number.isInteger(item) && item > 0);", "if (!rootPid) process.exit(43);", + "const reportedBrowserPid = Number.isInteger(data.browserPid) && data.browserPid > 0 ? data.browserPid : null;", "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", "const browserDescendant = () => {", " const rows = [];", @@ -86,14 +97,14 @@ export function webProbeGuardedLaunchShell( " const close = stat.lastIndexOf(')');", " const fields = stat.slice(close + 2).split(' ');", " const ppid = Number(fields[1]);", - " const command = fs.readFileSync('/proc/' + name + '/cmdline', 'utf8').replace(/\\0/g, ' ');", - " rows.push({ pid: Number(name), ppid, command });", + " const executable = fs.readlinkSync('/proc/' + name + '/exe');", + " rows.push({ pid: Number(name), ppid, executable });", " } catch {}", " }", " const descendants = new Set([rootPid]);", " let changed = true;", " while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }", - " return rows.some((row) => descendants.has(row.pid) && /(chromium|chrome|playwright)/i.test(row.command));", + " return rows.some((row) => descendants.has(row.pid) && (reportedBrowserPid === null || row.pid === reportedBrowserPid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable));", "};", "(async () => {", " const deadline = Date.now() + timeoutMs;", @@ -101,12 +112,24 @@ export function webProbeGuardedLaunchShell( " process.exit(43);", "})().catch(() => process.exit(43));", "UNIDESK_WEB_PROBE_STRUCTURED_READINESS", + "then flock -u 9; exit 43; fi", "flock -u 9", ].join("\n"); return [ ...common, - `setsid sh -c ${shellQuote(command)} &`, + `setsid sh -c ${shellQuote(`exec 9>&-\n${command}`)} &`, "browser_job_pid=$!", + "cleanup_browser_group() {", + " kill -TERM -- \"-$browser_job_pid\" 2>/dev/null || true", + " cleanup_attempt=0", + ` while kill -0 -- "-$browser_job_pid" 2>/dev/null && [ "$cleanup_attempt" -lt ${cleanupPollAttempts} ]; do cleanup_attempt=$((cleanup_attempt + 1)); sleep ${cleanupPollSeconds}; done`, + " if kill -0 -- \"-$browser_job_pid\" 2>/dev/null; then kill -KILL -- \"-$browser_job_pid\" 2>/dev/null || true; fi", + " wait \"$browser_job_pid\" 2>/dev/null || true", + "}", + "trap 'browser_exit=$?; trap - EXIT TERM INT HUP; cleanup_browser_group; exit \"$browser_exit\"' EXIT", + "trap 'exit 143' TERM", + "trap 'exit 130' INT", + "trap 'exit 129' HUP", "browser_ready=false", `browser_ready_deadline=$(( $(date +%s) + ${policy.launchReadyTimeoutSeconds} ))`, "while kill -0 \"$browser_job_pid\" 2>/dev/null; do", @@ -120,13 +143,13 @@ export function webProbeGuardedLaunchShell( " const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');", " const close = stat.lastIndexOf(')');", " const fields = stat.slice(close + 2).split(' ');", - " rows.push({ pid: Number(name), ppid: Number(fields[1]), command: fs.readFileSync('/proc/' + name + '/cmdline', 'utf8').replace(/\\0/g, ' ') });", + " rows.push({ pid: Number(name), ppid: Number(fields[1]), executable: fs.readlinkSync('/proc/' + name + '/exe') });", " } catch {}", "}", "const descendants = new Set([rootPid]);", "let changed = true;", "while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }", - "process.exit(rows.some((row) => descendants.has(row.pid) && /(chromium|chrome|playwright)/i.test(row.command)) ? 0 : 1);", + "process.exit(rows.some((row) => descendants.has(row.pid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable)) ? 0 : 1);", "UNIDESK_WEB_PROBE_CHILD_READINESS", " then browser_ready=true; break; fi", " if [ \"$(date +%s)\" -ge \"$browser_ready_deadline\" ]; then break; fi", @@ -134,14 +157,14 @@ export function webProbeGuardedLaunchShell( "done", "flock -u 9", "if [ \"$browser_ready\" != true ]; then", - " kill -TERM -- \"-$browser_job_pid\" 2>/dev/null || true", - " sleep 1", - " if kill -0 \"$browser_job_pid\" 2>/dev/null; then kill -KILL -- \"-$browser_job_pid\" 2>/dev/null || true; fi", - " wait \"$browser_job_pid\" 2>/dev/null || true", " printf '%s\\n' 'web-probe browser launch readiness was not confirmed' >&2", " exit 43", "fi", + "set +e", "wait \"$browser_job_pid\"", + "browser_exit=$?", + "set -e", + "exit \"$browser_exit\"", ].join("\n"); } diff --git a/scripts/src/hwlab-node-web-probe-runner-source.ts b/scripts/src/hwlab-node-web-probe-runner-source.ts index 16a10475..13d11855 100644 --- a/scripts/src/hwlab-node-web-probe-runner-source.ts +++ b/scripts/src/hwlab-node-web-probe-runner-source.ts @@ -111,6 +111,7 @@ try { }); process.exitCode = 2; } finally { + if (context) await context.close().catch(() => {}); if (browser) await browser.close().catch(() => {}); } diff --git a/scripts/src/hwlab-node-web-sentinel-p5.ts b/scripts/src/hwlab-node-web-sentinel-p5.ts index e0651305..c22e97ef 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5.ts @@ -9,7 +9,7 @@ import { startJob } from "./jobs"; import type { RenderedCliResult } from "./output"; import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact"; import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref"; -import { runWebProbeMemoryGuard, webProbeGuardedLaunchShell } from "./hwlab-node-web-probe-memory-guard"; +import { runWebProbeMemoryGuard, webProbeGuardedLaunchShell, webProbeScreenshotFullPage } from "./hwlab-node-web-probe-memory-guard"; import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd"; import { clipTail, @@ -1160,7 +1160,7 @@ export function probeSentinelDashboardBrowser(state: SentinelCicdState, options: `export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`, `export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`, `export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`, - `export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`, + `export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(webProbeScreenshotFullPage(state.spec, options.fullPage) ? "1" : "0")}`, `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID=${shellQuote(state.sentinelId)}`, `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX=${shellQuote(stringAtNullable(state.publicExposure, "routePrefix") ?? "/")}`, `export UNIDESK_SENTINEL_DASHBOARD_RUN_ID=${shellQuote(options.runId ?? "")}`, @@ -1279,12 +1279,15 @@ if (!url) throw new Error("missing dashboard URL"); const consoleMessages = []; const pageErrors = []; const requestFailures = []; -const browser = await chromium.launch({ +let browser = null; +let context = null; +try { +browser = await chromium.launch({ headless: true, args: ["--disable-gpu", "--no-sandbox"], ...(executablePath ? { executablePath } : {}), }); -const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 }); +context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 }); const page = await context.newPage(); page.on("console", (message) => { if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) }); @@ -1681,8 +1684,10 @@ console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({ valuesRedacted: true, })); -await context.close().catch(() => {}); -await browser.close().catch(() => {}); +} finally { + if (context) await context.close().catch(() => {}); + if (browser) await browser.close().catch(() => {}); +} `; } diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index f271fdb6..66d6198e 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -26,7 +26,7 @@ import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapp import { runWebProbeSentinelCommand, type WebProbeSentinelDashboardAction, type WebProbeSentinelOptions, type WebProbeSentinelReportView, type WebProbeSentinelSourceAuthority, type WebProbeSentinelSourceOverrideOptions } from "../hwlab-node-web-sentinel-cicd"; import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help"; import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary"; -import { runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell } from "../hwlab-node-web-probe-memory-guard"; +import { runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeScreenshotFullPage } from "../hwlab-node-web-probe-memory-guard"; import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql"; import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport"; import { runWebProbeRemoteArtifactJob } from "../web-probe-remote-artifact"; @@ -938,7 +938,7 @@ export function runNodeWebProbeScreenshot(options: NodeWebProbeScreenshotOptions const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start"); if (memoryGuard.status !== "allowed") return memoryGuard; const route = `${options.node}:${spec.workspace}`; - const script = webProbeGuardedLaunchShell(spec, "manual-start", webProbeScreenshotRemoteScript(options)); + const script = webProbeGuardedLaunchShell(spec, "manual-start", webProbeScreenshotRemoteScript(options, spec)); const job = runWebProbeRemoteArtifactJob({ route, localDir: options.localDir, @@ -1063,7 +1063,7 @@ function compactWebProbeScreenshotCleanup(value: unknown): Record/dev/null 2>&1; then", " export UNIDESK_WEB_PROBE_SCREENSHOT_EXECUTABLE_PATH=$(command -v chromium)", @@ -1113,8 +1113,11 @@ const launchOptions = { args: ["--disable-gpu", "--no-sandbox"], ...(executablePath ? { executablePath } : {}), }; -const browser = await chromium.launch(launchOptions); -const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 }); +let browser = null; +let context = null; +try { +browser = await chromium.launch(launchOptions); +context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 }); const page = await context.newPage(); page.on("console", (message) => { if (consoleMessages.length < 20) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 240) }); @@ -1124,7 +1127,6 @@ page.on("requestfailed", (request) => { }); let status = null; -try { const response = await page.goto(url, { timeout, waitUntil }); status = response?.status() ?? null; await page.waitForTimeout(350); @@ -1193,8 +1195,8 @@ try { valuesRedacted: true, })); } finally { - await context.close().catch(() => {}); - await browser.close().catch(() => {}); + if (context) await context.close().catch(() => {}); + if (browser) await browser.close().catch(() => {}); } `; }