diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 24a7e80f..de5ca62a 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -223,6 +223,25 @@ lanes: scrollJumpFromY: 250 scrollJumpToY: 40 sessionRailFallbackRatio: 0.5 + browserFreezePolicy: + enabled: true + blockerWindowMs: 30000 + memory: + totalRssBlockerMb: 500 + processRssBlockerMb: 500 + growthBlockerMb: 300 + responsiveness: + latencyBlockerMs: 5000 + eventBlockerCount: 2 + cdp: + metricsTimeoutBlockerCount: 2 + kill: + enabled: true + gracefulSignal: SIGTERM + forceSignal: SIGKILL + graceMs: 3000 + pollIntervalMs: 100 + exitCode: 7 projectManagement: enabled: true targetPaths: @@ -624,6 +643,25 @@ lanes: scrollJumpFromY: 250 scrollJumpToY: 40 sessionRailFallbackRatio: 0.5 + browserFreezePolicy: + enabled: true + blockerWindowMs: 30000 + memory: + totalRssBlockerMb: 500 + processRssBlockerMb: 500 + growthBlockerMb: 300 + responsiveness: + latencyBlockerMs: 5000 + eventBlockerCount: 2 + cdp: + metricsTimeoutBlockerCount: 2 + kill: + enabled: true + gracefulSignal: SIGTERM + forceSignal: SIGKILL + graceMs: 3000 + pollIntervalMs: 100 + exitCode: 7 projectManagement: enabled: true targetPaths: @@ -955,6 +993,25 @@ lanes: scrollJumpFromY: 250 scrollJumpToY: 40 sessionRailFallbackRatio: 0.5 + browserFreezePolicy: + enabled: true + blockerWindowMs: 30000 + memory: + totalRssBlockerMb: 500 + processRssBlockerMb: 500 + growthBlockerMb: 300 + responsiveness: + latencyBlockerMs: 5000 + eventBlockerCount: 2 + cdp: + metricsTimeoutBlockerCount: 2 + kill: + enabled: true + gracefulSignal: SIGTERM + forceSignal: SIGKILL + graceMs: 3000 + pollIntervalMs: 100 + exitCode: 7 tektonDir: tekton-v03 argoApplicationFile: application-v03.yaml registryPrefix: 127.0.0.1:5000/hwlab diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 38a5ceef..7971a7e0 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -141,6 +141,7 @@ export interface HwlabRuntimeWebProbeSpec { readonly defaultOrigin?: HwlabRuntimeWebProbeOriginSpec; readonly authLogin?: HwlabRuntimeWebProbeAuthLoginSpec; readonly alertThresholds?: HwlabRuntimeWebProbeAlertThresholdsSpec; + readonly browserFreezePolicy?: HwlabRuntimeWebProbeBrowserFreezePolicySpec; readonly projectManagement?: HwlabRuntimeWebProbeProjectManagementSpec; } @@ -151,6 +152,39 @@ export interface HwlabRuntimeWebProbeAuthLoginSpec { readonly maxDelayMs: number; } +export interface HwlabRuntimeWebProbeBrowserFreezeMemoryPolicySpec { + readonly totalRssBlockerMb: number; + readonly processRssBlockerMb: number; + readonly growthBlockerMb: number; +} + +export interface HwlabRuntimeWebProbeBrowserFreezeResponsivenessPolicySpec { + readonly latencyBlockerMs: number; + readonly eventBlockerCount: number; +} + +export interface HwlabRuntimeWebProbeBrowserFreezeCdpPolicySpec { + readonly metricsTimeoutBlockerCount: number; +} + +export interface HwlabRuntimeWebProbeBrowserFreezeKillPolicySpec { + readonly enabled: boolean; + readonly gracefulSignal: "SIGTERM"; + readonly forceSignal: "SIGKILL"; + readonly graceMs: number; + readonly pollIntervalMs: number; + readonly exitCode: number; +} + +export interface HwlabRuntimeWebProbeBrowserFreezePolicySpec { + readonly enabled: boolean; + readonly blockerWindowMs: number; + readonly memory: HwlabRuntimeWebProbeBrowserFreezeMemoryPolicySpec; + readonly responsiveness: HwlabRuntimeWebProbeBrowserFreezeResponsivenessPolicySpec; + readonly cdp: HwlabRuntimeWebProbeBrowserFreezeCdpPolicySpec; + readonly kill: HwlabRuntimeWebProbeBrowserFreezeKillPolicySpec; +} + export type HwlabRuntimeWebProbeSentinelConfigRefKey = "runtime" | "scenarios" | "promptSet" | "reportViews" | "publicExposure" | "cicd" | "secrets"; export const HWLAB_WEB_PROBE_SENTINEL_CONFIG_REF_KEYS = ["runtime", "scenarios", "promptSet", "reportViews", "publicExposure", "cicd", "secrets"] as const satisfies readonly HwlabRuntimeWebProbeSentinelConfigRefKey[]; @@ -1078,6 +1112,7 @@ function webProbeConfig(value: unknown, path: string): HwlabRuntimeWebProbeSpec ...(raw.defaultOrigin === undefined ? {} : { defaultOrigin: webProbeOriginConfig(raw.defaultOrigin, `${path}.defaultOrigin`) }), ...(raw.authLogin === undefined ? {} : { authLogin: webProbeAuthLoginConfig(raw.authLogin, `${path}.authLogin`) }), ...(raw.alertThresholds === undefined ? {} : { alertThresholds: webProbeAlertThresholdsConfig(raw.alertThresholds, `${path}.alertThresholds`) }), + ...(raw.browserFreezePolicy === undefined ? {} : { browserFreezePolicy: webProbeBrowserFreezePolicyConfig(raw.browserFreezePolicy, `${path}.browserFreezePolicy`) }), ...(raw.projectManagement === undefined ? {} : { projectManagement: webProbeProjectManagementConfig(raw.projectManagement, `${path}.projectManagement`) }), }; } @@ -1094,6 +1129,38 @@ function webProbeAuthLoginConfig(value: unknown, path: string): HwlabRuntimeWebP }; } +function webProbeBrowserFreezePolicyConfig(value: unknown, path: string): HwlabRuntimeWebProbeBrowserFreezePolicySpec { + const raw = asRecord(value, path); + const memory = asRecord(raw.memory, `${path}.memory`); + const responsiveness = asRecord(raw.responsiveness, `${path}.responsiveness`); + const cdp = asRecord(raw.cdp, `${path}.cdp`); + const kill = asRecord(raw.kill, `${path}.kill`); + return { + enabled: booleanField(raw, "enabled", path), + blockerWindowMs: positiveNumberField(raw, "blockerWindowMs", path), + memory: { + totalRssBlockerMb: positiveNumberField(memory, "totalRssBlockerMb", `${path}.memory`), + processRssBlockerMb: positiveNumberField(memory, "processRssBlockerMb", `${path}.memory`), + growthBlockerMb: positiveNumberField(memory, "growthBlockerMb", `${path}.memory`), + }, + responsiveness: { + latencyBlockerMs: positiveNumberField(responsiveness, "latencyBlockerMs", `${path}.responsiveness`), + eventBlockerCount: positiveNumberField(responsiveness, "eventBlockerCount", `${path}.responsiveness`), + }, + cdp: { + metricsTimeoutBlockerCount: positiveNumberField(cdp, "metricsTimeoutBlockerCount", `${path}.cdp`), + }, + kill: { + enabled: booleanField(kill, "enabled", `${path}.kill`), + gracefulSignal: enumStringField(kill, "gracefulSignal", `${path}.kill`, ["SIGTERM"] as const), + forceSignal: enumStringField(kill, "forceSignal", `${path}.kill`, ["SIGKILL"] as const), + graceMs: boundedIntegerField(kill, "graceMs", `${path}.kill`, 1, 120000), + pollIntervalMs: boundedIntegerField(kill, "pollIntervalMs", `${path}.kill`, 1, 10000), + exitCode: boundedIntegerField(kill, "exitCode", `${path}.kill`, 1, 125), + }, + }; +} + function webProbeSentinelConfig(value: unknown, path: string): HwlabRuntimeWebProbeSentinelSpec { const raw = asRecord(value, path); const allowed = new Set(["enabled", "configRefs"]); diff --git a/scripts/src/hwlab-node-web-observe-analyzer-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-source.ts index efd07443..4c2ffcec 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-source.ts @@ -19,6 +19,7 @@ const analyzeTailSamples = (() => { return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : 360; })(); const alertThresholds = parseAlertThresholds(process.env.UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON); +const browserFreezePolicy = parseBrowserFreezePolicy(process.env.UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON); const projectManagementConfig = parseProjectManagementConfig(process.env.UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON); const dataDir = archivePrefix ? path.join(stateDir, "archive") : stateDir; const dataFile = (name) => path.join(dataDir, archivePrefix ? archivePrefix + "-" + name : name); @@ -118,6 +119,7 @@ const report = { stateDir, jsonlScope: { mode: archivePrefix ? "archive" : "current", archivePrefix: archivePrefix || null, dataDir, analyzeTailSamples, sourceSampleCount: sourceSamples.length, effectiveSampleCount: samples.length, sourceControlCount: sourceControlAll.length, sampleWindow, focus: analysisFocus, valuesRedacted: true }, alertThresholds, + browserFreezePolicy, manifest: compactManifest(manifest), heartbeat: compactHeartbeat(heartbeat), counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length, browserProcess: browserProcessRows.length }, @@ -584,6 +586,85 @@ function parseAlertThresholds(value) { }; } +function parseBrowserFreezePolicy(value) { + if (!value) { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is required; configure config/hwlab-node-lanes.yaml webProbe.browserFreezePolicy for the selected node/lane"); + } + const raw = (() => { + try { return JSON.parse(value); } catch (error) { throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is invalid JSON: " + (error instanceof Error ? error.message : String(error))); } + })(); + const memory = requiredPolicyRecord(raw, "memory", "webProbe.browserFreezePolicy"); + const responsiveness = requiredPolicyRecord(raw, "responsiveness", "webProbe.browserFreezePolicy"); + const cdp = requiredPolicyRecord(raw, "cdp", "webProbe.browserFreezePolicy"); + const kill = requiredPolicyRecord(raw, "kill", "webProbe.browserFreezePolicy"); + return { + enabled: requiredPolicyBoolean(raw, "enabled", "webProbe.browserFreezePolicy"), + blockerWindowMs: requiredPolicyPositiveNumber(raw, "blockerWindowMs", "webProbe.browserFreezePolicy"), + memory: { + totalRssBlockerMb: requiredPolicyPositiveNumber(memory, "totalRssBlockerMb", "webProbe.browserFreezePolicy.memory"), + processRssBlockerMb: requiredPolicyPositiveNumber(memory, "processRssBlockerMb", "webProbe.browserFreezePolicy.memory"), + growthBlockerMb: requiredPolicyPositiveNumber(memory, "growthBlockerMb", "webProbe.browserFreezePolicy.memory"), + }, + responsiveness: { + latencyBlockerMs: requiredPolicyPositiveNumber(responsiveness, "latencyBlockerMs", "webProbe.browserFreezePolicy.responsiveness"), + eventBlockerCount: requiredPolicyPositiveNumber(responsiveness, "eventBlockerCount", "webProbe.browserFreezePolicy.responsiveness"), + }, + cdp: { + metricsTimeoutBlockerCount: requiredPolicyPositiveNumber(cdp, "metricsTimeoutBlockerCount", "webProbe.browserFreezePolicy.cdp"), + }, + kill: { + enabled: requiredPolicyBoolean(kill, "enabled", "webProbe.browserFreezePolicy.kill"), + gracefulSignal: requiredPolicySignal(kill, "gracefulSignal", "webProbe.browserFreezePolicy.kill", "SIGTERM"), + forceSignal: requiredPolicySignal(kill, "forceSignal", "webProbe.browserFreezePolicy.kill", "SIGKILL"), + graceMs: requiredPolicyIntegerInRange(kill, "graceMs", "webProbe.browserFreezePolicy.kill", 1, 120000), + pollIntervalMs: requiredPolicyIntegerInRange(kill, "pollIntervalMs", "webProbe.browserFreezePolicy.kill", 1, 10000), + exitCode: requiredPolicyIntegerInRange(kill, "exitCode", "webProbe.browserFreezePolicy.kill", 1, 125), + }, + source: "yaml-env", + valuesRedacted: true, + }; +} + +function requiredPolicyRecord(raw, key, pathLabel) { + const value = raw?.[key]; + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires object " + pathLabel + "." + key); + } + return value; +} + +function requiredPolicyBoolean(raw, key, pathLabel) { + const value = raw?.[key]; + if (typeof value !== "boolean") { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires boolean " + pathLabel + "." + key); + } + return value; +} + +function requiredPolicyPositiveNumber(raw, key, pathLabel) { + const value = Number(raw?.[key]); + if (!Number.isFinite(value) || value <= 0) { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires positive number " + pathLabel + "." + key); + } + return value; +} + +function requiredPolicyIntegerInRange(raw, key, pathLabel, min, max) { + const value = Number(raw?.[key]); + if (!Number.isInteger(value) || value < min || value > max) { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires integer " + pathLabel + "." + key + " between " + min + " and " + max); + } + return value; +} + +function requiredPolicySignal(raw, key, pathLabel, expected) { + const value = String(raw?.[key] || ""); + if (value !== expected) { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires " + pathLabel + "." + key + "=" + expected); + } + return value; +} + function parseProjectManagementConfig(value) { if (!value || value === "null") { return { @@ -3107,6 +3188,10 @@ function buildFrontendFreezeFindings(errors, control) { } function buildBrowserProcessReport(rows) { + const blockerEvents = (Array.isArray(rows) ? rows : []) + .filter((item) => item && item.type === "browser-freeze-blocker") + .map(compactBrowserFreezeBlockerEvent) + .filter(Boolean); const samples = (Array.isArray(rows) ? rows : []) .filter((item) => item && (item.type === "browser-process-sample" || item.process || item.pages)) .map((item) => ({ ...item, tsMs: Date.parse(String(item.ts || "")) })) @@ -3205,8 +3290,11 @@ function buildBrowserProcessReport(rows) { growthRedMb: alertThresholds.browserRssGrowthRedMb, growthWindowMs: alertThresholds.browserRssGrowthWindowMs, responsivenessRedMs: alertThresholds.playwrightResponsivenessRedMs, + freezeBlockerCount: blockerEvents.length, + browserFreezePolicy, valuesRedacted: true, }, + blockerEvents: blockerEvents.slice(-20), memorySamples: memorySamples.slice(-60), growthSamples: computedGrowth.slice(-60), maxTotalRssSample: maxTotal, @@ -3220,10 +3308,112 @@ function buildBrowserProcessReport(rows) { }; } +function compactBrowserFreezeBlockerEvent(item) { + if (!item || typeof item !== "object") return null; + const observed = objectValue(item.observed); + const threshold = objectValue(item.threshold); + const sample = objectValue(item.sample); + const page = objectValue(item.page); + const browserKill = objectValue(item.browserKill); + return { + ts: item.ts ?? null, + seq: item.seq ?? null, + sampleSeq: item.sampleSeq ?? sample.sampleSeq ?? null, + kind: item.kind ?? null, + severity: item.severity ?? "red", + blocking: item.blocking === true, + summary: item.summary ? String(item.summary).slice(0, 240) : null, + rootCause: item.rootCause ?? null, + rootCauseStatus: item.rootCauseStatus ?? null, + rootCauseConfidence: item.rootCauseConfidence ?? null, + policySource: item.policySource ?? null, + fallbackAllowed: item.fallbackAllowed === true, + observerRefreshAllowed: item.observerRefreshAllowed === true, + observed: { + totalRssMb: numberOrNull(observed.totalRssMb), + processRssMb: numberOrNull(observed.processRssMb), + totalGrowthMb: numberOrNull(observed.totalGrowthMb), + processGrowthMb: numberOrNull(observed.processGrowthMb), + responsivenessLatencyMs: numberOrNull(observed.responsivenessLatencyMs), + responsivenessTimeout: observed.responsivenessTimeout === true, + cdpMetricsTimeoutCount: numberOrNull(observed.cdpMetricsTimeoutCount), + methods: arrayStrings(observed.methods).slice(0, 8), + valuesRedacted: true, + }, + threshold: { + totalRssBlockerMb: numberOrNull(threshold.totalRssBlockerMb), + processRssBlockerMb: numberOrNull(threshold.processRssBlockerMb), + growthBlockerMb: numberOrNull(threshold.growthBlockerMb), + latencyBlockerMs: numberOrNull(threshold.latencyBlockerMs), + eventBlockerCount: numberOrNull(threshold.eventBlockerCount), + metricsTimeoutBlockerCount: numberOrNull(threshold.metricsTimeoutBlockerCount), + windowMs: numberOrNull(threshold.windowMs), + valuesRedacted: true, + }, + sample: { + ts: sample.ts ?? null, + seq: sample.seq ?? null, + sampleSeq: sample.sampleSeq ?? null, + browserPid: numberOrNull(sample.browserPid), + totalRssMb: numberOrNull(sample.totalRssMb), + maxProcessRssMb: numberOrNull(sample.maxProcessRssMb), + totalRssGrowthMb: numberOrNull(sample.totalRssGrowthMb), + maxProcessRssGrowthMb: numberOrNull(sample.maxProcessRssGrowthMb), + valuesRedacted: true, + }, + page: { + pageRole: page.pageRole ?? null, + pageId: page.pageId ?? null, + pageEpoch: numberOrNull(page.pageEpoch), + timeoutMs: numberOrNull(page.timeoutMs), + responsivenessLatencyMs: numberOrNull(page.responsivenessLatencyMs), + responsivenessTimeout: page.responsivenessTimeout === true, + cdpTimeoutCount: numberOrNull(page.cdpTimeoutCount), + cdpErrorCount: numberOrNull(page.cdpErrorCount), + valuesRedacted: true, + }, + browserKill: { + ok: browserKill.ok === true, + pending: browserKill.pending === true, + skipped: browserKill.skipped === true, + reason: browserKill.reason ?? null, + pid: numberOrNull(browserKill.pid), + gracefulSignal: browserKill.gracefulSignal ?? null, + forceSignal: browserKill.forceSignal ?? null, + gracefulSent: browserKill.gracefulSent === true, + forceSent: browserKill.forceSent === true, + exitedAfterGrace: browserKill.exitedAfterGrace === true, + exitedAfterForce: browserKill.exitedAfterForce === true, + valuesRedacted: true, + }, + valuesRedacted: true, + }; +} + function buildBrowserProcessFindings(report, runtimeAlerts = null) { const summary = report?.summary || {}; - if (!summary || Number(summary.sampleCount ?? 0) <= 0) return []; const findings = []; + const blockerEvents = Array.isArray(report?.blockerEvents) ? report.blockerEvents : []; + if (blockerEvents.length > 0) { + const first = blockerEvents[0] || {}; + findings.push({ + id: "frontend-browser-freeze-runner-blocker", + severity: "red", + summary: "web-probe runner matched YAML browserFreezePolicy, killed/stopped Chromium, and failed the observer run; do not clear this by refresh or fallback", + count: blockerEvents.length, + blocking: true, + rootCause: first.rootCause ?? "frontend_browser_freeze_policy_blocker", + rootCauseStatus: "confirmed-from-runner-browser-freeze-policy", + rootCauseConfidence: "high", + policySource: first.policySource ?? "config/hwlab-node-lanes.yaml#webProbe.browserFreezePolicy", + fallbackAllowed: false, + observerRefreshAllowed: false, + browserKilled: first.browserKill ?? null, + events: blockerEvents.slice(0, 20), + valuesRedacted: true, + }); + } + if (!summary || Number(summary.sampleCount ?? 0) <= 0) return findings; const rootCauseSignals = browserRootCauseSignals(report, runtimeAlerts); const maxTotalRssMb = Number(summary.maxTotalRssMb ?? 0); const maxProcessRssMb = Number(summary.maxProcessRssMb ?? 0); diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index c0218da0..7e0405bd 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -32,6 +32,7 @@ const authLoginInitialDelayMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_A const authLoginMaxDelayMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS, 10000, 0, 120000); const navigationMaxAttempts = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_NAVIGATION_MAX_ATTEMPTS, 4, 1, 10); const alertThresholds = parseAlertThresholds(process.env.UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON); +const browserFreezePolicy = parseBrowserFreezePolicy(process.env.UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON); const projectManagement = parseProjectManagementConfig(process.env.UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON); const playwrightProxy = proxyConfigFromEnv(baseUrl); const chromiumLaunchOptions = chromiumLaunchOptionsForProxy(playwrightProxy); @@ -79,7 +80,9 @@ let currentPageProvenance = null; let heartbeatPulseTimer = null; let browserProcessMonitorStop = null; let browserProcessMonitorSeq = 0; +let browserFreezeBlocker = null; const browserProcessHistory = []; +const browserFreezeSignalHistory = []; const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] }; try { @@ -117,17 +120,26 @@ try { await writeHeartbeat({ status: "running" }); while (!stopping) { await drainOneCommand(); + if (browserFreezeBlocker) break; await samplePage("interval"); + if (browserFreezeBlocker) break; if (maxSamples > 0 && sampleSeq >= maxSamples) { await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples })); break; } await sleep(sampleIntervalMs); } - terminalStatus = "completed"; - await writeHeartbeat({ status: "completed" }); - await writeManifest({ status: "completed", completedAt: new Date().toISOString() }); - process.exitCode = 0; + if (browserFreezeBlocker) { + terminalStatus = "failed"; + await writeHeartbeat({ status: "failed", blocker: browserFreezeBlocker }); + await writeManifest({ status: "failed", completedAt: new Date().toISOString(), blocker: browserFreezeBlocker }); + process.exitCode = browserFreezePolicy.kill.exitCode; + } else { + terminalStatus = "completed"; + await writeHeartbeat({ status: "completed" }); + await writeManifest({ status: "completed", completedAt: new Date().toISOString() }); + process.exitCode = 0; + } } catch (error) { terminalStatus = "failed"; await appendJsonl(files.errors, eventRecord("runner-error", { error: errorSummary(error) })).catch(() => {}); @@ -196,6 +208,7 @@ async function writeManifest(extra = {}) { pageProvenance: compactPageProvenance(currentPageProvenance), sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerRefreshIntervalMs, observerInitiatedDefault: false, responseBodyReadDefault: false }, alertThresholds, + browserFreezePolicy, projectManagement, jsonlRotation, commandDirs: dirs, @@ -247,7 +260,7 @@ function startBrowserProcessMonitor() { let stopped = false; let running = false; const tick = async (reason) => { - if (stopped || running) return; + if (stopped || running || stopping || browserFreezeBlocker) return; running = true; try { await collectBrowserProcessSample(reason || "interval"); @@ -281,7 +294,7 @@ async function collectBrowserProcessSample(reason) { if (observerPage && !observerPage.isClosed()) { pages.push(await collectBrowserPageRuntimeMetrics(observerPage, { pageRole: "observer", targetPageId: observerPageId, pageEpoch: observerPageEpoch }).catch((error) => browserPageRuntimeMetricError("observer", observerPageId, observerPageEpoch, error))); } - await appendJsonl(files.browserProcess, eventRecord("browser-process-sample", { + const sample = eventRecord("browser-process-sample", { seq: browserProcessMonitorSeq, reason, monitorIntervalMs: Math.max(250, Math.floor(Number(alertThresholds.browserProcessSampleIntervalMs) || 1000)), @@ -290,7 +303,260 @@ async function collectBrowserProcessSample(reason) { growth, pages, valuesRedacted: true, - })); + }); + await appendJsonl(files.browserProcess, sample); + await enforceBrowserFreezePolicy(sample); +} + +async function enforceBrowserFreezePolicy(sample) { + if (browserFreezePolicy.enabled !== true || browserFreezeBlocker) return; + const processSummary = sample && typeof sample.process === "object" ? sample.process : {}; + const growth = sample && typeof sample.growth === "object" ? sample.growth : {}; + const totalRssMb = Number(processSummary.totalRssMb); + const processRssMb = Number(processSummary.maxProcessRssMb); + const totalGrowthMb = Number(growth.totalRssGrowthMb); + const processGrowthMb = Number(growth.maxProcessRssGrowthMb); + if (Number.isFinite(totalRssMb) && totalRssMb >= browserFreezePolicy.memory.totalRssBlockerMb) { + await triggerBrowserFreezeBlocker({ + kind: "memory-total-rss", + rootCause: "frontend_browser_process_memory_pressure", + observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true }, + threshold: { totalRssBlockerMb: browserFreezePolicy.memory.totalRssBlockerMb, valuesRedacted: true }, + sample: browserProcessSampleRef(sample), + }); + return; + } + if (Number.isFinite(processRssMb) && processRssMb >= browserFreezePolicy.memory.processRssBlockerMb) { + await triggerBrowserFreezeBlocker({ + kind: "memory-process-rss", + rootCause: "frontend_browser_process_memory_pressure", + observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true }, + threshold: { processRssBlockerMb: browserFreezePolicy.memory.processRssBlockerMb, valuesRedacted: true }, + sample: browserProcessSampleRef(sample), + }); + return; + } + if ( + (Number.isFinite(totalGrowthMb) && totalGrowthMb >= browserFreezePolicy.memory.growthBlockerMb) + || (Number.isFinite(processGrowthMb) && processGrowthMb >= browserFreezePolicy.memory.growthBlockerMb) + ) { + await triggerBrowserFreezeBlocker({ + kind: "memory-rss-growth", + rootCause: "frontend_browser_process_memory_leak_or_unbounded_render_growth", + observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true }, + threshold: { growthBlockerMb: browserFreezePolicy.memory.growthBlockerMb, windowMs: browserFreezePolicy.blockerWindowMs, valuesRedacted: true }, + sample: browserProcessSampleRef(sample), + }); + return; + } + + for (const pageMetric of Array.isArray(sample.pages) ? sample.pages : []) { + const responsiveness = pageMetric?.responsiveness && typeof pageMetric.responsiveness === "object" ? pageMetric.responsiveness : {}; + const responsivenessLatencyMs = Number(responsiveness.latencyMs); + if (responsiveness.timeout === true || (Number.isFinite(responsivenessLatencyMs) && responsivenessLatencyMs >= browserFreezePolicy.responsiveness.latencyBlockerMs)) { + const signal = recordBrowserFreezeSignal("playwright-responsiveness", sample, pageMetric, { + rootCause: "frontend_browser_page_unresponsive_to_playwright", + observed: { + responsivenessLatencyMs: Number.isFinite(responsivenessLatencyMs) ? responsivenessLatencyMs : null, + responsivenessTimeout: responsiveness.timeout === true, + valuesRedacted: true, + }, + threshold: { + latencyBlockerMs: browserFreezePolicy.responsiveness.latencyBlockerMs, + eventBlockerCount: browserFreezePolicy.responsiveness.eventBlockerCount, + windowMs: browserFreezePolicy.blockerWindowMs, + valuesRedacted: true, + }, + }); + if (signal.burst.length >= browserFreezePolicy.responsiveness.eventBlockerCount) { + await triggerBrowserFreezeBlocker(signal); + return; + } + } + const cdp = pageMetric?.cdp && typeof pageMetric.cdp === "object" ? pageMetric.cdp : {}; + const calls = Array.isArray(cdp.calls) ? cdp.calls : []; + const metricTimeoutCalls = calls.filter((call) => call?.timeout === true && call?.method !== "Runtime.evaluate"); + const sessionTimeoutCount = calls.length === 0 ? Number(cdp.timeoutCount || 0) : 0; + const metricTimeoutCount = metricTimeoutCalls.length + (Number.isFinite(sessionTimeoutCount) ? sessionTimeoutCount : 0); + if (metricTimeoutCount > 0) { + const signal = recordBrowserFreezeSignal("cdp-metrics-timeout", sample, pageMetric, { + rootCause: "frontend_browser_cdp_metrics_unresponsive", + observed: { + cdpMetricsTimeoutCount: metricTimeoutCount, + methods: metricTimeoutCalls.map((call) => call.method || "unknown").slice(0, 8), + valuesRedacted: true, + }, + threshold: { + metricsTimeoutBlockerCount: browserFreezePolicy.cdp.metricsTimeoutBlockerCount, + windowMs: browserFreezePolicy.blockerWindowMs, + valuesRedacted: true, + }, + }); + if (signal.burst.length >= browserFreezePolicy.cdp.metricsTimeoutBlockerCount) { + await triggerBrowserFreezeBlocker(signal); + return; + } + } + } +} + +function recordBrowserFreezeSignal(kind, sample, pageMetric, detail) { + const tsMs = Date.parse(String(sample?.ts || "")); + const signal = { + kind, + ts: sample?.ts ?? new Date().toISOString(), + tsMs: Number.isFinite(tsMs) ? tsMs : Date.now(), + pageRole: pageMetric?.pageRole ?? null, + pageId: pageMetric?.pageId ?? null, + pageEpoch: pageMetric?.pageEpoch ?? null, + sample: browserProcessSampleRef(sample), + page: browserPageMetricRef(pageMetric), + ...detail, + valuesRedacted: true, + }; + browserFreezeSignalHistory.push(signal); + const windowMs = browserFreezePolicy.blockerWindowMs; + const cutoff = signal.tsMs - windowMs; + while (browserFreezeSignalHistory.length > 0 && Number(browserFreezeSignalHistory[0].tsMs || 0) < cutoff) browserFreezeSignalHistory.shift(); + return { + kind, + rootCause: detail.rootCause, + observed: detail.observed, + threshold: detail.threshold, + sample: browserProcessSampleRef(sample), + page: browserPageMetricRef(pageMetric), + burst: browserFreezeSignalHistory.filter((item) => item.kind === kind && item.tsMs >= cutoff).slice(-20), + valuesRedacted: true, + }; +} + +async function triggerBrowserFreezeBlocker(trigger) { + if (browserFreezeBlocker) return; + stopping = true; + terminalStatus = "failed"; + const base = { + id: "browser-freeze-policy-blocker", + severity: "red", + blocking: true, + kind: trigger.kind, + summary: "web-probe runner matched YAML browserFreezePolicy and stopped the browser; this run must stay red instead of refreshing or falling back", + rootCause: trigger.rootCause, + rootCauseStatus: "confirmed-from-runner-browser-freeze-policy", + rootCauseConfidence: "high", + fallbackAllowed: false, + observerRefreshAllowed: false, + policySource: "config/hwlab-node-lanes.yaml#webProbe.browserFreezePolicy via UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON", + observed: trigger.observed || null, + threshold: trigger.threshold || null, + sample: trigger.sample || null, + page: trigger.page || null, + burst: Array.isArray(trigger.burst) ? trigger.burst.slice(0, 20) : [], + valuesRedacted: true, + }; + browserFreezeBlocker = { ...base, browserKill: { ok: false, pending: true, valuesRedacted: true }, valuesRedacted: true }; + const browserKill = browserFreezePolicy.kill.enabled === true + ? await terminateBrowserForFreezeBlocker(base).catch((error) => ({ ok: false, error: errorSummary(error), valuesRedacted: true })) + : { ok: true, skipped: true, reason: "kill-disabled-by-yaml-policy", valuesRedacted: true }; + browserFreezeBlocker = { ...base, browserKill, valuesRedacted: true }; + await appendJsonl(files.browserProcess, eventRecord("browser-freeze-blocker", browserFreezeBlocker)).catch(() => {}); + await appendJsonl(files.errors, eventRecord("browser-freeze-blocker", { + ...browserFreezeBlocker, + error: { + name: "BrowserFreezeBlocker", + message: "YAML browserFreezePolicy matched: " + String(trigger.kind || "unknown"), + details: browserFreezeBlocker, + valuesRedacted: true, + }, + })).catch(() => {}); + await writeHeartbeat({ status: "failed", blocker: browserFreezeBlocker }).catch(() => {}); + await writeManifest({ status: "failed", blocker: browserFreezeBlocker }).catch(() => {}); +} + +async function terminateBrowserForFreezeBlocker(blocker) { + const childProcess = browser && typeof browser.process === "function" ? browser.process() : null; + const pid = Number(childProcess?.pid); + if (!Number.isFinite(pid) || pid <= 0) { + return { ok: false, reason: "browser-process-unavailable", blockerKind: blocker.kind, valuesRedacted: true }; + } + const result = { + ok: false, + pid: Math.floor(pid), + gracefulSignal: browserFreezePolicy.kill.gracefulSignal, + forceSignal: browserFreezePolicy.kill.forceSignal, + graceMs: browserFreezePolicy.kill.graceMs, + pollIntervalMs: browserFreezePolicy.kill.pollIntervalMs, + gracefulSent: false, + forceSent: false, + exitedAfterGrace: false, + exitedAfterForce: false, + valuesRedacted: true, + }; + try { + childProcess.kill(browserFreezePolicy.kill.gracefulSignal); + result.gracefulSent = true; + } catch (error) { + result.gracefulError = errorSummary(error); + } + result.exitedAfterGrace = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs); + if (!result.exitedAfterGrace) { + try { + childProcess.kill(browserFreezePolicy.kill.forceSignal); + result.forceSent = true; + } catch (error) { + result.forceError = errorSummary(error); + } + result.exitedAfterForce = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs); + } + result.ok = result.exitedAfterGrace || result.exitedAfterForce; + return result; +} + +async function waitForPidExit(pid, timeoutMs, pollIntervalMs) { + const deadline = Date.now() + timeoutMs; + while (Date.now() <= deadline) { + if (!pidAlive(pid)) return true; + await sleep(pollIntervalMs); + } + return !pidAlive(pid); +} + +function pidAlive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function browserProcessSampleRef(sample) { + return { + ts: sample?.ts ?? null, + seq: sample?.seq ?? null, + sampleSeq: sample?.sampleSeq ?? null, + browserPid: sample?.browserPid ?? null, + totalRssMb: sample?.process?.totalRssMb ?? null, + maxProcessRssMb: sample?.process?.maxProcessRssMb ?? null, + totalRssGrowthMb: sample?.growth?.totalRssGrowthMb ?? null, + maxProcessRssGrowthMb: sample?.growth?.maxProcessRssGrowthMb ?? null, + valuesRedacted: true, + }; +} + +function browserPageMetricRef(pageMetric) { + const responsiveness = pageMetric?.responsiveness && typeof pageMetric.responsiveness === "object" ? pageMetric.responsiveness : {}; + const cdp = pageMetric?.cdp && typeof pageMetric.cdp === "object" ? pageMetric.cdp : {}; + return { + pageRole: pageMetric?.pageRole ?? null, + pageId: pageMetric?.pageId ?? null, + pageEpoch: pageMetric?.pageEpoch ?? null, + timeoutMs: pageMetric?.timeoutMs ?? null, + responsivenessLatencyMs: responsiveness.latencyMs ?? null, + responsivenessTimeout: responsiveness.timeout === true, + cdpTimeoutCount: cdp.timeoutCount ?? null, + cdpErrorCount: cdp.errorCount ?? null, + valuesRedacted: true, + }; } function browserProcessPid() { @@ -5070,6 +5336,85 @@ function parseAlertThresholds(value) { }; } +function parseBrowserFreezePolicy(value) { + if (!value) { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is required; configure config/hwlab-node-lanes.yaml webProbe.browserFreezePolicy for the selected node/lane"); + } + const raw = (() => { + try { return JSON.parse(value); } catch (error) { throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is invalid JSON: " + (error instanceof Error ? error.message : String(error))); } + })(); + const memory = requiredPolicyRecord(raw, "memory", "webProbe.browserFreezePolicy"); + const responsiveness = requiredPolicyRecord(raw, "responsiveness", "webProbe.browserFreezePolicy"); + const cdp = requiredPolicyRecord(raw, "cdp", "webProbe.browserFreezePolicy"); + const kill = requiredPolicyRecord(raw, "kill", "webProbe.browserFreezePolicy"); + return { + enabled: requiredPolicyBoolean(raw, "enabled", "webProbe.browserFreezePolicy"), + blockerWindowMs: requiredPolicyPositiveNumber(raw, "blockerWindowMs", "webProbe.browserFreezePolicy"), + memory: { + totalRssBlockerMb: requiredPolicyPositiveNumber(memory, "totalRssBlockerMb", "webProbe.browserFreezePolicy.memory"), + processRssBlockerMb: requiredPolicyPositiveNumber(memory, "processRssBlockerMb", "webProbe.browserFreezePolicy.memory"), + growthBlockerMb: requiredPolicyPositiveNumber(memory, "growthBlockerMb", "webProbe.browserFreezePolicy.memory"), + }, + responsiveness: { + latencyBlockerMs: requiredPolicyPositiveNumber(responsiveness, "latencyBlockerMs", "webProbe.browserFreezePolicy.responsiveness"), + eventBlockerCount: requiredPolicyPositiveNumber(responsiveness, "eventBlockerCount", "webProbe.browserFreezePolicy.responsiveness"), + }, + cdp: { + metricsTimeoutBlockerCount: requiredPolicyPositiveNumber(cdp, "metricsTimeoutBlockerCount", "webProbe.browserFreezePolicy.cdp"), + }, + kill: { + enabled: requiredPolicyBoolean(kill, "enabled", "webProbe.browserFreezePolicy.kill"), + gracefulSignal: requiredPolicySignal(kill, "gracefulSignal", "webProbe.browserFreezePolicy.kill", "SIGTERM"), + forceSignal: requiredPolicySignal(kill, "forceSignal", "webProbe.browserFreezePolicy.kill", "SIGKILL"), + graceMs: requiredPolicyIntegerInRange(kill, "graceMs", "webProbe.browserFreezePolicy.kill", 1, 120000), + pollIntervalMs: requiredPolicyIntegerInRange(kill, "pollIntervalMs", "webProbe.browserFreezePolicy.kill", 1, 10000), + exitCode: requiredPolicyIntegerInRange(kill, "exitCode", "webProbe.browserFreezePolicy.kill", 1, 125), + }, + source: "yaml-env", + valuesRedacted: true, + }; +} + +function requiredPolicyRecord(raw, key, pathLabel) { + const value = raw?.[key]; + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires object " + pathLabel + "." + key); + } + return value; +} + +function requiredPolicyBoolean(raw, key, pathLabel) { + const value = raw?.[key]; + if (typeof value !== "boolean") { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires boolean " + pathLabel + "." + key); + } + return value; +} + +function requiredPolicyPositiveNumber(raw, key, pathLabel) { + const value = Number(raw?.[key]); + if (!Number.isFinite(value) || value <= 0) { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires positive number " + pathLabel + "." + key); + } + return value; +} + +function requiredPolicyIntegerInRange(raw, key, pathLabel, min, max) { + const value = Number(raw?.[key]); + if (!Number.isInteger(value) || value < min || value > max) { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires integer " + pathLabel + "." + key + " between " + min + " and " + max); + } + return value; +} + +function requiredPolicySignal(raw, key, pathLabel, expected) { + const value = String(raw?.[key] || ""); + if (value !== expected) { + throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires " + pathLabel + "." + key + "=" + expected); + } + return value; +} + function parseProjectManagementConfig(value) { if (!value || value === "null") { return { diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index b9e79b4d..5eff4a52 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -14,7 +14,7 @@ import { runCommand, type CommandResult } from "../command"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; -import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeAuthLoginSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; +import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeAuthLoginSpec, type HwlabRuntimeWebProbeBrowserFreezePolicySpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source"; import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; @@ -1141,6 +1141,14 @@ export function nodeWebProbeAlertThresholds(spec: HwlabRuntimeLaneSpec): HwlabRu return thresholds; } +export function nodeWebProbeBrowserFreezePolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeBrowserFreezePolicySpec { + const policy = spec.webProbe?.browserFreezePolicy; + if (policy === undefined) { + throw new Error(`${hwlabRuntimeLaneConfigPath()} node=${spec.nodeId} lane=${spec.lane} requires webProbe.browserFreezePolicy for web-probe observe`); + } + return policy; +} + export function nodeWebProbeProjectManagementConfig(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeProjectManagementSpec | null { return spec.webProbe?.projectManagement ?? null; } @@ -1311,6 +1319,7 @@ export function runNodeWebProbeObserveStart( const runnerB64Body = runnerB64.match(/.{1,76}/gu)?.join("\n") ?? runnerB64; const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode); const alertThresholds = nodeWebProbeAlertThresholds(spec); + const browserFreezePolicy = nodeWebProbeBrowserFreezePolicy(spec); const projectManagement = nodeWebProbeProjectManagementConfig(spec); const authLogin = nodeWebProbeAuthLoginConfig(spec); const runnerEnvAssignments = [ @@ -1332,6 +1341,7 @@ export function runNodeWebProbeObserveStart( `UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`, `UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`, `UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`, + `UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=${shellQuote(JSON.stringify(browserFreezePolicy))}`, `UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`, ...(authLogin === null ? [] @@ -1407,6 +1417,7 @@ export function runNodeWebProbeObserveStart( url: options.url, network: webProbeProxy.summary, alertThresholds, + browserFreezePolicy, projectManagement, targetPath: options.targetPath, id: observerId, @@ -1739,6 +1750,7 @@ function observeRecord(value: unknown): Record { export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record | RenderedCliResult { const analyzerB64 = Buffer.from(nodeWebObserveAnalyzerSource(), "utf8").toString("base64"); const alertThresholds = nodeWebProbeAlertThresholds(spec); + const browserFreezePolicy = nodeWebProbeBrowserFreezePolicy(spec); const projectManagement = nodeWebProbeProjectManagementConfig(spec); const script = [ "set -eu", @@ -1758,8 +1770,9 @@ export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOption `UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=${shellQuote(options.analyzeArchivePrefix ?? "")}`, `UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES=${shellQuote(options.analyzeTailSamples === null ? "" : String(options.analyzeTailSamples))}`, `UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`, + `UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=${shellQuote(JSON.stringify(browserFreezePolicy))}`, `UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`, - "UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=\"$UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX\" UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES=\"$UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES\" UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=\"$UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON\" UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=\"$UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON\" node \"$analyzer\" >\"$analysis_stdout\" 2>\"$analysis_stderr\"", + "UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=\"$UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX\" UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES=\"$UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES\" UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=\"$UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON\" UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=\"$UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON\" UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=\"$UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON\" node \"$analyzer\" >\"$analysis_stdout\" 2>\"$analysis_stderr\"", "analyzer_exit=$?", "set -e", "report_json=\"$state_dir/analysis/report.json\"", @@ -2169,6 +2182,7 @@ export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOption analysis: failureAnalysis, failure: analysisFailure, alertThresholds, + browserFreezePolicy, wrapper: buildWebObserveWrapperForObserveOptions("analyze", options, spec.workspace), result: analysis === null ? compactCommandResultWithStdoutTail(result) : compactCommandResult(result), valuesRedacted: true,