feat: block frozen web probe browsers from yaml policy

This commit is contained in:
Codex
2026-06-30 11:07:52 +00:00
parent 81c48b8126
commit 6168ec60f6
5 changed files with 683 additions and 10 deletions
+57
View File
@@ -223,6 +223,25 @@ lanes:
scrollJumpFromY: 250
scrollJumpToY: 40
sessionRailFallbackRatio: 0.5
browserFreezePolicy:
enabled: true
blockerWindowMs: 30000
memory:
totalRssBlockerMb: 500
processRssBlockerMb: 500
growthBlockerMb: 300
responsiveness:
latencyBlockerMs: 5000
eventBlockerCount: 2
cdp:
metricsTimeoutBlockerCount: 2
kill:
enabled: true
gracefulSignal: SIGTERM
forceSignal: SIGKILL
graceMs: 3000
pollIntervalMs: 100
exitCode: 7
projectManagement:
enabled: true
targetPaths:
@@ -624,6 +643,25 @@ lanes:
scrollJumpFromY: 250
scrollJumpToY: 40
sessionRailFallbackRatio: 0.5
browserFreezePolicy:
enabled: true
blockerWindowMs: 30000
memory:
totalRssBlockerMb: 500
processRssBlockerMb: 500
growthBlockerMb: 300
responsiveness:
latencyBlockerMs: 5000
eventBlockerCount: 2
cdp:
metricsTimeoutBlockerCount: 2
kill:
enabled: true
gracefulSignal: SIGTERM
forceSignal: SIGKILL
graceMs: 3000
pollIntervalMs: 100
exitCode: 7
projectManagement:
enabled: true
targetPaths:
@@ -955,6 +993,25 @@ lanes:
scrollJumpFromY: 250
scrollJumpToY: 40
sessionRailFallbackRatio: 0.5
browserFreezePolicy:
enabled: true
blockerWindowMs: 30000
memory:
totalRssBlockerMb: 500
processRssBlockerMb: 500
growthBlockerMb: 300
responsiveness:
latencyBlockerMs: 5000
eventBlockerCount: 2
cdp:
metricsTimeoutBlockerCount: 2
kill:
enabled: true
gracefulSignal: SIGTERM
forceSignal: SIGKILL
graceMs: 3000
pollIntervalMs: 100
exitCode: 7
tektonDir: tekton-v03
argoApplicationFile: application-v03.yaml
registryPrefix: 127.0.0.1:5000/hwlab
+67
View File
@@ -141,6 +141,7 @@ export interface HwlabRuntimeWebProbeSpec {
readonly defaultOrigin?: HwlabRuntimeWebProbeOriginSpec;
readonly authLogin?: HwlabRuntimeWebProbeAuthLoginSpec;
readonly alertThresholds?: HwlabRuntimeWebProbeAlertThresholdsSpec;
readonly browserFreezePolicy?: HwlabRuntimeWebProbeBrowserFreezePolicySpec;
readonly projectManagement?: HwlabRuntimeWebProbeProjectManagementSpec;
}
@@ -151,6 +152,39 @@ export interface HwlabRuntimeWebProbeAuthLoginSpec {
readonly maxDelayMs: number;
}
export interface HwlabRuntimeWebProbeBrowserFreezeMemoryPolicySpec {
readonly totalRssBlockerMb: number;
readonly processRssBlockerMb: number;
readonly growthBlockerMb: number;
}
export interface HwlabRuntimeWebProbeBrowserFreezeResponsivenessPolicySpec {
readonly latencyBlockerMs: number;
readonly eventBlockerCount: number;
}
export interface HwlabRuntimeWebProbeBrowserFreezeCdpPolicySpec {
readonly metricsTimeoutBlockerCount: number;
}
export interface HwlabRuntimeWebProbeBrowserFreezeKillPolicySpec {
readonly enabled: boolean;
readonly gracefulSignal: "SIGTERM";
readonly forceSignal: "SIGKILL";
readonly graceMs: number;
readonly pollIntervalMs: number;
readonly exitCode: number;
}
export interface HwlabRuntimeWebProbeBrowserFreezePolicySpec {
readonly enabled: boolean;
readonly blockerWindowMs: number;
readonly memory: HwlabRuntimeWebProbeBrowserFreezeMemoryPolicySpec;
readonly responsiveness: HwlabRuntimeWebProbeBrowserFreezeResponsivenessPolicySpec;
readonly cdp: HwlabRuntimeWebProbeBrowserFreezeCdpPolicySpec;
readonly kill: HwlabRuntimeWebProbeBrowserFreezeKillPolicySpec;
}
export type HwlabRuntimeWebProbeSentinelConfigRefKey = "runtime" | "scenarios" | "promptSet" | "reportViews" | "publicExposure" | "cicd" | "secrets";
export const HWLAB_WEB_PROBE_SENTINEL_CONFIG_REF_KEYS = ["runtime", "scenarios", "promptSet", "reportViews", "publicExposure", "cicd", "secrets"] as const satisfies readonly HwlabRuntimeWebProbeSentinelConfigRefKey[];
@@ -1078,6 +1112,7 @@ function webProbeConfig(value: unknown, path: string): HwlabRuntimeWebProbeSpec
...(raw.defaultOrigin === undefined ? {} : { defaultOrigin: webProbeOriginConfig(raw.defaultOrigin, `${path}.defaultOrigin`) }),
...(raw.authLogin === undefined ? {} : { authLogin: webProbeAuthLoginConfig(raw.authLogin, `${path}.authLogin`) }),
...(raw.alertThresholds === undefined ? {} : { alertThresholds: webProbeAlertThresholdsConfig(raw.alertThresholds, `${path}.alertThresholds`) }),
...(raw.browserFreezePolicy === undefined ? {} : { browserFreezePolicy: webProbeBrowserFreezePolicyConfig(raw.browserFreezePolicy, `${path}.browserFreezePolicy`) }),
...(raw.projectManagement === undefined ? {} : { projectManagement: webProbeProjectManagementConfig(raw.projectManagement, `${path}.projectManagement`) }),
};
}
@@ -1094,6 +1129,38 @@ function webProbeAuthLoginConfig(value: unknown, path: string): HwlabRuntimeWebP
};
}
function webProbeBrowserFreezePolicyConfig(value: unknown, path: string): HwlabRuntimeWebProbeBrowserFreezePolicySpec {
const raw = asRecord(value, path);
const memory = asRecord(raw.memory, `${path}.memory`);
const responsiveness = asRecord(raw.responsiveness, `${path}.responsiveness`);
const cdp = asRecord(raw.cdp, `${path}.cdp`);
const kill = asRecord(raw.kill, `${path}.kill`);
return {
enabled: booleanField(raw, "enabled", path),
blockerWindowMs: positiveNumberField(raw, "blockerWindowMs", path),
memory: {
totalRssBlockerMb: positiveNumberField(memory, "totalRssBlockerMb", `${path}.memory`),
processRssBlockerMb: positiveNumberField(memory, "processRssBlockerMb", `${path}.memory`),
growthBlockerMb: positiveNumberField(memory, "growthBlockerMb", `${path}.memory`),
},
responsiveness: {
latencyBlockerMs: positiveNumberField(responsiveness, "latencyBlockerMs", `${path}.responsiveness`),
eventBlockerCount: positiveNumberField(responsiveness, "eventBlockerCount", `${path}.responsiveness`),
},
cdp: {
metricsTimeoutBlockerCount: positiveNumberField(cdp, "metricsTimeoutBlockerCount", `${path}.cdp`),
},
kill: {
enabled: booleanField(kill, "enabled", `${path}.kill`),
gracefulSignal: enumStringField(kill, "gracefulSignal", `${path}.kill`, ["SIGTERM"] as const),
forceSignal: enumStringField(kill, "forceSignal", `${path}.kill`, ["SIGKILL"] as const),
graceMs: boundedIntegerField(kill, "graceMs", `${path}.kill`, 1, 120000),
pollIntervalMs: boundedIntegerField(kill, "pollIntervalMs", `${path}.kill`, 1, 10000),
exitCode: boundedIntegerField(kill, "exitCode", `${path}.kill`, 1, 125),
},
};
}
function webProbeSentinelConfig(value: unknown, path: string): HwlabRuntimeWebProbeSentinelSpec {
const raw = asRecord(value, path);
const allowed = new Set(["enabled", "configRefs"]);
@@ -19,6 +19,7 @@ const analyzeTailSamples = (() => {
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : 360;
})();
const alertThresholds = parseAlertThresholds(process.env.UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON);
const browserFreezePolicy = parseBrowserFreezePolicy(process.env.UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON);
const projectManagementConfig = parseProjectManagementConfig(process.env.UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON);
const dataDir = archivePrefix ? path.join(stateDir, "archive") : stateDir;
const dataFile = (name) => path.join(dataDir, archivePrefix ? archivePrefix + "-" + name : name);
@@ -118,6 +119,7 @@ const report = {
stateDir,
jsonlScope: { mode: archivePrefix ? "archive" : "current", archivePrefix: archivePrefix || null, dataDir, analyzeTailSamples, sourceSampleCount: sourceSamples.length, effectiveSampleCount: samples.length, sourceControlCount: sourceControlAll.length, sampleWindow, focus: analysisFocus, valuesRedacted: true },
alertThresholds,
browserFreezePolicy,
manifest: compactManifest(manifest),
heartbeat: compactHeartbeat(heartbeat),
counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length, browserProcess: browserProcessRows.length },
@@ -584,6 +586,85 @@ function parseAlertThresholds(value) {
};
}
function parseBrowserFreezePolicy(value) {
if (!value) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is required; configure config/hwlab-node-lanes.yaml webProbe.browserFreezePolicy for the selected node/lane");
}
const raw = (() => {
try { return JSON.parse(value); } catch (error) { throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is invalid JSON: " + (error instanceof Error ? error.message : String(error))); }
})();
const memory = requiredPolicyRecord(raw, "memory", "webProbe.browserFreezePolicy");
const responsiveness = requiredPolicyRecord(raw, "responsiveness", "webProbe.browserFreezePolicy");
const cdp = requiredPolicyRecord(raw, "cdp", "webProbe.browserFreezePolicy");
const kill = requiredPolicyRecord(raw, "kill", "webProbe.browserFreezePolicy");
return {
enabled: requiredPolicyBoolean(raw, "enabled", "webProbe.browserFreezePolicy"),
blockerWindowMs: requiredPolicyPositiveNumber(raw, "blockerWindowMs", "webProbe.browserFreezePolicy"),
memory: {
totalRssBlockerMb: requiredPolicyPositiveNumber(memory, "totalRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
processRssBlockerMb: requiredPolicyPositiveNumber(memory, "processRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
growthBlockerMb: requiredPolicyPositiveNumber(memory, "growthBlockerMb", "webProbe.browserFreezePolicy.memory"),
},
responsiveness: {
latencyBlockerMs: requiredPolicyPositiveNumber(responsiveness, "latencyBlockerMs", "webProbe.browserFreezePolicy.responsiveness"),
eventBlockerCount: requiredPolicyPositiveNumber(responsiveness, "eventBlockerCount", "webProbe.browserFreezePolicy.responsiveness"),
},
cdp: {
metricsTimeoutBlockerCount: requiredPolicyPositiveNumber(cdp, "metricsTimeoutBlockerCount", "webProbe.browserFreezePolicy.cdp"),
},
kill: {
enabled: requiredPolicyBoolean(kill, "enabled", "webProbe.browserFreezePolicy.kill"),
gracefulSignal: requiredPolicySignal(kill, "gracefulSignal", "webProbe.browserFreezePolicy.kill", "SIGTERM"),
forceSignal: requiredPolicySignal(kill, "forceSignal", "webProbe.browserFreezePolicy.kill", "SIGKILL"),
graceMs: requiredPolicyIntegerInRange(kill, "graceMs", "webProbe.browserFreezePolicy.kill", 1, 120000),
pollIntervalMs: requiredPolicyIntegerInRange(kill, "pollIntervalMs", "webProbe.browserFreezePolicy.kill", 1, 10000),
exitCode: requiredPolicyIntegerInRange(kill, "exitCode", "webProbe.browserFreezePolicy.kill", 1, 125),
},
source: "yaml-env",
valuesRedacted: true,
};
}
function requiredPolicyRecord(raw, key, pathLabel) {
const value = raw?.[key];
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires object " + pathLabel + "." + key);
}
return value;
}
function requiredPolicyBoolean(raw, key, pathLabel) {
const value = raw?.[key];
if (typeof value !== "boolean") {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires boolean " + pathLabel + "." + key);
}
return value;
}
function requiredPolicyPositiveNumber(raw, key, pathLabel) {
const value = Number(raw?.[key]);
if (!Number.isFinite(value) || value <= 0) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires positive number " + pathLabel + "." + key);
}
return value;
}
function requiredPolicyIntegerInRange(raw, key, pathLabel, min, max) {
const value = Number(raw?.[key]);
if (!Number.isInteger(value) || value < min || value > max) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires integer " + pathLabel + "." + key + " between " + min + " and " + max);
}
return value;
}
function requiredPolicySignal(raw, key, pathLabel, expected) {
const value = String(raw?.[key] || "");
if (value !== expected) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires " + pathLabel + "." + key + "=" + expected);
}
return value;
}
function parseProjectManagementConfig(value) {
if (!value || value === "null") {
return {
@@ -3107,6 +3188,10 @@ function buildFrontendFreezeFindings(errors, control) {
}
function buildBrowserProcessReport(rows) {
const blockerEvents = (Array.isArray(rows) ? rows : [])
.filter((item) => item && item.type === "browser-freeze-blocker")
.map(compactBrowserFreezeBlockerEvent)
.filter(Boolean);
const samples = (Array.isArray(rows) ? rows : [])
.filter((item) => item && (item.type === "browser-process-sample" || item.process || item.pages))
.map((item) => ({ ...item, tsMs: Date.parse(String(item.ts || "")) }))
@@ -3205,8 +3290,11 @@ function buildBrowserProcessReport(rows) {
growthRedMb: alertThresholds.browserRssGrowthRedMb,
growthWindowMs: alertThresholds.browserRssGrowthWindowMs,
responsivenessRedMs: alertThresholds.playwrightResponsivenessRedMs,
freezeBlockerCount: blockerEvents.length,
browserFreezePolicy,
valuesRedacted: true,
},
blockerEvents: blockerEvents.slice(-20),
memorySamples: memorySamples.slice(-60),
growthSamples: computedGrowth.slice(-60),
maxTotalRssSample: maxTotal,
@@ -3220,10 +3308,112 @@ function buildBrowserProcessReport(rows) {
};
}
function compactBrowserFreezeBlockerEvent(item) {
if (!item || typeof item !== "object") return null;
const observed = objectValue(item.observed);
const threshold = objectValue(item.threshold);
const sample = objectValue(item.sample);
const page = objectValue(item.page);
const browserKill = objectValue(item.browserKill);
return {
ts: item.ts ?? null,
seq: item.seq ?? null,
sampleSeq: item.sampleSeq ?? sample.sampleSeq ?? null,
kind: item.kind ?? null,
severity: item.severity ?? "red",
blocking: item.blocking === true,
summary: item.summary ? String(item.summary).slice(0, 240) : null,
rootCause: item.rootCause ?? null,
rootCauseStatus: item.rootCauseStatus ?? null,
rootCauseConfidence: item.rootCauseConfidence ?? null,
policySource: item.policySource ?? null,
fallbackAllowed: item.fallbackAllowed === true,
observerRefreshAllowed: item.observerRefreshAllowed === true,
observed: {
totalRssMb: numberOrNull(observed.totalRssMb),
processRssMb: numberOrNull(observed.processRssMb),
totalGrowthMb: numberOrNull(observed.totalGrowthMb),
processGrowthMb: numberOrNull(observed.processGrowthMb),
responsivenessLatencyMs: numberOrNull(observed.responsivenessLatencyMs),
responsivenessTimeout: observed.responsivenessTimeout === true,
cdpMetricsTimeoutCount: numberOrNull(observed.cdpMetricsTimeoutCount),
methods: arrayStrings(observed.methods).slice(0, 8),
valuesRedacted: true,
},
threshold: {
totalRssBlockerMb: numberOrNull(threshold.totalRssBlockerMb),
processRssBlockerMb: numberOrNull(threshold.processRssBlockerMb),
growthBlockerMb: numberOrNull(threshold.growthBlockerMb),
latencyBlockerMs: numberOrNull(threshold.latencyBlockerMs),
eventBlockerCount: numberOrNull(threshold.eventBlockerCount),
metricsTimeoutBlockerCount: numberOrNull(threshold.metricsTimeoutBlockerCount),
windowMs: numberOrNull(threshold.windowMs),
valuesRedacted: true,
},
sample: {
ts: sample.ts ?? null,
seq: sample.seq ?? null,
sampleSeq: sample.sampleSeq ?? null,
browserPid: numberOrNull(sample.browserPid),
totalRssMb: numberOrNull(sample.totalRssMb),
maxProcessRssMb: numberOrNull(sample.maxProcessRssMb),
totalRssGrowthMb: numberOrNull(sample.totalRssGrowthMb),
maxProcessRssGrowthMb: numberOrNull(sample.maxProcessRssGrowthMb),
valuesRedacted: true,
},
page: {
pageRole: page.pageRole ?? null,
pageId: page.pageId ?? null,
pageEpoch: numberOrNull(page.pageEpoch),
timeoutMs: numberOrNull(page.timeoutMs),
responsivenessLatencyMs: numberOrNull(page.responsivenessLatencyMs),
responsivenessTimeout: page.responsivenessTimeout === true,
cdpTimeoutCount: numberOrNull(page.cdpTimeoutCount),
cdpErrorCount: numberOrNull(page.cdpErrorCount),
valuesRedacted: true,
},
browserKill: {
ok: browserKill.ok === true,
pending: browserKill.pending === true,
skipped: browserKill.skipped === true,
reason: browserKill.reason ?? null,
pid: numberOrNull(browserKill.pid),
gracefulSignal: browserKill.gracefulSignal ?? null,
forceSignal: browserKill.forceSignal ?? null,
gracefulSent: browserKill.gracefulSent === true,
forceSent: browserKill.forceSent === true,
exitedAfterGrace: browserKill.exitedAfterGrace === true,
exitedAfterForce: browserKill.exitedAfterForce === true,
valuesRedacted: true,
},
valuesRedacted: true,
};
}
function buildBrowserProcessFindings(report, runtimeAlerts = null) {
const summary = report?.summary || {};
if (!summary || Number(summary.sampleCount ?? 0) <= 0) return [];
const findings = [];
const blockerEvents = Array.isArray(report?.blockerEvents) ? report.blockerEvents : [];
if (blockerEvents.length > 0) {
const first = blockerEvents[0] || {};
findings.push({
id: "frontend-browser-freeze-runner-blocker",
severity: "red",
summary: "web-probe runner matched YAML browserFreezePolicy, killed/stopped Chromium, and failed the observer run; do not clear this by refresh or fallback",
count: blockerEvents.length,
blocking: true,
rootCause: first.rootCause ?? "frontend_browser_freeze_policy_blocker",
rootCauseStatus: "confirmed-from-runner-browser-freeze-policy",
rootCauseConfidence: "high",
policySource: first.policySource ?? "config/hwlab-node-lanes.yaml#webProbe.browserFreezePolicy",
fallbackAllowed: false,
observerRefreshAllowed: false,
browserKilled: first.browserKill ?? null,
events: blockerEvents.slice(0, 20),
valuesRedacted: true,
});
}
if (!summary || Number(summary.sampleCount ?? 0) <= 0) return findings;
const rootCauseSignals = browserRootCauseSignals(report, runtimeAlerts);
const maxTotalRssMb = Number(summary.maxTotalRssMb ?? 0);
const maxProcessRssMb = Number(summary.maxProcessRssMb ?? 0);
@@ -32,6 +32,7 @@ const authLoginInitialDelayMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_A
const authLoginMaxDelayMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS, 10000, 0, 120000);
const navigationMaxAttempts = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_NAVIGATION_MAX_ATTEMPTS, 4, 1, 10);
const alertThresholds = parseAlertThresholds(process.env.UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON);
const browserFreezePolicy = parseBrowserFreezePolicy(process.env.UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON);
const projectManagement = parseProjectManagementConfig(process.env.UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON);
const playwrightProxy = proxyConfigFromEnv(baseUrl);
const chromiumLaunchOptions = chromiumLaunchOptionsForProxy(playwrightProxy);
@@ -79,7 +80,9 @@ let currentPageProvenance = null;
let heartbeatPulseTimer = null;
let browserProcessMonitorStop = null;
let browserProcessMonitorSeq = 0;
let browserFreezeBlocker = null;
const browserProcessHistory = [];
const browserFreezeSignalHistory = [];
const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] };
try {
@@ -117,17 +120,26 @@ try {
await writeHeartbeat({ status: "running" });
while (!stopping) {
await drainOneCommand();
if (browserFreezeBlocker) break;
await samplePage("interval");
if (browserFreezeBlocker) break;
if (maxSamples > 0 && sampleSeq >= maxSamples) {
await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples }));
break;
}
await sleep(sampleIntervalMs);
}
terminalStatus = "completed";
await writeHeartbeat({ status: "completed" });
await writeManifest({ status: "completed", completedAt: new Date().toISOString() });
process.exitCode = 0;
if (browserFreezeBlocker) {
terminalStatus = "failed";
await writeHeartbeat({ status: "failed", blocker: browserFreezeBlocker });
await writeManifest({ status: "failed", completedAt: new Date().toISOString(), blocker: browserFreezeBlocker });
process.exitCode = browserFreezePolicy.kill.exitCode;
} else {
terminalStatus = "completed";
await writeHeartbeat({ status: "completed" });
await writeManifest({ status: "completed", completedAt: new Date().toISOString() });
process.exitCode = 0;
}
} catch (error) {
terminalStatus = "failed";
await appendJsonl(files.errors, eventRecord("runner-error", { error: errorSummary(error) })).catch(() => {});
@@ -196,6 +208,7 @@ async function writeManifest(extra = {}) {
pageProvenance: compactPageProvenance(currentPageProvenance),
sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerRefreshIntervalMs, observerInitiatedDefault: false, responseBodyReadDefault: false },
alertThresholds,
browserFreezePolicy,
projectManagement,
jsonlRotation,
commandDirs: dirs,
@@ -247,7 +260,7 @@ function startBrowserProcessMonitor() {
let stopped = false;
let running = false;
const tick = async (reason) => {
if (stopped || running) return;
if (stopped || running || stopping || browserFreezeBlocker) return;
running = true;
try {
await collectBrowserProcessSample(reason || "interval");
@@ -281,7 +294,7 @@ async function collectBrowserProcessSample(reason) {
if (observerPage && !observerPage.isClosed()) {
pages.push(await collectBrowserPageRuntimeMetrics(observerPage, { pageRole: "observer", targetPageId: observerPageId, pageEpoch: observerPageEpoch }).catch((error) => browserPageRuntimeMetricError("observer", observerPageId, observerPageEpoch, error)));
}
await appendJsonl(files.browserProcess, eventRecord("browser-process-sample", {
const sample = eventRecord("browser-process-sample", {
seq: browserProcessMonitorSeq,
reason,
monitorIntervalMs: Math.max(250, Math.floor(Number(alertThresholds.browserProcessSampleIntervalMs) || 1000)),
@@ -290,7 +303,260 @@ async function collectBrowserProcessSample(reason) {
growth,
pages,
valuesRedacted: true,
}));
});
await appendJsonl(files.browserProcess, sample);
await enforceBrowserFreezePolicy(sample);
}
async function enforceBrowserFreezePolicy(sample) {
if (browserFreezePolicy.enabled !== true || browserFreezeBlocker) return;
const processSummary = sample && typeof sample.process === "object" ? sample.process : {};
const growth = sample && typeof sample.growth === "object" ? sample.growth : {};
const totalRssMb = Number(processSummary.totalRssMb);
const processRssMb = Number(processSummary.maxProcessRssMb);
const totalGrowthMb = Number(growth.totalRssGrowthMb);
const processGrowthMb = Number(growth.maxProcessRssGrowthMb);
if (Number.isFinite(totalRssMb) && totalRssMb >= browserFreezePolicy.memory.totalRssBlockerMb) {
await triggerBrowserFreezeBlocker({
kind: "memory-total-rss",
rootCause: "frontend_browser_process_memory_pressure",
observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true },
threshold: { totalRssBlockerMb: browserFreezePolicy.memory.totalRssBlockerMb, valuesRedacted: true },
sample: browserProcessSampleRef(sample),
});
return;
}
if (Number.isFinite(processRssMb) && processRssMb >= browserFreezePolicy.memory.processRssBlockerMb) {
await triggerBrowserFreezeBlocker({
kind: "memory-process-rss",
rootCause: "frontend_browser_process_memory_pressure",
observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true },
threshold: { processRssBlockerMb: browserFreezePolicy.memory.processRssBlockerMb, valuesRedacted: true },
sample: browserProcessSampleRef(sample),
});
return;
}
if (
(Number.isFinite(totalGrowthMb) && totalGrowthMb >= browserFreezePolicy.memory.growthBlockerMb)
|| (Number.isFinite(processGrowthMb) && processGrowthMb >= browserFreezePolicy.memory.growthBlockerMb)
) {
await triggerBrowserFreezeBlocker({
kind: "memory-rss-growth",
rootCause: "frontend_browser_process_memory_leak_or_unbounded_render_growth",
observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true },
threshold: { growthBlockerMb: browserFreezePolicy.memory.growthBlockerMb, windowMs: browserFreezePolicy.blockerWindowMs, valuesRedacted: true },
sample: browserProcessSampleRef(sample),
});
return;
}
for (const pageMetric of Array.isArray(sample.pages) ? sample.pages : []) {
const responsiveness = pageMetric?.responsiveness && typeof pageMetric.responsiveness === "object" ? pageMetric.responsiveness : {};
const responsivenessLatencyMs = Number(responsiveness.latencyMs);
if (responsiveness.timeout === true || (Number.isFinite(responsivenessLatencyMs) && responsivenessLatencyMs >= browserFreezePolicy.responsiveness.latencyBlockerMs)) {
const signal = recordBrowserFreezeSignal("playwright-responsiveness", sample, pageMetric, {
rootCause: "frontend_browser_page_unresponsive_to_playwright",
observed: {
responsivenessLatencyMs: Number.isFinite(responsivenessLatencyMs) ? responsivenessLatencyMs : null,
responsivenessTimeout: responsiveness.timeout === true,
valuesRedacted: true,
},
threshold: {
latencyBlockerMs: browserFreezePolicy.responsiveness.latencyBlockerMs,
eventBlockerCount: browserFreezePolicy.responsiveness.eventBlockerCount,
windowMs: browserFreezePolicy.blockerWindowMs,
valuesRedacted: true,
},
});
if (signal.burst.length >= browserFreezePolicy.responsiveness.eventBlockerCount) {
await triggerBrowserFreezeBlocker(signal);
return;
}
}
const cdp = pageMetric?.cdp && typeof pageMetric.cdp === "object" ? pageMetric.cdp : {};
const calls = Array.isArray(cdp.calls) ? cdp.calls : [];
const metricTimeoutCalls = calls.filter((call) => call?.timeout === true && call?.method !== "Runtime.evaluate");
const sessionTimeoutCount = calls.length === 0 ? Number(cdp.timeoutCount || 0) : 0;
const metricTimeoutCount = metricTimeoutCalls.length + (Number.isFinite(sessionTimeoutCount) ? sessionTimeoutCount : 0);
if (metricTimeoutCount > 0) {
const signal = recordBrowserFreezeSignal("cdp-metrics-timeout", sample, pageMetric, {
rootCause: "frontend_browser_cdp_metrics_unresponsive",
observed: {
cdpMetricsTimeoutCount: metricTimeoutCount,
methods: metricTimeoutCalls.map((call) => call.method || "unknown").slice(0, 8),
valuesRedacted: true,
},
threshold: {
metricsTimeoutBlockerCount: browserFreezePolicy.cdp.metricsTimeoutBlockerCount,
windowMs: browserFreezePolicy.blockerWindowMs,
valuesRedacted: true,
},
});
if (signal.burst.length >= browserFreezePolicy.cdp.metricsTimeoutBlockerCount) {
await triggerBrowserFreezeBlocker(signal);
return;
}
}
}
}
function recordBrowserFreezeSignal(kind, sample, pageMetric, detail) {
const tsMs = Date.parse(String(sample?.ts || ""));
const signal = {
kind,
ts: sample?.ts ?? new Date().toISOString(),
tsMs: Number.isFinite(tsMs) ? tsMs : Date.now(),
pageRole: pageMetric?.pageRole ?? null,
pageId: pageMetric?.pageId ?? null,
pageEpoch: pageMetric?.pageEpoch ?? null,
sample: browserProcessSampleRef(sample),
page: browserPageMetricRef(pageMetric),
...detail,
valuesRedacted: true,
};
browserFreezeSignalHistory.push(signal);
const windowMs = browserFreezePolicy.blockerWindowMs;
const cutoff = signal.tsMs - windowMs;
while (browserFreezeSignalHistory.length > 0 && Number(browserFreezeSignalHistory[0].tsMs || 0) < cutoff) browserFreezeSignalHistory.shift();
return {
kind,
rootCause: detail.rootCause,
observed: detail.observed,
threshold: detail.threshold,
sample: browserProcessSampleRef(sample),
page: browserPageMetricRef(pageMetric),
burst: browserFreezeSignalHistory.filter((item) => item.kind === kind && item.tsMs >= cutoff).slice(-20),
valuesRedacted: true,
};
}
async function triggerBrowserFreezeBlocker(trigger) {
if (browserFreezeBlocker) return;
stopping = true;
terminalStatus = "failed";
const base = {
id: "browser-freeze-policy-blocker",
severity: "red",
blocking: true,
kind: trigger.kind,
summary: "web-probe runner matched YAML browserFreezePolicy and stopped the browser; this run must stay red instead of refreshing or falling back",
rootCause: trigger.rootCause,
rootCauseStatus: "confirmed-from-runner-browser-freeze-policy",
rootCauseConfidence: "high",
fallbackAllowed: false,
observerRefreshAllowed: false,
policySource: "config/hwlab-node-lanes.yaml#webProbe.browserFreezePolicy via UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON",
observed: trigger.observed || null,
threshold: trigger.threshold || null,
sample: trigger.sample || null,
page: trigger.page || null,
burst: Array.isArray(trigger.burst) ? trigger.burst.slice(0, 20) : [],
valuesRedacted: true,
};
browserFreezeBlocker = { ...base, browserKill: { ok: false, pending: true, valuesRedacted: true }, valuesRedacted: true };
const browserKill = browserFreezePolicy.kill.enabled === true
? await terminateBrowserForFreezeBlocker(base).catch((error) => ({ ok: false, error: errorSummary(error), valuesRedacted: true }))
: { ok: true, skipped: true, reason: "kill-disabled-by-yaml-policy", valuesRedacted: true };
browserFreezeBlocker = { ...base, browserKill, valuesRedacted: true };
await appendJsonl(files.browserProcess, eventRecord("browser-freeze-blocker", browserFreezeBlocker)).catch(() => {});
await appendJsonl(files.errors, eventRecord("browser-freeze-blocker", {
...browserFreezeBlocker,
error: {
name: "BrowserFreezeBlocker",
message: "YAML browserFreezePolicy matched: " + String(trigger.kind || "unknown"),
details: browserFreezeBlocker,
valuesRedacted: true,
},
})).catch(() => {});
await writeHeartbeat({ status: "failed", blocker: browserFreezeBlocker }).catch(() => {});
await writeManifest({ status: "failed", blocker: browserFreezeBlocker }).catch(() => {});
}
async function terminateBrowserForFreezeBlocker(blocker) {
const childProcess = browser && typeof browser.process === "function" ? browser.process() : null;
const pid = Number(childProcess?.pid);
if (!Number.isFinite(pid) || pid <= 0) {
return { ok: false, reason: "browser-process-unavailable", blockerKind: blocker.kind, valuesRedacted: true };
}
const result = {
ok: false,
pid: Math.floor(pid),
gracefulSignal: browserFreezePolicy.kill.gracefulSignal,
forceSignal: browserFreezePolicy.kill.forceSignal,
graceMs: browserFreezePolicy.kill.graceMs,
pollIntervalMs: browserFreezePolicy.kill.pollIntervalMs,
gracefulSent: false,
forceSent: false,
exitedAfterGrace: false,
exitedAfterForce: false,
valuesRedacted: true,
};
try {
childProcess.kill(browserFreezePolicy.kill.gracefulSignal);
result.gracefulSent = true;
} catch (error) {
result.gracefulError = errorSummary(error);
}
result.exitedAfterGrace = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
if (!result.exitedAfterGrace) {
try {
childProcess.kill(browserFreezePolicy.kill.forceSignal);
result.forceSent = true;
} catch (error) {
result.forceError = errorSummary(error);
}
result.exitedAfterForce = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
}
result.ok = result.exitedAfterGrace || result.exitedAfterForce;
return result;
}
async function waitForPidExit(pid, timeoutMs, pollIntervalMs) {
const deadline = Date.now() + timeoutMs;
while (Date.now() <= deadline) {
if (!pidAlive(pid)) return true;
await sleep(pollIntervalMs);
}
return !pidAlive(pid);
}
function pidAlive(pid) {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
function browserProcessSampleRef(sample) {
return {
ts: sample?.ts ?? null,
seq: sample?.seq ?? null,
sampleSeq: sample?.sampleSeq ?? null,
browserPid: sample?.browserPid ?? null,
totalRssMb: sample?.process?.totalRssMb ?? null,
maxProcessRssMb: sample?.process?.maxProcessRssMb ?? null,
totalRssGrowthMb: sample?.growth?.totalRssGrowthMb ?? null,
maxProcessRssGrowthMb: sample?.growth?.maxProcessRssGrowthMb ?? null,
valuesRedacted: true,
};
}
function browserPageMetricRef(pageMetric) {
const responsiveness = pageMetric?.responsiveness && typeof pageMetric.responsiveness === "object" ? pageMetric.responsiveness : {};
const cdp = pageMetric?.cdp && typeof pageMetric.cdp === "object" ? pageMetric.cdp : {};
return {
pageRole: pageMetric?.pageRole ?? null,
pageId: pageMetric?.pageId ?? null,
pageEpoch: pageMetric?.pageEpoch ?? null,
timeoutMs: pageMetric?.timeoutMs ?? null,
responsivenessLatencyMs: responsiveness.latencyMs ?? null,
responsivenessTimeout: responsiveness.timeout === true,
cdpTimeoutCount: cdp.timeoutCount ?? null,
cdpErrorCount: cdp.errorCount ?? null,
valuesRedacted: true,
};
}
function browserProcessPid() {
@@ -5070,6 +5336,85 @@ function parseAlertThresholds(value) {
};
}
function parseBrowserFreezePolicy(value) {
if (!value) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is required; configure config/hwlab-node-lanes.yaml webProbe.browserFreezePolicy for the selected node/lane");
}
const raw = (() => {
try { return JSON.parse(value); } catch (error) { throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is invalid JSON: " + (error instanceof Error ? error.message : String(error))); }
})();
const memory = requiredPolicyRecord(raw, "memory", "webProbe.browserFreezePolicy");
const responsiveness = requiredPolicyRecord(raw, "responsiveness", "webProbe.browserFreezePolicy");
const cdp = requiredPolicyRecord(raw, "cdp", "webProbe.browserFreezePolicy");
const kill = requiredPolicyRecord(raw, "kill", "webProbe.browserFreezePolicy");
return {
enabled: requiredPolicyBoolean(raw, "enabled", "webProbe.browserFreezePolicy"),
blockerWindowMs: requiredPolicyPositiveNumber(raw, "blockerWindowMs", "webProbe.browserFreezePolicy"),
memory: {
totalRssBlockerMb: requiredPolicyPositiveNumber(memory, "totalRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
processRssBlockerMb: requiredPolicyPositiveNumber(memory, "processRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
growthBlockerMb: requiredPolicyPositiveNumber(memory, "growthBlockerMb", "webProbe.browserFreezePolicy.memory"),
},
responsiveness: {
latencyBlockerMs: requiredPolicyPositiveNumber(responsiveness, "latencyBlockerMs", "webProbe.browserFreezePolicy.responsiveness"),
eventBlockerCount: requiredPolicyPositiveNumber(responsiveness, "eventBlockerCount", "webProbe.browserFreezePolicy.responsiveness"),
},
cdp: {
metricsTimeoutBlockerCount: requiredPolicyPositiveNumber(cdp, "metricsTimeoutBlockerCount", "webProbe.browserFreezePolicy.cdp"),
},
kill: {
enabled: requiredPolicyBoolean(kill, "enabled", "webProbe.browserFreezePolicy.kill"),
gracefulSignal: requiredPolicySignal(kill, "gracefulSignal", "webProbe.browserFreezePolicy.kill", "SIGTERM"),
forceSignal: requiredPolicySignal(kill, "forceSignal", "webProbe.browserFreezePolicy.kill", "SIGKILL"),
graceMs: requiredPolicyIntegerInRange(kill, "graceMs", "webProbe.browserFreezePolicy.kill", 1, 120000),
pollIntervalMs: requiredPolicyIntegerInRange(kill, "pollIntervalMs", "webProbe.browserFreezePolicy.kill", 1, 10000),
exitCode: requiredPolicyIntegerInRange(kill, "exitCode", "webProbe.browserFreezePolicy.kill", 1, 125),
},
source: "yaml-env",
valuesRedacted: true,
};
}
function requiredPolicyRecord(raw, key, pathLabel) {
const value = raw?.[key];
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires object " + pathLabel + "." + key);
}
return value;
}
function requiredPolicyBoolean(raw, key, pathLabel) {
const value = raw?.[key];
if (typeof value !== "boolean") {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires boolean " + pathLabel + "." + key);
}
return value;
}
function requiredPolicyPositiveNumber(raw, key, pathLabel) {
const value = Number(raw?.[key]);
if (!Number.isFinite(value) || value <= 0) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires positive number " + pathLabel + "." + key);
}
return value;
}
function requiredPolicyIntegerInRange(raw, key, pathLabel, min, max) {
const value = Number(raw?.[key]);
if (!Number.isInteger(value) || value < min || value > max) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires integer " + pathLabel + "." + key + " between " + min + " and " + max);
}
return value;
}
function requiredPolicySignal(raw, key, pathLabel, expected) {
const value = String(raw?.[key] || "");
if (value !== expected) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires " + pathLabel + "." + key + "=" + expected);
}
return value;
}
function parseProjectManagementConfig(value) {
if (!value || value === "null") {
return {
+16 -2
View File
@@ -14,7 +14,7 @@ import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeAuthLoginSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeAuthLoginSpec, type HwlabRuntimeWebProbeBrowserFreezePolicySpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
@@ -1141,6 +1141,14 @@ export function nodeWebProbeAlertThresholds(spec: HwlabRuntimeLaneSpec): HwlabRu
return thresholds;
}
export function nodeWebProbeBrowserFreezePolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeBrowserFreezePolicySpec {
const policy = spec.webProbe?.browserFreezePolicy;
if (policy === undefined) {
throw new Error(`${hwlabRuntimeLaneConfigPath()} node=${spec.nodeId} lane=${spec.lane} requires webProbe.browserFreezePolicy for web-probe observe`);
}
return policy;
}
export function nodeWebProbeProjectManagementConfig(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeProjectManagementSpec | null {
return spec.webProbe?.projectManagement ?? null;
}
@@ -1311,6 +1319,7 @@ export function runNodeWebProbeObserveStart(
const runnerB64Body = runnerB64.match(/.{1,76}/gu)?.join("\n") ?? runnerB64;
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
const alertThresholds = nodeWebProbeAlertThresholds(spec);
const browserFreezePolicy = nodeWebProbeBrowserFreezePolicy(spec);
const projectManagement = nodeWebProbeProjectManagementConfig(spec);
const authLogin = nodeWebProbeAuthLoginConfig(spec);
const runnerEnvAssignments = [
@@ -1332,6 +1341,7 @@ export function runNodeWebProbeObserveStart(
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
`UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=${shellQuote(JSON.stringify(browserFreezePolicy))}`,
`UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`,
...(authLogin === null
? []
@@ -1407,6 +1417,7 @@ export function runNodeWebProbeObserveStart(
url: options.url,
network: webProbeProxy.summary,
alertThresholds,
browserFreezePolicy,
projectManagement,
targetPath: options.targetPath,
id: observerId,
@@ -1739,6 +1750,7 @@ function observeRecord(value: unknown): Record<string, unknown> {
export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
const analyzerB64 = Buffer.from(nodeWebObserveAnalyzerSource(), "utf8").toString("base64");
const alertThresholds = nodeWebProbeAlertThresholds(spec);
const browserFreezePolicy = nodeWebProbeBrowserFreezePolicy(spec);
const projectManagement = nodeWebProbeProjectManagementConfig(spec);
const script = [
"set -eu",
@@ -1758,8 +1770,9 @@ export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOption
`UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=${shellQuote(options.analyzeArchivePrefix ?? "")}`,
`UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES=${shellQuote(options.analyzeTailSamples === null ? "" : String(options.analyzeTailSamples))}`,
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
`UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=${shellQuote(JSON.stringify(browserFreezePolicy))}`,
`UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`,
"UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=\"$UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX\" UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES=\"$UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES\" UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=\"$UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON\" UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=\"$UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON\" node \"$analyzer\" >\"$analysis_stdout\" 2>\"$analysis_stderr\"",
"UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=\"$UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX\" UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES=\"$UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES\" UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=\"$UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON\" UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=\"$UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON\" UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=\"$UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON\" node \"$analyzer\" >\"$analysis_stdout\" 2>\"$analysis_stderr\"",
"analyzer_exit=$?",
"set -e",
"report_json=\"$state_dir/analysis/report.json\"",
@@ -2169,6 +2182,7 @@ export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOption
analysis: failureAnalysis,
failure: analysisFailure,
alertThresholds,
browserFreezePolicy,
wrapper: buildWebObserveWrapperForObserveOptions("analyze", options, spec.workspace),
result: analysis === null ? compactCommandResultWithStdoutTail(result) : compactCommandResult(result),
valuesRedacted: true,