feat: block frozen web probe browsers from yaml policy

This commit is contained in:
Codex
2026-06-30 11:07:52 +00:00
parent 81c48b8126
commit 6168ec60f6
5 changed files with 683 additions and 10 deletions
@@ -32,6 +32,7 @@ const authLoginInitialDelayMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_A
const authLoginMaxDelayMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS, 10000, 0, 120000);
const navigationMaxAttempts = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_NAVIGATION_MAX_ATTEMPTS, 4, 1, 10);
const alertThresholds = parseAlertThresholds(process.env.UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON);
const browserFreezePolicy = parseBrowserFreezePolicy(process.env.UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON);
const projectManagement = parseProjectManagementConfig(process.env.UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON);
const playwrightProxy = proxyConfigFromEnv(baseUrl);
const chromiumLaunchOptions = chromiumLaunchOptionsForProxy(playwrightProxy);
@@ -79,7 +80,9 @@ let currentPageProvenance = null;
let heartbeatPulseTimer = null;
let browserProcessMonitorStop = null;
let browserProcessMonitorSeq = 0;
let browserFreezeBlocker = null;
const browserProcessHistory = [];
const browserFreezeSignalHistory = [];
const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] };
try {
@@ -117,17 +120,26 @@ try {
await writeHeartbeat({ status: "running" });
while (!stopping) {
await drainOneCommand();
if (browserFreezeBlocker) break;
await samplePage("interval");
if (browserFreezeBlocker) break;
if (maxSamples > 0 && sampleSeq >= maxSamples) {
await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples }));
break;
}
await sleep(sampleIntervalMs);
}
terminalStatus = "completed";
await writeHeartbeat({ status: "completed" });
await writeManifest({ status: "completed", completedAt: new Date().toISOString() });
process.exitCode = 0;
if (browserFreezeBlocker) {
terminalStatus = "failed";
await writeHeartbeat({ status: "failed", blocker: browserFreezeBlocker });
await writeManifest({ status: "failed", completedAt: new Date().toISOString(), blocker: browserFreezeBlocker });
process.exitCode = browserFreezePolicy.kill.exitCode;
} else {
terminalStatus = "completed";
await writeHeartbeat({ status: "completed" });
await writeManifest({ status: "completed", completedAt: new Date().toISOString() });
process.exitCode = 0;
}
} catch (error) {
terminalStatus = "failed";
await appendJsonl(files.errors, eventRecord("runner-error", { error: errorSummary(error) })).catch(() => {});
@@ -196,6 +208,7 @@ async function writeManifest(extra = {}) {
pageProvenance: compactPageProvenance(currentPageProvenance),
sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerRefreshIntervalMs, observerInitiatedDefault: false, responseBodyReadDefault: false },
alertThresholds,
browserFreezePolicy,
projectManagement,
jsonlRotation,
commandDirs: dirs,
@@ -247,7 +260,7 @@ function startBrowserProcessMonitor() {
let stopped = false;
let running = false;
const tick = async (reason) => {
if (stopped || running) return;
if (stopped || running || stopping || browserFreezeBlocker) return;
running = true;
try {
await collectBrowserProcessSample(reason || "interval");
@@ -281,7 +294,7 @@ async function collectBrowserProcessSample(reason) {
if (observerPage && !observerPage.isClosed()) {
pages.push(await collectBrowserPageRuntimeMetrics(observerPage, { pageRole: "observer", targetPageId: observerPageId, pageEpoch: observerPageEpoch }).catch((error) => browserPageRuntimeMetricError("observer", observerPageId, observerPageEpoch, error)));
}
await appendJsonl(files.browserProcess, eventRecord("browser-process-sample", {
const sample = eventRecord("browser-process-sample", {
seq: browserProcessMonitorSeq,
reason,
monitorIntervalMs: Math.max(250, Math.floor(Number(alertThresholds.browserProcessSampleIntervalMs) || 1000)),
@@ -290,7 +303,260 @@ async function collectBrowserProcessSample(reason) {
growth,
pages,
valuesRedacted: true,
}));
});
await appendJsonl(files.browserProcess, sample);
await enforceBrowserFreezePolicy(sample);
}
async function enforceBrowserFreezePolicy(sample) {
if (browserFreezePolicy.enabled !== true || browserFreezeBlocker) return;
const processSummary = sample && typeof sample.process === "object" ? sample.process : {};
const growth = sample && typeof sample.growth === "object" ? sample.growth : {};
const totalRssMb = Number(processSummary.totalRssMb);
const processRssMb = Number(processSummary.maxProcessRssMb);
const totalGrowthMb = Number(growth.totalRssGrowthMb);
const processGrowthMb = Number(growth.maxProcessRssGrowthMb);
if (Number.isFinite(totalRssMb) && totalRssMb >= browserFreezePolicy.memory.totalRssBlockerMb) {
await triggerBrowserFreezeBlocker({
kind: "memory-total-rss",
rootCause: "frontend_browser_process_memory_pressure",
observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true },
threshold: { totalRssBlockerMb: browserFreezePolicy.memory.totalRssBlockerMb, valuesRedacted: true },
sample: browserProcessSampleRef(sample),
});
return;
}
if (Number.isFinite(processRssMb) && processRssMb >= browserFreezePolicy.memory.processRssBlockerMb) {
await triggerBrowserFreezeBlocker({
kind: "memory-process-rss",
rootCause: "frontend_browser_process_memory_pressure",
observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true },
threshold: { processRssBlockerMb: browserFreezePolicy.memory.processRssBlockerMb, valuesRedacted: true },
sample: browserProcessSampleRef(sample),
});
return;
}
if (
(Number.isFinite(totalGrowthMb) && totalGrowthMb >= browserFreezePolicy.memory.growthBlockerMb)
|| (Number.isFinite(processGrowthMb) && processGrowthMb >= browserFreezePolicy.memory.growthBlockerMb)
) {
await triggerBrowserFreezeBlocker({
kind: "memory-rss-growth",
rootCause: "frontend_browser_process_memory_leak_or_unbounded_render_growth",
observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true },
threshold: { growthBlockerMb: browserFreezePolicy.memory.growthBlockerMb, windowMs: browserFreezePolicy.blockerWindowMs, valuesRedacted: true },
sample: browserProcessSampleRef(sample),
});
return;
}
for (const pageMetric of Array.isArray(sample.pages) ? sample.pages : []) {
const responsiveness = pageMetric?.responsiveness && typeof pageMetric.responsiveness === "object" ? pageMetric.responsiveness : {};
const responsivenessLatencyMs = Number(responsiveness.latencyMs);
if (responsiveness.timeout === true || (Number.isFinite(responsivenessLatencyMs) && responsivenessLatencyMs >= browserFreezePolicy.responsiveness.latencyBlockerMs)) {
const signal = recordBrowserFreezeSignal("playwright-responsiveness", sample, pageMetric, {
rootCause: "frontend_browser_page_unresponsive_to_playwright",
observed: {
responsivenessLatencyMs: Number.isFinite(responsivenessLatencyMs) ? responsivenessLatencyMs : null,
responsivenessTimeout: responsiveness.timeout === true,
valuesRedacted: true,
},
threshold: {
latencyBlockerMs: browserFreezePolicy.responsiveness.latencyBlockerMs,
eventBlockerCount: browserFreezePolicy.responsiveness.eventBlockerCount,
windowMs: browserFreezePolicy.blockerWindowMs,
valuesRedacted: true,
},
});
if (signal.burst.length >= browserFreezePolicy.responsiveness.eventBlockerCount) {
await triggerBrowserFreezeBlocker(signal);
return;
}
}
const cdp = pageMetric?.cdp && typeof pageMetric.cdp === "object" ? pageMetric.cdp : {};
const calls = Array.isArray(cdp.calls) ? cdp.calls : [];
const metricTimeoutCalls = calls.filter((call) => call?.timeout === true && call?.method !== "Runtime.evaluate");
const sessionTimeoutCount = calls.length === 0 ? Number(cdp.timeoutCount || 0) : 0;
const metricTimeoutCount = metricTimeoutCalls.length + (Number.isFinite(sessionTimeoutCount) ? sessionTimeoutCount : 0);
if (metricTimeoutCount > 0) {
const signal = recordBrowserFreezeSignal("cdp-metrics-timeout", sample, pageMetric, {
rootCause: "frontend_browser_cdp_metrics_unresponsive",
observed: {
cdpMetricsTimeoutCount: metricTimeoutCount,
methods: metricTimeoutCalls.map((call) => call.method || "unknown").slice(0, 8),
valuesRedacted: true,
},
threshold: {
metricsTimeoutBlockerCount: browserFreezePolicy.cdp.metricsTimeoutBlockerCount,
windowMs: browserFreezePolicy.blockerWindowMs,
valuesRedacted: true,
},
});
if (signal.burst.length >= browserFreezePolicy.cdp.metricsTimeoutBlockerCount) {
await triggerBrowserFreezeBlocker(signal);
return;
}
}
}
}
function recordBrowserFreezeSignal(kind, sample, pageMetric, detail) {
const tsMs = Date.parse(String(sample?.ts || ""));
const signal = {
kind,
ts: sample?.ts ?? new Date().toISOString(),
tsMs: Number.isFinite(tsMs) ? tsMs : Date.now(),
pageRole: pageMetric?.pageRole ?? null,
pageId: pageMetric?.pageId ?? null,
pageEpoch: pageMetric?.pageEpoch ?? null,
sample: browserProcessSampleRef(sample),
page: browserPageMetricRef(pageMetric),
...detail,
valuesRedacted: true,
};
browserFreezeSignalHistory.push(signal);
const windowMs = browserFreezePolicy.blockerWindowMs;
const cutoff = signal.tsMs - windowMs;
while (browserFreezeSignalHistory.length > 0 && Number(browserFreezeSignalHistory[0].tsMs || 0) < cutoff) browserFreezeSignalHistory.shift();
return {
kind,
rootCause: detail.rootCause,
observed: detail.observed,
threshold: detail.threshold,
sample: browserProcessSampleRef(sample),
page: browserPageMetricRef(pageMetric),
burst: browserFreezeSignalHistory.filter((item) => item.kind === kind && item.tsMs >= cutoff).slice(-20),
valuesRedacted: true,
};
}
async function triggerBrowserFreezeBlocker(trigger) {
if (browserFreezeBlocker) return;
stopping = true;
terminalStatus = "failed";
const base = {
id: "browser-freeze-policy-blocker",
severity: "red",
blocking: true,
kind: trigger.kind,
summary: "web-probe runner matched YAML browserFreezePolicy and stopped the browser; this run must stay red instead of refreshing or falling back",
rootCause: trigger.rootCause,
rootCauseStatus: "confirmed-from-runner-browser-freeze-policy",
rootCauseConfidence: "high",
fallbackAllowed: false,
observerRefreshAllowed: false,
policySource: "config/hwlab-node-lanes.yaml#webProbe.browserFreezePolicy via UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON",
observed: trigger.observed || null,
threshold: trigger.threshold || null,
sample: trigger.sample || null,
page: trigger.page || null,
burst: Array.isArray(trigger.burst) ? trigger.burst.slice(0, 20) : [],
valuesRedacted: true,
};
browserFreezeBlocker = { ...base, browserKill: { ok: false, pending: true, valuesRedacted: true }, valuesRedacted: true };
const browserKill = browserFreezePolicy.kill.enabled === true
? await terminateBrowserForFreezeBlocker(base).catch((error) => ({ ok: false, error: errorSummary(error), valuesRedacted: true }))
: { ok: true, skipped: true, reason: "kill-disabled-by-yaml-policy", valuesRedacted: true };
browserFreezeBlocker = { ...base, browserKill, valuesRedacted: true };
await appendJsonl(files.browserProcess, eventRecord("browser-freeze-blocker", browserFreezeBlocker)).catch(() => {});
await appendJsonl(files.errors, eventRecord("browser-freeze-blocker", {
...browserFreezeBlocker,
error: {
name: "BrowserFreezeBlocker",
message: "YAML browserFreezePolicy matched: " + String(trigger.kind || "unknown"),
details: browserFreezeBlocker,
valuesRedacted: true,
},
})).catch(() => {});
await writeHeartbeat({ status: "failed", blocker: browserFreezeBlocker }).catch(() => {});
await writeManifest({ status: "failed", blocker: browserFreezeBlocker }).catch(() => {});
}
async function terminateBrowserForFreezeBlocker(blocker) {
const childProcess = browser && typeof browser.process === "function" ? browser.process() : null;
const pid = Number(childProcess?.pid);
if (!Number.isFinite(pid) || pid <= 0) {
return { ok: false, reason: "browser-process-unavailable", blockerKind: blocker.kind, valuesRedacted: true };
}
const result = {
ok: false,
pid: Math.floor(pid),
gracefulSignal: browserFreezePolicy.kill.gracefulSignal,
forceSignal: browserFreezePolicy.kill.forceSignal,
graceMs: browserFreezePolicy.kill.graceMs,
pollIntervalMs: browserFreezePolicy.kill.pollIntervalMs,
gracefulSent: false,
forceSent: false,
exitedAfterGrace: false,
exitedAfterForce: false,
valuesRedacted: true,
};
try {
childProcess.kill(browserFreezePolicy.kill.gracefulSignal);
result.gracefulSent = true;
} catch (error) {
result.gracefulError = errorSummary(error);
}
result.exitedAfterGrace = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
if (!result.exitedAfterGrace) {
try {
childProcess.kill(browserFreezePolicy.kill.forceSignal);
result.forceSent = true;
} catch (error) {
result.forceError = errorSummary(error);
}
result.exitedAfterForce = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
}
result.ok = result.exitedAfterGrace || result.exitedAfterForce;
return result;
}
async function waitForPidExit(pid, timeoutMs, pollIntervalMs) {
const deadline = Date.now() + timeoutMs;
while (Date.now() <= deadline) {
if (!pidAlive(pid)) return true;
await sleep(pollIntervalMs);
}
return !pidAlive(pid);
}
function pidAlive(pid) {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
function browserProcessSampleRef(sample) {
return {
ts: sample?.ts ?? null,
seq: sample?.seq ?? null,
sampleSeq: sample?.sampleSeq ?? null,
browserPid: sample?.browserPid ?? null,
totalRssMb: sample?.process?.totalRssMb ?? null,
maxProcessRssMb: sample?.process?.maxProcessRssMb ?? null,
totalRssGrowthMb: sample?.growth?.totalRssGrowthMb ?? null,
maxProcessRssGrowthMb: sample?.growth?.maxProcessRssGrowthMb ?? null,
valuesRedacted: true,
};
}
function browserPageMetricRef(pageMetric) {
const responsiveness = pageMetric?.responsiveness && typeof pageMetric.responsiveness === "object" ? pageMetric.responsiveness : {};
const cdp = pageMetric?.cdp && typeof pageMetric.cdp === "object" ? pageMetric.cdp : {};
return {
pageRole: pageMetric?.pageRole ?? null,
pageId: pageMetric?.pageId ?? null,
pageEpoch: pageMetric?.pageEpoch ?? null,
timeoutMs: pageMetric?.timeoutMs ?? null,
responsivenessLatencyMs: responsiveness.latencyMs ?? null,
responsivenessTimeout: responsiveness.timeout === true,
cdpTimeoutCount: cdp.timeoutCount ?? null,
cdpErrorCount: cdp.errorCount ?? null,
valuesRedacted: true,
};
}
function browserProcessPid() {
@@ -5070,6 +5336,85 @@ function parseAlertThresholds(value) {
};
}
function parseBrowserFreezePolicy(value) {
if (!value) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is required; configure config/hwlab-node-lanes.yaml webProbe.browserFreezePolicy for the selected node/lane");
}
const raw = (() => {
try { return JSON.parse(value); } catch (error) { throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is invalid JSON: " + (error instanceof Error ? error.message : String(error))); }
})();
const memory = requiredPolicyRecord(raw, "memory", "webProbe.browserFreezePolicy");
const responsiveness = requiredPolicyRecord(raw, "responsiveness", "webProbe.browserFreezePolicy");
const cdp = requiredPolicyRecord(raw, "cdp", "webProbe.browserFreezePolicy");
const kill = requiredPolicyRecord(raw, "kill", "webProbe.browserFreezePolicy");
return {
enabled: requiredPolicyBoolean(raw, "enabled", "webProbe.browserFreezePolicy"),
blockerWindowMs: requiredPolicyPositiveNumber(raw, "blockerWindowMs", "webProbe.browserFreezePolicy"),
memory: {
totalRssBlockerMb: requiredPolicyPositiveNumber(memory, "totalRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
processRssBlockerMb: requiredPolicyPositiveNumber(memory, "processRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
growthBlockerMb: requiredPolicyPositiveNumber(memory, "growthBlockerMb", "webProbe.browserFreezePolicy.memory"),
},
responsiveness: {
latencyBlockerMs: requiredPolicyPositiveNumber(responsiveness, "latencyBlockerMs", "webProbe.browserFreezePolicy.responsiveness"),
eventBlockerCount: requiredPolicyPositiveNumber(responsiveness, "eventBlockerCount", "webProbe.browserFreezePolicy.responsiveness"),
},
cdp: {
metricsTimeoutBlockerCount: requiredPolicyPositiveNumber(cdp, "metricsTimeoutBlockerCount", "webProbe.browserFreezePolicy.cdp"),
},
kill: {
enabled: requiredPolicyBoolean(kill, "enabled", "webProbe.browserFreezePolicy.kill"),
gracefulSignal: requiredPolicySignal(kill, "gracefulSignal", "webProbe.browserFreezePolicy.kill", "SIGTERM"),
forceSignal: requiredPolicySignal(kill, "forceSignal", "webProbe.browserFreezePolicy.kill", "SIGKILL"),
graceMs: requiredPolicyIntegerInRange(kill, "graceMs", "webProbe.browserFreezePolicy.kill", 1, 120000),
pollIntervalMs: requiredPolicyIntegerInRange(kill, "pollIntervalMs", "webProbe.browserFreezePolicy.kill", 1, 10000),
exitCode: requiredPolicyIntegerInRange(kill, "exitCode", "webProbe.browserFreezePolicy.kill", 1, 125),
},
source: "yaml-env",
valuesRedacted: true,
};
}
function requiredPolicyRecord(raw, key, pathLabel) {
const value = raw?.[key];
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires object " + pathLabel + "." + key);
}
return value;
}
function requiredPolicyBoolean(raw, key, pathLabel) {
const value = raw?.[key];
if (typeof value !== "boolean") {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires boolean " + pathLabel + "." + key);
}
return value;
}
function requiredPolicyPositiveNumber(raw, key, pathLabel) {
const value = Number(raw?.[key]);
if (!Number.isFinite(value) || value <= 0) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires positive number " + pathLabel + "." + key);
}
return value;
}
function requiredPolicyIntegerInRange(raw, key, pathLabel, min, max) {
const value = Number(raw?.[key]);
if (!Number.isInteger(value) || value < min || value > max) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires integer " + pathLabel + "." + key + " between " + min + " and " + max);
}
return value;
}
function requiredPolicySignal(raw, key, pathLabel, expected) {
const value = String(raw?.[key] || "");
if (value !== expected) {
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires " + pathLabel + "." + key + "=" + expected);
}
return value;
}
function parseProjectManagementConfig(value) {
if (!value || value === "null") {
return {