fix: mark browser memory freezes red
This commit is contained in:
Codex
@@ -55,6 +55,7 @@ const files = {
|
||||
console: path.join(stateDir, "console.jsonl"),
|
||||
errors: path.join(stateDir, "errors.jsonl"),
|
||||
artifacts: path.join(stateDir, "artifacts.jsonl"),
|
||||
browserProcess: path.join(stateDir, "browser-process.jsonl"),
|
||||
};
|
||||
|
||||
let browser;
|
||||
@@ -76,6 +77,9 @@ let controlPageEpoch = 0;
|
||||
let observerPageEpoch = 0;
|
||||
let currentPageProvenance = null;
|
||||
let heartbeatPulseTimer = null;
|
||||
let browserProcessMonitorStop = null;
|
||||
let browserProcessMonitorSeq = 0;
|
||||
const browserProcessHistory = [];
|
||||
const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] };
|
||||
|
||||
try {
|
||||
@@ -107,6 +111,7 @@ try {
|
||||
}
|
||||
return result ?? { ok: false, reason: "observer-not-ready", pageRole: "observer", pageId: observerPageId, valuesRedacted: true };
|
||||
});
|
||||
browserProcessMonitorStop = startBrowserProcessMonitor();
|
||||
terminalStatus = "running";
|
||||
await writeManifest({ status: "running", auth: publicAuth(auth) });
|
||||
await writeHeartbeat({ status: "running" });
|
||||
@@ -130,6 +135,7 @@ try {
|
||||
await writeManifest({ status: "failed", error: errorSummary(error) }).catch(() => {});
|
||||
process.exitCode = 2;
|
||||
} finally {
|
||||
if (browserProcessMonitorStop) browserProcessMonitorStop();
|
||||
if (heartbeatPulseTimer) clearInterval(heartbeatPulseTimer);
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
}
|
||||
@@ -236,6 +242,339 @@ function startHeartbeatPulse() {
|
||||
return timer;
|
||||
}
|
||||
|
||||
function startBrowserProcessMonitor() {
|
||||
const intervalMs = Math.max(250, Math.floor(Number(alertThresholds.browserProcessSampleIntervalMs) || 1000));
|
||||
let stopped = false;
|
||||
let running = false;
|
||||
const tick = async (reason) => {
|
||||
if (stopped || running) return;
|
||||
running = true;
|
||||
try {
|
||||
await collectBrowserProcessSample(reason || "interval");
|
||||
} catch (error) {
|
||||
await appendJsonl(files.errors, eventRecord("browser-process-monitor-error", { error: errorSummary(error), valuesRedacted: true })).catch(() => {});
|
||||
} finally {
|
||||
running = false;
|
||||
}
|
||||
};
|
||||
void tick("startup");
|
||||
const timer = setInterval(() => {
|
||||
void tick("interval");
|
||||
}, intervalMs);
|
||||
if (timer && typeof timer.unref === "function") timer.unref();
|
||||
return () => {
|
||||
stopped = true;
|
||||
clearInterval(timer);
|
||||
};
|
||||
}
|
||||
|
||||
async function collectBrowserProcessSample(reason) {
|
||||
browserProcessMonitorSeq += 1;
|
||||
const tsMs = Date.now();
|
||||
const browserPid = browserProcessPid();
|
||||
const processSummary = await collectChromiumProcessSummary(browserPid);
|
||||
const growth = updateBrowserProcessHistory(tsMs, processSummary);
|
||||
const pages = [];
|
||||
if (page && !page.isClosed()) {
|
||||
pages.push(await collectBrowserPageRuntimeMetrics(page, { pageRole: "control", targetPageId: pageId, pageEpoch: controlPageEpoch }).catch((error) => browserPageRuntimeMetricError("control", pageId, controlPageEpoch, error)));
|
||||
}
|
||||
if (observerPage && !observerPage.isClosed()) {
|
||||
pages.push(await collectBrowserPageRuntimeMetrics(observerPage, { pageRole: "observer", targetPageId: observerPageId, pageEpoch: observerPageEpoch }).catch((error) => browserPageRuntimeMetricError("observer", observerPageId, observerPageEpoch, error)));
|
||||
}
|
||||
await appendJsonl(files.browserProcess, eventRecord("browser-process-sample", {
|
||||
seq: browserProcessMonitorSeq,
|
||||
reason,
|
||||
monitorIntervalMs: Math.max(250, Math.floor(Number(alertThresholds.browserProcessSampleIntervalMs) || 1000)),
|
||||
browserPid,
|
||||
process: processSummary,
|
||||
growth,
|
||||
pages,
|
||||
valuesRedacted: true,
|
||||
}));
|
||||
}
|
||||
|
||||
function browserProcessPid() {
|
||||
try {
|
||||
const childProcess = browser && typeof browser.process === "function" ? browser.process() : null;
|
||||
const pid = Number(childProcess?.pid);
|
||||
return Number.isFinite(pid) && pid > 0 ? Math.floor(pid) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function collectChromiumProcessSummary(browserPid) {
|
||||
const all = await readProcProcessTable();
|
||||
const childrenByPpid = new Map();
|
||||
for (const item of all) {
|
||||
if (!Number.isFinite(item.ppid)) continue;
|
||||
const list = childrenByPpid.get(item.ppid) || [];
|
||||
list.push(item.pid);
|
||||
childrenByPpid.set(item.ppid, list);
|
||||
}
|
||||
const roots = [process.pid, browserPid].filter((pid, index, items) => Number.isFinite(Number(pid)) && Number(pid) > 0 && items.indexOf(pid) === index).map((pid) => Number(pid));
|
||||
const descendants = new Set(roots);
|
||||
const queue = roots.slice();
|
||||
while (queue.length > 0) {
|
||||
const pid = queue.shift();
|
||||
for (const child of childrenByPpid.get(pid) || []) {
|
||||
if (descendants.has(child)) continue;
|
||||
descendants.add(child);
|
||||
queue.push(child);
|
||||
}
|
||||
}
|
||||
const processes = all
|
||||
.filter((item) => descendants.has(item.pid))
|
||||
.map((item) => ({ ...item, role: classifyChromiumProcess(item) }))
|
||||
.filter((item) => item.role)
|
||||
.map((item) => ({
|
||||
pid: item.pid,
|
||||
ppid: item.ppid,
|
||||
name: item.name || null,
|
||||
role: item.role,
|
||||
rssBytes: item.rssBytes,
|
||||
vmSizeBytes: item.vmSizeBytes,
|
||||
commandHash: item.cmdline ? sha256Text(item.cmdline) : null,
|
||||
commandPreview: redactProcessCommandPreview(item.cmdline),
|
||||
valuesRedacted: true,
|
||||
}))
|
||||
.sort((a, b) => Number(b.rssBytes || 0) - Number(a.rssBytes || 0));
|
||||
const roles = {};
|
||||
for (const item of processes) {
|
||||
const role = item.role || "unknown";
|
||||
const summary = roles[role] || { count: 0, rssBytes: 0, maxRssBytes: 0 };
|
||||
summary.count += 1;
|
||||
summary.rssBytes += Number(item.rssBytes || 0);
|
||||
summary.maxRssBytes = Math.max(summary.maxRssBytes, Number(item.rssBytes || 0));
|
||||
roles[role] = summary;
|
||||
}
|
||||
const totalRssBytes = processes.reduce((sum, item) => sum + Number(item.rssBytes || 0), 0);
|
||||
const maxProcess = processes[0] || null;
|
||||
return {
|
||||
sampledRootPids: roots,
|
||||
chromiumProcessCount: processes.length,
|
||||
totalRssBytes,
|
||||
totalRssMb: bytesToMb(totalRssBytes),
|
||||
maxProcessRssBytes: maxProcess ? Number(maxProcess.rssBytes || 0) : 0,
|
||||
maxProcessRssMb: maxProcess ? bytesToMb(maxProcess.rssBytes) : 0,
|
||||
maxProcess,
|
||||
roles,
|
||||
processes: processes.slice(0, 20),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function readProcProcessTable() {
|
||||
let entries = [];
|
||||
try {
|
||||
entries = await readdir("/proc");
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const numeric = entries.map((name) => Number(name)).filter((pid) => Number.isInteger(pid) && pid > 0);
|
||||
const rows = await Promise.all(numeric.map((pid) => readOneProcProcess(pid).catch(() => null)));
|
||||
return rows.filter(Boolean);
|
||||
}
|
||||
|
||||
async function readOneProcProcess(pid) {
|
||||
const [statText, statusText, cmdlineText] = await Promise.all([
|
||||
readFile(path.join("/proc", String(pid), "stat"), "utf8").catch(() => ""),
|
||||
readFile(path.join("/proc", String(pid), "status"), "utf8").catch(() => ""),
|
||||
readFile(path.join("/proc", String(pid), "cmdline"), "utf8").catch(() => ""),
|
||||
]);
|
||||
if (!statText && !statusText) return null;
|
||||
const ppid = procPpidFromStat(statText);
|
||||
const name = procStatusField(statusText, "Name") || null;
|
||||
const rssKb = procStatusKb(statusText, "VmRSS");
|
||||
const vmSizeKb = procStatusKb(statusText, "VmSize");
|
||||
return {
|
||||
pid,
|
||||
ppid,
|
||||
name,
|
||||
cmdline: String(cmdlineText || "").replace(/\0/gu, " ").replace(/\s+/gu, " ").trim(),
|
||||
rssBytes: Number.isFinite(rssKb) ? rssKb * 1024 : 0,
|
||||
vmSizeBytes: Number.isFinite(vmSizeKb) ? vmSizeKb * 1024 : 0,
|
||||
};
|
||||
}
|
||||
|
||||
function procPpidFromStat(value) {
|
||||
const text = String(value || "");
|
||||
const end = text.lastIndexOf(")");
|
||||
if (end < 0) return null;
|
||||
const parts = text.slice(end + 1).trim().split(/\s+/u);
|
||||
const ppid = Number(parts[1]);
|
||||
return Number.isFinite(ppid) ? ppid : null;
|
||||
}
|
||||
|
||||
function procStatusField(value, key) {
|
||||
const prefix = String(key || "") + ":";
|
||||
for (const line of String(value || "").split(/\n/u)) {
|
||||
if (line.startsWith(prefix)) return line.slice(prefix.length).trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function procStatusKb(value, key) {
|
||||
const raw = procStatusField(value, key);
|
||||
const match = String(raw || "").match(/^(\d+)\s+kB$/u);
|
||||
return match ? Number(match[1]) : null;
|
||||
}
|
||||
|
||||
function classifyChromiumProcess(item) {
|
||||
const cmdline = String(item?.cmdline || "");
|
||||
const combined = (cmdline + " " + String(item?.name || "")).toLowerCase();
|
||||
if (!/(?:chromium|chrome|headless_shell)/u.test(combined)) return null;
|
||||
if (/--type=renderer\b/u.test(cmdline)) return "renderer";
|
||||
if (/--type=gpu-process\b/u.test(cmdline)) return "gpu";
|
||||
if (/--type=utility\b/u.test(cmdline)) return "utility";
|
||||
if (/--type=zygote\b/u.test(cmdline)) return "zygote";
|
||||
if (/--type=broker\b/u.test(cmdline)) return "broker";
|
||||
if (/--type=/u.test(cmdline)) return "other";
|
||||
return "browser";
|
||||
}
|
||||
|
||||
function redactProcessCommandPreview(value) {
|
||||
const text = String(value || "");
|
||||
if (!text) return null;
|
||||
const parts = text.split(/\s+/u).slice(0, 18).map((part) => {
|
||||
if (/--(?:proxy|token|secret|password|cookie|auth|key)[^=]*=/iu.test(part)) return part.replace(/=.*/u, "=[redacted]");
|
||||
return part.replace(/:\/\/[^/@\s]+@/u, "://[redacted]@");
|
||||
});
|
||||
return truncate(parts.join(" "), 260);
|
||||
}
|
||||
|
||||
function updateBrowserProcessHistory(tsMs, processSummary) {
|
||||
const windowMs = Math.max(1000, Number(alertThresholds.browserRssGrowthWindowMs) || 30000);
|
||||
const totalRssBytes = Number(processSummary?.totalRssBytes || 0);
|
||||
const maxProcessRssBytes = Number(processSummary?.maxProcessRssBytes || 0);
|
||||
browserProcessHistory.push({ tsMs, totalRssBytes, maxProcessRssBytes });
|
||||
const retentionMs = Math.max(windowMs * 3, 180000);
|
||||
while (browserProcessHistory.length > 0 && tsMs - browserProcessHistory[0].tsMs > retentionMs) browserProcessHistory.shift();
|
||||
const windowStartMs = tsMs - windowMs;
|
||||
const candidates = browserProcessHistory.filter((item) => item.tsMs >= windowStartMs && item.tsMs <= tsMs);
|
||||
const baseline = candidates[0] || browserProcessHistory[0] || null;
|
||||
const totalRssGrowthBytes = baseline ? totalRssBytes - Number(baseline.totalRssBytes || 0) : 0;
|
||||
const maxProcessRssGrowthBytes = baseline ? maxProcessRssBytes - Number(baseline.maxProcessRssBytes || 0) : 0;
|
||||
return {
|
||||
windowMs,
|
||||
baselineAt: baseline ? new Date(baseline.tsMs).toISOString() : null,
|
||||
baselineTotalRssBytes: baseline ? baseline.totalRssBytes : null,
|
||||
baselineMaxProcessRssBytes: baseline ? baseline.maxProcessRssBytes : null,
|
||||
totalRssGrowthBytes,
|
||||
totalRssGrowthMb: bytesToMb(totalRssGrowthBytes),
|
||||
maxProcessRssGrowthBytes,
|
||||
maxProcessRssGrowthMb: bytesToMb(maxProcessRssGrowthBytes),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function collectBrowserPageRuntimeMetrics(targetPage, { pageRole, targetPageId, pageEpoch }) {
|
||||
const timeoutMs = Math.max(1000, Math.floor(Number(alertThresholds.playwrightResponsivenessRedMs) || 5000));
|
||||
const startedAtMs = Date.now();
|
||||
let session = null;
|
||||
const result = {
|
||||
pageRole,
|
||||
pageId: targetPageId,
|
||||
pageEpoch: Number.isFinite(Number(pageEpoch)) ? Number(pageEpoch) : 0,
|
||||
url: pageUrl(targetPage),
|
||||
timeoutMs,
|
||||
responsiveness: null,
|
||||
cdp: { timeoutCount: 0, errorCount: 0, calls: [] },
|
||||
valuesRedacted: true,
|
||||
};
|
||||
try {
|
||||
session = await withHardTimeout(targetPage.context().newCDPSession(targetPage), timeoutMs, "newCDPSession exceeded " + timeoutMs + "ms");
|
||||
const enable = await timedCdpSend(session, "Performance.enable", {}, timeoutMs);
|
||||
if (enable.ok !== true) result.cdp.calls.push({ method: "Performance.enable", ...enable });
|
||||
result.responsiveness = await timedCdpSend(session, "Runtime.evaluate", { expression: "1", returnByValue: true }, timeoutMs);
|
||||
result.cdp.calls.push({ method: "Runtime.evaluate", ...result.responsiveness });
|
||||
const performance = await timedCdpSend(session, "Performance.getMetrics", {}, timeoutMs);
|
||||
result.cdp.calls.push({ method: "Performance.getMetrics", ...performance });
|
||||
result.performance = performance.value;
|
||||
const heap = await timedCdpSend(session, "Runtime.getHeapUsage", {}, timeoutMs);
|
||||
result.cdp.calls.push({ method: "Runtime.getHeapUsage", ...heap });
|
||||
result.heapUsage = heap.value;
|
||||
const domCounters = await timedCdpSend(session, "Memory.getDOMCounters", {}, timeoutMs);
|
||||
result.cdp.calls.push({ method: "Memory.getDOMCounters", ...domCounters });
|
||||
result.domCounters = domCounters.value;
|
||||
result.cdp.timeoutCount = result.cdp.calls.filter((item) => item.timeout === true).length;
|
||||
result.cdp.errorCount = result.cdp.calls.filter((item) => item.ok !== true).length;
|
||||
} catch (error) {
|
||||
result.sessionError = errorSummary(error);
|
||||
result.cdp.timeoutCount = isTimeoutErrorMessage(error?.message) ? 1 : 0;
|
||||
result.cdp.errorCount = 1;
|
||||
} finally {
|
||||
result.latencyMs = Date.now() - startedAtMs;
|
||||
if (session) await withHardTimeout(session.detach(), 1000, "CDP session detach exceeded 1000ms").catch(() => {});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function browserPageRuntimeMetricError(pageRole, targetPageId, pageEpoch, error) {
|
||||
return {
|
||||
pageRole,
|
||||
pageId: targetPageId,
|
||||
pageEpoch: Number.isFinite(Number(pageEpoch)) ? Number(pageEpoch) : 0,
|
||||
url: null,
|
||||
timeoutMs: Math.max(1000, Math.floor(Number(alertThresholds.playwrightResponsivenessRedMs) || 5000)),
|
||||
responsiveness: { ok: false, timeout: isTimeoutErrorMessage(error?.message), error: errorSummary(error), valuesRedacted: true },
|
||||
cdp: { timeoutCount: isTimeoutErrorMessage(error?.message) ? 1 : 0, errorCount: 1, calls: [] },
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function timedCdpSend(session, method, params, timeoutMs) {
|
||||
const startedAtMs = Date.now();
|
||||
try {
|
||||
const value = await withHardTimeout(session.send(method, params || {}), timeoutMs, method + " exceeded " + timeoutMs + "ms");
|
||||
return { ok: true, timeout: false, latencyMs: Date.now() - startedAtMs, value: compactCdpValue(method, value), valuesRedacted: true };
|
||||
} catch (error) {
|
||||
return { ok: false, timeout: isTimeoutErrorMessage(error?.message), latencyMs: Date.now() - startedAtMs, error: errorSummary(error), valuesRedacted: true };
|
||||
}
|
||||
}
|
||||
|
||||
function compactCdpValue(method, value) {
|
||||
if (!value || typeof value !== "object") return value ?? null;
|
||||
if (method === "Performance.getMetrics") {
|
||||
const wanted = new Set(["Timestamp", "Documents", "Frames", "JSEventListeners", "Nodes", "LayoutCount", "RecalcStyleCount", "LayoutDuration", "RecalcStyleDuration", "ScriptDuration", "TaskDuration", "JSHeapUsedSize", "JSHeapTotalSize"]);
|
||||
const metrics = {};
|
||||
for (const item of Array.isArray(value.metrics) ? value.metrics : []) {
|
||||
if (wanted.has(item?.name)) metrics[item.name] = Number.isFinite(Number(item?.value)) ? Number(item.value) : null;
|
||||
}
|
||||
return { metricCount: Array.isArray(value.metrics) ? value.metrics.length : 0, metrics, valuesRedacted: true };
|
||||
}
|
||||
if (method === "Runtime.getHeapUsage") {
|
||||
return {
|
||||
usedSize: Number.isFinite(Number(value.usedSize)) ? Number(value.usedSize) : null,
|
||||
totalSize: Number.isFinite(Number(value.totalSize)) ? Number(value.totalSize) : null,
|
||||
embedderHeapUsedSize: Number.isFinite(Number(value.embedderHeapUsedSize)) ? Number(value.embedderHeapUsedSize) : null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
if (method === "Memory.getDOMCounters") {
|
||||
return {
|
||||
documents: Number.isFinite(Number(value.documents)) ? Number(value.documents) : null,
|
||||
nodes: Number.isFinite(Number(value.nodes)) ? Number(value.nodes) : null,
|
||||
jsEventListeners: Number.isFinite(Number(value.jsEventListeners)) ? Number(value.jsEventListeners) : null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
return { resultType: value.result?.type ?? null, unserializableValue: value.result?.unserializableValue ?? null, exceptionDetails: value.exceptionDetails ? { text: truncate(value.exceptionDetails.text || "", 200), valuesRedacted: true } : null, valuesRedacted: true };
|
||||
}
|
||||
return { valuesRedacted: true };
|
||||
}
|
||||
|
||||
function isTimeoutErrorMessage(value) {
|
||||
return /timeout|timed\s*out|exceeded\s+\d+\s*ms/iu.test(String(value || ""));
|
||||
}
|
||||
|
||||
function bytesToMb(value) {
|
||||
const numeric = Number(value);
|
||||
if (!Number.isFinite(numeric)) return null;
|
||||
return Number((numeric / 1024 / 1024).toFixed(1));
|
||||
}
|
||||
|
||||
function attachPassiveListeners(targetPage, pageRole = "control", targetPageId = pageId) {
|
||||
targetPage.on("request", (request) => {
|
||||
void appendJsonl(files.network, eventRecord("request", {
|
||||
@@ -4713,6 +5052,14 @@ function parseAlertThresholds(value) {
|
||||
domEvaluateTimeoutRedWindowMs: requiredPositiveThreshold(raw, "domEvaluateTimeoutRedWindowMs"),
|
||||
screenshotTimeoutRedCount: requiredPositiveThreshold(raw, "screenshotTimeoutRedCount"),
|
||||
pageErrorRedCount: requiredPositiveThreshold(raw, "pageErrorRedCount"),
|
||||
browserProcessSampleIntervalMs: requiredPositiveThreshold(raw, "browserProcessSampleIntervalMs"),
|
||||
browserTotalRssRedMb: requiredPositiveThreshold(raw, "browserTotalRssRedMb"),
|
||||
browserProcessRssRedMb: requiredPositiveThreshold(raw, "browserProcessRssRedMb"),
|
||||
browserRssGrowthRedMb: requiredPositiveThreshold(raw, "browserRssGrowthRedMb"),
|
||||
browserRssGrowthWindowMs: requiredPositiveThreshold(raw, "browserRssGrowthWindowMs"),
|
||||
playwrightResponsivenessRedMs: requiredPositiveThreshold(raw, "playwrightResponsivenessRedMs"),
|
||||
playwrightResponsivenessTimeoutRedCount: requiredPositiveThreshold(raw, "playwrightResponsivenessTimeoutRedCount"),
|
||||
cdpMetricsTimeoutRedCount: requiredPositiveThreshold(raw, "cdpMetricsTimeoutRedCount"),
|
||||
uncommandedStateChangeCommandWindowMs: requiredPositiveThreshold(raw, "uncommandedStateChangeCommandWindowMs"),
|
||||
scrollJumpCommandWindowMs: requiredPositiveThreshold(raw, "scrollJumpCommandWindowMs"),
|
||||
scrollJumpFromY: requiredPositiveThreshold(raw, "scrollJumpFromY"),
|
||||
|
||||
Reference in New Issue
Block a user