// SPEC: PJ2026-01040111 long-running Workbench observation. // Responsibility: Runner filesystem setup, heartbeat, browser process monitoring, freeze policy, and passive page listeners source fragment. export function nodeWebObserveRunnerRuntimeSource(): string { return String.raw`async function prepareDirs() { await mkdir(stateDir, { recursive: true, mode: 0o700 }); await Promise.all(Object.values(dirs).map((dir) => mkdir(dir, { recursive: true, mode: 0o700 }))); } async function rotateExistingJsonlArtifacts() { for (const [key, file] of Object.entries(files)) { if (!file.endsWith(".jsonl")) continue; let meta = null; try { meta = await stat(file); } catch (error) { if (error?.code === "ENOENT") continue; throw error; } if (!meta?.isFile()) continue; const archiveFile = await uniqueArchiveFile(jsonlRotation.stamp + "-" + path.basename(file)); await rename(file, archiveFile); jsonlRotation.files.push({ key, from: path.relative(stateDir, file), to: path.relative(stateDir, archiveFile), byteCount: meta.size }); } } async function uniqueArchiveFile(name) { let candidate = path.join(dirs.archive, name); const parsed = path.parse(name); for (let index = 1; ; index += 1) { try { await stat(candidate); candidate = path.join(dirs.archive, parsed.name + "-" + index + parsed.ext); } catch (error) { if (error?.code === "ENOENT") return candidate; throw error; } } } function compactFileTimestamp(value) { return new Date(value).toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z"); } async function writeManifest(extra = {}) { const manifest = { ok: extra.status !== "failed", command: "web-probe-observe", specRef, jobId, pid: process.pid, stateDir, baseUrl, targetPath, network: publicNetwork(playwrightProxy), navigation: { maxAttempts: navigationMaxAttempts, valuesRedacted: true }, pageAuthority: { browser: "chromium", context: "shared-auth", pageMode: "dual-control-observer", controlPageId: pageId, observerPageId, continuityBreaksRecorded: true }, pageProvenance: compactPageProvenance(currentPageProvenance), sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerRefreshIntervalMs, observerInitiatedDefault: false, responseBodyReadDefault: false }, alertThresholds, browserFreezePolicy, projectManagement, jsonlRotation, commandDirs: dirs, artifacts: files, safety: { pureClient: true, inboundApi: false, database: false, queueConsumer: false, k8sWorkload: false, valuesRedacted: true, secretValuesPrinted: false }, startedAt, ...extra, }; await writeFile(files.manifest, JSON.stringify(manifest, null, 2) + "\n", { mode: 0o600 }); } async function writeHeartbeat(extra = {}) { const heartbeat = { ok: terminalStatus !== "failed", jobId, pid: process.pid, stateDir, status: terminalStatus, pageId, observerPageId, baseUrl, currentUrl: currentPageUrl(), observerUrl: pageUrl(observerPage), observerRefreshIntervalMs, lastObserverRefreshAt: Number.isFinite(lastObserverRefreshAtMs) ? new Date(lastObserverRefreshAtMs).toISOString() : null, pageProvenance: compactPageProvenance(currentPageProvenance), sampleSeq, commandSeq, activeCommandId, updatedAt: new Date().toISOString(), uptimeMs: Date.now() - startedAtMs, ...extra, }; await writeFile(files.heartbeat, JSON.stringify(heartbeat, null, 2) + "\n", { mode: 0o600 }); } function startHeartbeatPulse() { const timer = setInterval(() => { if (stopping) return; void writeHeartbeat({ status: terminalStatus, heartbeatPulse: true }) .catch((error) => appendJsonl(files.errors, eventRecord("heartbeat-pulse-error", { error: errorSummary(error) }))); }, 5000); if (timer && typeof timer.unref === "function") timer.unref(); return timer; } function startBrowserProcessMonitor() { const intervalMs = Math.max(250, Math.floor(Number(alertThresholds.browserProcessSampleIntervalMs) || 1000)); let stopped = false; let running = false; const tick = async (reason) => { if (stopped || running || stopping || browserFreezeBlocker) return; running = true; try { await collectBrowserProcessSample(reason || "interval"); } catch (error) { await appendJsonl(files.errors, eventRecord("browser-process-monitor-error", { error: errorSummary(error), valuesRedacted: true })).catch(() => {}); } finally { running = false; } }; void tick("startup"); const timer = setInterval(() => { void tick("interval"); }, intervalMs); if (timer && typeof timer.unref === "function") timer.unref(); return () => { stopped = true; clearInterval(timer); }; } async function collectBrowserProcessSample(reason) { browserProcessMonitorSeq += 1; const tsMs = Date.now(); const browserPid = browserProcessPid(); const processSummary = await collectChromiumProcessSummary(browserPid); const growth = updateBrowserProcessHistory(tsMs, processSummary); const pages = []; if (page && !page.isClosed()) { pages.push(await collectBrowserPageRuntimeMetrics(page, { pageRole: "control", targetPageId: pageId, pageEpoch: controlPageEpoch }).catch((error) => browserPageRuntimeMetricError("control", pageId, controlPageEpoch, error))); } if (observerPage && !observerPage.isClosed()) { pages.push(await collectBrowserPageRuntimeMetrics(observerPage, { pageRole: "observer", targetPageId: observerPageId, pageEpoch: observerPageEpoch }).catch((error) => browserPageRuntimeMetricError("observer", observerPageId, observerPageEpoch, error))); } const sample = eventRecord("browser-process-sample", { seq: browserProcessMonitorSeq, reason, monitorIntervalMs: Math.max(250, Math.floor(Number(alertThresholds.browserProcessSampleIntervalMs) || 1000)), browserPid, process: processSummary, growth, pages, valuesRedacted: true, }); await appendJsonl(files.browserProcess, sample); await enforceBrowserFreezePolicy(sample); } async function enforceBrowserFreezePolicy(sample) { if (browserFreezePolicy.enabled !== true || browserFreezeBlocker) return; const suppressRuntimeProbeFreeze = activeCommandType === "performanceCapture"; const processSummary = sample && typeof sample.process === "object" ? sample.process : {}; const growth = sample && typeof sample.growth === "object" ? sample.growth : {}; const totalRssMb = Number(processSummary.totalRssMb); const processRssMb = Number(processSummary.maxProcessRssMb); const totalGrowthMb = Number(growth.totalRssGrowthMb); const processGrowthMb = Number(growth.maxProcessRssGrowthMb); for (const pageMetric of Array.isArray(sample.pages) ? sample.pages : []) { const effectiveMemory = pageMetric?.effectiveMemory && typeof pageMetric.effectiveMemory === "object" ? pageMetric.effectiveMemory : {}; const effectiveHeapUsedMb = Number(effectiveMemory.effectiveHeapUsedMb); const effectiveJsHeapUsedMb = Number(effectiveMemory.effectiveJsHeapUsedMb); const heapGrowthMb = Number(effectiveMemory.heapUsedGrowthMb); const jsHeapGrowthMb = Number(effectiveMemory.jsHeapUsedGrowthMb); if ( (Number.isFinite(effectiveHeapUsedMb) && effectiveHeapUsedMb >= browserFreezePolicy.memory.processRssBlockerMb) || (Number.isFinite(effectiveJsHeapUsedMb) && effectiveJsHeapUsedMb >= browserFreezePolicy.memory.processRssBlockerMb) ) { await triggerBrowserFreezeBlocker({ kind: "memory-page-effective", rootCause: "frontend_browser_page_effective_memory_pressure", observed: { pageRole: pageMetric?.pageRole ?? null, pageId: pageMetric?.pageId ?? null, pageEpoch: pageMetric?.pageEpoch ?? null, totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, effectiveHeapUsedMb: Number.isFinite(effectiveHeapUsedMb) ? effectiveHeapUsedMb : null, effectiveJsHeapUsedMb: Number.isFinite(effectiveJsHeapUsedMb) ? effectiveJsHeapUsedMb : null, baseline: pageMetric?.baseline ?? null, valuesRedacted: true, }, threshold: { processRssBlockerMb: browserFreezePolicy.memory.processRssBlockerMb, policyScope: "per-page-effective-memory", valuesRedacted: true }, sample: browserProcessSampleRef(sample), page: browserPageMetricRef(pageMetric), }); return; } if ( (Number.isFinite(heapGrowthMb) && heapGrowthMb >= browserFreezePolicy.memory.growthBlockerMb) || (Number.isFinite(jsHeapGrowthMb) && jsHeapGrowthMb >= browserFreezePolicy.memory.growthBlockerMb) ) { await triggerBrowserFreezeBlocker({ kind: "memory-page-effective-growth", rootCause: "frontend_browser_page_memory_leak_or_unbounded_render_growth", observed: { pageRole: pageMetric?.pageRole ?? null, pageId: pageMetric?.pageId ?? null, pageEpoch: pageMetric?.pageEpoch ?? null, totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, heapGrowthMb: Number.isFinite(heapGrowthMb) ? heapGrowthMb : null, jsHeapGrowthMb: Number.isFinite(jsHeapGrowthMb) ? jsHeapGrowthMb : null, baseline: pageMetric?.baseline ?? null, valuesRedacted: true, }, threshold: { growthBlockerMb: browserFreezePolicy.memory.growthBlockerMb, windowMs: browserFreezePolicy.blockerWindowMs, policyScope: "per-page-effective-memory", valuesRedacted: true }, sample: browserProcessSampleRef(sample), page: browserPageMetricRef(pageMetric), }); return; } const responsiveness = pageMetric?.responsiveness && typeof pageMetric.responsiveness === "object" ? pageMetric.responsiveness : {}; const responsivenessLatencyMs = Number(responsiveness.latencyMs); if (responsiveness.timeout === true || (Number.isFinite(responsivenessLatencyMs) && responsivenessLatencyMs >= browserFreezePolicy.responsiveness.latencyBlockerMs)) { const detail = { rootCause: "frontend_browser_page_unresponsive_to_playwright", observed: { responsivenessLatencyMs: Number.isFinite(responsivenessLatencyMs) ? responsivenessLatencyMs : null, responsivenessTimeout: responsiveness.timeout === true, valuesRedacted: true, }, threshold: { latencyBlockerMs: browserFreezePolicy.responsiveness.latencyBlockerMs, eventBlockerCount: browserFreezePolicy.responsiveness.eventBlockerCount, windowMs: browserFreezePolicy.blockerWindowMs, valuesRedacted: true, }, }; if (suppressRuntimeProbeFreeze) { await appendBrowserFreezeSuppressedSignal("playwright-responsiveness", sample, pageMetric, detail, "performance-capture-active"); } else { const signal = recordBrowserFreezeSignal("playwright-responsiveness", sample, pageMetric, detail); if (signal.burst.length >= browserFreezePolicy.responsiveness.eventBlockerCount) { await triggerBrowserFreezeBlocker(signal); return; } } } const cdp = pageMetric?.cdp && typeof pageMetric.cdp === "object" ? pageMetric.cdp : {}; const calls = Array.isArray(cdp.calls) ? cdp.calls : []; const metricTimeoutCalls = calls.filter((call) => call?.timeout === true && call?.method !== "Runtime.evaluate"); const sessionTimeoutCount = calls.length === 0 ? Number(cdp.timeoutCount || 0) : 0; const metricTimeoutCount = metricTimeoutCalls.length + (Number.isFinite(sessionTimeoutCount) ? sessionTimeoutCount : 0); if (metricTimeoutCount > 0) { const detail = { rootCause: "frontend_browser_cdp_metrics_unresponsive", observed: { cdpMetricsTimeoutCount: metricTimeoutCount, methods: metricTimeoutCalls.map((call) => call.method || "unknown").slice(0, 8), valuesRedacted: true, }, threshold: { metricsTimeoutBlockerCount: browserFreezePolicy.cdp.metricsTimeoutBlockerCount, windowMs: browserFreezePolicy.blockerWindowMs, valuesRedacted: true, }, }; if (suppressRuntimeProbeFreeze) { await appendBrowserFreezeSuppressedSignal("cdp-metrics-timeout", sample, pageMetric, detail, "performance-capture-active"); } else { const signal = recordBrowserFreezeSignal("cdp-metrics-timeout", sample, pageMetric, detail); if (signal.burst.length >= browserFreezePolicy.cdp.metricsTimeoutBlockerCount) { await triggerBrowserFreezeBlocker(signal); return; } } } } } async function appendBrowserFreezeSuppressedSignal(kind, sample, pageMetric, detail, reason) { await appendJsonl(files.browserProcess, eventRecord("browser-freeze-signal-suppressed", { kind, reason, activeCommandId, activeCommandType, rootCause: detail?.rootCause ?? null, observed: detail?.observed ?? null, threshold: detail?.threshold ?? null, sample: browserProcessSampleRef(sample), page: browserPageMetricRef(pageMetric), valuesRedacted: true, })).catch(() => {}); } function recordBrowserFreezeSignal(kind, sample, pageMetric, detail) { const tsMs = Date.parse(String(sample?.ts || "")); const signal = { kind, ts: sample?.ts ?? new Date().toISOString(), tsMs: Number.isFinite(tsMs) ? tsMs : Date.now(), pageRole: pageMetric?.pageRole ?? null, pageId: pageMetric?.pageId ?? null, pageEpoch: pageMetric?.pageEpoch ?? null, sample: browserProcessSampleRef(sample), page: browserPageMetricRef(pageMetric), ...detail, valuesRedacted: true, }; browserFreezeSignalHistory.push(signal); const windowMs = browserFreezePolicy.blockerWindowMs; const cutoff = signal.tsMs - windowMs; while (browserFreezeSignalHistory.length > 0 && Number(browserFreezeSignalHistory[0].tsMs || 0) < cutoff) browserFreezeSignalHistory.shift(); return { kind, rootCause: detail.rootCause, observed: detail.observed, threshold: detail.threshold, sample: browserProcessSampleRef(sample), page: browserPageMetricRef(pageMetric), burst: browserFreezeSignalHistory.filter((item) => item.kind === kind && item.tsMs >= cutoff).slice(-20), valuesRedacted: true, }; } async function triggerBrowserFreezeBlocker(trigger) { if (browserFreezeBlocker) return; stopping = true; terminalStatus = "failed"; const base = { id: "browser-freeze-policy-blocker", severity: "red", blocking: true, kind: trigger.kind, summary: "web-probe runner matched YAML browserFreezePolicy and stopped the browser; this run must stay red instead of refreshing or falling back", rootCause: trigger.rootCause, rootCauseStatus: "confirmed-from-runner-browser-freeze-policy", rootCauseConfidence: "high", fallbackAllowed: false, observerRefreshAllowed: false, policySource: "config/hwlab-node-lanes.yaml#webProbe.browserFreezePolicy via UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON", observed: trigger.observed || null, threshold: trigger.threshold || null, sample: trigger.sample || null, page: trigger.page || null, burst: Array.isArray(trigger.burst) ? trigger.burst.slice(0, 20) : [], valuesRedacted: true, }; browserFreezeBlocker = { ...base, browserKill: { ok: false, pending: true, valuesRedacted: true }, valuesRedacted: true }; const browserKill = browserFreezePolicy.kill.enabled === true ? await terminateBrowserForFreezeBlocker(base).catch((error) => ({ ok: false, error: errorSummary(error), valuesRedacted: true })) : { ok: true, skipped: true, reason: "kill-disabled-by-yaml-policy", valuesRedacted: true }; browserFreezeBlocker = { ...base, browserKill, valuesRedacted: true }; await appendJsonl(files.browserProcess, eventRecord("browser-freeze-blocker", browserFreezeBlocker)).catch(() => {}); await appendJsonl(files.errors, eventRecord("browser-freeze-blocker", { ...browserFreezeBlocker, error: { name: "BrowserFreezeBlocker", message: "YAML browserFreezePolicy matched: " + String(trigger.kind || "unknown"), details: browserFreezeBlocker, valuesRedacted: true, }, })).catch(() => {}); await writeHeartbeat({ status: "failed", blocker: browserFreezeBlocker }).catch(() => {}); await writeManifest({ status: "failed", blocker: browserFreezeBlocker }).catch(() => {}); } async function terminateBrowserForFreezeBlocker(blocker) { const childProcess = browser && typeof browser.process === "function" ? browser.process() : null; const pid = Number(childProcess?.pid); if (!Number.isFinite(pid) || pid <= 0) { return { ok: false, reason: "browser-process-unavailable", blockerKind: blocker.kind, valuesRedacted: true }; } const result = { ok: false, pid: Math.floor(pid), gracefulSignal: browserFreezePolicy.kill.gracefulSignal, forceSignal: browserFreezePolicy.kill.forceSignal, graceMs: browserFreezePolicy.kill.graceMs, pollIntervalMs: browserFreezePolicy.kill.pollIntervalMs, gracefulSent: false, forceSent: false, exitedAfterGrace: false, exitedAfterForce: false, valuesRedacted: true, }; try { childProcess.kill(browserFreezePolicy.kill.gracefulSignal); result.gracefulSent = true; } catch (error) { result.gracefulError = errorSummary(error); } result.exitedAfterGrace = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs); if (!result.exitedAfterGrace) { try { childProcess.kill(browserFreezePolicy.kill.forceSignal); result.forceSent = true; } catch (error) { result.forceError = errorSummary(error); } result.exitedAfterForce = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs); } result.ok = result.exitedAfterGrace || result.exitedAfterForce; return result; } async function waitForPidExit(pid, timeoutMs, pollIntervalMs) { const deadline = Date.now() + timeoutMs; while (Date.now() <= deadline) { if (!pidAlive(pid)) return true; await sleep(pollIntervalMs); } return !pidAlive(pid); } function pidAlive(pid) { try { process.kill(pid, 0); return true; } catch { return false; } } function browserProcessSampleRef(sample) { return { ts: sample?.ts ?? null, seq: sample?.seq ?? null, sampleSeq: sample?.sampleSeq ?? null, browserPid: sample?.browserPid ?? null, totalRssMb: sample?.process?.totalRssMb ?? null, maxProcessRssMb: sample?.process?.maxProcessRssMb ?? null, totalRssGrowthMb: sample?.growth?.totalRssGrowthMb ?? null, maxProcessRssGrowthMb: sample?.growth?.maxProcessRssGrowthMb ?? null, valuesRedacted: true, }; } function browserPageMetricRef(pageMetric) { const responsiveness = pageMetric?.responsiveness && typeof pageMetric.responsiveness === "object" ? pageMetric.responsiveness : {}; const cdp = pageMetric?.cdp && typeof pageMetric.cdp === "object" ? pageMetric.cdp : {}; const effectiveMemory = pageMetric?.effectiveMemory && typeof pageMetric.effectiveMemory === "object" ? pageMetric.effectiveMemory : {}; return { pageRole: pageMetric?.pageRole ?? null, pageId: pageMetric?.pageId ?? null, pageEpoch: pageMetric?.pageEpoch ?? null, timeoutMs: pageMetric?.timeoutMs ?? null, responsivenessLatencyMs: responsiveness.latencyMs ?? null, responsivenessTimeout: responsiveness.timeout === true, cdpTimeoutCount: cdp.timeoutCount ?? null, cdpErrorCount: cdp.errorCount ?? null, effectiveHeapUsedMb: Number.isFinite(Number(effectiveMemory.effectiveHeapUsedMb)) ? Number(effectiveMemory.effectiveHeapUsedMb) : null, effectiveJsHeapUsedMb: Number.isFinite(Number(effectiveMemory.effectiveJsHeapUsedMb)) ? Number(effectiveMemory.effectiveJsHeapUsedMb) : null, heapUsedGrowthMb: Number.isFinite(Number(effectiveMemory.heapUsedGrowthMb)) ? Number(effectiveMemory.heapUsedGrowthMb) : null, jsHeapUsedGrowthMb: Number.isFinite(Number(effectiveMemory.jsHeapUsedGrowthMb)) ? Number(effectiveMemory.jsHeapUsedGrowthMb) : null, baselineCapturedAt: pageMetric?.baseline?.capturedAt ?? null, valuesRedacted: true, }; } function browserProcessPid() { try { const childProcess = browser && typeof browser.process === "function" ? browser.process() : null; const pid = Number(childProcess?.pid); return Number.isFinite(pid) && pid > 0 ? Math.floor(pid) : null; } catch { return null; } } async function collectChromiumProcessSummary(browserPid) { const all = await readProcProcessTable(); const childrenByPpid = new Map(); for (const item of all) { if (!Number.isFinite(item.ppid)) continue; const list = childrenByPpid.get(item.ppid) || []; list.push(item.pid); childrenByPpid.set(item.ppid, list); } const roots = [process.pid, browserPid].filter((pid, index, items) => Number.isFinite(Number(pid)) && Number(pid) > 0 && items.indexOf(pid) === index).map((pid) => Number(pid)); const descendants = new Set(roots); const queue = roots.slice(); while (queue.length > 0) { const pid = queue.shift(); for (const child of childrenByPpid.get(pid) || []) { if (descendants.has(child)) continue; descendants.add(child); queue.push(child); } } const processes = all .filter((item) => descendants.has(item.pid)) .map((item) => ({ ...item, role: classifyChromiumProcess(item) })) .filter((item) => item.role) .map((item) => ({ pid: item.pid, ppid: item.ppid, name: item.name || null, role: item.role, rssBytes: item.rssBytes, vmSizeBytes: item.vmSizeBytes, commandHash: item.cmdline ? sha256Text(item.cmdline) : null, commandPreview: redactProcessCommandPreview(item.cmdline), valuesRedacted: true, })) .sort((a, b) => Number(b.rssBytes || 0) - Number(a.rssBytes || 0)); const roles = {}; for (const item of processes) { const role = item.role || "unknown"; const summary = roles[role] || { count: 0, rssBytes: 0, maxRssBytes: 0 }; summary.count += 1; summary.rssBytes += Number(item.rssBytes || 0); summary.maxRssBytes = Math.max(summary.maxRssBytes, Number(item.rssBytes || 0)); roles[role] = summary; } const totalRssBytes = processes.reduce((sum, item) => sum + Number(item.rssBytes || 0), 0); const maxProcess = processes[0] || null; return { sampledRootPids: roots, chromiumProcessCount: processes.length, totalRssBytes, totalRssMb: bytesToMb(totalRssBytes), maxProcessRssBytes: maxProcess ? Number(maxProcess.rssBytes || 0) : 0, maxProcessRssMb: maxProcess ? bytesToMb(maxProcess.rssBytes) : 0, maxProcess, roles, processes: processes.slice(0, 20), valuesRedacted: true, }; } async function readProcProcessTable() { let entries = []; try { entries = await readdir("/proc"); } catch { return []; } const numeric = entries.map((name) => Number(name)).filter((pid) => Number.isInteger(pid) && pid > 0); const rows = await Promise.all(numeric.map((pid) => readOneProcProcess(pid).catch(() => null))); return rows.filter(Boolean); } async function readOneProcProcess(pid) { const [statText, statusText, cmdlineText] = await Promise.all([ readFile(path.join("/proc", String(pid), "stat"), "utf8").catch(() => ""), readFile(path.join("/proc", String(pid), "status"), "utf8").catch(() => ""), readFile(path.join("/proc", String(pid), "cmdline"), "utf8").catch(() => ""), ]); if (!statText && !statusText) return null; const ppid = procPpidFromStat(statText); const name = procStatusField(statusText, "Name") || null; const rssKb = procStatusKb(statusText, "VmRSS"); const vmSizeKb = procStatusKb(statusText, "VmSize"); return { pid, ppid, name, cmdline: String(cmdlineText || "").replace(/\0/gu, " ").replace(/\s+/gu, " ").trim(), rssBytes: Number.isFinite(rssKb) ? rssKb * 1024 : 0, vmSizeBytes: Number.isFinite(vmSizeKb) ? vmSizeKb * 1024 : 0, }; } function procPpidFromStat(value) { const text = String(value || ""); const end = text.lastIndexOf(")"); if (end < 0) return null; const parts = text.slice(end + 1).trim().split(/\s+/u); const ppid = Number(parts[1]); return Number.isFinite(ppid) ? ppid : null; } function procStatusField(value, key) { const prefix = String(key || "") + ":"; for (const line of String(value || "").split(/\n/u)) { if (line.startsWith(prefix)) return line.slice(prefix.length).trim(); } return null; } function procStatusKb(value, key) { const raw = procStatusField(value, key); const match = String(raw || "").match(/^(\d+)\s+kB$/u); return match ? Number(match[1]) : null; } function classifyChromiumProcess(item) { const cmdline = String(item?.cmdline || ""); const combined = (cmdline + " " + String(item?.name || "")).toLowerCase(); if (!/(?:chromium|chrome|headless_shell)/u.test(combined)) return null; if (/--type=renderer\b/u.test(cmdline)) return "renderer"; if (/--type=gpu-process\b/u.test(cmdline)) return "gpu"; if (/--type=utility\b/u.test(cmdline)) return "utility"; if (/--type=zygote\b/u.test(cmdline)) return "zygote"; if (/--type=broker\b/u.test(cmdline)) return "broker"; if (/--type=/u.test(cmdline)) return "other"; return "browser"; } function redactProcessCommandPreview(value) { const text = String(value || ""); if (!text) return null; const parts = text.split(/\s+/u).slice(0, 18).map((part) => { if (/--(?:proxy|token|secret|password|cookie|auth|key)[^=]*=/iu.test(part)) return part.replace(/=.*/u, "=[redacted]"); return part.replace(/:\/\/[^/@\s]+@/u, "://[redacted]@"); }); return truncate(parts.join(" "), 260); } function updateBrowserProcessHistory(tsMs, processSummary) { const windowMs = Math.max(1000, Number(alertThresholds.browserRssGrowthWindowMs) || 30000); const totalRssBytes = Number(processSummary?.totalRssBytes || 0); const maxProcessRssBytes = Number(processSummary?.maxProcessRssBytes || 0); browserProcessHistory.push({ tsMs, totalRssBytes, maxProcessRssBytes }); const retentionMs = Math.max(windowMs * 3, 180000); while (browserProcessHistory.length > 0 && tsMs - browserProcessHistory[0].tsMs > retentionMs) browserProcessHistory.shift(); const windowStartMs = tsMs - windowMs; const candidates = browserProcessHistory.filter((item) => item.tsMs >= windowStartMs && item.tsMs <= tsMs); const baseline = candidates[0] || browserProcessHistory[0] || null; const totalRssGrowthBytes = baseline ? totalRssBytes - Number(baseline.totalRssBytes || 0) : 0; const maxProcessRssGrowthBytes = baseline ? maxProcessRssBytes - Number(baseline.maxProcessRssBytes || 0) : 0; return { windowMs, baselineAt: baseline ? new Date(baseline.tsMs).toISOString() : null, baselineTotalRssBytes: baseline ? baseline.totalRssBytes : null, baselineMaxProcessRssBytes: baseline ? baseline.maxProcessRssBytes : null, totalRssGrowthBytes, totalRssGrowthMb: bytesToMb(totalRssGrowthBytes), maxProcessRssGrowthBytes, maxProcessRssGrowthMb: bytesToMb(maxProcessRssGrowthBytes), valuesRedacted: true, }; } async function collectBrowserPageRuntimeMetrics(targetPage, { pageRole, targetPageId, pageEpoch }) { const timeoutMs = Math.max(1000, Math.floor(Number(alertThresholds.playwrightResponsivenessRedMs) || 5000)); const startedAtMs = Date.now(); let session = null; const result = { pageRole, pageId: targetPageId, pageEpoch: Number.isFinite(Number(pageEpoch)) ? Number(pageEpoch) : 0, url: pageUrl(targetPage), timeoutMs, responsiveness: null, cdp: { timeoutCount: 0, errorCount: 0, calls: [] }, valuesRedacted: true, }; try { session = await withHardTimeout(targetPage.context().newCDPSession(targetPage), timeoutMs, "newCDPSession exceeded " + timeoutMs + "ms"); const enable = await timedCdpSend(session, "Performance.enable", {}, timeoutMs); if (enable.ok !== true) result.cdp.calls.push({ method: "Performance.enable", ...enable }); result.responsiveness = await timedCdpSend(session, "Runtime.evaluate", { expression: "1", returnByValue: true }, timeoutMs); result.cdp.calls.push({ method: "Runtime.evaluate", ...result.responsiveness }); const performance = await timedCdpSend(session, "Performance.getMetrics", {}, timeoutMs); result.cdp.calls.push({ method: "Performance.getMetrics", ...performance }); result.performance = performance.value; const heap = await timedCdpSend(session, "Runtime.getHeapUsage", {}, timeoutMs); result.cdp.calls.push({ method: "Runtime.getHeapUsage", ...heap }); result.heapUsage = heap.value; const domCounters = await timedCdpSend(session, "Memory.getDOMCounters", {}, timeoutMs); result.cdp.calls.push({ method: "Memory.getDOMCounters", ...domCounters }); result.domCounters = domCounters.value; result.cdp.timeoutCount = result.cdp.calls.filter((item) => item.timeout === true).length; result.cdp.errorCount = result.cdp.calls.filter((item) => item.ok !== true).length; } catch (error) { result.sessionError = errorSummary(error); result.cdp.timeoutCount = isTimeoutErrorMessage(error?.message) ? 1 : 0; result.cdp.errorCount = 1; } finally { result.latencyMs = Date.now() - startedAtMs; applyBrowserPageRuntimeBaseline(result); if (session) await withHardTimeout(session.detach(), 1000, "CDP session detach exceeded 1000ms").catch(() => {}); } return result; } function applyBrowserPageRuntimeBaseline(result) { const key = [result.pageRole || "unknown", result.pageId || "unknown", Number.isFinite(Number(result.pageEpoch)) ? Number(result.pageEpoch) : 0].join(":"); const current = browserPageRuntimeMemorySnapshot(result); const existing = browserPageRuntimeBaselines.get(key); if (!existing && current) browserPageRuntimeBaselines.set(key, { ...current, capturedAt: new Date().toISOString(), source: "first-page-runtime-sample", valuesRedacted: true }); const baseline = browserPageRuntimeBaselines.get(key) || null; result.baseline = baseline ? { capturedAt: baseline.capturedAt, source: baseline.source, heapUsedMb: baseline.heapUsedMb, jsHeapUsedMb: baseline.jsHeapUsedMb, domNodes: baseline.domNodes, valuesRedacted: true, } : null; result.effectiveMemory = browserPageEffectiveMemory(current, baseline); } function browserPageRuntimeMemorySnapshot(result) { const heapUsedBytes = Number(result?.heapUsage?.usedSize); const metrics = result?.performance?.metrics && typeof result.performance.metrics === "object" ? result.performance.metrics : {}; const jsHeapUsedBytes = Number(metrics.JSHeapUsedSize); const domNodes = Number(result?.domCounters?.nodes ?? metrics.Nodes); if (!Number.isFinite(heapUsedBytes) && !Number.isFinite(jsHeapUsedBytes) && !Number.isFinite(domNodes)) return null; return { heapUsedBytes: Number.isFinite(heapUsedBytes) ? heapUsedBytes : null, heapUsedMb: Number.isFinite(heapUsedBytes) ? bytesToMb(heapUsedBytes) : null, jsHeapUsedBytes: Number.isFinite(jsHeapUsedBytes) ? jsHeapUsedBytes : null, jsHeapUsedMb: Number.isFinite(jsHeapUsedBytes) ? bytesToMb(jsHeapUsedBytes) : null, domNodes: Number.isFinite(domNodes) ? domNodes : null, valuesRedacted: true, }; } function browserPageEffectiveMemory(current, baseline) { if (!current) return { available: false, baselineAvailable: Boolean(baseline), valuesRedacted: true }; const heapUsedGrowthBytes = numericDelta(current.heapUsedBytes, baseline?.heapUsedBytes); const jsHeapUsedGrowthBytes = numericDelta(current.jsHeapUsedBytes, baseline?.jsHeapUsedBytes); const domNodesGrowth = numericDelta(current.domNodes, baseline?.domNodes); return { available: true, baselineAvailable: Boolean(baseline), heapUsedMb: current.heapUsedMb, jsHeapUsedMb: current.jsHeapUsedMb, effectiveHeapUsedMb: Number.isFinite(heapUsedGrowthBytes) ? bytesToMb(heapUsedGrowthBytes) : current.heapUsedMb, effectiveJsHeapUsedMb: Number.isFinite(jsHeapUsedGrowthBytes) ? bytesToMb(jsHeapUsedGrowthBytes) : current.jsHeapUsedMb, heapUsedGrowthMb: Number.isFinite(heapUsedGrowthBytes) ? bytesToMb(heapUsedGrowthBytes) : null, jsHeapUsedGrowthMb: Number.isFinite(jsHeapUsedGrowthBytes) ? bytesToMb(jsHeapUsedGrowthBytes) : null, domNodes: current.domNodes, domNodesGrowth: Number.isFinite(domNodesGrowth) ? domNodesGrowth : null, valuesRedacted: true, }; } function numericDelta(current, baseline) { const value = Number(current); const base = Number(baseline); if (!Number.isFinite(value) || !Number.isFinite(base)) return null; return value - base; } function browserPageRuntimeMetricError(pageRole, targetPageId, pageEpoch, error) { return { pageRole, pageId: targetPageId, pageEpoch: Number.isFinite(Number(pageEpoch)) ? Number(pageEpoch) : 0, url: null, timeoutMs: Math.max(1000, Math.floor(Number(alertThresholds.playwrightResponsivenessRedMs) || 5000)), responsiveness: { ok: false, timeout: isTimeoutErrorMessage(error?.message), error: errorSummary(error), valuesRedacted: true }, cdp: { timeoutCount: isTimeoutErrorMessage(error?.message) ? 1 : 0, errorCount: 1, calls: [] }, valuesRedacted: true, }; } async function timedCdpSend(session, method, params, timeoutMs) { const startedAtMs = Date.now(); try { const value = await withHardTimeout(session.send(method, params || {}), timeoutMs, method + " exceeded " + timeoutMs + "ms"); return { ok: true, timeout: false, latencyMs: Date.now() - startedAtMs, value: compactCdpValue(method, value), valuesRedacted: true }; } catch (error) { return { ok: false, timeout: isTimeoutErrorMessage(error?.message), latencyMs: Date.now() - startedAtMs, error: errorSummary(error), valuesRedacted: true }; } } function compactCdpValue(method, value) { if (!value || typeof value !== "object") return value ?? null; if (method === "Performance.getMetrics") { const wanted = new Set(["Timestamp", "Documents", "Frames", "JSEventListeners", "Nodes", "LayoutCount", "RecalcStyleCount", "LayoutDuration", "RecalcStyleDuration", "ScriptDuration", "TaskDuration", "JSHeapUsedSize", "JSHeapTotalSize"]); const metrics = {}; for (const item of Array.isArray(value.metrics) ? value.metrics : []) { if (wanted.has(item?.name)) metrics[item.name] = Number.isFinite(Number(item?.value)) ? Number(item.value) : null; } return { metricCount: Array.isArray(value.metrics) ? value.metrics.length : 0, metrics, valuesRedacted: true }; } if (method === "Runtime.getHeapUsage") { return { usedSize: Number.isFinite(Number(value.usedSize)) ? Number(value.usedSize) : null, totalSize: Number.isFinite(Number(value.totalSize)) ? Number(value.totalSize) : null, embedderHeapUsedSize: Number.isFinite(Number(value.embedderHeapUsedSize)) ? Number(value.embedderHeapUsedSize) : null, valuesRedacted: true, }; } if (method === "Memory.getDOMCounters") { return { documents: Number.isFinite(Number(value.documents)) ? Number(value.documents) : null, nodes: Number.isFinite(Number(value.nodes)) ? Number(value.nodes) : null, jsEventListeners: Number.isFinite(Number(value.jsEventListeners)) ? Number(value.jsEventListeners) : null, valuesRedacted: true, }; } if (method === "Runtime.evaluate") { return { resultType: value.result?.type ?? null, unserializableValue: value.result?.unserializableValue ?? null, exceptionDetails: value.exceptionDetails ? { text: truncate(value.exceptionDetails.text || "", 200), valuesRedacted: true } : null, valuesRedacted: true }; } return { valuesRedacted: true }; } function isTimeoutErrorMessage(value) { return /timeout|timed\s*out|exceeded\s+\d+\s*ms/iu.test(String(value || "")); } function bytesToMb(value) { const numeric = Number(value); if (!Number.isFinite(numeric)) return null; return Number((numeric / 1024 / 1024).toFixed(1)); } function attachPassiveListeners(targetPage, pageRole = "control", targetPageId = pageId) { void installPagePerformanceProbe(targetPage, pageRole, targetPageId) .catch((error) => appendJsonl(files.errors, eventRecord("performance-probe-install-error", { pageRole, pageId: targetPageId, error: errorSummary(error), valuesRedacted: true }))); targetPage.on("request", (request) => { const webPerformancePayload = summarizeWebPerformanceRequestPayload(request); void appendJsonl(files.network, eventRecord("request", { pageRole, pageId: targetPageId, observerInitiated: false, commandId: activeCommandId, method: request.method(), url: safeUrl(request.url()), resourceType: request.resourceType(), frameUrl: safeFrameUrl(request.frame()), ...(webPerformancePayload ? { webPerformancePayload } : {}), })); }); targetPage.on("response", (response) => { const request = response.request(); const base = { pageRole, pageId: targetPageId, sampleSeq, observerInitiated: false, commandId: activeCommandId, method: request.method(), url: safeUrl(response.url()), resourceType: request.resourceType(), status: response.status(), statusText: response.statusText(), fromServiceWorker: response.fromServiceWorker(), }; void (async () => { const bodyFields = await summarizeWorkbenchResponseBody(response, request); await appendJsonl(files.network, eventRecord("response", { ...base, ...bodyFields })); })().catch((error) => appendJsonl(files.errors, eventRecord("response-body-summary-error", { pageRole, pageId: targetPageId, sampleSeq, commandId: activeCommandId, method: request.method(), url: safeUrl(response.url()), error: errorSummary(error), valuesRedacted: true }))); }); targetPage.on("requestfailed", (request) => { void appendJsonl(files.network, eventRecord("requestfailed", { pageRole, pageId: targetPageId, observerInitiated: false, commandId: activeCommandId, method: request.method(), url: safeUrl(request.url()), resourceType: request.resourceType(), failure: request.failure()?.errorText ?? null, })); }); targetPage.on("console", (message) => { void appendJsonl(files.console, eventRecord("console", { pageRole, pageId: targetPageId, type: message.type(), text: truncate(message.text(), 1000), location: message.location() })); }); targetPage.on("pageerror", (error) => { void appendJsonl(files.errors, eventRecord("pageerror", { pageRole, pageId: targetPageId, error: errorSummary(error) })); }); targetPage.on("crash", () => { void appendJsonl(files.errors, eventRecord("page-crash", { pageRole, pageId: targetPageId })); }); targetPage.on("close", () => { void appendJsonl(files.control, eventRecord("continuity-break", { pageRole, pageId: targetPageId, reason: "page-closed" })); }); } function summarizeWebPerformanceRequestPayload(request) { if (!request || request.method() !== "POST" || !isSameOriginWebPerformanceRequestUrl(request.url())) return null; let raw = null; try { raw = request.postData(); } catch (error) { return { captureStatus: "post-data-read-failed", parseStatus: "post-data-read-failed", byteCount: null, byteLimit: webPerformanceRequestBodyMaxBytes, error: errorSummary(error), eventCount: null, events: [], valuesRedacted: true, }; } if (raw === null || raw === undefined || raw === "") { return { captureStatus: "missing-body", parseStatus: "missing-body", byteCount: 0, byteLimit: webPerformanceRequestBodyMaxBytes, eventCount: 0, events: [], valuesRedacted: true, }; } const byteCount = Buffer.byteLength(raw, "utf8"); const bodyHash = sha256Text(raw); if (byteCount > webPerformanceRequestBodyMaxBytes) { return { captureStatus: "skipped-over-limit", parseStatus: "not-parsed-over-limit", byteCount, byteLimit: webPerformanceRequestBodyMaxBytes, bodyHash, truncated: true, eventCount: null, events: [], valuesRedacted: true, }; } try { const parsed = JSON.parse(raw); const schemaVersion = typeof parsed?.schemaVersion === "string" ? parsed.schemaVersion : null; if (schemaVersion !== "hwlab-web-performance-v2") { return { captureStatus: "captured", parseStatus: "unsupported-schema", schemaVersion, byteCount, byteLimit: webPerformanceRequestBodyMaxBytes, bodyHash, eventCount: Array.isArray(parsed?.events) ? parsed.events.length : null, events: [], valuesRedacted: true, }; } const events = (Array.isArray(parsed.events) ? parsed.events : []).slice(0, 80).map(compactWebPerformancePayloadEvent).filter(Boolean); return { captureStatus: "captured", parseStatus: "parsed", schemaVersion, byteCount, byteLimit: webPerformanceRequestBodyMaxBytes, bodyHash, truncated: false, eventCount: Array.isArray(parsed.events) ? parsed.events.length : 0, storedEventCount: events.length, events, valuesRedacted: true, }; } catch (error) { return { captureStatus: "captured", parseStatus: "invalid-json", byteCount, byteLimit: webPerformanceRequestBodyMaxBytes, bodyHash, parseError: truncate(error?.message || String(error), 180), eventCount: null, events: [], valuesRedacted: true, }; } } function isSameOriginWebPerformanceRequestUrl(value) { try { const url = new URL(String(value || ""), baseUrl); return url.origin === baseUrl && url.pathname === "/v1/web-performance"; } catch { return false; } } function compactWebPerformancePayloadEvent(event) { if (!event || typeof event !== "object") return null; const detail = firstObject(event.detail, event.details, event.payload, event.data, event.metrics); const read = (key) => event?.[key] ?? detail?.[key] ?? null; const diagnosticCode = compactString(read("diagnosticCode") ?? read("code") ?? read("name"), 96); const reason = compactString(read("reason"), 96); const module = compactString(read("module"), 96); const eventType = compactString(read("eventType") ?? read("type") ?? read("kind"), 96); const traceId = compactTraceId(read("traceId")); const sessionId = compactString(read("sessionId"), 120); const replacedByKey = compactScalar(read("replacedByKey")); const eventId = compactString(read("eventId") ?? read("id"), 120); return { ts: compactString(read("ts") ?? read("timestamp") ?? read("time"), 64), eventType, diagnosticCode, reason, module, traceId, sessionIdHash: sessionId ? sha256Text(sessionId) : null, eventIdHash: eventId ? sha256Text(eventId) : null, eventCount: numberOrNull(read("eventCount")), deliveredCount: numberOrNull(read("deliveredCount")), chunkCount: numberOrNull(read("chunkCount")), flushDurationMs: numberOrNull(read("flushDurationMs")), droppedCount: numberOrNull(read("droppedCount")), maxItemsPerChunk: numberOrNull(read("maxItemsPerChunk")), maxChunkMs: numberOrNull(read("maxChunkMs")), replacedByKey: typeof replacedByKey === "boolean" || typeof replacedByKey === "number" ? replacedByKey : null, replacedByKeyHash: typeof replacedByKey === "string" ? sha256Text(replacedByKey) : null, valuesRedacted: true, }; } function firstObject(...values) { for (const value of values) if (value && typeof value === "object" && !Array.isArray(value)) return value; return {}; } function compactString(value, limit) { if (value === null || value === undefined) return null; return truncate(String(value), limit); } function compactTraceId(value) { const text = String(value || ""); return /^trc_[A-Za-z0-9_-]+$/u.test(text) ? text : null; } function compactScalar(value) { if (value === null || value === undefined) return null; if (typeof value === "boolean") return value; const numeric = Number(value); if (Number.isFinite(numeric) && String(value).trim() !== "") return numeric; return truncate(String(value), 120); } `; }