diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 334fec72..24a7e80f 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -210,6 +210,14 @@ lanes: domEvaluateTimeoutRedWindowMs: 30000 screenshotTimeoutRedCount: 2 pageErrorRedCount: 2 + browserProcessSampleIntervalMs: 1000 + browserTotalRssRedMb: 800 + browserProcessRssRedMb: 600 + browserRssGrowthRedMb: 300 + browserRssGrowthWindowMs: 30000 + playwrightResponsivenessRedMs: 5000 + playwrightResponsivenessTimeoutRedCount: 2 + cdpMetricsTimeoutRedCount: 2 uncommandedStateChangeCommandWindowMs: 10000 scrollJumpCommandWindowMs: 8000 scrollJumpFromY: 250 @@ -603,6 +611,14 @@ lanes: domEvaluateTimeoutRedWindowMs: 30000 screenshotTimeoutRedCount: 2 pageErrorRedCount: 2 + browserProcessSampleIntervalMs: 1000 + browserTotalRssRedMb: 800 + browserProcessRssRedMb: 600 + browserRssGrowthRedMb: 300 + browserRssGrowthWindowMs: 30000 + playwrightResponsivenessRedMs: 5000 + playwrightResponsivenessTimeoutRedCount: 2 + cdpMetricsTimeoutRedCount: 2 uncommandedStateChangeCommandWindowMs: 10000 scrollJumpCommandWindowMs: 8000 scrollJumpFromY: 250 @@ -926,6 +942,14 @@ lanes: domEvaluateTimeoutRedWindowMs: 30000 screenshotTimeoutRedCount: 2 pageErrorRedCount: 2 + browserProcessSampleIntervalMs: 1000 + browserTotalRssRedMb: 800 + browserProcessRssRedMb: 600 + browserRssGrowthRedMb: 300 + browserRssGrowthWindowMs: 30000 + playwrightResponsivenessRedMs: 5000 + playwrightResponsivenessTimeoutRedCount: 2 + cdpMetricsTimeoutRedCount: 2 uncommandedStateChangeCommandWindowMs: 10000 scrollJumpCommandWindowMs: 8000 scrollJumpFromY: 250 diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index d32e4a64..d21066dc 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -184,6 +184,14 @@ export interface HwlabRuntimeWebProbeAlertThresholdsSpec { readonly domEvaluateTimeoutRedWindowMs: number; readonly screenshotTimeoutRedCount: number; readonly pageErrorRedCount: number; + readonly browserProcessSampleIntervalMs: number; + readonly browserTotalRssRedMb: number; + readonly browserProcessRssRedMb: number; + readonly browserRssGrowthRedMb: number; + readonly browserRssGrowthWindowMs: number; + readonly playwrightResponsivenessRedMs: number; + readonly playwrightResponsivenessTimeoutRedCount: number; + readonly cdpMetricsTimeoutRedCount: number; readonly uncommandedStateChangeCommandWindowMs: number; readonly scrollJumpCommandWindowMs: number; readonly scrollJumpFromY: number; @@ -1184,6 +1192,14 @@ function webProbeAlertThresholdsConfig(value: unknown, path: string): HwlabRunti domEvaluateTimeoutRedWindowMs: positiveNumberField(raw, "domEvaluateTimeoutRedWindowMs", path), screenshotTimeoutRedCount: positiveNumberField(raw, "screenshotTimeoutRedCount", path), pageErrorRedCount: positiveNumberField(raw, "pageErrorRedCount", path), + browserProcessSampleIntervalMs: positiveNumberField(raw, "browserProcessSampleIntervalMs", path), + browserTotalRssRedMb: positiveNumberField(raw, "browserTotalRssRedMb", path), + browserProcessRssRedMb: positiveNumberField(raw, "browserProcessRssRedMb", path), + browserRssGrowthRedMb: positiveNumberField(raw, "browserRssGrowthRedMb", path), + browserRssGrowthWindowMs: positiveNumberField(raw, "browserRssGrowthWindowMs", path), + playwrightResponsivenessRedMs: positiveNumberField(raw, "playwrightResponsivenessRedMs", path), + playwrightResponsivenessTimeoutRedCount: positiveNumberField(raw, "playwrightResponsivenessTimeoutRedCount", path), + cdpMetricsTimeoutRedCount: positiveNumberField(raw, "cdpMetricsTimeoutRedCount", path), uncommandedStateChangeCommandWindowMs: positiveNumberField(raw, "uncommandedStateChangeCommandWindowMs", path), scrollJumpCommandWindowMs: positiveNumberField(raw, "scrollJumpCommandWindowMs", path), scrollJumpFromY: positiveNumberField(raw, "scrollJumpFromY", path), diff --git a/scripts/src/hwlab-node-web-observe-analyzer-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-source.ts index fd5c5496..54bede8f 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-source.ts @@ -43,6 +43,7 @@ const promptNetworkRows = applyAnalysisFocus(filterRowsByTimeWindow(sourceNetwor const consoleEvents = applyAnalysisFocus(filterRowsByTimeWindow(await readJsonl(dataFile("console.jsonl"), { tail: smallJsonlTailLimit }), sampleWindow), analysisFocus); const errors = applyAnalysisFocus(filterRowsByTimeWindow(await readJsonl(dataFile("errors.jsonl"), { tail: smallJsonlTailLimit }), sampleWindow), analysisFocus); const artifacts = applyAnalysisFocus(filterRowsByTimeWindow(await readJsonl(dataFile("artifacts.jsonl"), { tail: smallJsonlTailLimit }), sampleWindow), analysisFocus); +const browserProcessRows = applyAnalysisFocus(filterRowsByTimeWindow(await readJsonl(dataFile("browser-process.jsonl"), { tail: smallJsonlTailLimit }), sampleWindow), analysisFocus); const manifest = await readJson(path.join(stateDir, "manifest.json")); const heartbeat = await readJson(path.join(stateDir, "heartbeat.json")); const commandState = await readCommandState(stateDir); @@ -55,6 +56,7 @@ const pagePerformance = buildPagePerformanceReport(samples, manifest); const promptNetwork = buildPromptNetworkReport(control, promptNetworkRows); const runtimeAlerts = buildRuntimeAlerts(samples, control, network, consoleEvents, errors); const apiDomLag = buildApiDomLagReport(samples, network); +const browserProcess = buildBrowserProcessReport(browserProcessRows); const projectManagement = buildProjectManagementReport(samples, control, network, pagePerformance, projectManagementConfig); const runnerErrors = errors.slice(-8).map((item) => { const details = item.error?.details && typeof item.error.details === "object" ? item.error.details : {}; @@ -105,9 +107,9 @@ const runnerErrors = errors.slice(-8).map((item) => { }); const commandFailures = summarizeCommandFailures(control); const toolFindings = buildToolFindings({ manifest, heartbeat, commandState }); -const findings = [...toolFindings, ...buildProjectManagementFindings(projectManagement), ...buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures, manifest, apiDomLag)]; +const findings = [...toolFindings, ...buildProjectManagementFindings(projectManagement), ...buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures, manifest, apiDomLag, browserProcess)]; if (jsonlReadIssues.length > 0) findings.unshift({ id: "jsonl-read-issues", severity: "red", summary: "observer analyzer hit JSONL read/parse issues", count: jsonlReadIssues.length, issues: jsonlReadIssues.slice(0, 20) }); -const recentWindow = buildRecentAnalysisWindow({ samples, control, network, consoleEvents, errors, manifest }); +const recentWindow = buildRecentAnalysisWindow({ samples, control, network, consoleEvents, errors, browserProcessRows, manifest }); const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl })); const report = { ok: findings.filter((item) => item.severity === "red").length === 0, @@ -118,7 +120,7 @@ const report = { alertThresholds, manifest: compactManifest(manifest), heartbeat: compactHeartbeat(heartbeat), - counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length }, + counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length, browserProcess: browserProcessRows.length }, commandTimeline, transitions, sampleMetrics, @@ -128,6 +130,7 @@ const report = { promptNetwork, runtimeAlerts, apiDomLag, + browserProcess, runnerErrors, commandFailures, commandState, @@ -157,6 +160,7 @@ console.log(JSON.stringify({ projectManagement: projectManagement.summary, runtimeAlerts: runtimeAlerts.summary, apiDomLag: apiDomLag.summary, + browserProcess: browserProcess.summary, findingCount: findings.length, redFindingCount: findings.filter((item) => item.severity === "red").length, redFindings: prioritizeFindings(findings.filter((item) => item.severity === "red")).slice(0, 12).map((item) => ({ kind: item.id ?? item.kind ?? item.code, severity: item.severity, count: item.count ?? item.sampleCount ?? null, summary: String(item.summary ?? item.message ?? "").slice(0, 180) })), @@ -177,6 +181,7 @@ console.log(JSON.stringify({ promptNetwork: recentWindow.promptNetwork.summary, runtimeAlerts: recentWindow.runtimeAlerts.summary, apiDomLag: compactApiDomLagForOutput(apiDomLag), + browserProcess: recentWindow.browserProcess.summary, runnerErrors, commandFailures: commandFailures.slice(-8), commandState, @@ -561,6 +566,14 @@ function parseAlertThresholds(value) { domEvaluateTimeoutRedWindowMs: requiredPositiveThreshold(raw, "domEvaluateTimeoutRedWindowMs"), screenshotTimeoutRedCount: requiredPositiveThreshold(raw, "screenshotTimeoutRedCount"), pageErrorRedCount: requiredPositiveThreshold(raw, "pageErrorRedCount"), + browserProcessSampleIntervalMs: requiredPositiveThreshold(raw, "browserProcessSampleIntervalMs"), + browserTotalRssRedMb: requiredPositiveThreshold(raw, "browserTotalRssRedMb"), + browserProcessRssRedMb: requiredPositiveThreshold(raw, "browserProcessRssRedMb"), + browserRssGrowthRedMb: requiredPositiveThreshold(raw, "browserRssGrowthRedMb"), + browserRssGrowthWindowMs: requiredPositiveThreshold(raw, "browserRssGrowthWindowMs"), + playwrightResponsivenessRedMs: requiredPositiveThreshold(raw, "playwrightResponsivenessRedMs"), + playwrightResponsivenessTimeoutRedCount: requiredPositiveThreshold(raw, "playwrightResponsivenessTimeoutRedCount"), + cdpMetricsTimeoutRedCount: requiredPositiveThreshold(raw, "cdpMetricsTimeoutRedCount"), uncommandedStateChangeCommandWindowMs: requiredPositiveThreshold(raw, "uncommandedStateChangeCommandWindowMs"), scrollJumpCommandWindowMs: requiredPositiveThreshold(raw, "scrollJumpCommandWindowMs"), scrollJumpFromY: requiredPositiveThreshold(raw, "scrollJumpFromY"), @@ -2725,11 +2738,12 @@ function promptCommandHasAuthoritativeSubmitSideEffect(control, promptRound) { || Number(sideEffect.messageCountDelta ?? 0) > 0; } -function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures = [], manifest = {}, apiDomLag = null) { +function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures = [], manifest = {}, apiDomLag = null, browserProcess = null) { const findings = []; const effectiveApiDomLag = apiDomLag || buildApiDomLagReport(samples, network); if (commandFailures.length > 0) findings.push({ id: "observer-command-failed", severity: "red", summary: "observer control commands failed; analyze must surface command failure instead of hiding it in command artifacts", count: commandFailures.length, commands: commandFailures.slice(0, 20) }); findings.push(...buildFrontendFreezeFindings(errors, control)); + findings.push(...buildBrowserProcessFindings(browserProcess)); findings.push(...buildControlledNavigationRootCauseFindings(control, manifest)); findings.push(...buildSessionInvariantFindings(control, manifest)); const commandTimes = control @@ -3092,6 +3106,314 @@ function buildFrontendFreezeFindings(errors, control) { return findings; } +function buildBrowserProcessReport(rows) { + const samples = (Array.isArray(rows) ? rows : []) + .filter((item) => item && (item.type === "browser-process-sample" || item.process || item.pages)) + .map((item) => ({ ...item, tsMs: Date.parse(String(item.ts || "")) })) + .filter((item) => Number.isFinite(item.tsMs)) + .sort((a, b) => a.tsMs - b.tsMs); + const memorySamples = samples.map((item) => { + const process = item.process && typeof item.process === "object" ? item.process : {}; + const growth = item.growth && typeof item.growth === "object" ? item.growth : {}; + const totalRssBytes = browserMetricNumber(process.totalRssBytes); + const maxProcessRssBytes = browserMetricNumber(process.maxProcessRssBytes); + return { + ts: item.ts, + tsMs: item.tsMs, + seq: item.seq ?? null, + sampleSeq: item.sampleSeq ?? null, + commandId: item.commandId ?? null, + processCount: browserMetricNumber(process.chromiumProcessCount), + totalRssBytes, + totalRssMb: bytesToMb(totalRssBytes), + maxProcessRssBytes, + maxProcessRssMb: bytesToMb(maxProcessRssBytes), + maxProcess: compactBrowserProcessRow(process.maxProcess), + totalRssGrowthBytes: browserMetricNumber(growth.totalRssGrowthBytes), + totalRssGrowthMb: bytesToMb(growth.totalRssGrowthBytes), + maxProcessRssGrowthBytes: browserMetricNumber(growth.maxProcessRssGrowthBytes), + maxProcessRssGrowthMb: bytesToMb(growth.maxProcessRssGrowthBytes), + roles: process.roles && typeof process.roles === "object" ? process.roles : {}, + valuesRedacted: true, + }; + }); + const computedGrowth = computeBrowserProcessGrowth(memorySamples, alertThresholds.browserRssGrowthWindowMs); + const pageEvents = []; + const cdpTimeoutEvents = []; + for (const sample of samples) { + for (const page of Array.isArray(sample.pages) ? sample.pages : []) { + const responsiveness = page?.responsiveness && typeof page.responsiveness === "object" ? page.responsiveness : {}; + const cdp = page?.cdp && typeof page.cdp === "object" ? page.cdp : {}; + const calls = Array.isArray(cdp.calls) ? cdp.calls : []; + const event = { + ts: sample.ts, + tsMs: sample.tsMs, + seq: sample.seq ?? null, + sampleSeq: sample.sampleSeq ?? null, + commandId: sample.commandId ?? null, + pageRole: page?.pageRole ?? null, + pageId: page?.pageId ?? null, + pageEpoch: page?.pageEpoch ?? null, + url: safeReportUrl(page?.url), + timeoutMs: browserMetricNumber(page?.timeoutMs), + responsivenessOk: responsiveness.ok === true, + responsivenessTimeout: responsiveness.timeout === true, + responsivenessLatencyMs: browserMetricNumber(responsiveness.latencyMs), + cdpTimeoutCount: browserMetricNumber(cdp.timeoutCount), + cdpErrorCount: browserMetricNumber(cdp.errorCount), + heapUsedMb: page?.heapUsage ? bytesToMb(page.heapUsage.usedSize) : null, + heapTotalMb: page?.heapUsage ? bytesToMb(page.heapUsage.totalSize) : null, + domNodes: page?.domCounters ? browserMetricNumber(page.domCounters.nodes) : null, + valuesRedacted: true, + }; + pageEvents.push(event); + for (const call of calls) { + if (call?.timeout !== true || call?.method === "Runtime.evaluate") continue; + cdpTimeoutEvents.push({ + ...event, + method: call.method ?? null, + latencyMs: browserMetricNumber(call.latencyMs), + errorMessage: limitText(call.error?.message ?? "", 180), + valuesRedacted: true, + }); + } + const callTimeoutCount = calls.filter((call) => call?.timeout === true && call?.method !== "Runtime.evaluate").length; + if (Number(event.cdpTimeoutCount || 0) > callTimeoutCount && calls.length === 0) { + cdpTimeoutEvents.push({ ...event, method: "cdp-session", latencyMs: event.responsivenessLatencyMs, errorMessage: "CDP session or metric collection timed out", valuesRedacted: true }); + } + } + } + const responsivenessEvents = pageEvents.filter((item) => item.responsivenessTimeout === true || Number(item.responsivenessLatencyMs ?? 0) >= alertThresholds.playwrightResponsivenessRedMs); + const maxTotal = maxByNumber(memorySamples, (item) => item.totalRssBytes); + const maxProcess = maxByNumber(memorySamples, (item) => item.maxProcessRssBytes); + const maxGrowth = maxByNumber(computedGrowth, (item) => item.totalRssGrowthBytes); + const maxProcessGrowth = maxByNumber(computedGrowth, (item) => item.maxProcessRssGrowthBytes); + return { + summary: { + sampleCount: samples.length, + memorySampleCount: memorySamples.length, + pageMetricCount: pageEvents.length, + cdpMetricsTimeoutCount: cdpTimeoutEvents.length, + responsivenessSlowCount: responsivenessEvents.length, + responsivenessTimeoutCount: responsivenessEvents.filter((item) => item.responsivenessTimeout === true).length, + maxTotalRssMb: maxTotal ? bytesToMb(maxTotal.totalRssBytes) : null, + maxProcessRssMb: maxProcess ? bytesToMb(maxProcess.maxProcessRssBytes) : null, + maxTotalRssGrowthMb: maxGrowth ? bytesToMb(maxGrowth.totalRssGrowthBytes) : null, + maxProcessRssGrowthMb: maxProcessGrowth ? bytesToMb(maxProcessGrowth.maxProcessRssGrowthBytes) : null, + totalRssRedMb: alertThresholds.browserTotalRssRedMb, + processRssRedMb: alertThresholds.browserProcessRssRedMb, + growthRedMb: alertThresholds.browserRssGrowthRedMb, + growthWindowMs: alertThresholds.browserRssGrowthWindowMs, + responsivenessRedMs: alertThresholds.playwrightResponsivenessRedMs, + valuesRedacted: true, + }, + memorySamples: memorySamples.slice(-60), + growthSamples: computedGrowth.slice(-60), + maxTotalRssSample: maxTotal, + maxProcessRssSample: maxProcess, + maxGrowthSample: maxGrowth, + maxProcessGrowthSample: maxProcessGrowth, + responsivenessEvents: responsivenessEvents.slice(-80), + cdpTimeoutEvents: cdpTimeoutEvents.slice(-80), + latestPageEvents: pageEvents.slice(-20), + valuesRedacted: true, + }; +} + +function buildBrowserProcessFindings(report) { + const summary = report?.summary || {}; + if (!summary || Number(summary.sampleCount ?? 0) <= 0) return []; + const findings = []; + const maxTotalRssMb = Number(summary.maxTotalRssMb ?? 0); + const maxProcessRssMb = Number(summary.maxProcessRssMb ?? 0); + if (maxTotalRssMb >= alertThresholds.browserTotalRssRedMb || maxProcessRssMb >= alertThresholds.browserProcessRssRedMb) { + findings.push({ + id: "frontend-browser-memory-rss-red", + severity: "red", + summary: "Chromium RSS exceeded YAML red threshold during observation; browser memory growth is freeze evidence and must not be hidden by page refresh/fallback", + maxTotalRssMb, + maxProcessRssMb, + totalRssRedMb: alertThresholds.browserTotalRssRedMb, + processRssRedMb: alertThresholds.browserProcessRssRedMb, + maxTotalRssSample: report.maxTotalRssSample, + maxProcessRssSample: report.maxProcessRssSample, + rootCause: "frontend_browser_process_memory_pressure", + rootCauseStatus: "confirmed-from-runner-process-rss", + rootCauseConfidence: "high", + fallbackAllowed: false, + valuesRedacted: true, + }); + } + const maxTotalRssGrowthMb = Number(summary.maxTotalRssGrowthMb ?? 0); + const maxProcessRssGrowthMb = Number(summary.maxProcessRssGrowthMb ?? 0); + if (maxTotalRssGrowthMb >= alertThresholds.browserRssGrowthRedMb || maxProcessRssGrowthMb >= alertThresholds.browserRssGrowthRedMb) { + findings.push({ + id: "frontend-browser-memory-growth-red", + severity: "red", + summary: "Chromium RSS grew beyond YAML window budget; this matches the reported freeze pattern of browser memory rapidly climbing", + maxTotalRssGrowthMb, + maxProcessRssGrowthMb, + growthRedMb: alertThresholds.browserRssGrowthRedMb, + windowMs: alertThresholds.browserRssGrowthWindowMs, + maxGrowthSample: report.maxGrowthSample, + maxProcessGrowthSample: report.maxProcessGrowthSample, + rootCause: "frontend_browser_process_memory_leak_or_unbounded_render_growth", + rootCauseStatus: "confirmed-from-runner-process-rss-growth", + rootCauseConfidence: "high", + fallbackAllowed: false, + valuesRedacted: true, + }); + } + const responsivenessEvents = Array.isArray(report.responsivenessEvents) ? report.responsivenessEvents : []; + const severeLatencyEvents = responsivenessEvents.filter((item) => item.responsivenessTimeout !== true && Number(item.responsivenessLatencyMs ?? 0) >= alertThresholds.playwrightResponsivenessRedMs); + const responsivenessTimeoutBurst = firstBurst( + responsivenessEvents.filter((item) => item.responsivenessTimeout === true), + alertThresholds.playwrightResponsivenessTimeoutRedCount, + alertThresholds.domEvaluateTimeoutRedWindowMs, + ); + if (severeLatencyEvents.length > 0 || responsivenessTimeoutBurst) { + const events = severeLatencyEvents.length > 0 ? severeLatencyEvents : responsivenessTimeoutBurst; + findings.push({ + id: "frontend-playwright-responsiveness-red", + severity: "red", + summary: "Playwright/CDP responsiveness probe exceeded YAML budget; treat the browser page as frozen instead of refreshing or falling back", + count: events.length, + latencyRedMs: alertThresholds.playwrightResponsivenessRedMs, + timeoutThresholdCount: alertThresholds.playwrightResponsivenessTimeoutRedCount, + windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs, + events: events.slice(0, 20).map(browserProcessEventRef), + rootCause: "frontend_browser_page_unresponsive_to_playwright", + rootCauseStatus: "confirmed-from-cdp-runtime-evaluate", + rootCauseConfidence: "high", + fallbackAllowed: false, + valuesRedacted: true, + }); + } + const cdpTimeoutEvents = Array.isArray(report.cdpTimeoutEvents) ? report.cdpTimeoutEvents : []; + const cdpTimeoutBurst = firstBurst(cdpTimeoutEvents, alertThresholds.cdpMetricsTimeoutRedCount, alertThresholds.domEvaluateTimeoutRedWindowMs); + if (cdpTimeoutBurst) { + findings.push({ + id: "frontend-cdp-metrics-timeout-red", + severity: "red", + summary: "CDP browser metrics timed out repeatedly; the sentinel has direct browser-side evidence of a hung page/runtime", + count: cdpTimeoutBurst.length, + thresholdCount: alertThresholds.cdpMetricsTimeoutRedCount, + windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs, + events: cdpTimeoutBurst.slice(0, 20).map(browserProcessEventRef), + rootCause: "frontend_browser_cdp_metrics_unresponsive", + rootCauseStatus: "confirmed-from-cdp-metrics-timeouts", + rootCauseConfidence: "high", + fallbackAllowed: false, + valuesRedacted: true, + }); + } + return findings; +} + +function computeBrowserProcessGrowth(samples, windowMs) { + const budgetMs = Math.max(1000, Number(windowMs || 0)); + const sorted = (samples || []).filter((item) => Number.isFinite(item.tsMs)).sort((a, b) => a.tsMs - b.tsMs); + return sorted.map((item, index) => { + const candidates = sorted.slice(0, index + 1).filter((candidate) => item.tsMs - candidate.tsMs <= budgetMs); + const totalBaseline = minByNumber(candidates, (candidate) => candidate.totalRssBytes); + const processBaseline = minByNumber(candidates, (candidate) => candidate.maxProcessRssBytes); + const totalGrowth = Number(item.totalRssBytes || 0) - Number(totalBaseline?.totalRssBytes || 0); + const processGrowth = Number(item.maxProcessRssBytes || 0) - Number(processBaseline?.maxProcessRssBytes || 0); + return { + ...item, + windowMs: budgetMs, + totalBaselineAt: totalBaseline?.ts ?? null, + processBaselineAt: processBaseline?.ts ?? null, + totalRssGrowthBytes: totalGrowth, + totalRssGrowthMb: bytesToMb(totalGrowth), + maxProcessRssGrowthBytes: processGrowth, + maxProcessRssGrowthMb: bytesToMb(processGrowth), + valuesRedacted: true, + }; + }); +} + +function browserProcessEventRef(item) { + return { + ts: item?.ts ?? null, + seq: item?.seq ?? null, + sampleSeq: item?.sampleSeq ?? null, + commandId: item?.commandId ?? null, + pageRole: item?.pageRole ?? null, + pageId: item?.pageId ?? null, + pageEpoch: item?.pageEpoch ?? null, + timeoutMs: item?.timeoutMs ?? null, + responsivenessLatencyMs: item?.responsivenessLatencyMs ?? item?.latencyMs ?? null, + responsivenessTimeout: item?.responsivenessTimeout === true, + cdpTimeoutCount: item?.cdpTimeoutCount ?? null, + method: item?.method ?? null, + errorMessage: item?.errorMessage ?? null, + valuesRedacted: true, + }; +} + +function compactBrowserProcessRow(item) { + if (!item || typeof item !== "object") return null; + return { + pid: item.pid ?? null, + role: item.role ?? null, + name: item.name ?? null, + rssMb: bytesToMb(item.rssBytes), + vmSizeMb: bytesToMb(item.vmSizeBytes), + commandHash: item.commandHash ?? null, + valuesRedacted: true, + }; +} + +function browserMetricNumber(value) { + const numeric = Number(value); + return Number.isFinite(numeric) ? numeric : null; +} + +function bytesToMb(value) { + const numeric = Number(value); + if (!Number.isFinite(numeric)) return null; + return Number((numeric / 1024 / 1024).toFixed(1)); +} + +function maxByNumber(items, getter) { + let selected = null; + let selectedValue = Number.NEGATIVE_INFINITY; + for (const item of Array.isArray(items) ? items : []) { + const value = Number(getter(item)); + if (!Number.isFinite(value)) continue; + if (!selected || value > selectedValue) { + selected = item; + selectedValue = value; + } + } + return selected; +} + +function minByNumber(items, getter) { + let selected = null; + let selectedValue = Number.POSITIVE_INFINITY; + for (const item of Array.isArray(items) ? items : []) { + const value = Number(getter(item)); + if (!Number.isFinite(value)) continue; + if (!selected || value < selectedValue) { + selected = item; + selectedValue = value; + } + } + return selected; +} + +function safeReportUrl(value) { + if (!value) return null; + try { + const url = new URL(String(value)); + return url.origin + url.pathname; + } catch { + return limitText(String(value), 200); + } +} + function frontendFreezeErrorEvent(item, promptTimes) { const details = objectValue(item?.error?.details); const message = String(item?.error?.message ?? item?.message ?? item?.error ?? ""); @@ -3220,7 +3542,7 @@ function maxPresentNumber(values) { return numbers.length > 0 ? Math.max(...numbers) : null; } -function buildRecentAnalysisWindow({ samples, control, network, consoleEvents, errors, manifest }) { +function buildRecentAnalysisWindow({ samples, control, network, consoleEvents, errors, browserProcessRows, manifest }) { const latestSampleMs = latestTimestampMs(samples); const windowMs = 5 * 60 * 1000; const fromMs = Number.isFinite(latestSampleMs) ? latestSampleMs - windowMs : Number.NEGATIVE_INFINITY; @@ -3234,13 +3556,15 @@ function buildRecentAnalysisWindow({ samples, control, network, consoleEvents, e const windowNetwork = network.filter(inWindow); const windowConsole = consoleEvents.filter(inWindow); const windowErrors = errors.filter(inWindow); + const windowBrowserProcessRows = (browserProcessRows || []).filter(inWindow); const sampleMetrics = buildSampleMetrics(windowSamples, control); const pageProvenance = buildPageProvenanceReport(windowSamples, windowControl, manifest); const pagePerformance = buildPagePerformanceReport(windowSamples, manifest); const promptNetwork = buildPromptNetworkReport(windowControl, windowNetwork); const runtimeAlerts = buildRuntimeAlerts(windowSamples, control, windowNetwork, windowConsole, windowErrors); const apiDomLag = buildApiDomLagReport(windowSamples, windowNetwork); - const findings = buildFindings(windowSamples, control, windowNetwork, windowErrors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, [], {}, apiDomLag); + const browserProcess = buildBrowserProcessReport(windowBrowserProcessRows); + const findings = buildFindings(windowSamples, control, windowNetwork, windowErrors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, [], {}, apiDomLag, browserProcess); return { summary: { name: "recent-5m", @@ -3252,6 +3576,7 @@ function buildRecentAnalysisWindow({ samples, control, network, consoleEvents, e network: windowNetwork.length, console: windowConsole.length, errors: windowErrors.length, + browserProcess: windowBrowserProcessRows.length, valuesRedacted: true }, sampleMetrics, @@ -3260,6 +3585,7 @@ function buildRecentAnalysisWindow({ samples, control, network, consoleEvents, e promptNetwork, runtimeAlerts, apiDomLag, + browserProcess, findings, valuesRedacted: true }; diff --git a/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts index 33d8a2f6..6df9340a 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-timing-source.ts @@ -2227,6 +2227,14 @@ function renderMarkdown(report) { const alertSummary = report.runtimeAlerts?.summary || {}; const apiDomLag = report.apiDomLag || {}; const apiDomLagSummary = apiDomLag.summary || {}; + const browserProcess = report.browserProcess || {}; + const browserProcessSummary = browserProcess.summary || {}; + const browserProcessLines = Array.isArray(browserProcess.memorySamples) && browserProcess.memorySamples.length > 0 + ? browserProcess.memorySamples.slice(-40).map((item) => "- " + (item.ts || "-") + " seq=" + (item.seq ?? "-") + " totalRssMb=" + (item.totalRssMb ?? "-") + " maxProcessRssMb=" + (item.maxProcessRssMb ?? "-") + " totalGrowthMb=" + (item.totalRssGrowthMb ?? "-") + " processGrowthMb=" + (item.maxProcessRssGrowthMb ?? "-") + " processes=" + (item.processCount ?? "-")).join("\n") + : "- 无 browser-process RSS 样本。"; + const browserResponsivenessLines = Array.isArray(browserProcess.responsivenessEvents) && browserProcess.responsivenessEvents.length > 0 + ? browserProcess.responsivenessEvents.slice(-40).map((item) => "- " + (item.ts || "-") + " role=" + (item.pageRole || "-") + " pageId=" + (item.pageId || "-") + " latencyMs=" + (item.responsivenessLatencyMs ?? "-") + " timeout=" + String(item.responsivenessTimeout === true) + " cdpTimeouts=" + (item.cdpTimeoutCount ?? 0)).join("\n") + : "- 无 Playwright/CDP 响应性超阈值样本。"; const projectManagement = report.projectManagement || {}; const projectSummary = projectManagement.summary || {}; const projectCommandLines = Array.isArray(projectManagement.commands) && projectManagement.commands.length > 0 @@ -2336,6 +2344,16 @@ function renderMarkdown(report) { + "- console: " + (report.counts.console ?? 0) + "\n" + "- errors: " + report.counts.errors + "\n\n" + "## Findings\n\n" + findingLines + "\n\n" + + "## Browser process\n\n" + + "- browserProcessSamples: " + (browserProcessSummary.sampleCount ?? 0) + "\n" + + "- pageMetricCount: " + (browserProcessSummary.pageMetricCount ?? 0) + "\n" + + "- maxTotalRssMb: " + (browserProcessSummary.maxTotalRssMb ?? "-") + " / red=" + (browserProcessSummary.totalRssRedMb ?? "-") + "\n" + + "- maxProcessRssMb: " + (browserProcessSummary.maxProcessRssMb ?? "-") + " / red=" + (browserProcessSummary.processRssRedMb ?? "-") + "\n" + + "- maxTotalRssGrowthMb: " + (browserProcessSummary.maxTotalRssGrowthMb ?? "-") + " / red=" + (browserProcessSummary.growthRedMb ?? "-") + " windowMs=" + (browserProcessSummary.growthWindowMs ?? "-") + "\n" + + "- responsivenessSlowCount: " + (browserProcessSummary.responsivenessSlowCount ?? 0) + " timeoutCount=" + (browserProcessSummary.responsivenessTimeoutCount ?? 0) + " redMs=" + (browserProcessSummary.responsivenessRedMs ?? "-") + "\n" + + "- cdpMetricsTimeoutCount: " + (browserProcessSummary.cdpMetricsTimeoutCount ?? 0) + "\n\n" + + "### Browser RSS samples\n\n" + browserProcessLines + "\n\n" + + "### Browser responsiveness samples\n\n" + browserResponsivenessLines + "\n\n" + "## Project management\n\n" + "- enabled: " + String(projectSummary.enabled === true) + "\n" + "- projectSampleCount: " + (projectSummary.projectSampleCount ?? 0) + "\n" diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index fa595f21..c0218da0 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -55,6 +55,7 @@ const files = { console: path.join(stateDir, "console.jsonl"), errors: path.join(stateDir, "errors.jsonl"), artifacts: path.join(stateDir, "artifacts.jsonl"), + browserProcess: path.join(stateDir, "browser-process.jsonl"), }; let browser; @@ -76,6 +77,9 @@ let controlPageEpoch = 0; let observerPageEpoch = 0; let currentPageProvenance = null; let heartbeatPulseTimer = null; +let browserProcessMonitorStop = null; +let browserProcessMonitorSeq = 0; +const browserProcessHistory = []; const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] }; try { @@ -107,6 +111,7 @@ try { } return result ?? { ok: false, reason: "observer-not-ready", pageRole: "observer", pageId: observerPageId, valuesRedacted: true }; }); + browserProcessMonitorStop = startBrowserProcessMonitor(); terminalStatus = "running"; await writeManifest({ status: "running", auth: publicAuth(auth) }); await writeHeartbeat({ status: "running" }); @@ -130,6 +135,7 @@ try { await writeManifest({ status: "failed", error: errorSummary(error) }).catch(() => {}); process.exitCode = 2; } finally { + if (browserProcessMonitorStop) browserProcessMonitorStop(); if (heartbeatPulseTimer) clearInterval(heartbeatPulseTimer); if (browser) await browser.close().catch(() => {}); } @@ -236,6 +242,339 @@ function startHeartbeatPulse() { return timer; } +function startBrowserProcessMonitor() { + const intervalMs = Math.max(250, Math.floor(Number(alertThresholds.browserProcessSampleIntervalMs) || 1000)); + let stopped = false; + let running = false; + const tick = async (reason) => { + if (stopped || running) return; + running = true; + try { + await collectBrowserProcessSample(reason || "interval"); + } catch (error) { + await appendJsonl(files.errors, eventRecord("browser-process-monitor-error", { error: errorSummary(error), valuesRedacted: true })).catch(() => {}); + } finally { + running = false; + } + }; + void tick("startup"); + const timer = setInterval(() => { + void tick("interval"); + }, intervalMs); + if (timer && typeof timer.unref === "function") timer.unref(); + return () => { + stopped = true; + clearInterval(timer); + }; +} + +async function collectBrowserProcessSample(reason) { + browserProcessMonitorSeq += 1; + const tsMs = Date.now(); + const browserPid = browserProcessPid(); + const processSummary = await collectChromiumProcessSummary(browserPid); + const growth = updateBrowserProcessHistory(tsMs, processSummary); + const pages = []; + if (page && !page.isClosed()) { + pages.push(await collectBrowserPageRuntimeMetrics(page, { pageRole: "control", targetPageId: pageId, pageEpoch: controlPageEpoch }).catch((error) => browserPageRuntimeMetricError("control", pageId, controlPageEpoch, error))); + } + if (observerPage && !observerPage.isClosed()) { + pages.push(await collectBrowserPageRuntimeMetrics(observerPage, { pageRole: "observer", targetPageId: observerPageId, pageEpoch: observerPageEpoch }).catch((error) => browserPageRuntimeMetricError("observer", observerPageId, observerPageEpoch, error))); + } + await appendJsonl(files.browserProcess, eventRecord("browser-process-sample", { + seq: browserProcessMonitorSeq, + reason, + monitorIntervalMs: Math.max(250, Math.floor(Number(alertThresholds.browserProcessSampleIntervalMs) || 1000)), + browserPid, + process: processSummary, + growth, + pages, + valuesRedacted: true, + })); +} + +function browserProcessPid() { + try { + const childProcess = browser && typeof browser.process === "function" ? browser.process() : null; + const pid = Number(childProcess?.pid); + return Number.isFinite(pid) && pid > 0 ? Math.floor(pid) : null; + } catch { + return null; + } +} + +async function collectChromiumProcessSummary(browserPid) { + const all = await readProcProcessTable(); + const childrenByPpid = new Map(); + for (const item of all) { + if (!Number.isFinite(item.ppid)) continue; + const list = childrenByPpid.get(item.ppid) || []; + list.push(item.pid); + childrenByPpid.set(item.ppid, list); + } + const roots = [process.pid, browserPid].filter((pid, index, items) => Number.isFinite(Number(pid)) && Number(pid) > 0 && items.indexOf(pid) === index).map((pid) => Number(pid)); + const descendants = new Set(roots); + const queue = roots.slice(); + while (queue.length > 0) { + const pid = queue.shift(); + for (const child of childrenByPpid.get(pid) || []) { + if (descendants.has(child)) continue; + descendants.add(child); + queue.push(child); + } + } + const processes = all + .filter((item) => descendants.has(item.pid)) + .map((item) => ({ ...item, role: classifyChromiumProcess(item) })) + .filter((item) => item.role) + .map((item) => ({ + pid: item.pid, + ppid: item.ppid, + name: item.name || null, + role: item.role, + rssBytes: item.rssBytes, + vmSizeBytes: item.vmSizeBytes, + commandHash: item.cmdline ? sha256Text(item.cmdline) : null, + commandPreview: redactProcessCommandPreview(item.cmdline), + valuesRedacted: true, + })) + .sort((a, b) => Number(b.rssBytes || 0) - Number(a.rssBytes || 0)); + const roles = {}; + for (const item of processes) { + const role = item.role || "unknown"; + const summary = roles[role] || { count: 0, rssBytes: 0, maxRssBytes: 0 }; + summary.count += 1; + summary.rssBytes += Number(item.rssBytes || 0); + summary.maxRssBytes = Math.max(summary.maxRssBytes, Number(item.rssBytes || 0)); + roles[role] = summary; + } + const totalRssBytes = processes.reduce((sum, item) => sum + Number(item.rssBytes || 0), 0); + const maxProcess = processes[0] || null; + return { + sampledRootPids: roots, + chromiumProcessCount: processes.length, + totalRssBytes, + totalRssMb: bytesToMb(totalRssBytes), + maxProcessRssBytes: maxProcess ? Number(maxProcess.rssBytes || 0) : 0, + maxProcessRssMb: maxProcess ? bytesToMb(maxProcess.rssBytes) : 0, + maxProcess, + roles, + processes: processes.slice(0, 20), + valuesRedacted: true, + }; +} + +async function readProcProcessTable() { + let entries = []; + try { + entries = await readdir("/proc"); + } catch { + return []; + } + const numeric = entries.map((name) => Number(name)).filter((pid) => Number.isInteger(pid) && pid > 0); + const rows = await Promise.all(numeric.map((pid) => readOneProcProcess(pid).catch(() => null))); + return rows.filter(Boolean); +} + +async function readOneProcProcess(pid) { + const [statText, statusText, cmdlineText] = await Promise.all([ + readFile(path.join("/proc", String(pid), "stat"), "utf8").catch(() => ""), + readFile(path.join("/proc", String(pid), "status"), "utf8").catch(() => ""), + readFile(path.join("/proc", String(pid), "cmdline"), "utf8").catch(() => ""), + ]); + if (!statText && !statusText) return null; + const ppid = procPpidFromStat(statText); + const name = procStatusField(statusText, "Name") || null; + const rssKb = procStatusKb(statusText, "VmRSS"); + const vmSizeKb = procStatusKb(statusText, "VmSize"); + return { + pid, + ppid, + name, + cmdline: String(cmdlineText || "").replace(/\0/gu, " ").replace(/\s+/gu, " ").trim(), + rssBytes: Number.isFinite(rssKb) ? rssKb * 1024 : 0, + vmSizeBytes: Number.isFinite(vmSizeKb) ? vmSizeKb * 1024 : 0, + }; +} + +function procPpidFromStat(value) { + const text = String(value || ""); + const end = text.lastIndexOf(")"); + if (end < 0) return null; + const parts = text.slice(end + 1).trim().split(/\s+/u); + const ppid = Number(parts[1]); + return Number.isFinite(ppid) ? ppid : null; +} + +function procStatusField(value, key) { + const prefix = String(key || "") + ":"; + for (const line of String(value || "").split(/\n/u)) { + if (line.startsWith(prefix)) return line.slice(prefix.length).trim(); + } + return null; +} + +function procStatusKb(value, key) { + const raw = procStatusField(value, key); + const match = String(raw || "").match(/^(\d+)\s+kB$/u); + return match ? Number(match[1]) : null; +} + +function classifyChromiumProcess(item) { + const cmdline = String(item?.cmdline || ""); + const combined = (cmdline + " " + String(item?.name || "")).toLowerCase(); + if (!/(?:chromium|chrome|headless_shell)/u.test(combined)) return null; + if (/--type=renderer\b/u.test(cmdline)) return "renderer"; + if (/--type=gpu-process\b/u.test(cmdline)) return "gpu"; + if (/--type=utility\b/u.test(cmdline)) return "utility"; + if (/--type=zygote\b/u.test(cmdline)) return "zygote"; + if (/--type=broker\b/u.test(cmdline)) return "broker"; + if (/--type=/u.test(cmdline)) return "other"; + return "browser"; +} + +function redactProcessCommandPreview(value) { + const text = String(value || ""); + if (!text) return null; + const parts = text.split(/\s+/u).slice(0, 18).map((part) => { + if (/--(?:proxy|token|secret|password|cookie|auth|key)[^=]*=/iu.test(part)) return part.replace(/=.*/u, "=[redacted]"); + return part.replace(/:\/\/[^/@\s]+@/u, "://[redacted]@"); + }); + return truncate(parts.join(" "), 260); +} + +function updateBrowserProcessHistory(tsMs, processSummary) { + const windowMs = Math.max(1000, Number(alertThresholds.browserRssGrowthWindowMs) || 30000); + const totalRssBytes = Number(processSummary?.totalRssBytes || 0); + const maxProcessRssBytes = Number(processSummary?.maxProcessRssBytes || 0); + browserProcessHistory.push({ tsMs, totalRssBytes, maxProcessRssBytes }); + const retentionMs = Math.max(windowMs * 3, 180000); + while (browserProcessHistory.length > 0 && tsMs - browserProcessHistory[0].tsMs > retentionMs) browserProcessHistory.shift(); + const windowStartMs = tsMs - windowMs; + const candidates = browserProcessHistory.filter((item) => item.tsMs >= windowStartMs && item.tsMs <= tsMs); + const baseline = candidates[0] || browserProcessHistory[0] || null; + const totalRssGrowthBytes = baseline ? totalRssBytes - Number(baseline.totalRssBytes || 0) : 0; + const maxProcessRssGrowthBytes = baseline ? maxProcessRssBytes - Number(baseline.maxProcessRssBytes || 0) : 0; + return { + windowMs, + baselineAt: baseline ? new Date(baseline.tsMs).toISOString() : null, + baselineTotalRssBytes: baseline ? baseline.totalRssBytes : null, + baselineMaxProcessRssBytes: baseline ? baseline.maxProcessRssBytes : null, + totalRssGrowthBytes, + totalRssGrowthMb: bytesToMb(totalRssGrowthBytes), + maxProcessRssGrowthBytes, + maxProcessRssGrowthMb: bytesToMb(maxProcessRssGrowthBytes), + valuesRedacted: true, + }; +} + +async function collectBrowserPageRuntimeMetrics(targetPage, { pageRole, targetPageId, pageEpoch }) { + const timeoutMs = Math.max(1000, Math.floor(Number(alertThresholds.playwrightResponsivenessRedMs) || 5000)); + const startedAtMs = Date.now(); + let session = null; + const result = { + pageRole, + pageId: targetPageId, + pageEpoch: Number.isFinite(Number(pageEpoch)) ? Number(pageEpoch) : 0, + url: pageUrl(targetPage), + timeoutMs, + responsiveness: null, + cdp: { timeoutCount: 0, errorCount: 0, calls: [] }, + valuesRedacted: true, + }; + try { + session = await withHardTimeout(targetPage.context().newCDPSession(targetPage), timeoutMs, "newCDPSession exceeded " + timeoutMs + "ms"); + const enable = await timedCdpSend(session, "Performance.enable", {}, timeoutMs); + if (enable.ok !== true) result.cdp.calls.push({ method: "Performance.enable", ...enable }); + result.responsiveness = await timedCdpSend(session, "Runtime.evaluate", { expression: "1", returnByValue: true }, timeoutMs); + result.cdp.calls.push({ method: "Runtime.evaluate", ...result.responsiveness }); + const performance = await timedCdpSend(session, "Performance.getMetrics", {}, timeoutMs); + result.cdp.calls.push({ method: "Performance.getMetrics", ...performance }); + result.performance = performance.value; + const heap = await timedCdpSend(session, "Runtime.getHeapUsage", {}, timeoutMs); + result.cdp.calls.push({ method: "Runtime.getHeapUsage", ...heap }); + result.heapUsage = heap.value; + const domCounters = await timedCdpSend(session, "Memory.getDOMCounters", {}, timeoutMs); + result.cdp.calls.push({ method: "Memory.getDOMCounters", ...domCounters }); + result.domCounters = domCounters.value; + result.cdp.timeoutCount = result.cdp.calls.filter((item) => item.timeout === true).length; + result.cdp.errorCount = result.cdp.calls.filter((item) => item.ok !== true).length; + } catch (error) { + result.sessionError = errorSummary(error); + result.cdp.timeoutCount = isTimeoutErrorMessage(error?.message) ? 1 : 0; + result.cdp.errorCount = 1; + } finally { + result.latencyMs = Date.now() - startedAtMs; + if (session) await withHardTimeout(session.detach(), 1000, "CDP session detach exceeded 1000ms").catch(() => {}); + } + return result; +} + +function browserPageRuntimeMetricError(pageRole, targetPageId, pageEpoch, error) { + return { + pageRole, + pageId: targetPageId, + pageEpoch: Number.isFinite(Number(pageEpoch)) ? Number(pageEpoch) : 0, + url: null, + timeoutMs: Math.max(1000, Math.floor(Number(alertThresholds.playwrightResponsivenessRedMs) || 5000)), + responsiveness: { ok: false, timeout: isTimeoutErrorMessage(error?.message), error: errorSummary(error), valuesRedacted: true }, + cdp: { timeoutCount: isTimeoutErrorMessage(error?.message) ? 1 : 0, errorCount: 1, calls: [] }, + valuesRedacted: true, + }; +} + +async function timedCdpSend(session, method, params, timeoutMs) { + const startedAtMs = Date.now(); + try { + const value = await withHardTimeout(session.send(method, params || {}), timeoutMs, method + " exceeded " + timeoutMs + "ms"); + return { ok: true, timeout: false, latencyMs: Date.now() - startedAtMs, value: compactCdpValue(method, value), valuesRedacted: true }; + } catch (error) { + return { ok: false, timeout: isTimeoutErrorMessage(error?.message), latencyMs: Date.now() - startedAtMs, error: errorSummary(error), valuesRedacted: true }; + } +} + +function compactCdpValue(method, value) { + if (!value || typeof value !== "object") return value ?? null; + if (method === "Performance.getMetrics") { + const wanted = new Set(["Timestamp", "Documents", "Frames", "JSEventListeners", "Nodes", "LayoutCount", "RecalcStyleCount", "LayoutDuration", "RecalcStyleDuration", "ScriptDuration", "TaskDuration", "JSHeapUsedSize", "JSHeapTotalSize"]); + const metrics = {}; + for (const item of Array.isArray(value.metrics) ? value.metrics : []) { + if (wanted.has(item?.name)) metrics[item.name] = Number.isFinite(Number(item?.value)) ? Number(item.value) : null; + } + return { metricCount: Array.isArray(value.metrics) ? value.metrics.length : 0, metrics, valuesRedacted: true }; + } + if (method === "Runtime.getHeapUsage") { + return { + usedSize: Number.isFinite(Number(value.usedSize)) ? Number(value.usedSize) : null, + totalSize: Number.isFinite(Number(value.totalSize)) ? Number(value.totalSize) : null, + embedderHeapUsedSize: Number.isFinite(Number(value.embedderHeapUsedSize)) ? Number(value.embedderHeapUsedSize) : null, + valuesRedacted: true, + }; + } + if (method === "Memory.getDOMCounters") { + return { + documents: Number.isFinite(Number(value.documents)) ? Number(value.documents) : null, + nodes: Number.isFinite(Number(value.nodes)) ? Number(value.nodes) : null, + jsEventListeners: Number.isFinite(Number(value.jsEventListeners)) ? Number(value.jsEventListeners) : null, + valuesRedacted: true, + }; + } + if (method === "Runtime.evaluate") { + return { resultType: value.result?.type ?? null, unserializableValue: value.result?.unserializableValue ?? null, exceptionDetails: value.exceptionDetails ? { text: truncate(value.exceptionDetails.text || "", 200), valuesRedacted: true } : null, valuesRedacted: true }; + } + return { valuesRedacted: true }; +} + +function isTimeoutErrorMessage(value) { + return /timeout|timed\s*out|exceeded\s+\d+\s*ms/iu.test(String(value || "")); +} + +function bytesToMb(value) { + const numeric = Number(value); + if (!Number.isFinite(numeric)) return null; + return Number((numeric / 1024 / 1024).toFixed(1)); +} + function attachPassiveListeners(targetPage, pageRole = "control", targetPageId = pageId) { targetPage.on("request", (request) => { void appendJsonl(files.network, eventRecord("request", { @@ -4713,6 +5052,14 @@ function parseAlertThresholds(value) { domEvaluateTimeoutRedWindowMs: requiredPositiveThreshold(raw, "domEvaluateTimeoutRedWindowMs"), screenshotTimeoutRedCount: requiredPositiveThreshold(raw, "screenshotTimeoutRedCount"), pageErrorRedCount: requiredPositiveThreshold(raw, "pageErrorRedCount"), + browserProcessSampleIntervalMs: requiredPositiveThreshold(raw, "browserProcessSampleIntervalMs"), + browserTotalRssRedMb: requiredPositiveThreshold(raw, "browserTotalRssRedMb"), + browserProcessRssRedMb: requiredPositiveThreshold(raw, "browserProcessRssRedMb"), + browserRssGrowthRedMb: requiredPositiveThreshold(raw, "browserRssGrowthRedMb"), + browserRssGrowthWindowMs: requiredPositiveThreshold(raw, "browserRssGrowthWindowMs"), + playwrightResponsivenessRedMs: requiredPositiveThreshold(raw, "playwrightResponsivenessRedMs"), + playwrightResponsivenessTimeoutRedCount: requiredPositiveThreshold(raw, "playwrightResponsivenessTimeoutRedCount"), + cdpMetricsTimeoutRedCount: requiredPositiveThreshold(raw, "cdpMetricsTimeoutRedCount"), uncommandedStateChangeCommandWindowMs: requiredPositiveThreshold(raw, "uncommandedStateChangeCommandWindowMs"), scrollJumpCommandWindowMs: requiredPositiveThreshold(raw, "scrollJumpCommandWindowMs"), scrollJumpFromY: requiredPositiveThreshold(raw, "scrollJumpFromY"), diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index 6ce69124..f3d02037 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -2214,11 +2214,12 @@ export function recoverWebObserveAnalyzeFromArtifacts(options: NodeWebProbeObser "const srcMetrics = objectOrNull(source.sampleMetrics) || objectOrNull(recent.sampleMetrics) || {};", "const pagePerformance = objectOrNull(source.pagePerformance) || objectOrNull(recent.pagePerformance) || {};", "const runtimeAlerts = objectOrNull(source.runtimeAlerts) || objectOrNull(recent.runtimeAlerts) || {};", + "const browserProcess = objectOrNull(source.browserProcess) || objectOrNull(recent.browserProcess) || {};", "const promptNetwork = objectOrNull(source.promptNetwork) || objectOrNull(recent.promptNetwork) || {};", "const archiveSummary = objectOrNull(source.archiveSummary) || {};", "const archiveSampleMetrics = objectOrNull(archiveSummary.sampleMetrics) || objectOrNull(source.sampleMetrics?.summary) || objectOrNull(srcMetrics.summary) || {};", "const slowApis = arr(source.pagePerformanceSlowApi).length > 0 ? arr(source.pagePerformanceSlowApi) : arr(pagePerformance.sameOriginApiByPath).filter((item) => Number(item?.overBudgetCount ?? item?.overFiveSecondCount ?? 0) > 0);", - "const compact = { ok: source.ok === true, command: source.command ?? 'web-probe-observe analyze', stateDir: source.stateDir ?? stateDir, jsonlScope: source.jsonlScope ?? null, alertThresholds: source.alertThresholds ?? null, counts: source.counts ?? null, archiveSummary: { ...archiveSummary, sampleMetrics: archiveSampleMetrics, pagePerformance: objectOrNull(archiveSummary.pagePerformance) || objectOrNull(pagePerformance.summary) || {}, runtimeAlerts: objectOrNull(archiveSummary.runtimeAlerts) || objectOrNull(runtimeAlerts.summary) || {}, redFindings: arr(archiveSummary.redFindings).slice(0, 12).map(slimFinding) }, analysisWindow: source.analysisWindow ?? objectOrNull(recent.summary), sampleMetrics: compactMetrics(srcMetrics), pageProvenance: objectOrNull(source.pageProvenance?.summary) || source.pageProvenance ?? null, pagePerformance: objectOrNull(pagePerformance.summary) || pagePerformance, projectManagement: objectOrNull(source.projectManagement) || null, promptNetwork: objectOrNull(promptNetwork.summary) || promptNetwork, runtimeAlerts: objectOrNull(runtimeAlerts.summary) || runtimeAlerts, runnerErrors: arr(source.runnerErrors).slice(-8), commandFailures: arr(source.commandFailures).slice(-8), commandState: objectOrNull(source.commandState) || null, toolFindings: arr(source.toolFindings).slice(0, 8).map(slimFinding), httpErrorGroups: arr(source.httpErrorGroups ?? runtimeAlerts.networkHttpErrorsByPath).slice(0, 8).map(slimGroup), requestFailedGroups: arr(source.requestFailedGroups ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 8).map(slimGroup), domDiagnosticGroups: arr(source.domDiagnosticGroups ?? runtimeAlerts.domDiagnosticsByText).slice(0, 5).map(slimGroup), domDiagnosticSamples: arr(source.domDiagnosticSamples ?? runtimeAlerts.domDiagnostics).slice(0, 8).map(slimGroup), consoleAlertGroups: arr(source.consoleAlertGroups ?? runtimeAlerts.consoleAlertsByPath).slice(0, 8).map(slimGroup), consoleAlertSamples: arr(source.consoleAlertSamples ?? runtimeAlerts.consoleAlerts).slice(0, 8).map(slimGroup), turnTimingRecentUpdateJumps: arr(source.turnTimingRecentUpdateJumps ?? srcMetrics.turnTimingRecentUpdateSawtoothJumps).slice(0, 8), turnTimingElapsedZeroResets: arr(source.turnTimingElapsedZeroResets ?? srcMetrics.turnTimingElapsedZeroResets).slice(0, 8), turnTimingTotalElapsedForwardJumps: arr(source.turnTimingTotalElapsedForwardJumps ?? srcMetrics.turnTimingTotalElapsedForwardJumps).slice(0, 8), pagePerformanceSlowApi: slowApis.slice(0, 8).map(slimSlowApi), archivePagePerformanceSlowApi: arr(source.archivePagePerformanceSlowApi).slice(0, 8).map(slimSlowApi), pagePerformancePartialApi: arr(source.pagePerformancePartialApi).slice(0, 8), pagePerformanceSseStreams: arr(source.pagePerformanceSseStreams).slice(0, 8), findings: arr(source.findings).slice(0, 12).map(slimFinding), archiveRedFindings: arr(source.archiveRedFindings ?? archiveSummary.redFindings).slice(0, 12).map(slimFinding), reportJsonPath: source.reportJsonPath ?? reportJsonPath, reportJsonSha256: source.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: source.reportMdPath ?? reportMdPath, reportMdSha256: source.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(objectOrNull(source.analyzer) || {}), recoveredFrom: 'analysis-artifact-after-transport-timeout', stdoutPath, stderrPath, stdoutBytes: statSize(stdoutPath), stderrBytes: statSize(stderrPath), reportJsonBytes: statSize(reportJsonPath), reportMdBytes: statSize(reportMdPath), transportExitCode: Number(transportExitRaw), transportTimedOut: transportTimedOutRaw === 'true', valuesRedacted: true }, valuesRedacted: true };", + "const compact = { ok: source.ok === true, command: source.command ?? 'web-probe-observe analyze', stateDir: source.stateDir ?? stateDir, jsonlScope: source.jsonlScope ?? null, alertThresholds: source.alertThresholds ?? null, counts: source.counts ?? null, archiveSummary: { ...archiveSummary, sampleMetrics: archiveSampleMetrics, pagePerformance: objectOrNull(archiveSummary.pagePerformance) || objectOrNull(pagePerformance.summary) || {}, runtimeAlerts: objectOrNull(archiveSummary.runtimeAlerts) || objectOrNull(runtimeAlerts.summary) || {}, browserProcess: objectOrNull(archiveSummary.browserProcess) || objectOrNull(browserProcess.summary) || {}, redFindings: arr(archiveSummary.redFindings).slice(0, 12).map(slimFinding) }, analysisWindow: source.analysisWindow ?? objectOrNull(recent.summary), sampleMetrics: compactMetrics(srcMetrics), pageProvenance: objectOrNull(source.pageProvenance?.summary) || source.pageProvenance ?? null, pagePerformance: objectOrNull(pagePerformance.summary) || pagePerformance, projectManagement: objectOrNull(source.projectManagement) || null, promptNetwork: objectOrNull(promptNetwork.summary) || promptNetwork, runtimeAlerts: objectOrNull(runtimeAlerts.summary) || runtimeAlerts, browserProcess: objectOrNull(browserProcess.summary) || browserProcess, runnerErrors: arr(source.runnerErrors).slice(-8), commandFailures: arr(source.commandFailures).slice(-8), commandState: objectOrNull(source.commandState) || null, toolFindings: arr(source.toolFindings).slice(0, 8).map(slimFinding), httpErrorGroups: arr(source.httpErrorGroups ?? runtimeAlerts.networkHttpErrorsByPath).slice(0, 8).map(slimGroup), requestFailedGroups: arr(source.requestFailedGroups ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 8).map(slimGroup), domDiagnosticGroups: arr(source.domDiagnosticGroups ?? runtimeAlerts.domDiagnosticsByText).slice(0, 5).map(slimGroup), domDiagnosticSamples: arr(source.domDiagnosticSamples ?? runtimeAlerts.domDiagnostics).slice(0, 8).map(slimGroup), consoleAlertGroups: arr(source.consoleAlertGroups ?? runtimeAlerts.consoleAlertsByPath).slice(0, 8).map(slimGroup), consoleAlertSamples: arr(source.consoleAlertSamples ?? runtimeAlerts.consoleAlerts).slice(0, 8).map(slimGroup), turnTimingRecentUpdateJumps: arr(source.turnTimingRecentUpdateJumps ?? srcMetrics.turnTimingRecentUpdateSawtoothJumps).slice(0, 8), turnTimingElapsedZeroResets: arr(source.turnTimingElapsedZeroResets ?? srcMetrics.turnTimingElapsedZeroResets).slice(0, 8), turnTimingTotalElapsedForwardJumps: arr(source.turnTimingTotalElapsedForwardJumps ?? srcMetrics.turnTimingTotalElapsedForwardJumps).slice(0, 8), pagePerformanceSlowApi: slowApis.slice(0, 8).map(slimSlowApi), archivePagePerformanceSlowApi: arr(source.archivePagePerformanceSlowApi).slice(0, 8).map(slimSlowApi), pagePerformancePartialApi: arr(source.pagePerformancePartialApi).slice(0, 8), pagePerformanceSseStreams: arr(source.pagePerformanceSseStreams).slice(0, 8), findings: arr(source.findings).slice(0, 12).map(slimFinding), archiveRedFindings: arr(source.archiveRedFindings ?? archiveSummary.redFindings).slice(0, 12).map(slimFinding), reportJsonPath: source.reportJsonPath ?? reportJsonPath, reportJsonSha256: source.reportJsonSha256 ?? sha256(reportJsonPath), reportMdPath: source.reportMdPath ?? reportMdPath, reportMdSha256: source.reportMdSha256 ?? sha256(reportMdPath), analyzer: { ...(objectOrNull(source.analyzer) || {}), recoveredFrom: 'analysis-artifact-after-transport-timeout', stdoutPath, stderrPath, stdoutBytes: statSize(stdoutPath), stderrBytes: statSize(stderrPath), reportJsonBytes: statSize(reportJsonPath), reportMdBytes: statSize(reportMdPath), transportExitCode: Number(transportExitRaw), transportTimedOut: transportTimedOutRaw === 'true', valuesRedacted: true }, valuesRedacted: true };", "console.log(JSON.stringify(compact));", "UNIDESK_WEB_OBSERVE_RECOVER_ANALYZE_ARTIFACT", ].join("\n");