Codex
599 lines
28 KiB
TypeScript
599 lines
28 KiB
TypeScript
// SPEC: PJ2026-01040111 long-running Workbench observation.
|
|
// Responsibility: Analyzer browser process, memory, responsiveness, and root-cause signal source fragment.
|
|
|
|
export function nodeWebObserveAnalyzerBrowserProcessSource(): string {
|
|
return String.raw`function buildBrowserProcessReport(rows) {
|
|
const blockerEvents = (Array.isArray(rows) ? rows : [])
|
|
.filter((item) => item && item.type === "browser-freeze-blocker")
|
|
.map(compactBrowserFreezeBlockerEvent)
|
|
.filter(Boolean);
|
|
const samples = (Array.isArray(rows) ? rows : [])
|
|
.filter((item) => item && (item.type === "browser-process-sample" || item.process || item.pages))
|
|
.map((item) => ({ ...item, tsMs: Date.parse(String(item.ts || "")) }))
|
|
.filter((item) => Number.isFinite(item.tsMs))
|
|
.sort((a, b) => a.tsMs - b.tsMs);
|
|
const memorySamples = samples.map((item) => {
|
|
const process = item.process && typeof item.process === "object" ? item.process : {};
|
|
const growth = item.growth && typeof item.growth === "object" ? item.growth : {};
|
|
const totalRssBytes = browserMetricNumber(process.totalRssBytes);
|
|
const maxProcessRssBytes = browserMetricNumber(process.maxProcessRssBytes);
|
|
return {
|
|
ts: item.ts,
|
|
tsMs: item.tsMs,
|
|
seq: item.seq ?? null,
|
|
sampleSeq: item.sampleSeq ?? null,
|
|
commandId: item.commandId ?? null,
|
|
processCount: browserMetricNumber(process.chromiumProcessCount),
|
|
totalRssBytes,
|
|
totalRssMb: bytesToMb(totalRssBytes),
|
|
maxProcessRssBytes,
|
|
maxProcessRssMb: bytesToMb(maxProcessRssBytes),
|
|
maxProcess: compactBrowserProcessRow(process.maxProcess),
|
|
totalRssGrowthBytes: browserMetricNumber(growth.totalRssGrowthBytes),
|
|
totalRssGrowthMb: bytesToMb(growth.totalRssGrowthBytes),
|
|
maxProcessRssGrowthBytes: browserMetricNumber(growth.maxProcessRssGrowthBytes),
|
|
maxProcessRssGrowthMb: bytesToMb(growth.maxProcessRssGrowthBytes),
|
|
roles: process.roles && typeof process.roles === "object" ? process.roles : {},
|
|
valuesRedacted: true,
|
|
};
|
|
});
|
|
const computedGrowth = computeBrowserProcessGrowth(memorySamples, alertThresholds.browserRssGrowthWindowMs);
|
|
const pageEvents = [];
|
|
const cdpTimeoutEvents = [];
|
|
for (const sample of samples) {
|
|
for (const page of Array.isArray(sample.pages) ? sample.pages : []) {
|
|
const responsiveness = page?.responsiveness && typeof page.responsiveness === "object" ? page.responsiveness : {};
|
|
const cdp = page?.cdp && typeof page.cdp === "object" ? page.cdp : {};
|
|
const calls = Array.isArray(cdp.calls) ? cdp.calls : [];
|
|
const event = {
|
|
ts: sample.ts,
|
|
tsMs: sample.tsMs,
|
|
seq: sample.seq ?? null,
|
|
sampleSeq: sample.sampleSeq ?? null,
|
|
commandId: sample.commandId ?? null,
|
|
pageRole: page?.pageRole ?? null,
|
|
pageId: page?.pageId ?? null,
|
|
pageEpoch: page?.pageEpoch ?? null,
|
|
url: safeReportUrl(page?.url),
|
|
timeoutMs: browserMetricNumber(page?.timeoutMs),
|
|
responsivenessOk: responsiveness.ok === true,
|
|
responsivenessTimeout: responsiveness.timeout === true,
|
|
responsivenessLatencyMs: browserMetricNumber(responsiveness.latencyMs),
|
|
cdpTimeoutCount: browserMetricNumber(cdp.timeoutCount),
|
|
cdpErrorCount: browserMetricNumber(cdp.errorCount),
|
|
heapUsedMb: page?.heapUsage ? bytesToMb(page.heapUsage.usedSize) : null,
|
|
heapTotalMb: page?.heapUsage ? bytesToMb(page.heapUsage.totalSize) : null,
|
|
effectiveHeapUsedMb: page?.effectiveMemory ? browserMetricNumber(page.effectiveMemory.effectiveHeapUsedMb) : null,
|
|
effectiveJsHeapUsedMb: page?.effectiveMemory ? browserMetricNumber(page.effectiveMemory.effectiveJsHeapUsedMb) : null,
|
|
heapUsedGrowthMb: page?.effectiveMemory ? browserMetricNumber(page.effectiveMemory.heapUsedGrowthMb) : null,
|
|
jsHeapUsedGrowthMb: page?.effectiveMemory ? browserMetricNumber(page.effectiveMemory.jsHeapUsedGrowthMb) : null,
|
|
baselineCapturedAt: page?.baseline?.capturedAt ?? null,
|
|
domNodes: page?.domCounters ? browserMetricNumber(page.domCounters.nodes) : null,
|
|
valuesRedacted: true,
|
|
};
|
|
pageEvents.push(event);
|
|
for (const call of calls) {
|
|
if (call?.timeout !== true || call?.method === "Runtime.evaluate") continue;
|
|
cdpTimeoutEvents.push({
|
|
...event,
|
|
method: call.method ?? null,
|
|
latencyMs: browserMetricNumber(call.latencyMs),
|
|
errorMessage: limitText(call.error?.message ?? "", 180),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
const callTimeoutCount = calls.filter((call) => call?.timeout === true && call?.method !== "Runtime.evaluate").length;
|
|
if (Number(event.cdpTimeoutCount || 0) > callTimeoutCount && calls.length === 0) {
|
|
cdpTimeoutEvents.push({ ...event, method: "cdp-session", latencyMs: event.responsivenessLatencyMs, errorMessage: "CDP session or metric collection timed out", valuesRedacted: true });
|
|
}
|
|
}
|
|
}
|
|
const responsivenessEvents = pageEvents.filter((item) => item.responsivenessTimeout === true || Number(item.responsivenessLatencyMs ?? 0) >= alertThresholds.playwrightResponsivenessRedMs);
|
|
const maxTotal = maxByNumber(memorySamples, (item) => item.totalRssBytes);
|
|
const maxProcess = maxByNumber(memorySamples, (item) => item.maxProcessRssBytes);
|
|
const maxGrowth = maxByNumber(computedGrowth, (item) => item.totalRssGrowthBytes);
|
|
const maxProcessGrowth = maxByNumber(computedGrowth, (item) => item.maxProcessRssGrowthBytes);
|
|
return {
|
|
summary: {
|
|
sampleCount: samples.length,
|
|
memorySampleCount: memorySamples.length,
|
|
pageMetricCount: pageEvents.length,
|
|
cdpMetricsTimeoutCount: cdpTimeoutEvents.length,
|
|
responsivenessSlowCount: responsivenessEvents.length,
|
|
responsivenessTimeoutCount: responsivenessEvents.filter((item) => item.responsivenessTimeout === true).length,
|
|
maxTotalRssMb: maxTotal ? bytesToMb(maxTotal.totalRssBytes) : null,
|
|
maxProcessRssMb: maxProcess ? bytesToMb(maxProcess.maxProcessRssBytes) : null,
|
|
maxTotalRssGrowthMb: maxGrowth ? bytesToMb(maxGrowth.totalRssGrowthBytes) : null,
|
|
maxProcessRssGrowthMb: maxProcessGrowth ? bytesToMb(maxProcessGrowth.maxProcessRssGrowthBytes) : null,
|
|
totalRssRedMb: alertThresholds.browserTotalRssRedMb,
|
|
processRssRedMb: alertThresholds.browserProcessRssRedMb,
|
|
growthRedMb: alertThresholds.browserRssGrowthRedMb,
|
|
growthWindowMs: alertThresholds.browserRssGrowthWindowMs,
|
|
responsivenessRedMs: alertThresholds.playwrightResponsivenessRedMs,
|
|
memoryRedPolicyScope: "per-page-effective-memory",
|
|
freezeBlockerCount: blockerEvents.length,
|
|
browserFreezePolicy,
|
|
valuesRedacted: true,
|
|
},
|
|
blockerEvents: blockerEvents.slice(-20),
|
|
memorySamples: memorySamples.slice(-60),
|
|
growthSamples: computedGrowth.slice(-60),
|
|
maxTotalRssSample: maxTotal,
|
|
maxProcessRssSample: maxProcess,
|
|
maxGrowthSample: maxGrowth,
|
|
maxProcessGrowthSample: maxProcessGrowth,
|
|
responsivenessEvents: responsivenessEvents.slice(-80),
|
|
cdpTimeoutEvents: cdpTimeoutEvents.slice(-80),
|
|
latestPageEvents: pageEvents.slice(-20),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactBrowserFreezeBlockerEvent(item) {
|
|
if (!item || typeof item !== "object") return null;
|
|
const observed = objectValue(item.observed);
|
|
const threshold = objectValue(item.threshold);
|
|
const sample = objectValue(item.sample);
|
|
const page = objectValue(item.page);
|
|
const browserKill = objectValue(item.browserKill);
|
|
return {
|
|
ts: item.ts ?? null,
|
|
seq: item.seq ?? null,
|
|
sampleSeq: item.sampleSeq ?? sample.sampleSeq ?? null,
|
|
kind: item.kind ?? null,
|
|
severity: item.severity ?? "red",
|
|
blocking: item.blocking === true,
|
|
summary: item.summary ? String(item.summary).slice(0, 240) : null,
|
|
rootCause: item.rootCause ?? null,
|
|
rootCauseStatus: item.rootCauseStatus ?? null,
|
|
rootCauseConfidence: item.rootCauseConfidence ?? null,
|
|
policySource: item.policySource ?? null,
|
|
fallbackAllowed: item.fallbackAllowed === true,
|
|
observerRefreshAllowed: item.observerRefreshAllowed === true,
|
|
observed: {
|
|
totalRssMb: numberOrNull(observed.totalRssMb),
|
|
processRssMb: numberOrNull(observed.processRssMb),
|
|
totalGrowthMb: numberOrNull(observed.totalGrowthMb),
|
|
processGrowthMb: numberOrNull(observed.processGrowthMb),
|
|
effectiveHeapUsedMb: numberOrNull(observed.effectiveHeapUsedMb),
|
|
effectiveJsHeapUsedMb: numberOrNull(observed.effectiveJsHeapUsedMb),
|
|
heapGrowthMb: numberOrNull(observed.heapGrowthMb),
|
|
jsHeapGrowthMb: numberOrNull(observed.jsHeapGrowthMb),
|
|
responsivenessLatencyMs: numberOrNull(observed.responsivenessLatencyMs),
|
|
responsivenessTimeout: observed.responsivenessTimeout === true,
|
|
cdpMetricsTimeoutCount: numberOrNull(observed.cdpMetricsTimeoutCount),
|
|
methods: arrayStrings(observed.methods).slice(0, 8),
|
|
valuesRedacted: true,
|
|
},
|
|
threshold: {
|
|
totalRssBlockerMb: numberOrNull(threshold.totalRssBlockerMb),
|
|
processRssBlockerMb: numberOrNull(threshold.processRssBlockerMb),
|
|
growthBlockerMb: numberOrNull(threshold.growthBlockerMb),
|
|
latencyBlockerMs: numberOrNull(threshold.latencyBlockerMs),
|
|
eventBlockerCount: numberOrNull(threshold.eventBlockerCount),
|
|
metricsTimeoutBlockerCount: numberOrNull(threshold.metricsTimeoutBlockerCount),
|
|
windowMs: numberOrNull(threshold.windowMs),
|
|
valuesRedacted: true,
|
|
},
|
|
sample: {
|
|
ts: sample.ts ?? null,
|
|
seq: sample.seq ?? null,
|
|
sampleSeq: sample.sampleSeq ?? null,
|
|
browserPid: numberOrNull(sample.browserPid),
|
|
totalRssMb: numberOrNull(sample.totalRssMb),
|
|
maxProcessRssMb: numberOrNull(sample.maxProcessRssMb),
|
|
totalRssGrowthMb: numberOrNull(sample.totalRssGrowthMb),
|
|
maxProcessRssGrowthMb: numberOrNull(sample.maxProcessRssGrowthMb),
|
|
valuesRedacted: true,
|
|
},
|
|
page: {
|
|
pageRole: page.pageRole ?? null,
|
|
pageId: page.pageId ?? null,
|
|
pageEpoch: numberOrNull(page.pageEpoch),
|
|
timeoutMs: numberOrNull(page.timeoutMs),
|
|
responsivenessLatencyMs: numberOrNull(page.responsivenessLatencyMs),
|
|
responsivenessTimeout: page.responsivenessTimeout === true,
|
|
cdpTimeoutCount: numberOrNull(page.cdpTimeoutCount),
|
|
cdpErrorCount: numberOrNull(page.cdpErrorCount),
|
|
effectiveHeapUsedMb: numberOrNull(page.effectiveHeapUsedMb),
|
|
effectiveJsHeapUsedMb: numberOrNull(page.effectiveJsHeapUsedMb),
|
|
heapUsedGrowthMb: numberOrNull(page.heapUsedGrowthMb),
|
|
jsHeapUsedGrowthMb: numberOrNull(page.jsHeapUsedGrowthMb),
|
|
baselineCapturedAt: page.baselineCapturedAt ?? null,
|
|
valuesRedacted: true,
|
|
},
|
|
browserKill: {
|
|
ok: browserKill.ok === true,
|
|
pending: browserKill.pending === true,
|
|
skipped: browserKill.skipped === true,
|
|
reason: browserKill.reason ?? null,
|
|
pid: numberOrNull(browserKill.pid),
|
|
gracefulSignal: browserKill.gracefulSignal ?? null,
|
|
forceSignal: browserKill.forceSignal ?? null,
|
|
gracefulSent: browserKill.gracefulSent === true,
|
|
forceSent: browserKill.forceSent === true,
|
|
exitedAfterGrace: browserKill.exitedAfterGrace === true,
|
|
exitedAfterForce: browserKill.exitedAfterForce === true,
|
|
valuesRedacted: true,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function buildBrowserProcessFindings(report, runtimeAlerts = null) {
|
|
const summary = report?.summary || {};
|
|
const findings = [];
|
|
const blockerEvents = Array.isArray(report?.blockerEvents) ? report.blockerEvents : [];
|
|
if (blockerEvents.length > 0) {
|
|
const first = blockerEvents[0] || {};
|
|
findings.push({
|
|
id: "frontend-browser-freeze-runner-blocker",
|
|
severity: "red",
|
|
summary: "web-probe runner matched YAML browserFreezePolicy, killed/stopped Chromium, and failed the observer run; do not clear this by refresh or fallback",
|
|
count: blockerEvents.length,
|
|
blocking: true,
|
|
rootCause: first.rootCause ?? "frontend_browser_freeze_policy_blocker",
|
|
rootCauseStatus: "confirmed-from-runner-browser-freeze-policy",
|
|
rootCauseConfidence: "high",
|
|
policySource: first.policySource ?? "config/hwlab-node-lanes.yaml#webProbe.browserFreezePolicy",
|
|
fallbackAllowed: false,
|
|
observerRefreshAllowed: false,
|
|
browserKilled: first.browserKill ?? null,
|
|
events: blockerEvents.slice(0, 20),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
if (!summary || Number(summary.sampleCount ?? 0) <= 0) return findings;
|
|
const rootCauseSignals = browserRootCauseSignals(report, runtimeAlerts);
|
|
const pageEvents = Array.isArray(report.latestPageEvents) ? report.latestPageEvents : [];
|
|
const maxEffectiveHeapEvent = maxByNumber(pageEvents, (item) => Math.max(Number(item.effectiveHeapUsedMb ?? 0), Number(item.effectiveJsHeapUsedMb ?? 0)));
|
|
const maxEffectiveHeapMb = maxEffectiveHeapEvent ? Math.max(Number(maxEffectiveHeapEvent.effectiveHeapUsedMb ?? 0), Number(maxEffectiveHeapEvent.effectiveJsHeapUsedMb ?? 0)) : 0;
|
|
if (maxEffectiveHeapMb >= alertThresholds.browserProcessRssRedMb) {
|
|
findings.push({
|
|
id: "frontend-browser-memory-rss-red",
|
|
severity: "red",
|
|
summary: "Page effective memory exceeded YAML red threshold after subtracting page baseline; process RSS remains diagnostic evidence only",
|
|
maxEffectiveHeapMb,
|
|
processRssRedMb: alertThresholds.browserProcessRssRedMb,
|
|
memoryRedPolicyScope: "per-page-effective-memory",
|
|
maxEffectiveHeapEvent,
|
|
maxTotalRssSample: report.maxTotalRssSample,
|
|
maxProcessRssSample: report.maxProcessRssSample,
|
|
rootCause: "frontend_browser_page_effective_memory_pressure",
|
|
rootCauseStatus: "confirmed-from-runner-page-effective-memory",
|
|
rootCauseConfidence: "high",
|
|
rootCauseSignals,
|
|
fallbackAllowed: false,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
const maxEffectiveGrowthEvent = maxByNumber(pageEvents, (item) => Math.max(Number(item.heapUsedGrowthMb ?? 0), Number(item.jsHeapUsedGrowthMb ?? 0)));
|
|
const maxEffectiveGrowthMb = maxEffectiveGrowthEvent ? Math.max(Number(maxEffectiveGrowthEvent.heapUsedGrowthMb ?? 0), Number(maxEffectiveGrowthEvent.jsHeapUsedGrowthMb ?? 0)) : 0;
|
|
if (maxEffectiveGrowthMb >= alertThresholds.browserRssGrowthRedMb) {
|
|
findings.push({
|
|
id: "frontend-browser-memory-growth-red",
|
|
severity: "red",
|
|
summary: "Page effective memory grew beyond YAML window budget; this matches the reported freeze pattern without counting multi-page startup RSS",
|
|
maxEffectiveGrowthMb,
|
|
growthRedMb: alertThresholds.browserRssGrowthRedMb,
|
|
windowMs: alertThresholds.browserRssGrowthWindowMs,
|
|
memoryRedPolicyScope: "per-page-effective-memory",
|
|
maxEffectiveGrowthEvent,
|
|
maxGrowthSample: report.maxGrowthSample,
|
|
maxProcessGrowthSample: report.maxProcessGrowthSample,
|
|
rootCause: "frontend_browser_page_memory_leak_or_unbounded_render_growth",
|
|
rootCauseStatus: "confirmed-from-runner-page-effective-memory-growth",
|
|
rootCauseConfidence: "high",
|
|
rootCauseSignals,
|
|
fallbackAllowed: false,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
const responsivenessEvents = Array.isArray(report.responsivenessEvents) ? report.responsivenessEvents : [];
|
|
const severeLatencyEvents = responsivenessEvents.filter((item) => item.responsivenessTimeout !== true && Number(item.responsivenessLatencyMs ?? 0) >= alertThresholds.playwrightResponsivenessRedMs);
|
|
const responsivenessTimeoutBurst = firstBurst(
|
|
responsivenessEvents.filter((item) => item.responsivenessTimeout === true),
|
|
alertThresholds.playwrightResponsivenessTimeoutRedCount,
|
|
alertThresholds.domEvaluateTimeoutRedWindowMs,
|
|
);
|
|
if (severeLatencyEvents.length > 0 || responsivenessTimeoutBurst) {
|
|
const events = severeLatencyEvents.length > 0 ? severeLatencyEvents : responsivenessTimeoutBurst;
|
|
findings.push({
|
|
id: "frontend-playwright-responsiveness-red",
|
|
severity: "red",
|
|
summary: "Playwright/CDP responsiveness probe exceeded YAML budget; treat the browser page as frozen instead of refreshing or falling back",
|
|
count: events.length,
|
|
latencyRedMs: alertThresholds.playwrightResponsivenessRedMs,
|
|
timeoutThresholdCount: alertThresholds.playwrightResponsivenessTimeoutRedCount,
|
|
windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs,
|
|
events: events.slice(0, 20).map(browserProcessEventRef),
|
|
rootCause: "frontend_browser_page_unresponsive_to_playwright",
|
|
rootCauseStatus: "confirmed-from-cdp-runtime-evaluate",
|
|
rootCauseConfidence: "high",
|
|
rootCauseSignals,
|
|
fallbackAllowed: false,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
const cdpTimeoutEvents = Array.isArray(report.cdpTimeoutEvents) ? report.cdpTimeoutEvents : [];
|
|
const cdpTimeoutBurst = firstBurst(cdpTimeoutEvents, alertThresholds.cdpMetricsTimeoutRedCount, alertThresholds.domEvaluateTimeoutRedWindowMs);
|
|
if (cdpTimeoutBurst) {
|
|
findings.push({
|
|
id: "frontend-cdp-metrics-timeout-red",
|
|
severity: "red",
|
|
summary: "CDP browser metrics timed out repeatedly; the sentinel has direct browser-side evidence of a hung page/runtime",
|
|
count: cdpTimeoutBurst.length,
|
|
thresholdCount: alertThresholds.cdpMetricsTimeoutRedCount,
|
|
windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs,
|
|
events: cdpTimeoutBurst.slice(0, 20).map(browserProcessEventRef),
|
|
rootCause: "frontend_browser_cdp_metrics_unresponsive",
|
|
rootCauseStatus: "confirmed-from-cdp-metrics-timeouts",
|
|
rootCauseConfidence: "high",
|
|
rootCauseSignals,
|
|
fallbackAllowed: false,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
return findings;
|
|
}
|
|
|
|
function browserRootCauseSignals(report, runtimeAlerts) {
|
|
const browserSummary = report?.summary || {};
|
|
const runtimeSummary = runtimeAlerts?.summary || {};
|
|
const sampleCount = Number(browserSummary.sampleCount ?? 0);
|
|
const sessionListReadCount = Number(runtimeSummary.workbenchSessionListReadCount ?? 0);
|
|
const traceEventsReadCount = Number(runtimeSummary.workbenchTraceEventsReadCount ?? 0);
|
|
const webPerformanceBeaconFailureCount = Number(runtimeSummary.webPerformanceBeaconFailureCount ?? 0);
|
|
const eventSourceFailureCount = Number(runtimeSummary.workbenchEventSourceFailureCount ?? 0);
|
|
const suspectedFrontendRefreshStorm = sessionListReadCount >= Math.max(20, sampleCount * 2);
|
|
return {
|
|
suspectedFrontendRefreshStorm,
|
|
sessionListReadCount,
|
|
traceEventsReadCount,
|
|
webPerformanceBeaconFailureCount,
|
|
eventSourceFailureCount,
|
|
requestFailedCount: runtimeSummary.significantRequestFailedCount ?? runtimeSummary.requestFailedCount ?? 0,
|
|
httpErrorCount: runtimeSummary.httpErrorCount ?? 0,
|
|
topRequestFailedPaths: (runtimeAlerts?.networkSignificantRequestFailedByPath ?? runtimeAlerts?.networkRequestFailedByPath ?? []).slice(0, 5),
|
|
topHttpErrorPaths: (runtimeAlerts?.networkHttpErrorsByPath ?? []).slice(0, 5),
|
|
note: suspectedFrontendRefreshStorm
|
|
? "suspected frontend refresh storm: session list reads exceed the sample-derived budget during a browser red finding"
|
|
: "root-cause signals are included so memory/responsiveness/CDP red findings can be correlated without manual grep",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function computeBrowserProcessGrowth(samples, windowMs) {
|
|
const budgetMs = Math.max(1000, Number(windowMs || 0));
|
|
const sorted = (samples || []).filter((item) => Number.isFinite(item.tsMs)).sort((a, b) => a.tsMs - b.tsMs);
|
|
return sorted.map((item, index) => {
|
|
const candidates = sorted.slice(0, index + 1).filter((candidate) => item.tsMs - candidate.tsMs <= budgetMs);
|
|
const totalBaseline = minByNumber(candidates, (candidate) => candidate.totalRssBytes);
|
|
const processBaseline = minByNumber(candidates, (candidate) => candidate.maxProcessRssBytes);
|
|
const totalGrowth = Number(item.totalRssBytes || 0) - Number(totalBaseline?.totalRssBytes || 0);
|
|
const processGrowth = Number(item.maxProcessRssBytes || 0) - Number(processBaseline?.maxProcessRssBytes || 0);
|
|
return {
|
|
...item,
|
|
windowMs: budgetMs,
|
|
totalBaselineAt: totalBaseline?.ts ?? null,
|
|
processBaselineAt: processBaseline?.ts ?? null,
|
|
totalRssGrowthBytes: totalGrowth,
|
|
totalRssGrowthMb: bytesToMb(totalGrowth),
|
|
maxProcessRssGrowthBytes: processGrowth,
|
|
maxProcessRssGrowthMb: bytesToMb(processGrowth),
|
|
valuesRedacted: true,
|
|
};
|
|
});
|
|
}
|
|
|
|
function browserProcessEventRef(item) {
|
|
return {
|
|
ts: item?.ts ?? null,
|
|
seq: item?.seq ?? null,
|
|
sampleSeq: item?.sampleSeq ?? null,
|
|
commandId: item?.commandId ?? null,
|
|
pageRole: item?.pageRole ?? null,
|
|
pageId: item?.pageId ?? null,
|
|
pageEpoch: item?.pageEpoch ?? null,
|
|
timeoutMs: item?.timeoutMs ?? null,
|
|
responsivenessLatencyMs: item?.responsivenessLatencyMs ?? item?.latencyMs ?? null,
|
|
responsivenessTimeout: item?.responsivenessTimeout === true,
|
|
cdpTimeoutCount: item?.cdpTimeoutCount ?? null,
|
|
method: item?.method ?? null,
|
|
errorMessage: item?.errorMessage ?? null,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactBrowserProcessRow(item) {
|
|
if (!item || typeof item !== "object") return null;
|
|
return {
|
|
pid: item.pid ?? null,
|
|
role: item.role ?? null,
|
|
name: item.name ?? null,
|
|
rssMb: bytesToMb(item.rssBytes),
|
|
vmSizeMb: bytesToMb(item.vmSizeBytes),
|
|
commandHash: item.commandHash ?? null,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function browserMetricNumber(value) {
|
|
const numeric = Number(value);
|
|
return Number.isFinite(numeric) ? numeric : null;
|
|
}
|
|
|
|
function bytesToMb(value) {
|
|
const numeric = Number(value);
|
|
if (!Number.isFinite(numeric)) return null;
|
|
return Number((numeric / 1024 / 1024).toFixed(1));
|
|
}
|
|
|
|
function maxByNumber(items, getter) {
|
|
let selected = null;
|
|
let selectedValue = Number.NEGATIVE_INFINITY;
|
|
for (const item of Array.isArray(items) ? items : []) {
|
|
const value = Number(getter(item));
|
|
if (!Number.isFinite(value)) continue;
|
|
if (!selected || value > selectedValue) {
|
|
selected = item;
|
|
selectedValue = value;
|
|
}
|
|
}
|
|
return selected;
|
|
}
|
|
|
|
function minByNumber(items, getter) {
|
|
let selected = null;
|
|
let selectedValue = Number.POSITIVE_INFINITY;
|
|
for (const item of Array.isArray(items) ? items : []) {
|
|
const value = Number(getter(item));
|
|
if (!Number.isFinite(value)) continue;
|
|
if (!selected || value < selectedValue) {
|
|
selected = item;
|
|
selectedValue = value;
|
|
}
|
|
}
|
|
return selected;
|
|
}
|
|
|
|
function safeReportUrl(value) {
|
|
if (!value) return null;
|
|
try {
|
|
const url = new URL(String(value));
|
|
return url.origin + url.pathname;
|
|
} catch {
|
|
return limitText(String(value), 200);
|
|
}
|
|
}
|
|
|
|
function frontendFreezeErrorEvent(item, promptTimes) {
|
|
const details = objectValue(item?.error?.details);
|
|
const message = String(item?.error?.message ?? item?.message ?? item?.error ?? "");
|
|
const type = String(item?.type || "");
|
|
const tsMs = Date.parse(String(item?.ts || ""));
|
|
if (!Number.isFinite(tsMs)) return null;
|
|
const kind = classifyFrontendFreezeError(type, message);
|
|
if (!kind) return null;
|
|
return {
|
|
ts: item.ts ?? null,
|
|
tsMs,
|
|
promptIndex: promptIndexForTs(promptTimes, item.ts),
|
|
kind,
|
|
type: item.type ?? null,
|
|
pageRole: stringOrNull(item?.pageRole) ?? stringOrNull(details.pageRole) ?? pageRoleFromErrorType(type),
|
|
pageId: stringOrNull(item?.pageId) ?? stringOrNull(details.pageId),
|
|
routeSessionId: stringOrNull(item?.routeSessionId) ?? stringOrNull(details.routeSessionId),
|
|
activeSessionId: stringOrNull(item?.activeSessionId) ?? stringOrNull(details.activeSessionId),
|
|
commandId: stringOrNull(item?.commandId) ?? stringOrNull(details.commandId),
|
|
sampleSeq: numberOrNull(item?.sampleSeq ?? details.sampleSeq),
|
|
timeoutMs: timeoutMsFromMessage(message),
|
|
messageHash: message ? sha256(message) : null,
|
|
preview: limitText(message, 240),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function pageRoleFromErrorType(type) {
|
|
const value = String(type || "");
|
|
if (/^control-/iu.test(value)) return "control";
|
|
if (/^observer-/iu.test(value)) return "observer";
|
|
return null;
|
|
}
|
|
|
|
function classifyFrontendFreezeError(type, message) {
|
|
const value = String(message || "");
|
|
if (/sampleOnePage\s+DOM\s+evaluate\s+exceeded/iu.test(value) && /(?:control|observer)-sample-error/iu.test(type)) return "dom-evaluate-timeout";
|
|
if (/screenshot|captureScreenshot|page\.screenshot/iu.test(type + " " + value) && /timeout|timed\s*out|exceeded/iu.test(value)) return "screenshot-timeout";
|
|
if (/pageerror|uncaught|unhandledrejection/iu.test(type) || /^(?:Error|TypeError|ReferenceError|RangeError|SyntaxError):/u.test(value)) return "page-error";
|
|
return null;
|
|
}
|
|
|
|
function firstBurst(events, thresholdCount, windowMs) {
|
|
const count = Math.max(1, Math.floor(Number(thresholdCount || 0)));
|
|
const budgetMs = Math.max(1, Number(windowMs || 0));
|
|
const sorted = (events || []).filter((item) => Number.isFinite(item?.tsMs)).sort((a, b) => a.tsMs - b.tsMs);
|
|
if (sorted.length < count) return null;
|
|
for (let start = 0; start <= sorted.length - count; start += 1) {
|
|
const end = start + count - 1;
|
|
if (sorted[end].tsMs - sorted[start].tsMs <= budgetMs) return sorted.slice(start, end + 1);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function frontendFreezeBurstFinding({ id, summary, burst, thresholdCount, windowMs, pageRole }) {
|
|
const first = burst[0];
|
|
const last = burst[burst.length - 1];
|
|
const pageIds = uniqueStrings(burst.map((item) => item.pageId));
|
|
const routeSessionIds = uniqueStrings(burst.map((item) => item.routeSessionId));
|
|
const activeSessionIds = uniqueStrings(burst.map((item) => item.activeSessionId));
|
|
return {
|
|
id,
|
|
severity: "red",
|
|
summary,
|
|
count: burst.length,
|
|
thresholdCount,
|
|
windowMs,
|
|
firstAt: first?.ts ?? null,
|
|
lastAt: last?.ts ?? null,
|
|
pageRole,
|
|
pageIds,
|
|
routeSessionIds,
|
|
activeSessionIds,
|
|
timeoutMsMax: maxPresentNumber(burst.map((item) => item.timeoutMs)),
|
|
rootCause: "frontend_page_freeze_or_runtime_exception",
|
|
rootCauseStatus: "confirmed-from-browser-observer-errors",
|
|
rootCauseConfidence: "high",
|
|
fallbackAllowed: false,
|
|
observerRefreshMayNotClear: true,
|
|
nextAction: "Keep this run red; do not auto-refresh, fallback, or mark healthy until OTel/browser evidence explains why the page stopped responding.",
|
|
events: burst.map((item) => ({
|
|
ts: item.ts,
|
|
promptIndex: item.promptIndex,
|
|
type: item.type,
|
|
pageRole: item.pageRole,
|
|
pageId: item.pageId,
|
|
routeSessionId: item.routeSessionId,
|
|
activeSessionId: item.activeSessionId,
|
|
commandId: item.commandId,
|
|
sampleSeq: item.sampleSeq,
|
|
timeoutMs: item.timeoutMs,
|
|
messageHash: item.messageHash,
|
|
preview: item.preview,
|
|
valuesRedacted: true,
|
|
})),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function stopCommandWindows(control) {
|
|
return (control || [])
|
|
.filter((item) => /^(?:stop|forceStop|cancel|close)$/iu.test(String(item?.type || item?.command || "")))
|
|
.map((item) => {
|
|
const tsMs = Date.parse(String(item?.ts || ""));
|
|
return Number.isFinite(tsMs) ? { fromMs: tsMs - 1000, toMs: tsMs + 10000 } : null;
|
|
})
|
|
.filter(Boolean);
|
|
}
|
|
|
|
function errorInsideStopWindow(event, windows) {
|
|
return (windows || []).some((window) => event.tsMs >= window.fromMs && event.tsMs <= window.toMs);
|
|
}
|
|
|
|
function timeoutMsFromMessage(value) {
|
|
const match = String(value || "").match(/\b(?:exceeded|timeout|timed\s*out\s*after)\s+(\d{2,})\s*ms\b/iu)
|
|
|| String(value || "").match(/\b(\d{2,})\s*ms\b/iu);
|
|
return match ? Number(match[1]) : null;
|
|
}
|
|
|
|
function uniqueStrings(values) {
|
|
return Array.from(new Set((values || []).filter((item) => typeof item === "string" && item.length > 0))).slice(0, 12);
|
|
}
|
|
|
|
function maxPresentNumber(values) {
|
|
const numbers = (values || []).filter((item) => item !== null && item !== undefined && Number.isFinite(Number(item))).map((item) => Number(item));
|
|
return numbers.length > 0 ? Math.max(...numbers) : null;
|
|
}
|
|
`;
|
|
}
|