feat: add web probe performance hotspot analysis
This commit is contained in:
Codex
@@ -0,0 +1,598 @@
|
||||
// SPEC: PJ2026-01040111 long-running Workbench observation.
|
||||
// Responsibility: Analyzer browser process, memory, responsiveness, and root-cause signal source fragment.
|
||||
|
||||
export function nodeWebObserveAnalyzerBrowserProcessSource(): string {
|
||||
return String.raw`function buildBrowserProcessReport(rows) {
|
||||
const blockerEvents = (Array.isArray(rows) ? rows : [])
|
||||
.filter((item) => item && item.type === "browser-freeze-blocker")
|
||||
.map(compactBrowserFreezeBlockerEvent)
|
||||
.filter(Boolean);
|
||||
const samples = (Array.isArray(rows) ? rows : [])
|
||||
.filter((item) => item && (item.type === "browser-process-sample" || item.process || item.pages))
|
||||
.map((item) => ({ ...item, tsMs: Date.parse(String(item.ts || "")) }))
|
||||
.filter((item) => Number.isFinite(item.tsMs))
|
||||
.sort((a, b) => a.tsMs - b.tsMs);
|
||||
const memorySamples = samples.map((item) => {
|
||||
const process = item.process && typeof item.process === "object" ? item.process : {};
|
||||
const growth = item.growth && typeof item.growth === "object" ? item.growth : {};
|
||||
const totalRssBytes = browserMetricNumber(process.totalRssBytes);
|
||||
const maxProcessRssBytes = browserMetricNumber(process.maxProcessRssBytes);
|
||||
return {
|
||||
ts: item.ts,
|
||||
tsMs: item.tsMs,
|
||||
seq: item.seq ?? null,
|
||||
sampleSeq: item.sampleSeq ?? null,
|
||||
commandId: item.commandId ?? null,
|
||||
processCount: browserMetricNumber(process.chromiumProcessCount),
|
||||
totalRssBytes,
|
||||
totalRssMb: bytesToMb(totalRssBytes),
|
||||
maxProcessRssBytes,
|
||||
maxProcessRssMb: bytesToMb(maxProcessRssBytes),
|
||||
maxProcess: compactBrowserProcessRow(process.maxProcess),
|
||||
totalRssGrowthBytes: browserMetricNumber(growth.totalRssGrowthBytes),
|
||||
totalRssGrowthMb: bytesToMb(growth.totalRssGrowthBytes),
|
||||
maxProcessRssGrowthBytes: browserMetricNumber(growth.maxProcessRssGrowthBytes),
|
||||
maxProcessRssGrowthMb: bytesToMb(growth.maxProcessRssGrowthBytes),
|
||||
roles: process.roles && typeof process.roles === "object" ? process.roles : {},
|
||||
valuesRedacted: true,
|
||||
};
|
||||
});
|
||||
const computedGrowth = computeBrowserProcessGrowth(memorySamples, alertThresholds.browserRssGrowthWindowMs);
|
||||
const pageEvents = [];
|
||||
const cdpTimeoutEvents = [];
|
||||
for (const sample of samples) {
|
||||
for (const page of Array.isArray(sample.pages) ? sample.pages : []) {
|
||||
const responsiveness = page?.responsiveness && typeof page.responsiveness === "object" ? page.responsiveness : {};
|
||||
const cdp = page?.cdp && typeof page.cdp === "object" ? page.cdp : {};
|
||||
const calls = Array.isArray(cdp.calls) ? cdp.calls : [];
|
||||
const event = {
|
||||
ts: sample.ts,
|
||||
tsMs: sample.tsMs,
|
||||
seq: sample.seq ?? null,
|
||||
sampleSeq: sample.sampleSeq ?? null,
|
||||
commandId: sample.commandId ?? null,
|
||||
pageRole: page?.pageRole ?? null,
|
||||
pageId: page?.pageId ?? null,
|
||||
pageEpoch: page?.pageEpoch ?? null,
|
||||
url: safeReportUrl(page?.url),
|
||||
timeoutMs: browserMetricNumber(page?.timeoutMs),
|
||||
responsivenessOk: responsiveness.ok === true,
|
||||
responsivenessTimeout: responsiveness.timeout === true,
|
||||
responsivenessLatencyMs: browserMetricNumber(responsiveness.latencyMs),
|
||||
cdpTimeoutCount: browserMetricNumber(cdp.timeoutCount),
|
||||
cdpErrorCount: browserMetricNumber(cdp.errorCount),
|
||||
heapUsedMb: page?.heapUsage ? bytesToMb(page.heapUsage.usedSize) : null,
|
||||
heapTotalMb: page?.heapUsage ? bytesToMb(page.heapUsage.totalSize) : null,
|
||||
effectiveHeapUsedMb: page?.effectiveMemory ? browserMetricNumber(page.effectiveMemory.effectiveHeapUsedMb) : null,
|
||||
effectiveJsHeapUsedMb: page?.effectiveMemory ? browserMetricNumber(page.effectiveMemory.effectiveJsHeapUsedMb) : null,
|
||||
heapUsedGrowthMb: page?.effectiveMemory ? browserMetricNumber(page.effectiveMemory.heapUsedGrowthMb) : null,
|
||||
jsHeapUsedGrowthMb: page?.effectiveMemory ? browserMetricNumber(page.effectiveMemory.jsHeapUsedGrowthMb) : null,
|
||||
baselineCapturedAt: page?.baseline?.capturedAt ?? null,
|
||||
domNodes: page?.domCounters ? browserMetricNumber(page.domCounters.nodes) : null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
pageEvents.push(event);
|
||||
for (const call of calls) {
|
||||
if (call?.timeout !== true || call?.method === "Runtime.evaluate") continue;
|
||||
cdpTimeoutEvents.push({
|
||||
...event,
|
||||
method: call.method ?? null,
|
||||
latencyMs: browserMetricNumber(call.latencyMs),
|
||||
errorMessage: limitText(call.error?.message ?? "", 180),
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
const callTimeoutCount = calls.filter((call) => call?.timeout === true && call?.method !== "Runtime.evaluate").length;
|
||||
if (Number(event.cdpTimeoutCount || 0) > callTimeoutCount && calls.length === 0) {
|
||||
cdpTimeoutEvents.push({ ...event, method: "cdp-session", latencyMs: event.responsivenessLatencyMs, errorMessage: "CDP session or metric collection timed out", valuesRedacted: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
const responsivenessEvents = pageEvents.filter((item) => item.responsivenessTimeout === true || Number(item.responsivenessLatencyMs ?? 0) >= alertThresholds.playwrightResponsivenessRedMs);
|
||||
const maxTotal = maxByNumber(memorySamples, (item) => item.totalRssBytes);
|
||||
const maxProcess = maxByNumber(memorySamples, (item) => item.maxProcessRssBytes);
|
||||
const maxGrowth = maxByNumber(computedGrowth, (item) => item.totalRssGrowthBytes);
|
||||
const maxProcessGrowth = maxByNumber(computedGrowth, (item) => item.maxProcessRssGrowthBytes);
|
||||
return {
|
||||
summary: {
|
||||
sampleCount: samples.length,
|
||||
memorySampleCount: memorySamples.length,
|
||||
pageMetricCount: pageEvents.length,
|
||||
cdpMetricsTimeoutCount: cdpTimeoutEvents.length,
|
||||
responsivenessSlowCount: responsivenessEvents.length,
|
||||
responsivenessTimeoutCount: responsivenessEvents.filter((item) => item.responsivenessTimeout === true).length,
|
||||
maxTotalRssMb: maxTotal ? bytesToMb(maxTotal.totalRssBytes) : null,
|
||||
maxProcessRssMb: maxProcess ? bytesToMb(maxProcess.maxProcessRssBytes) : null,
|
||||
maxTotalRssGrowthMb: maxGrowth ? bytesToMb(maxGrowth.totalRssGrowthBytes) : null,
|
||||
maxProcessRssGrowthMb: maxProcessGrowth ? bytesToMb(maxProcessGrowth.maxProcessRssGrowthBytes) : null,
|
||||
totalRssRedMb: alertThresholds.browserTotalRssRedMb,
|
||||
processRssRedMb: alertThresholds.browserProcessRssRedMb,
|
||||
growthRedMb: alertThresholds.browserRssGrowthRedMb,
|
||||
growthWindowMs: alertThresholds.browserRssGrowthWindowMs,
|
||||
responsivenessRedMs: alertThresholds.playwrightResponsivenessRedMs,
|
||||
memoryRedPolicyScope: "per-page-effective-memory",
|
||||
freezeBlockerCount: blockerEvents.length,
|
||||
browserFreezePolicy,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
blockerEvents: blockerEvents.slice(-20),
|
||||
memorySamples: memorySamples.slice(-60),
|
||||
growthSamples: computedGrowth.slice(-60),
|
||||
maxTotalRssSample: maxTotal,
|
||||
maxProcessRssSample: maxProcess,
|
||||
maxGrowthSample: maxGrowth,
|
||||
maxProcessGrowthSample: maxProcessGrowth,
|
||||
responsivenessEvents: responsivenessEvents.slice(-80),
|
||||
cdpTimeoutEvents: cdpTimeoutEvents.slice(-80),
|
||||
latestPageEvents: pageEvents.slice(-20),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function compactBrowserFreezeBlockerEvent(item) {
|
||||
if (!item || typeof item !== "object") return null;
|
||||
const observed = objectValue(item.observed);
|
||||
const threshold = objectValue(item.threshold);
|
||||
const sample = objectValue(item.sample);
|
||||
const page = objectValue(item.page);
|
||||
const browserKill = objectValue(item.browserKill);
|
||||
return {
|
||||
ts: item.ts ?? null,
|
||||
seq: item.seq ?? null,
|
||||
sampleSeq: item.sampleSeq ?? sample.sampleSeq ?? null,
|
||||
kind: item.kind ?? null,
|
||||
severity: item.severity ?? "red",
|
||||
blocking: item.blocking === true,
|
||||
summary: item.summary ? String(item.summary).slice(0, 240) : null,
|
||||
rootCause: item.rootCause ?? null,
|
||||
rootCauseStatus: item.rootCauseStatus ?? null,
|
||||
rootCauseConfidence: item.rootCauseConfidence ?? null,
|
||||
policySource: item.policySource ?? null,
|
||||
fallbackAllowed: item.fallbackAllowed === true,
|
||||
observerRefreshAllowed: item.observerRefreshAllowed === true,
|
||||
observed: {
|
||||
totalRssMb: numberOrNull(observed.totalRssMb),
|
||||
processRssMb: numberOrNull(observed.processRssMb),
|
||||
totalGrowthMb: numberOrNull(observed.totalGrowthMb),
|
||||
processGrowthMb: numberOrNull(observed.processGrowthMb),
|
||||
effectiveHeapUsedMb: numberOrNull(observed.effectiveHeapUsedMb),
|
||||
effectiveJsHeapUsedMb: numberOrNull(observed.effectiveJsHeapUsedMb),
|
||||
heapGrowthMb: numberOrNull(observed.heapGrowthMb),
|
||||
jsHeapGrowthMb: numberOrNull(observed.jsHeapGrowthMb),
|
||||
responsivenessLatencyMs: numberOrNull(observed.responsivenessLatencyMs),
|
||||
responsivenessTimeout: observed.responsivenessTimeout === true,
|
||||
cdpMetricsTimeoutCount: numberOrNull(observed.cdpMetricsTimeoutCount),
|
||||
methods: arrayStrings(observed.methods).slice(0, 8),
|
||||
valuesRedacted: true,
|
||||
},
|
||||
threshold: {
|
||||
totalRssBlockerMb: numberOrNull(threshold.totalRssBlockerMb),
|
||||
processRssBlockerMb: numberOrNull(threshold.processRssBlockerMb),
|
||||
growthBlockerMb: numberOrNull(threshold.growthBlockerMb),
|
||||
latencyBlockerMs: numberOrNull(threshold.latencyBlockerMs),
|
||||
eventBlockerCount: numberOrNull(threshold.eventBlockerCount),
|
||||
metricsTimeoutBlockerCount: numberOrNull(threshold.metricsTimeoutBlockerCount),
|
||||
windowMs: numberOrNull(threshold.windowMs),
|
||||
valuesRedacted: true,
|
||||
},
|
||||
sample: {
|
||||
ts: sample.ts ?? null,
|
||||
seq: sample.seq ?? null,
|
||||
sampleSeq: sample.sampleSeq ?? null,
|
||||
browserPid: numberOrNull(sample.browserPid),
|
||||
totalRssMb: numberOrNull(sample.totalRssMb),
|
||||
maxProcessRssMb: numberOrNull(sample.maxProcessRssMb),
|
||||
totalRssGrowthMb: numberOrNull(sample.totalRssGrowthMb),
|
||||
maxProcessRssGrowthMb: numberOrNull(sample.maxProcessRssGrowthMb),
|
||||
valuesRedacted: true,
|
||||
},
|
||||
page: {
|
||||
pageRole: page.pageRole ?? null,
|
||||
pageId: page.pageId ?? null,
|
||||
pageEpoch: numberOrNull(page.pageEpoch),
|
||||
timeoutMs: numberOrNull(page.timeoutMs),
|
||||
responsivenessLatencyMs: numberOrNull(page.responsivenessLatencyMs),
|
||||
responsivenessTimeout: page.responsivenessTimeout === true,
|
||||
cdpTimeoutCount: numberOrNull(page.cdpTimeoutCount),
|
||||
cdpErrorCount: numberOrNull(page.cdpErrorCount),
|
||||
effectiveHeapUsedMb: numberOrNull(page.effectiveHeapUsedMb),
|
||||
effectiveJsHeapUsedMb: numberOrNull(page.effectiveJsHeapUsedMb),
|
||||
heapUsedGrowthMb: numberOrNull(page.heapUsedGrowthMb),
|
||||
jsHeapUsedGrowthMb: numberOrNull(page.jsHeapUsedGrowthMb),
|
||||
baselineCapturedAt: page.baselineCapturedAt ?? null,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
browserKill: {
|
||||
ok: browserKill.ok === true,
|
||||
pending: browserKill.pending === true,
|
||||
skipped: browserKill.skipped === true,
|
||||
reason: browserKill.reason ?? null,
|
||||
pid: numberOrNull(browserKill.pid),
|
||||
gracefulSignal: browserKill.gracefulSignal ?? null,
|
||||
forceSignal: browserKill.forceSignal ?? null,
|
||||
gracefulSent: browserKill.gracefulSent === true,
|
||||
forceSent: browserKill.forceSent === true,
|
||||
exitedAfterGrace: browserKill.exitedAfterGrace === true,
|
||||
exitedAfterForce: browserKill.exitedAfterForce === true,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function buildBrowserProcessFindings(report, runtimeAlerts = null) {
|
||||
const summary = report?.summary || {};
|
||||
const findings = [];
|
||||
const blockerEvents = Array.isArray(report?.blockerEvents) ? report.blockerEvents : [];
|
||||
if (blockerEvents.length > 0) {
|
||||
const first = blockerEvents[0] || {};
|
||||
findings.push({
|
||||
id: "frontend-browser-freeze-runner-blocker",
|
||||
severity: "red",
|
||||
summary: "web-probe runner matched YAML browserFreezePolicy, killed/stopped Chromium, and failed the observer run; do not clear this by refresh or fallback",
|
||||
count: blockerEvents.length,
|
||||
blocking: true,
|
||||
rootCause: first.rootCause ?? "frontend_browser_freeze_policy_blocker",
|
||||
rootCauseStatus: "confirmed-from-runner-browser-freeze-policy",
|
||||
rootCauseConfidence: "high",
|
||||
policySource: first.policySource ?? "config/hwlab-node-lanes.yaml#webProbe.browserFreezePolicy",
|
||||
fallbackAllowed: false,
|
||||
observerRefreshAllowed: false,
|
||||
browserKilled: first.browserKill ?? null,
|
||||
events: blockerEvents.slice(0, 20),
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
if (!summary || Number(summary.sampleCount ?? 0) <= 0) return findings;
|
||||
const rootCauseSignals = browserRootCauseSignals(report, runtimeAlerts);
|
||||
const pageEvents = Array.isArray(report.latestPageEvents) ? report.latestPageEvents : [];
|
||||
const maxEffectiveHeapEvent = maxByNumber(pageEvents, (item) => Math.max(Number(item.effectiveHeapUsedMb ?? 0), Number(item.effectiveJsHeapUsedMb ?? 0)));
|
||||
const maxEffectiveHeapMb = maxEffectiveHeapEvent ? Math.max(Number(maxEffectiveHeapEvent.effectiveHeapUsedMb ?? 0), Number(maxEffectiveHeapEvent.effectiveJsHeapUsedMb ?? 0)) : 0;
|
||||
if (maxEffectiveHeapMb >= alertThresholds.browserProcessRssRedMb) {
|
||||
findings.push({
|
||||
id: "frontend-browser-memory-rss-red",
|
||||
severity: "red",
|
||||
summary: "Page effective memory exceeded YAML red threshold after subtracting page baseline; process RSS remains diagnostic evidence only",
|
||||
maxEffectiveHeapMb,
|
||||
processRssRedMb: alertThresholds.browserProcessRssRedMb,
|
||||
memoryRedPolicyScope: "per-page-effective-memory",
|
||||
maxEffectiveHeapEvent,
|
||||
maxTotalRssSample: report.maxTotalRssSample,
|
||||
maxProcessRssSample: report.maxProcessRssSample,
|
||||
rootCause: "frontend_browser_page_effective_memory_pressure",
|
||||
rootCauseStatus: "confirmed-from-runner-page-effective-memory",
|
||||
rootCauseConfidence: "high",
|
||||
rootCauseSignals,
|
||||
fallbackAllowed: false,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
const maxEffectiveGrowthEvent = maxByNumber(pageEvents, (item) => Math.max(Number(item.heapUsedGrowthMb ?? 0), Number(item.jsHeapUsedGrowthMb ?? 0)));
|
||||
const maxEffectiveGrowthMb = maxEffectiveGrowthEvent ? Math.max(Number(maxEffectiveGrowthEvent.heapUsedGrowthMb ?? 0), Number(maxEffectiveGrowthEvent.jsHeapUsedGrowthMb ?? 0)) : 0;
|
||||
if (maxEffectiveGrowthMb >= alertThresholds.browserRssGrowthRedMb) {
|
||||
findings.push({
|
||||
id: "frontend-browser-memory-growth-red",
|
||||
severity: "red",
|
||||
summary: "Page effective memory grew beyond YAML window budget; this matches the reported freeze pattern without counting multi-page startup RSS",
|
||||
maxEffectiveGrowthMb,
|
||||
growthRedMb: alertThresholds.browserRssGrowthRedMb,
|
||||
windowMs: alertThresholds.browserRssGrowthWindowMs,
|
||||
memoryRedPolicyScope: "per-page-effective-memory",
|
||||
maxEffectiveGrowthEvent,
|
||||
maxGrowthSample: report.maxGrowthSample,
|
||||
maxProcessGrowthSample: report.maxProcessGrowthSample,
|
||||
rootCause: "frontend_browser_page_memory_leak_or_unbounded_render_growth",
|
||||
rootCauseStatus: "confirmed-from-runner-page-effective-memory-growth",
|
||||
rootCauseConfidence: "high",
|
||||
rootCauseSignals,
|
||||
fallbackAllowed: false,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
const responsivenessEvents = Array.isArray(report.responsivenessEvents) ? report.responsivenessEvents : [];
|
||||
const severeLatencyEvents = responsivenessEvents.filter((item) => item.responsivenessTimeout !== true && Number(item.responsivenessLatencyMs ?? 0) >= alertThresholds.playwrightResponsivenessRedMs);
|
||||
const responsivenessTimeoutBurst = firstBurst(
|
||||
responsivenessEvents.filter((item) => item.responsivenessTimeout === true),
|
||||
alertThresholds.playwrightResponsivenessTimeoutRedCount,
|
||||
alertThresholds.domEvaluateTimeoutRedWindowMs,
|
||||
);
|
||||
if (severeLatencyEvents.length > 0 || responsivenessTimeoutBurst) {
|
||||
const events = severeLatencyEvents.length > 0 ? severeLatencyEvents : responsivenessTimeoutBurst;
|
||||
findings.push({
|
||||
id: "frontend-playwright-responsiveness-red",
|
||||
severity: "red",
|
||||
summary: "Playwright/CDP responsiveness probe exceeded YAML budget; treat the browser page as frozen instead of refreshing or falling back",
|
||||
count: events.length,
|
||||
latencyRedMs: alertThresholds.playwrightResponsivenessRedMs,
|
||||
timeoutThresholdCount: alertThresholds.playwrightResponsivenessTimeoutRedCount,
|
||||
windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs,
|
||||
events: events.slice(0, 20).map(browserProcessEventRef),
|
||||
rootCause: "frontend_browser_page_unresponsive_to_playwright",
|
||||
rootCauseStatus: "confirmed-from-cdp-runtime-evaluate",
|
||||
rootCauseConfidence: "high",
|
||||
rootCauseSignals,
|
||||
fallbackAllowed: false,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
const cdpTimeoutEvents = Array.isArray(report.cdpTimeoutEvents) ? report.cdpTimeoutEvents : [];
|
||||
const cdpTimeoutBurst = firstBurst(cdpTimeoutEvents, alertThresholds.cdpMetricsTimeoutRedCount, alertThresholds.domEvaluateTimeoutRedWindowMs);
|
||||
if (cdpTimeoutBurst) {
|
||||
findings.push({
|
||||
id: "frontend-cdp-metrics-timeout-red",
|
||||
severity: "red",
|
||||
summary: "CDP browser metrics timed out repeatedly; the sentinel has direct browser-side evidence of a hung page/runtime",
|
||||
count: cdpTimeoutBurst.length,
|
||||
thresholdCount: alertThresholds.cdpMetricsTimeoutRedCount,
|
||||
windowMs: alertThresholds.domEvaluateTimeoutRedWindowMs,
|
||||
events: cdpTimeoutBurst.slice(0, 20).map(browserProcessEventRef),
|
||||
rootCause: "frontend_browser_cdp_metrics_unresponsive",
|
||||
rootCauseStatus: "confirmed-from-cdp-metrics-timeouts",
|
||||
rootCauseConfidence: "high",
|
||||
rootCauseSignals,
|
||||
fallbackAllowed: false,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
function browserRootCauseSignals(report, runtimeAlerts) {
|
||||
const browserSummary = report?.summary || {};
|
||||
const runtimeSummary = runtimeAlerts?.summary || {};
|
||||
const sampleCount = Number(browserSummary.sampleCount ?? 0);
|
||||
const sessionListReadCount = Number(runtimeSummary.workbenchSessionListReadCount ?? 0);
|
||||
const traceEventsReadCount = Number(runtimeSummary.workbenchTraceEventsReadCount ?? 0);
|
||||
const webPerformanceBeaconFailureCount = Number(runtimeSummary.webPerformanceBeaconFailureCount ?? 0);
|
||||
const eventSourceFailureCount = Number(runtimeSummary.workbenchEventSourceFailureCount ?? 0);
|
||||
const suspectedFrontendRefreshStorm = sessionListReadCount >= Math.max(20, sampleCount * 2);
|
||||
return {
|
||||
suspectedFrontendRefreshStorm,
|
||||
sessionListReadCount,
|
||||
traceEventsReadCount,
|
||||
webPerformanceBeaconFailureCount,
|
||||
eventSourceFailureCount,
|
||||
requestFailedCount: runtimeSummary.significantRequestFailedCount ?? runtimeSummary.requestFailedCount ?? 0,
|
||||
httpErrorCount: runtimeSummary.httpErrorCount ?? 0,
|
||||
topRequestFailedPaths: (runtimeAlerts?.networkSignificantRequestFailedByPath ?? runtimeAlerts?.networkRequestFailedByPath ?? []).slice(0, 5),
|
||||
topHttpErrorPaths: (runtimeAlerts?.networkHttpErrorsByPath ?? []).slice(0, 5),
|
||||
note: suspectedFrontendRefreshStorm
|
||||
? "suspected frontend refresh storm: session list reads exceed the sample-derived budget during a browser red finding"
|
||||
: "root-cause signals are included so memory/responsiveness/CDP red findings can be correlated without manual grep",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function computeBrowserProcessGrowth(samples, windowMs) {
|
||||
const budgetMs = Math.max(1000, Number(windowMs || 0));
|
||||
const sorted = (samples || []).filter((item) => Number.isFinite(item.tsMs)).sort((a, b) => a.tsMs - b.tsMs);
|
||||
return sorted.map((item, index) => {
|
||||
const candidates = sorted.slice(0, index + 1).filter((candidate) => item.tsMs - candidate.tsMs <= budgetMs);
|
||||
const totalBaseline = minByNumber(candidates, (candidate) => candidate.totalRssBytes);
|
||||
const processBaseline = minByNumber(candidates, (candidate) => candidate.maxProcessRssBytes);
|
||||
const totalGrowth = Number(item.totalRssBytes || 0) - Number(totalBaseline?.totalRssBytes || 0);
|
||||
const processGrowth = Number(item.maxProcessRssBytes || 0) - Number(processBaseline?.maxProcessRssBytes || 0);
|
||||
return {
|
||||
...item,
|
||||
windowMs: budgetMs,
|
||||
totalBaselineAt: totalBaseline?.ts ?? null,
|
||||
processBaselineAt: processBaseline?.ts ?? null,
|
||||
totalRssGrowthBytes: totalGrowth,
|
||||
totalRssGrowthMb: bytesToMb(totalGrowth),
|
||||
maxProcessRssGrowthBytes: processGrowth,
|
||||
maxProcessRssGrowthMb: bytesToMb(processGrowth),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function browserProcessEventRef(item) {
|
||||
return {
|
||||
ts: item?.ts ?? null,
|
||||
seq: item?.seq ?? null,
|
||||
sampleSeq: item?.sampleSeq ?? null,
|
||||
commandId: item?.commandId ?? null,
|
||||
pageRole: item?.pageRole ?? null,
|
||||
pageId: item?.pageId ?? null,
|
||||
pageEpoch: item?.pageEpoch ?? null,
|
||||
timeoutMs: item?.timeoutMs ?? null,
|
||||
responsivenessLatencyMs: item?.responsivenessLatencyMs ?? item?.latencyMs ?? null,
|
||||
responsivenessTimeout: item?.responsivenessTimeout === true,
|
||||
cdpTimeoutCount: item?.cdpTimeoutCount ?? null,
|
||||
method: item?.method ?? null,
|
||||
errorMessage: item?.errorMessage ?? null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function compactBrowserProcessRow(item) {
|
||||
if (!item || typeof item !== "object") return null;
|
||||
return {
|
||||
pid: item.pid ?? null,
|
||||
role: item.role ?? null,
|
||||
name: item.name ?? null,
|
||||
rssMb: bytesToMb(item.rssBytes),
|
||||
vmSizeMb: bytesToMb(item.vmSizeBytes),
|
||||
commandHash: item.commandHash ?? null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function browserMetricNumber(value) {
|
||||
const numeric = Number(value);
|
||||
return Number.isFinite(numeric) ? numeric : null;
|
||||
}
|
||||
|
||||
function bytesToMb(value) {
|
||||
const numeric = Number(value);
|
||||
if (!Number.isFinite(numeric)) return null;
|
||||
return Number((numeric / 1024 / 1024).toFixed(1));
|
||||
}
|
||||
|
||||
function maxByNumber(items, getter) {
|
||||
let selected = null;
|
||||
let selectedValue = Number.NEGATIVE_INFINITY;
|
||||
for (const item of Array.isArray(items) ? items : []) {
|
||||
const value = Number(getter(item));
|
||||
if (!Number.isFinite(value)) continue;
|
||||
if (!selected || value > selectedValue) {
|
||||
selected = item;
|
||||
selectedValue = value;
|
||||
}
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
function minByNumber(items, getter) {
|
||||
let selected = null;
|
||||
let selectedValue = Number.POSITIVE_INFINITY;
|
||||
for (const item of Array.isArray(items) ? items : []) {
|
||||
const value = Number(getter(item));
|
||||
if (!Number.isFinite(value)) continue;
|
||||
if (!selected || value < selectedValue) {
|
||||
selected = item;
|
||||
selectedValue = value;
|
||||
}
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
function safeReportUrl(value) {
|
||||
if (!value) return null;
|
||||
try {
|
||||
const url = new URL(String(value));
|
||||
return url.origin + url.pathname;
|
||||
} catch {
|
||||
return limitText(String(value), 200);
|
||||
}
|
||||
}
|
||||
|
||||
function frontendFreezeErrorEvent(item, promptTimes) {
|
||||
const details = objectValue(item?.error?.details);
|
||||
const message = String(item?.error?.message ?? item?.message ?? item?.error ?? "");
|
||||
const type = String(item?.type || "");
|
||||
const tsMs = Date.parse(String(item?.ts || ""));
|
||||
if (!Number.isFinite(tsMs)) return null;
|
||||
const kind = classifyFrontendFreezeError(type, message);
|
||||
if (!kind) return null;
|
||||
return {
|
||||
ts: item.ts ?? null,
|
||||
tsMs,
|
||||
promptIndex: promptIndexForTs(promptTimes, item.ts),
|
||||
kind,
|
||||
type: item.type ?? null,
|
||||
pageRole: stringOrNull(item?.pageRole) ?? stringOrNull(details.pageRole) ?? pageRoleFromErrorType(type),
|
||||
pageId: stringOrNull(item?.pageId) ?? stringOrNull(details.pageId),
|
||||
routeSessionId: stringOrNull(item?.routeSessionId) ?? stringOrNull(details.routeSessionId),
|
||||
activeSessionId: stringOrNull(item?.activeSessionId) ?? stringOrNull(details.activeSessionId),
|
||||
commandId: stringOrNull(item?.commandId) ?? stringOrNull(details.commandId),
|
||||
sampleSeq: numberOrNull(item?.sampleSeq ?? details.sampleSeq),
|
||||
timeoutMs: timeoutMsFromMessage(message),
|
||||
messageHash: message ? sha256(message) : null,
|
||||
preview: limitText(message, 240),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function pageRoleFromErrorType(type) {
|
||||
const value = String(type || "");
|
||||
if (/^control-/iu.test(value)) return "control";
|
||||
if (/^observer-/iu.test(value)) return "observer";
|
||||
return null;
|
||||
}
|
||||
|
||||
function classifyFrontendFreezeError(type, message) {
|
||||
const value = String(message || "");
|
||||
if (/sampleOnePage\s+DOM\s+evaluate\s+exceeded/iu.test(value) && /(?:control|observer)-sample-error/iu.test(type)) return "dom-evaluate-timeout";
|
||||
if (/screenshot|captureScreenshot|page\.screenshot/iu.test(type + " " + value) && /timeout|timed\s*out|exceeded/iu.test(value)) return "screenshot-timeout";
|
||||
if (/pageerror|uncaught|unhandledrejection/iu.test(type) || /^(?:Error|TypeError|ReferenceError|RangeError|SyntaxError):/u.test(value)) return "page-error";
|
||||
return null;
|
||||
}
|
||||
|
||||
function firstBurst(events, thresholdCount, windowMs) {
|
||||
const count = Math.max(1, Math.floor(Number(thresholdCount || 0)));
|
||||
const budgetMs = Math.max(1, Number(windowMs || 0));
|
||||
const sorted = (events || []).filter((item) => Number.isFinite(item?.tsMs)).sort((a, b) => a.tsMs - b.tsMs);
|
||||
if (sorted.length < count) return null;
|
||||
for (let start = 0; start <= sorted.length - count; start += 1) {
|
||||
const end = start + count - 1;
|
||||
if (sorted[end].tsMs - sorted[start].tsMs <= budgetMs) return sorted.slice(start, end + 1);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function frontendFreezeBurstFinding({ id, summary, burst, thresholdCount, windowMs, pageRole }) {
|
||||
const first = burst[0];
|
||||
const last = burst[burst.length - 1];
|
||||
const pageIds = uniqueStrings(burst.map((item) => item.pageId));
|
||||
const routeSessionIds = uniqueStrings(burst.map((item) => item.routeSessionId));
|
||||
const activeSessionIds = uniqueStrings(burst.map((item) => item.activeSessionId));
|
||||
return {
|
||||
id,
|
||||
severity: "red",
|
||||
summary,
|
||||
count: burst.length,
|
||||
thresholdCount,
|
||||
windowMs,
|
||||
firstAt: first?.ts ?? null,
|
||||
lastAt: last?.ts ?? null,
|
||||
pageRole,
|
||||
pageIds,
|
||||
routeSessionIds,
|
||||
activeSessionIds,
|
||||
timeoutMsMax: maxPresentNumber(burst.map((item) => item.timeoutMs)),
|
||||
rootCause: "frontend_page_freeze_or_runtime_exception",
|
||||
rootCauseStatus: "confirmed-from-browser-observer-errors",
|
||||
rootCauseConfidence: "high",
|
||||
fallbackAllowed: false,
|
||||
observerRefreshMayNotClear: true,
|
||||
nextAction: "Keep this run red; do not auto-refresh, fallback, or mark healthy until OTel/browser evidence explains why the page stopped responding.",
|
||||
events: burst.map((item) => ({
|
||||
ts: item.ts,
|
||||
promptIndex: item.promptIndex,
|
||||
type: item.type,
|
||||
pageRole: item.pageRole,
|
||||
pageId: item.pageId,
|
||||
routeSessionId: item.routeSessionId,
|
||||
activeSessionId: item.activeSessionId,
|
||||
commandId: item.commandId,
|
||||
sampleSeq: item.sampleSeq,
|
||||
timeoutMs: item.timeoutMs,
|
||||
messageHash: item.messageHash,
|
||||
preview: item.preview,
|
||||
valuesRedacted: true,
|
||||
})),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function stopCommandWindows(control) {
|
||||
return (control || [])
|
||||
.filter((item) => /^(?:stop|forceStop|cancel|close)$/iu.test(String(item?.type || item?.command || "")))
|
||||
.map((item) => {
|
||||
const tsMs = Date.parse(String(item?.ts || ""));
|
||||
return Number.isFinite(tsMs) ? { fromMs: tsMs - 1000, toMs: tsMs + 10000 } : null;
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function errorInsideStopWindow(event, windows) {
|
||||
return (windows || []).some((window) => event.tsMs >= window.fromMs && event.tsMs <= window.toMs);
|
||||
}
|
||||
|
||||
function timeoutMsFromMessage(value) {
|
||||
const match = String(value || "").match(/\b(?:exceeded|timeout|timed\s*out\s*after)\s+(\d{2,})\s*ms\b/iu)
|
||||
|| String(value || "").match(/\b(\d{2,})\s*ms\b/iu);
|
||||
return match ? Number(match[1]) : null;
|
||||
}
|
||||
|
||||
function uniqueStrings(values) {
|
||||
return Array.from(new Set((values || []).filter((item) => typeof item === "string" && item.length > 0))).slice(0, 12);
|
||||
}
|
||||
|
||||
function maxPresentNumber(values) {
|
||||
const numbers = (values || []).filter((item) => item !== null && item !== undefined && Number.isFinite(Number(item))).map((item) => Number(item));
|
||||
return numbers.length > 0 ? Math.max(...numbers) : null;
|
||||
}
|
||||
`;
|
||||
}
|
||||
Reference in New Issue
Block a user