fix: 收敛 WebProbe 物理内存与资源上限

2026-07-13 08:42:44 +02:00
parent 0c22a05cee
commit 2e38d690e7
31 changed files with 3882 additions and 1772 deletions
@@ -73,14 +73,34 @@ async function writeManifest(extra = {}) {
      maxPerformanceEntries,
      maxBrowserProcessHistoryEntries,
      maxBrowserFreezeSignalHistoryEntries,
+      maxRealtimeEventEntries,
+      maxRealtimeEventBodyBytes,
+      maxDomEvidenceRows,
+      maxProcessScanEntries,
+      maxPerformanceCaptureMs,
      responseBodyMaxBytes,
      webPerformanceRequestBodyMaxBytes,
+      maxJsonlQueueEntries,
+      maxJsonlQueueBytes,
+      maxJsonlLineBytes,
+      jsonlFlushTimeoutMs,
+      maxResponseBodyInFlight,
+      maxPassiveListenerTasks,
+      passiveTaskFlushTimeoutMs,
+      closeStepTimeoutMs,
+      maxProcessReadConcurrency,
+      maxRealtimeTotalEntries,
+      maxRealtimeTotalBodyBytes,
      valuesRedacted: true,
    },
    alertThresholds,
    browserFreezePolicy,
    projectManagement,
    jsonlRotation,
+    jsonlWriter: jsonlWriterStats(),
+    responseBodies: responseBodyStats(),
+    passiveTasks: passiveTaskStats(),
+    realtimeAggregate: realtimeAggregateLast,
    commandDirs: dirs,
    artifacts: files,
    safety: { pureClient: true, inboundApi: false, database: false, queueConsumer: false, k8sWorkload: false, valuesRedacted: true, secretValuesPrinted: false },
@@ -115,8 +135,24 @@ async function writeHeartbeat(extra = {}) {
      maxPerformanceEntries,
      maxBrowserProcessHistoryEntries,
      maxBrowserFreezeSignalHistoryEntries,
+      maxRealtimeEventEntries,
+      maxRealtimeEventBodyBytes,
+      maxDomEvidenceRows,
+      maxProcessScanEntries,
+      maxPerformanceCaptureMs,
      responseBodyMaxBytes,
      webPerformanceRequestBodyMaxBytes,
+      maxJsonlQueueEntries,
+      maxJsonlQueueBytes,
+      maxJsonlLineBytes,
+      jsonlFlushTimeoutMs,
+      maxResponseBodyInFlight,
+      maxPassiveListenerTasks,
+      passiveTaskFlushTimeoutMs,
+      closeStepTimeoutMs,
+      maxProcessReadConcurrency,
+      maxRealtimeTotalEntries,
+      maxRealtimeTotalBodyBytes,
      valuesRedacted: true,
    },
    lastObserverRefreshAt: Number.isFinite(lastObserverRefreshAtMs) ? new Date(lastObserverRefreshAtMs).toISOString() : null,
@@ -124,6 +160,10 @@ async function writeHeartbeat(extra = {}) {
    sampleSeq,
    commandSeq,
    activeCommandId,
+    jsonlWriter: jsonlWriterStats(),
+    responseBodies: responseBodyStats(),
+    passiveTasks: passiveTaskStats(),
+    realtimeAggregate: realtimeAggregateLast,
    updatedAt: new Date().toISOString(),
    uptimeMs: Date.now() - startedAtMs,
    ...extra,
@@ -131,6 +171,22 @@ async function writeHeartbeat(extra = {}) {
  await writeFile(files.heartbeat, JSON.stringify(heartbeat, null, 2) + "\n", { mode: 0o600 });
 }

+async function writeTerminalResourceStats(extra = {}) {
+  for (const file of [files.manifest, files.heartbeat]) {
+    let existing = {};
+    try { existing = JSON.parse(await readFile(file, "utf8")); } catch {}
+    await writeFile(file, JSON.stringify({
+      ...existing,
+      jsonlWriter: jsonlWriterStats(),
+      responseBodies: responseBodyStats(),
+      passiveTasks: passiveTaskStats(),
+      terminalResources: sanitize(extra),
+      updatedAt: new Date().toISOString(),
+      valuesRedacted: true,
+    }, null, 2) + "\n", { mode: 0o600 });
+  }
+}
+
 function startHeartbeatPulse() {
  const timer = setInterval(() => {
    if (stopping) return;
@@ -410,62 +466,96 @@ async function triggerBrowserFreezeBlocker(trigger) {
 }

 async function terminateBrowserForFreezeBlocker(blocker) {
-  const childProcess = browser && typeof browser.process === "function" ? browser.process() : null;
-  const pid = Number(childProcess?.pid);
-  if (!Number.isFinite(pid) || pid <= 0) {
-    return { ok: false, reason: "browser-process-unavailable", blockerKind: blocker.kind, valuesRedacted: true };
+  const result = await terminateTrackedBrowserProcesses("freeze-blocker:" + String(blocker.kind || "unknown"));
+  return { ...result, blockerKind: blocker.kind, valuesRedacted: true };
+}
+
+function rememberTrackedBrowserProcesses(processes) {
+  const merged = new Map(trackedBrowserProcessIdentities.map((item) => [item.pid + ":" + item.startTicks, item]));
+  for (const item of Array.isArray(processes) ? processes : []) {
+    if (!Number.isInteger(item.pid) || !Number.isInteger(item.startTicks) || !item.role) continue;
+    merged.set(item.pid + ":" + item.startTicks, {
+      pid: item.pid,
+      ppid: item.ppid,
+      pgid: item.pgid,
+      sid: item.sid,
+      startTicks: item.startTicks,
+      role: item.role,
+      commandHash: item.commandHash || null,
+      valuesRedacted: true,
+    });
  }
+  trackedBrowserProcessIdentities = [...merged.values()].slice(-Math.min(maxProcessScanEntries, 1024));
+}
+
+async function matchingTrackedBrowserProcesses() {
+  const rows = await Promise.all(trackedBrowserProcessIdentities.map(async (identity) => {
+    const current = await readOneProcProcessStat(identity.pid).catch(() => null);
+    return current && current.startTicks === identity.startTicks ? { ...identity, currentPpid: current.ppid, currentPgid: current.pgid, currentSid: current.sid } : null;
+  }));
+  return rows.filter(Boolean);
+}
+
+async function waitForTrackedBrowserExit(timeoutMs, pollIntervalMs) {
+  const deadline = Date.now() + timeoutMs;
+  let remaining = await matchingTrackedBrowserProcesses();
+  while (remaining.length > 0 && Date.now() <= deadline) {
+    await boundedCleanupSleep(pollIntervalMs);
+    remaining = await matchingTrackedBrowserProcesses();
+  }
+  return remaining;
+}
+
+async function terminateTrackedBrowserProcesses(reason, requiredDiscovery = null) {
+  let refreshDiscovery;
+  try {
+    const summary = await collectChromiumProcessSummary(null);
+    refreshDiscovery = { ok: true, processScan: summary.processScan, chromiumProcessCount: summary.chromiumProcessCount, valuesRedacted: true };
+  } catch (error) {
+    refreshDiscovery = { ok: false, processScan: lastProcessScan, error: errorSummary(error), valuesRedacted: true };
+  }
+  const discovery = requiredDiscovery || refreshDiscovery;
+  const identityScanUnavailable = browserLaunchAttempted === true
+    && (discovery?.ok !== true || discovery?.processScan?.complete !== true);
+  const initial = await matchingTrackedBrowserProcesses();
  const result = {
-    ok: false,
-    pid: Math.floor(pid),
+    ok: initial.length === 0 && !identityScanUnavailable,
+    reason,
+    browserLaunchAttempted,
+    identityScanUnavailable,
+    requiredDiscovery: discovery,
+    refreshDiscovery,
+    trackedIdentityCount: trackedBrowserProcessIdentities.length,
+    initialLiveCount: initial.length,
    gracefulSignal: browserFreezePolicy.kill.gracefulSignal,
    forceSignal: browserFreezePolicy.kill.forceSignal,
    graceMs: browserFreezePolicy.kill.graceMs,
    pollIntervalMs: browserFreezePolicy.kill.pollIntervalMs,
-    gracefulSent: false,
-    forceSent: false,
-    exitedAfterGrace: false,
-    exitedAfterForce: false,
+    gracefulSentCount: 0,
+    forceSentCount: 0,
+    remainingIdentityCount: initial.length,
    valuesRedacted: true,
  };
-  try {
-    childProcess.kill(browserFreezePolicy.kill.gracefulSignal);
-    result.gracefulSent = true;
-  } catch (error) {
-    result.gracefulError = errorSummary(error);
-  }
-  result.exitedAfterGrace = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
-  if (!result.exitedAfterGrace) {
+  for (const item of [...initial].sort((a, b) => b.pid - a.pid)) {
    try {
-      childProcess.kill(browserFreezePolicy.kill.forceSignal);
-      result.forceSent = true;
-    } catch (error) {
-      result.forceError = errorSummary(error);
-    }
-    result.exitedAfterForce = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
+      process.kill(item.pid, browserFreezePolicy.kill.gracefulSignal);
+      result.gracefulSentCount += 1;
+    } catch {}
  }
-  result.ok = result.exitedAfterGrace || result.exitedAfterForce;
+  let remaining = await waitForTrackedBrowserExit(browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
+  for (const item of [...remaining].sort((a, b) => b.pid - a.pid)) {
+    try {
+      process.kill(item.pid, browserFreezePolicy.kill.forceSignal);
+      result.forceSentCount += 1;
+    } catch {}
+  }
+  remaining = await waitForTrackedBrowserExit(browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
+  result.remainingIdentityCount = remaining.length;
+  result.remainingIdentities = remaining.slice(0, 50).map((item) => ({ pid: item.pid, startTicks: item.startTicks, role: item.role, originalPpid: item.ppid, currentPpid: item.currentPpid, originalSid: item.sid, currentSid: item.currentSid, valuesRedacted: true }));
+  result.ok = remaining.length === 0 && !identityScanUnavailable;
  return result;
 }

-async function waitForPidExit(pid, timeoutMs, pollIntervalMs) {
-  const deadline = Date.now() + timeoutMs;
-  while (Date.now() <= deadline) {
-    if (!pidAlive(pid)) return true;
-    await sleep(pollIntervalMs);
-  }
-  return !pidAlive(pid);
-}
-
-function pidAlive(pid) {
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
 function browserProcessSampleRef(sample) {
  return {
    ts: sample?.ts ?? null,
@@ -532,13 +622,16 @@ async function collectChromiumProcessSummary(browserPid) {
      queue.push(child);
    }
  }
-  const processes = all
-    .filter((item) => descendants.has(item.pid))
+  const detailed = await readProcProcessDetails(all.filter((item) => descendants.has(item.pid)));
+  const processes = detailed
    .map((item) => ({ ...item, role: classifyChromiumProcess(item) }))
    .filter((item) => item.role)
    .map((item) => ({
      pid: item.pid,
      ppid: item.ppid,
+      pgid: item.pgid,
+      sid: item.sid,
+      startTicks: item.startTicks,
      name: item.name || null,
      role: item.role,
      rssBytes: item.rssBytes,
@@ -548,6 +641,7 @@ async function collectChromiumProcessSummary(browserPid) {
      valuesRedacted: true,
    }))
    .sort((a, b) => Number(b.rssBytes || 0) - Number(a.rssBytes || 0));
+  rememberTrackedBrowserProcesses(processes);
  const roles = {};
  for (const item of processes) {
    const role = item.role || "unknown";
@@ -561,6 +655,7 @@ async function collectChromiumProcessSummary(browserPid) {
  const maxProcess = processes[0] || null;
  return {
    sampledRootPids: roots,
+    processScan: lastProcessScan,
    chromiumProcessCount: processes.length,
    totalRssBytes,
    totalRssMb: bytesToMb(totalRssBytes),
@@ -577,28 +672,68 @@ async function readProcProcessTable() {
  let entries = [];
  try {
    entries = await readdir("/proc");
-  } catch {
-    return [];
+  } catch (error) {
+    lastProcessScan = { ok: false, complete: false, totalPidEntries: 0, scannedPidEntries: 0, failedPidEntries: 0, truncated: false, readConcurrency: maxProcessReadConcurrency, error: errorSummary(error), valuesRedacted: true };
+    throw new Error("proc-process-table-unavailable");
  }
-  const numeric = entries.map((name) => Number(name)).filter((pid) => Number.isInteger(pid) && pid > 0);
-  const rows = await Promise.all(numeric.map((pid) => readOneProcProcess(pid).catch(() => null)));
-  return rows.filter(Boolean);
+  const allNumeric = entries.map((name) => Number(name)).filter((pid) => Number.isInteger(pid) && pid > 0).sort((a, b) => b - a);
+  const numeric = allNumeric.slice(0, maxProcessScanEntries);
+  const rows = [];
+  for (let offset = 0; offset < numeric.length; offset += maxProcessReadConcurrency) {
+    const batch = await Promise.all(numeric.slice(offset, offset + maxProcessReadConcurrency).map((pid) => readOneProcProcessStat(pid).catch(() => null)));
+    rows.push(...batch.filter(Boolean));
+  }
+  const failedPidEntries = numeric.length - rows.length;
+  const truncated = numeric.length < allNumeric.length;
+  const unusable = numeric.length > 0 && rows.length === 0;
+  lastProcessScan = {
+    ok: !unusable,
+    complete: !unusable && !truncated,
+    totalPidEntries: allNumeric.length,
+    scannedPidEntries: numeric.length,
+    readablePidEntries: rows.length,
+    failedPidEntries,
+    truncated,
+    readConcurrency: maxProcessReadConcurrency,
+    error: unusable ? "all-proc-stat-reads-failed" : null,
+    valuesRedacted: true,
+  };
+  if (unusable) throw new Error("proc-process-table-unreadable");
+  return rows;
 }

-async function readOneProcProcess(pid) {
-  const [statText, statusText, cmdlineText] = await Promise.all([
-    readFile(path.join("/proc", String(pid), "stat"), "utf8").catch(() => ""),
+async function readOneProcProcessStat(pid) {
+  const statText = await readFile(path.join("/proc", String(pid), "stat"), "utf8");
+  if (!statText) return null;
+  return { pid, ...procIdentityFromStat(statText) };
+}
+
+async function readProcProcessDetails(rows) {
+  const detailed = [];
+  for (let offset = 0; offset < rows.length; offset += maxProcessReadConcurrency) {
+    const batch = await Promise.all(rows.slice(offset, offset + maxProcessReadConcurrency).map((row) => readOneProcProcessDetails(row).catch(() => null)));
+    detailed.push(...batch.filter(Boolean));
+  }
+  return detailed;
+}
+
+async function readOneProcProcessDetails(row) {
+  const pid = row.pid;
+  const [statusText, cmdlineText] = await Promise.all([
    readFile(path.join("/proc", String(pid), "status"), "utf8").catch(() => ""),
    readFile(path.join("/proc", String(pid), "cmdline"), "utf8").catch(() => ""),
  ]);
-  if (!statText && !statusText) return null;
-  const ppid = procPpidFromStat(statText);
+  if (!statusText && !cmdlineText) return null;
+  const ppid = row.ppid;
  const name = procStatusField(statusText, "Name") || null;
  const rssKb = procStatusKb(statusText, "VmRSS");
  const vmSizeKb = procStatusKb(statusText, "VmSize");
  return {
    pid,
    ppid,
+    pgid: row.pgid,
+    sid: row.sid,
+    startTicks: row.startTicks,
    name,
    cmdline: String(cmdlineText || "").replace(/\0/gu, " ").replace(/\s+/gu, " ").trim(),
    rssBytes: Number.isFinite(rssKb) ? rssKb * 1024 : 0,
@@ -606,13 +741,13 @@ async function readOneProcProcess(pid) {
  };
 }

-function procPpidFromStat(value) {
+function procIdentityFromStat(value) {
  const text = String(value || "");
  const end = text.lastIndexOf(")");
-  if (end < 0) return null;
+  if (end < 0) return { ppid: null, pgid: null, sid: null, startTicks: null };
  const parts = text.slice(end + 1).trim().split(/\s+/u);
-  const ppid = Number(parts[1]);
-  return Number.isFinite(ppid) ? ppid : null;
+  const integerAt = (index) => Number.isInteger(Number(parts[index])) ? Number(parts[index]) : null;
+  return { ppid: integerAt(1), pgid: integerAt(2), sid: integerAt(3), startTicks: integerAt(19) };
 }

 function procStatusField(value, key) {
@@ -725,7 +860,13 @@ function applyBrowserPageRuntimeBaseline(result) {
  const key = [result.pageRole || "unknown", result.pageId || "unknown", Number.isFinite(Number(result.pageEpoch)) ? Number(result.pageEpoch) : 0].join(":");
  const current = browserPageRuntimeMemorySnapshot(result);
  const existing = browserPageRuntimeBaselines.get(key);
-  if (!existing && current) browserPageRuntimeBaselines.set(key, { ...current, capturedAt: new Date().toISOString(), source: "first-page-runtime-sample", valuesRedacted: true });
+  if (!existing && current) {
+    const prefix = [result.pageRole || "unknown", result.pageId || "unknown"].join(":") + ":";
+    for (const existingKey of browserPageRuntimeBaselines.keys()) {
+      if (existingKey.startsWith(prefix)) browserPageRuntimeBaselines.delete(existingKey);
+    }
+    browserPageRuntimeBaselines.set(key, { ...current, capturedAt: new Date().toISOString(), source: "first-page-runtime-sample", valuesRedacted: true });
+  }
  const baseline = browserPageRuntimeBaselines.get(key) || null;
  result.baseline = baseline ? {
    capturedAt: baseline.capturedAt,
@@ -847,8 +988,10 @@ function bytesToMb(value) {
 }

 function attachPassiveListeners(targetPage, pageRole = "control", targetPageId = pageId) {
-  void installPagePerformanceProbe(targetPage, pageRole, targetPageId)
-    .catch((error) => appendJsonl(files.errors, eventRecord("performance-probe-install-error", { pageRole, pageId: targetPageId, error: errorSummary(error), valuesRedacted: true })));
+  trackPassiveTask(
+    () => installPagePerformanceProbe(targetPage, pageRole, targetPageId),
+    "performance-probe-install:" + pageRole,
+  );
  targetPage.on("request", (request) => {
    const webPerformancePayload = summarizeWebPerformanceRequestPayload(request);
    void appendJsonl(files.network, eventRecord("request", {
@@ -878,19 +1021,10 @@ function attachPassiveListeners(targetPage, pageRole = "control", targetPageId =
      statusText: response.statusText(),
      fromServiceWorker: response.fromServiceWorker(),
    };
-    void (async () => {
+    trackPassiveTask(async () => {
      const bodyFields = await summarizeWorkbenchResponseBody(response, request);
      await appendJsonl(files.network, eventRecord("response", { ...base, ...bodyFields }));
-    })().catch((error) => appendJsonl(files.errors, eventRecord("response-body-summary-error", {
-      pageRole,
-      pageId: targetPageId,
-      sampleSeq,
-      commandId: activeCommandId,
-      method: request.method(),
-      url: safeUrl(response.url()),
-      error: errorSummary(error),
-      valuesRedacted: true
-    })));
+    }, "response-body-summary:" + pageRole);
  });
  targetPage.on("requestfailed", (request) => {
    void appendJsonl(files.network, eventRecord("requestfailed", {