feat: block frozen web probe browsers from yaml policy

2026-06-30 11:07:52 +00:00
parent 81c48b8126
commit 6168ec60f6
5 changed files with 683 additions and 10 deletions
@@ -19,6 +19,7 @@ const analyzeTailSamples = (() => {
  return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : 360;
 })();
 const alertThresholds = parseAlertThresholds(process.env.UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON);
+const browserFreezePolicy = parseBrowserFreezePolicy(process.env.UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON);
 const projectManagementConfig = parseProjectManagementConfig(process.env.UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON);
 const dataDir = archivePrefix ? path.join(stateDir, "archive") : stateDir;
 const dataFile = (name) => path.join(dataDir, archivePrefix ? archivePrefix + "-" + name : name);
@@ -118,6 +119,7 @@ const report = {
  stateDir,
  jsonlScope: { mode: archivePrefix ? "archive" : "current", archivePrefix: archivePrefix || null, dataDir, analyzeTailSamples, sourceSampleCount: sourceSamples.length, effectiveSampleCount: samples.length, sourceControlCount: sourceControlAll.length, sampleWindow, focus: analysisFocus, valuesRedacted: true },
  alertThresholds,
+  browserFreezePolicy,
  manifest: compactManifest(manifest),
  heartbeat: compactHeartbeat(heartbeat),
  counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length, browserProcess: browserProcessRows.length },
@@ -584,6 +586,85 @@ function parseAlertThresholds(value) {
  };
 }

+function parseBrowserFreezePolicy(value) {
+  if (!value) {
+    throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is required; configure config/hwlab-node-lanes.yaml webProbe.browserFreezePolicy for the selected node/lane");
+  }
+  const raw = (() => {
+    try { return JSON.parse(value); } catch (error) { throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is invalid JSON: " + (error instanceof Error ? error.message : String(error))); }
+  })();
+  const memory = requiredPolicyRecord(raw, "memory", "webProbe.browserFreezePolicy");
+  const responsiveness = requiredPolicyRecord(raw, "responsiveness", "webProbe.browserFreezePolicy");
+  const cdp = requiredPolicyRecord(raw, "cdp", "webProbe.browserFreezePolicy");
+  const kill = requiredPolicyRecord(raw, "kill", "webProbe.browserFreezePolicy");
+  return {
+    enabled: requiredPolicyBoolean(raw, "enabled", "webProbe.browserFreezePolicy"),
+    blockerWindowMs: requiredPolicyPositiveNumber(raw, "blockerWindowMs", "webProbe.browserFreezePolicy"),
+    memory: {
+      totalRssBlockerMb: requiredPolicyPositiveNumber(memory, "totalRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
+      processRssBlockerMb: requiredPolicyPositiveNumber(memory, "processRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
+      growthBlockerMb: requiredPolicyPositiveNumber(memory, "growthBlockerMb", "webProbe.browserFreezePolicy.memory"),
+    },
+    responsiveness: {
+      latencyBlockerMs: requiredPolicyPositiveNumber(responsiveness, "latencyBlockerMs", "webProbe.browserFreezePolicy.responsiveness"),
+      eventBlockerCount: requiredPolicyPositiveNumber(responsiveness, "eventBlockerCount", "webProbe.browserFreezePolicy.responsiveness"),
+    },
+    cdp: {
+      metricsTimeoutBlockerCount: requiredPolicyPositiveNumber(cdp, "metricsTimeoutBlockerCount", "webProbe.browserFreezePolicy.cdp"),
+    },
+    kill: {
+      enabled: requiredPolicyBoolean(kill, "enabled", "webProbe.browserFreezePolicy.kill"),
+      gracefulSignal: requiredPolicySignal(kill, "gracefulSignal", "webProbe.browserFreezePolicy.kill", "SIGTERM"),
+      forceSignal: requiredPolicySignal(kill, "forceSignal", "webProbe.browserFreezePolicy.kill", "SIGKILL"),
+      graceMs: requiredPolicyIntegerInRange(kill, "graceMs", "webProbe.browserFreezePolicy.kill", 1, 120000),
+      pollIntervalMs: requiredPolicyIntegerInRange(kill, "pollIntervalMs", "webProbe.browserFreezePolicy.kill", 1, 10000),
+      exitCode: requiredPolicyIntegerInRange(kill, "exitCode", "webProbe.browserFreezePolicy.kill", 1, 125),
+    },
+    source: "yaml-env",
+    valuesRedacted: true,
+  };
+}
+
+function requiredPolicyRecord(raw, key, pathLabel) {
+  const value = raw?.[key];
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires object " + pathLabel + "." + key);
+  }
+  return value;
+}
+
+function requiredPolicyBoolean(raw, key, pathLabel) {
+  const value = raw?.[key];
+  if (typeof value !== "boolean") {
+    throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires boolean " + pathLabel + "." + key);
+  }
+  return value;
+}
+
+function requiredPolicyPositiveNumber(raw, key, pathLabel) {
+  const value = Number(raw?.[key]);
+  if (!Number.isFinite(value) || value <= 0) {
+    throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires positive number " + pathLabel + "." + key);
+  }
+  return value;
+}
+
+function requiredPolicyIntegerInRange(raw, key, pathLabel, min, max) {
+  const value = Number(raw?.[key]);
+  if (!Number.isInteger(value) || value < min || value > max) {
+    throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires integer " + pathLabel + "." + key + " between " + min + " and " + max);
+  }
+  return value;
+}
+
+function requiredPolicySignal(raw, key, pathLabel, expected) {
+  const value = String(raw?.[key] || "");
+  if (value !== expected) {
+    throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires " + pathLabel + "." + key + "=" + expected);
+  }
+  return value;
+}
+
 function parseProjectManagementConfig(value) {
  if (!value || value === "null") {
    return {
@@ -3107,6 +3188,10 @@ function buildFrontendFreezeFindings(errors, control) {
 }

 function buildBrowserProcessReport(rows) {
+  const blockerEvents = (Array.isArray(rows) ? rows : [])
+    .filter((item) => item && item.type === "browser-freeze-blocker")
+    .map(compactBrowserFreezeBlockerEvent)
+    .filter(Boolean);
  const samples = (Array.isArray(rows) ? rows : [])
    .filter((item) => item && (item.type === "browser-process-sample" || item.process || item.pages))
    .map((item) => ({ ...item, tsMs: Date.parse(String(item.ts || "")) }))
@@ -3205,8 +3290,11 @@ function buildBrowserProcessReport(rows) {
      growthRedMb: alertThresholds.browserRssGrowthRedMb,
      growthWindowMs: alertThresholds.browserRssGrowthWindowMs,
      responsivenessRedMs: alertThresholds.playwrightResponsivenessRedMs,
+      freezeBlockerCount: blockerEvents.length,
+      browserFreezePolicy,
      valuesRedacted: true,
    },
+    blockerEvents: blockerEvents.slice(-20),
    memorySamples: memorySamples.slice(-60),
    growthSamples: computedGrowth.slice(-60),
    maxTotalRssSample: maxTotal,
@@ -3220,10 +3308,112 @@ function buildBrowserProcessReport(rows) {
  };
 }

+function compactBrowserFreezeBlockerEvent(item) {
+  if (!item || typeof item !== "object") return null;
+  const observed = objectValue(item.observed);
+  const threshold = objectValue(item.threshold);
+  const sample = objectValue(item.sample);
+  const page = objectValue(item.page);
+  const browserKill = objectValue(item.browserKill);
+  return {
+    ts: item.ts ?? null,
+    seq: item.seq ?? null,
+    sampleSeq: item.sampleSeq ?? sample.sampleSeq ?? null,
+    kind: item.kind ?? null,
+    severity: item.severity ?? "red",
+    blocking: item.blocking === true,
+    summary: item.summary ? String(item.summary).slice(0, 240) : null,
+    rootCause: item.rootCause ?? null,
+    rootCauseStatus: item.rootCauseStatus ?? null,
+    rootCauseConfidence: item.rootCauseConfidence ?? null,
+    policySource: item.policySource ?? null,
+    fallbackAllowed: item.fallbackAllowed === true,
+    observerRefreshAllowed: item.observerRefreshAllowed === true,
+    observed: {
+      totalRssMb: numberOrNull(observed.totalRssMb),
+      processRssMb: numberOrNull(observed.processRssMb),
+      totalGrowthMb: numberOrNull(observed.totalGrowthMb),
+      processGrowthMb: numberOrNull(observed.processGrowthMb),
+      responsivenessLatencyMs: numberOrNull(observed.responsivenessLatencyMs),
+      responsivenessTimeout: observed.responsivenessTimeout === true,
+      cdpMetricsTimeoutCount: numberOrNull(observed.cdpMetricsTimeoutCount),
+      methods: arrayStrings(observed.methods).slice(0, 8),
+      valuesRedacted: true,
+    },
+    threshold: {
+      totalRssBlockerMb: numberOrNull(threshold.totalRssBlockerMb),
+      processRssBlockerMb: numberOrNull(threshold.processRssBlockerMb),
+      growthBlockerMb: numberOrNull(threshold.growthBlockerMb),
+      latencyBlockerMs: numberOrNull(threshold.latencyBlockerMs),
+      eventBlockerCount: numberOrNull(threshold.eventBlockerCount),
+      metricsTimeoutBlockerCount: numberOrNull(threshold.metricsTimeoutBlockerCount),
+      windowMs: numberOrNull(threshold.windowMs),
+      valuesRedacted: true,
+    },
+    sample: {
+      ts: sample.ts ?? null,
+      seq: sample.seq ?? null,
+      sampleSeq: sample.sampleSeq ?? null,
+      browserPid: numberOrNull(sample.browserPid),
+      totalRssMb: numberOrNull(sample.totalRssMb),
+      maxProcessRssMb: numberOrNull(sample.maxProcessRssMb),
+      totalRssGrowthMb: numberOrNull(sample.totalRssGrowthMb),
+      maxProcessRssGrowthMb: numberOrNull(sample.maxProcessRssGrowthMb),
+      valuesRedacted: true,
+    },
+    page: {
+      pageRole: page.pageRole ?? null,
+      pageId: page.pageId ?? null,
+      pageEpoch: numberOrNull(page.pageEpoch),
+      timeoutMs: numberOrNull(page.timeoutMs),
+      responsivenessLatencyMs: numberOrNull(page.responsivenessLatencyMs),
+      responsivenessTimeout: page.responsivenessTimeout === true,
+      cdpTimeoutCount: numberOrNull(page.cdpTimeoutCount),
+      cdpErrorCount: numberOrNull(page.cdpErrorCount),
+      valuesRedacted: true,
+    },
+    browserKill: {
+      ok: browserKill.ok === true,
+      pending: browserKill.pending === true,
+      skipped: browserKill.skipped === true,
+      reason: browserKill.reason ?? null,
+      pid: numberOrNull(browserKill.pid),
+      gracefulSignal: browserKill.gracefulSignal ?? null,
+      forceSignal: browserKill.forceSignal ?? null,
+      gracefulSent: browserKill.gracefulSent === true,
+      forceSent: browserKill.forceSent === true,
+      exitedAfterGrace: browserKill.exitedAfterGrace === true,
+      exitedAfterForce: browserKill.exitedAfterForce === true,
+      valuesRedacted: true,
+    },
+    valuesRedacted: true,
+  };
+}
+
 function buildBrowserProcessFindings(report, runtimeAlerts = null) {
  const summary = report?.summary || {};
-  if (!summary || Number(summary.sampleCount ?? 0) <= 0) return [];
  const findings = [];
+  const blockerEvents = Array.isArray(report?.blockerEvents) ? report.blockerEvents : [];
+  if (blockerEvents.length > 0) {
+    const first = blockerEvents[0] || {};
+    findings.push({
+      id: "frontend-browser-freeze-runner-blocker",
+      severity: "red",
+      summary: "web-probe runner matched YAML browserFreezePolicy, killed/stopped Chromium, and failed the observer run; do not clear this by refresh or fallback",
+      count: blockerEvents.length,
+      blocking: true,
+      rootCause: first.rootCause ?? "frontend_browser_freeze_policy_blocker",
+      rootCauseStatus: "confirmed-from-runner-browser-freeze-policy",
+      rootCauseConfidence: "high",
+      policySource: first.policySource ?? "config/hwlab-node-lanes.yaml#webProbe.browserFreezePolicy",
+      fallbackAllowed: false,
+      observerRefreshAllowed: false,
+      browserKilled: first.browserKill ?? null,
+      events: blockerEvents.slice(0, 20),
+      valuesRedacted: true,
+    });
+  }
+  if (!summary || Number(summary.sampleCount ?? 0) <= 0) return findings;
  const rootCauseSignals = browserRootCauseSignals(report, runtimeAlerts);
  const maxTotalRssMb = Number(summary.maxTotalRssMb ?? 0);
  const maxProcessRssMb = Number(summary.maxProcessRssMb ?? 0);