Merge pull request #1312 from pikasTech/issue-1306-sentinel-freeze-blocker
feat: block frozen web probe browsers from yaml policy
This commit is contained in:
@@ -223,6 +223,25 @@ lanes:
|
||||
scrollJumpFromY: 250
|
||||
scrollJumpToY: 40
|
||||
sessionRailFallbackRatio: 0.5
|
||||
browserFreezePolicy:
|
||||
enabled: true
|
||||
blockerWindowMs: 30000
|
||||
memory:
|
||||
totalRssBlockerMb: 500
|
||||
processRssBlockerMb: 500
|
||||
growthBlockerMb: 300
|
||||
responsiveness:
|
||||
latencyBlockerMs: 5000
|
||||
eventBlockerCount: 2
|
||||
cdp:
|
||||
metricsTimeoutBlockerCount: 2
|
||||
kill:
|
||||
enabled: true
|
||||
gracefulSignal: SIGTERM
|
||||
forceSignal: SIGKILL
|
||||
graceMs: 3000
|
||||
pollIntervalMs: 100
|
||||
exitCode: 7
|
||||
projectManagement:
|
||||
enabled: true
|
||||
targetPaths:
|
||||
@@ -624,6 +643,25 @@ lanes:
|
||||
scrollJumpFromY: 250
|
||||
scrollJumpToY: 40
|
||||
sessionRailFallbackRatio: 0.5
|
||||
browserFreezePolicy:
|
||||
enabled: true
|
||||
blockerWindowMs: 30000
|
||||
memory:
|
||||
totalRssBlockerMb: 500
|
||||
processRssBlockerMb: 500
|
||||
growthBlockerMb: 300
|
||||
responsiveness:
|
||||
latencyBlockerMs: 5000
|
||||
eventBlockerCount: 2
|
||||
cdp:
|
||||
metricsTimeoutBlockerCount: 2
|
||||
kill:
|
||||
enabled: true
|
||||
gracefulSignal: SIGTERM
|
||||
forceSignal: SIGKILL
|
||||
graceMs: 3000
|
||||
pollIntervalMs: 100
|
||||
exitCode: 7
|
||||
projectManagement:
|
||||
enabled: true
|
||||
targetPaths:
|
||||
@@ -955,6 +993,25 @@ lanes:
|
||||
scrollJumpFromY: 250
|
||||
scrollJumpToY: 40
|
||||
sessionRailFallbackRatio: 0.5
|
||||
browserFreezePolicy:
|
||||
enabled: true
|
||||
blockerWindowMs: 30000
|
||||
memory:
|
||||
totalRssBlockerMb: 500
|
||||
processRssBlockerMb: 500
|
||||
growthBlockerMb: 300
|
||||
responsiveness:
|
||||
latencyBlockerMs: 5000
|
||||
eventBlockerCount: 2
|
||||
cdp:
|
||||
metricsTimeoutBlockerCount: 2
|
||||
kill:
|
||||
enabled: true
|
||||
gracefulSignal: SIGTERM
|
||||
forceSignal: SIGKILL
|
||||
graceMs: 3000
|
||||
pollIntervalMs: 100
|
||||
exitCode: 7
|
||||
tektonDir: tekton-v03
|
||||
argoApplicationFile: application-v03.yaml
|
||||
registryPrefix: 127.0.0.1:5000/hwlab
|
||||
|
||||
@@ -141,6 +141,7 @@ export interface HwlabRuntimeWebProbeSpec {
|
||||
readonly defaultOrigin?: HwlabRuntimeWebProbeOriginSpec;
|
||||
readonly authLogin?: HwlabRuntimeWebProbeAuthLoginSpec;
|
||||
readonly alertThresholds?: HwlabRuntimeWebProbeAlertThresholdsSpec;
|
||||
readonly browserFreezePolicy?: HwlabRuntimeWebProbeBrowserFreezePolicySpec;
|
||||
readonly projectManagement?: HwlabRuntimeWebProbeProjectManagementSpec;
|
||||
}
|
||||
|
||||
@@ -151,6 +152,39 @@ export interface HwlabRuntimeWebProbeAuthLoginSpec {
|
||||
readonly maxDelayMs: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeBrowserFreezeMemoryPolicySpec {
|
||||
readonly totalRssBlockerMb: number;
|
||||
readonly processRssBlockerMb: number;
|
||||
readonly growthBlockerMb: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeBrowserFreezeResponsivenessPolicySpec {
|
||||
readonly latencyBlockerMs: number;
|
||||
readonly eventBlockerCount: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeBrowserFreezeCdpPolicySpec {
|
||||
readonly metricsTimeoutBlockerCount: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeBrowserFreezeKillPolicySpec {
|
||||
readonly enabled: boolean;
|
||||
readonly gracefulSignal: "SIGTERM";
|
||||
readonly forceSignal: "SIGKILL";
|
||||
readonly graceMs: number;
|
||||
readonly pollIntervalMs: number;
|
||||
readonly exitCode: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeBrowserFreezePolicySpec {
|
||||
readonly enabled: boolean;
|
||||
readonly blockerWindowMs: number;
|
||||
readonly memory: HwlabRuntimeWebProbeBrowserFreezeMemoryPolicySpec;
|
||||
readonly responsiveness: HwlabRuntimeWebProbeBrowserFreezeResponsivenessPolicySpec;
|
||||
readonly cdp: HwlabRuntimeWebProbeBrowserFreezeCdpPolicySpec;
|
||||
readonly kill: HwlabRuntimeWebProbeBrowserFreezeKillPolicySpec;
|
||||
}
|
||||
|
||||
export type HwlabRuntimeWebProbeSentinelConfigRefKey = "runtime" | "scenarios" | "promptSet" | "reportViews" | "publicExposure" | "cicd" | "secrets";
|
||||
|
||||
export const HWLAB_WEB_PROBE_SENTINEL_CONFIG_REF_KEYS = ["runtime", "scenarios", "promptSet", "reportViews", "publicExposure", "cicd", "secrets"] as const satisfies readonly HwlabRuntimeWebProbeSentinelConfigRefKey[];
|
||||
@@ -1078,6 +1112,7 @@ function webProbeConfig(value: unknown, path: string): HwlabRuntimeWebProbeSpec
|
||||
...(raw.defaultOrigin === undefined ? {} : { defaultOrigin: webProbeOriginConfig(raw.defaultOrigin, `${path}.defaultOrigin`) }),
|
||||
...(raw.authLogin === undefined ? {} : { authLogin: webProbeAuthLoginConfig(raw.authLogin, `${path}.authLogin`) }),
|
||||
...(raw.alertThresholds === undefined ? {} : { alertThresholds: webProbeAlertThresholdsConfig(raw.alertThresholds, `${path}.alertThresholds`) }),
|
||||
...(raw.browserFreezePolicy === undefined ? {} : { browserFreezePolicy: webProbeBrowserFreezePolicyConfig(raw.browserFreezePolicy, `${path}.browserFreezePolicy`) }),
|
||||
...(raw.projectManagement === undefined ? {} : { projectManagement: webProbeProjectManagementConfig(raw.projectManagement, `${path}.projectManagement`) }),
|
||||
};
|
||||
}
|
||||
@@ -1094,6 +1129,38 @@ function webProbeAuthLoginConfig(value: unknown, path: string): HwlabRuntimeWebP
|
||||
};
|
||||
}
|
||||
|
||||
function webProbeBrowserFreezePolicyConfig(value: unknown, path: string): HwlabRuntimeWebProbeBrowserFreezePolicySpec {
|
||||
const raw = asRecord(value, path);
|
||||
const memory = asRecord(raw.memory, `${path}.memory`);
|
||||
const responsiveness = asRecord(raw.responsiveness, `${path}.responsiveness`);
|
||||
const cdp = asRecord(raw.cdp, `${path}.cdp`);
|
||||
const kill = asRecord(raw.kill, `${path}.kill`);
|
||||
return {
|
||||
enabled: booleanField(raw, "enabled", path),
|
||||
blockerWindowMs: positiveNumberField(raw, "blockerWindowMs", path),
|
||||
memory: {
|
||||
totalRssBlockerMb: positiveNumberField(memory, "totalRssBlockerMb", `${path}.memory`),
|
||||
processRssBlockerMb: positiveNumberField(memory, "processRssBlockerMb", `${path}.memory`),
|
||||
growthBlockerMb: positiveNumberField(memory, "growthBlockerMb", `${path}.memory`),
|
||||
},
|
||||
responsiveness: {
|
||||
latencyBlockerMs: positiveNumberField(responsiveness, "latencyBlockerMs", `${path}.responsiveness`),
|
||||
eventBlockerCount: positiveNumberField(responsiveness, "eventBlockerCount", `${path}.responsiveness`),
|
||||
},
|
||||
cdp: {
|
||||
metricsTimeoutBlockerCount: positiveNumberField(cdp, "metricsTimeoutBlockerCount", `${path}.cdp`),
|
||||
},
|
||||
kill: {
|
||||
enabled: booleanField(kill, "enabled", `${path}.kill`),
|
||||
gracefulSignal: enumStringField(kill, "gracefulSignal", `${path}.kill`, ["SIGTERM"] as const),
|
||||
forceSignal: enumStringField(kill, "forceSignal", `${path}.kill`, ["SIGKILL"] as const),
|
||||
graceMs: boundedIntegerField(kill, "graceMs", `${path}.kill`, 1, 120000),
|
||||
pollIntervalMs: boundedIntegerField(kill, "pollIntervalMs", `${path}.kill`, 1, 10000),
|
||||
exitCode: boundedIntegerField(kill, "exitCode", `${path}.kill`, 1, 125),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function webProbeSentinelConfig(value: unknown, path: string): HwlabRuntimeWebProbeSentinelSpec {
|
||||
const raw = asRecord(value, path);
|
||||
const allowed = new Set(["enabled", "configRefs"]);
|
||||
|
||||
@@ -19,6 +19,7 @@ const analyzeTailSamples = (() => {
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : 360;
|
||||
})();
|
||||
const alertThresholds = parseAlertThresholds(process.env.UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON);
|
||||
const browserFreezePolicy = parseBrowserFreezePolicy(process.env.UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON);
|
||||
const projectManagementConfig = parseProjectManagementConfig(process.env.UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON);
|
||||
const dataDir = archivePrefix ? path.join(stateDir, "archive") : stateDir;
|
||||
const dataFile = (name) => path.join(dataDir, archivePrefix ? archivePrefix + "-" + name : name);
|
||||
@@ -118,6 +119,7 @@ const report = {
|
||||
stateDir,
|
||||
jsonlScope: { mode: archivePrefix ? "archive" : "current", archivePrefix: archivePrefix || null, dataDir, analyzeTailSamples, sourceSampleCount: sourceSamples.length, effectiveSampleCount: samples.length, sourceControlCount: sourceControlAll.length, sampleWindow, focus: analysisFocus, valuesRedacted: true },
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
manifest: compactManifest(manifest),
|
||||
heartbeat: compactHeartbeat(heartbeat),
|
||||
counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length, browserProcess: browserProcessRows.length },
|
||||
@@ -584,6 +586,85 @@ function parseAlertThresholds(value) {
|
||||
};
|
||||
}
|
||||
|
||||
function parseBrowserFreezePolicy(value) {
|
||||
if (!value) {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is required; configure config/hwlab-node-lanes.yaml webProbe.browserFreezePolicy for the selected node/lane");
|
||||
}
|
||||
const raw = (() => {
|
||||
try { return JSON.parse(value); } catch (error) { throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is invalid JSON: " + (error instanceof Error ? error.message : String(error))); }
|
||||
})();
|
||||
const memory = requiredPolicyRecord(raw, "memory", "webProbe.browserFreezePolicy");
|
||||
const responsiveness = requiredPolicyRecord(raw, "responsiveness", "webProbe.browserFreezePolicy");
|
||||
const cdp = requiredPolicyRecord(raw, "cdp", "webProbe.browserFreezePolicy");
|
||||
const kill = requiredPolicyRecord(raw, "kill", "webProbe.browserFreezePolicy");
|
||||
return {
|
||||
enabled: requiredPolicyBoolean(raw, "enabled", "webProbe.browserFreezePolicy"),
|
||||
blockerWindowMs: requiredPolicyPositiveNumber(raw, "blockerWindowMs", "webProbe.browserFreezePolicy"),
|
||||
memory: {
|
||||
totalRssBlockerMb: requiredPolicyPositiveNumber(memory, "totalRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
|
||||
processRssBlockerMb: requiredPolicyPositiveNumber(memory, "processRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
|
||||
growthBlockerMb: requiredPolicyPositiveNumber(memory, "growthBlockerMb", "webProbe.browserFreezePolicy.memory"),
|
||||
},
|
||||
responsiveness: {
|
||||
latencyBlockerMs: requiredPolicyPositiveNumber(responsiveness, "latencyBlockerMs", "webProbe.browserFreezePolicy.responsiveness"),
|
||||
eventBlockerCount: requiredPolicyPositiveNumber(responsiveness, "eventBlockerCount", "webProbe.browserFreezePolicy.responsiveness"),
|
||||
},
|
||||
cdp: {
|
||||
metricsTimeoutBlockerCount: requiredPolicyPositiveNumber(cdp, "metricsTimeoutBlockerCount", "webProbe.browserFreezePolicy.cdp"),
|
||||
},
|
||||
kill: {
|
||||
enabled: requiredPolicyBoolean(kill, "enabled", "webProbe.browserFreezePolicy.kill"),
|
||||
gracefulSignal: requiredPolicySignal(kill, "gracefulSignal", "webProbe.browserFreezePolicy.kill", "SIGTERM"),
|
||||
forceSignal: requiredPolicySignal(kill, "forceSignal", "webProbe.browserFreezePolicy.kill", "SIGKILL"),
|
||||
graceMs: requiredPolicyIntegerInRange(kill, "graceMs", "webProbe.browserFreezePolicy.kill", 1, 120000),
|
||||
pollIntervalMs: requiredPolicyIntegerInRange(kill, "pollIntervalMs", "webProbe.browserFreezePolicy.kill", 1, 10000),
|
||||
exitCode: requiredPolicyIntegerInRange(kill, "exitCode", "webProbe.browserFreezePolicy.kill", 1, 125),
|
||||
},
|
||||
source: "yaml-env",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function requiredPolicyRecord(raw, key, pathLabel) {
|
||||
const value = raw?.[key];
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires object " + pathLabel + "." + key);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredPolicyBoolean(raw, key, pathLabel) {
|
||||
const value = raw?.[key];
|
||||
if (typeof value !== "boolean") {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires boolean " + pathLabel + "." + key);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredPolicyPositiveNumber(raw, key, pathLabel) {
|
||||
const value = Number(raw?.[key]);
|
||||
if (!Number.isFinite(value) || value <= 0) {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires positive number " + pathLabel + "." + key);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredPolicyIntegerInRange(raw, key, pathLabel, min, max) {
|
||||
const value = Number(raw?.[key]);
|
||||
if (!Number.isInteger(value) || value < min || value > max) {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires integer " + pathLabel + "." + key + " between " + min + " and " + max);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredPolicySignal(raw, key, pathLabel, expected) {
|
||||
const value = String(raw?.[key] || "");
|
||||
if (value !== expected) {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires " + pathLabel + "." + key + "=" + expected);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseProjectManagementConfig(value) {
|
||||
if (!value || value === "null") {
|
||||
return {
|
||||
@@ -3107,6 +3188,10 @@ function buildFrontendFreezeFindings(errors, control) {
|
||||
}
|
||||
|
||||
function buildBrowserProcessReport(rows) {
|
||||
const blockerEvents = (Array.isArray(rows) ? rows : [])
|
||||
.filter((item) => item && item.type === "browser-freeze-blocker")
|
||||
.map(compactBrowserFreezeBlockerEvent)
|
||||
.filter(Boolean);
|
||||
const samples = (Array.isArray(rows) ? rows : [])
|
||||
.filter((item) => item && (item.type === "browser-process-sample" || item.process || item.pages))
|
||||
.map((item) => ({ ...item, tsMs: Date.parse(String(item.ts || "")) }))
|
||||
@@ -3205,8 +3290,11 @@ function buildBrowserProcessReport(rows) {
|
||||
growthRedMb: alertThresholds.browserRssGrowthRedMb,
|
||||
growthWindowMs: alertThresholds.browserRssGrowthWindowMs,
|
||||
responsivenessRedMs: alertThresholds.playwrightResponsivenessRedMs,
|
||||
freezeBlockerCount: blockerEvents.length,
|
||||
browserFreezePolicy,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
blockerEvents: blockerEvents.slice(-20),
|
||||
memorySamples: memorySamples.slice(-60),
|
||||
growthSamples: computedGrowth.slice(-60),
|
||||
maxTotalRssSample: maxTotal,
|
||||
@@ -3220,10 +3308,112 @@ function buildBrowserProcessReport(rows) {
|
||||
};
|
||||
}
|
||||
|
||||
function compactBrowserFreezeBlockerEvent(item) {
|
||||
if (!item || typeof item !== "object") return null;
|
||||
const observed = objectValue(item.observed);
|
||||
const threshold = objectValue(item.threshold);
|
||||
const sample = objectValue(item.sample);
|
||||
const page = objectValue(item.page);
|
||||
const browserKill = objectValue(item.browserKill);
|
||||
return {
|
||||
ts: item.ts ?? null,
|
||||
seq: item.seq ?? null,
|
||||
sampleSeq: item.sampleSeq ?? sample.sampleSeq ?? null,
|
||||
kind: item.kind ?? null,
|
||||
severity: item.severity ?? "red",
|
||||
blocking: item.blocking === true,
|
||||
summary: item.summary ? String(item.summary).slice(0, 240) : null,
|
||||
rootCause: item.rootCause ?? null,
|
||||
rootCauseStatus: item.rootCauseStatus ?? null,
|
||||
rootCauseConfidence: item.rootCauseConfidence ?? null,
|
||||
policySource: item.policySource ?? null,
|
||||
fallbackAllowed: item.fallbackAllowed === true,
|
||||
observerRefreshAllowed: item.observerRefreshAllowed === true,
|
||||
observed: {
|
||||
totalRssMb: numberOrNull(observed.totalRssMb),
|
||||
processRssMb: numberOrNull(observed.processRssMb),
|
||||
totalGrowthMb: numberOrNull(observed.totalGrowthMb),
|
||||
processGrowthMb: numberOrNull(observed.processGrowthMb),
|
||||
responsivenessLatencyMs: numberOrNull(observed.responsivenessLatencyMs),
|
||||
responsivenessTimeout: observed.responsivenessTimeout === true,
|
||||
cdpMetricsTimeoutCount: numberOrNull(observed.cdpMetricsTimeoutCount),
|
||||
methods: arrayStrings(observed.methods).slice(0, 8),
|
||||
valuesRedacted: true,
|
||||
},
|
||||
threshold: {
|
||||
totalRssBlockerMb: numberOrNull(threshold.totalRssBlockerMb),
|
||||
processRssBlockerMb: numberOrNull(threshold.processRssBlockerMb),
|
||||
growthBlockerMb: numberOrNull(threshold.growthBlockerMb),
|
||||
latencyBlockerMs: numberOrNull(threshold.latencyBlockerMs),
|
||||
eventBlockerCount: numberOrNull(threshold.eventBlockerCount),
|
||||
metricsTimeoutBlockerCount: numberOrNull(threshold.metricsTimeoutBlockerCount),
|
||||
windowMs: numberOrNull(threshold.windowMs),
|
||||
valuesRedacted: true,
|
||||
},
|
||||
sample: {
|
||||
ts: sample.ts ?? null,
|
||||
seq: sample.seq ?? null,
|
||||
sampleSeq: sample.sampleSeq ?? null,
|
||||
browserPid: numberOrNull(sample.browserPid),
|
||||
totalRssMb: numberOrNull(sample.totalRssMb),
|
||||
maxProcessRssMb: numberOrNull(sample.maxProcessRssMb),
|
||||
totalRssGrowthMb: numberOrNull(sample.totalRssGrowthMb),
|
||||
maxProcessRssGrowthMb: numberOrNull(sample.maxProcessRssGrowthMb),
|
||||
valuesRedacted: true,
|
||||
},
|
||||
page: {
|
||||
pageRole: page.pageRole ?? null,
|
||||
pageId: page.pageId ?? null,
|
||||
pageEpoch: numberOrNull(page.pageEpoch),
|
||||
timeoutMs: numberOrNull(page.timeoutMs),
|
||||
responsivenessLatencyMs: numberOrNull(page.responsivenessLatencyMs),
|
||||
responsivenessTimeout: page.responsivenessTimeout === true,
|
||||
cdpTimeoutCount: numberOrNull(page.cdpTimeoutCount),
|
||||
cdpErrorCount: numberOrNull(page.cdpErrorCount),
|
||||
valuesRedacted: true,
|
||||
},
|
||||
browserKill: {
|
||||
ok: browserKill.ok === true,
|
||||
pending: browserKill.pending === true,
|
||||
skipped: browserKill.skipped === true,
|
||||
reason: browserKill.reason ?? null,
|
||||
pid: numberOrNull(browserKill.pid),
|
||||
gracefulSignal: browserKill.gracefulSignal ?? null,
|
||||
forceSignal: browserKill.forceSignal ?? null,
|
||||
gracefulSent: browserKill.gracefulSent === true,
|
||||
forceSent: browserKill.forceSent === true,
|
||||
exitedAfterGrace: browserKill.exitedAfterGrace === true,
|
||||
exitedAfterForce: browserKill.exitedAfterForce === true,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function buildBrowserProcessFindings(report, runtimeAlerts = null) {
|
||||
const summary = report?.summary || {};
|
||||
if (!summary || Number(summary.sampleCount ?? 0) <= 0) return [];
|
||||
const findings = [];
|
||||
const blockerEvents = Array.isArray(report?.blockerEvents) ? report.blockerEvents : [];
|
||||
if (blockerEvents.length > 0) {
|
||||
const first = blockerEvents[0] || {};
|
||||
findings.push({
|
||||
id: "frontend-browser-freeze-runner-blocker",
|
||||
severity: "red",
|
||||
summary: "web-probe runner matched YAML browserFreezePolicy, killed/stopped Chromium, and failed the observer run; do not clear this by refresh or fallback",
|
||||
count: blockerEvents.length,
|
||||
blocking: true,
|
||||
rootCause: first.rootCause ?? "frontend_browser_freeze_policy_blocker",
|
||||
rootCauseStatus: "confirmed-from-runner-browser-freeze-policy",
|
||||
rootCauseConfidence: "high",
|
||||
policySource: first.policySource ?? "config/hwlab-node-lanes.yaml#webProbe.browserFreezePolicy",
|
||||
fallbackAllowed: false,
|
||||
observerRefreshAllowed: false,
|
||||
browserKilled: first.browserKill ?? null,
|
||||
events: blockerEvents.slice(0, 20),
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
if (!summary || Number(summary.sampleCount ?? 0) <= 0) return findings;
|
||||
const rootCauseSignals = browserRootCauseSignals(report, runtimeAlerts);
|
||||
const maxTotalRssMb = Number(summary.maxTotalRssMb ?? 0);
|
||||
const maxProcessRssMb = Number(summary.maxProcessRssMb ?? 0);
|
||||
|
||||
@@ -32,6 +32,7 @@ const authLoginInitialDelayMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_A
|
||||
const authLoginMaxDelayMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS, 10000, 0, 120000);
|
||||
const navigationMaxAttempts = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_NAVIGATION_MAX_ATTEMPTS, 4, 1, 10);
|
||||
const alertThresholds = parseAlertThresholds(process.env.UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON);
|
||||
const browserFreezePolicy = parseBrowserFreezePolicy(process.env.UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON);
|
||||
const projectManagement = parseProjectManagementConfig(process.env.UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON);
|
||||
const playwrightProxy = proxyConfigFromEnv(baseUrl);
|
||||
const chromiumLaunchOptions = chromiumLaunchOptionsForProxy(playwrightProxy);
|
||||
@@ -79,7 +80,9 @@ let currentPageProvenance = null;
|
||||
let heartbeatPulseTimer = null;
|
||||
let browserProcessMonitorStop = null;
|
||||
let browserProcessMonitorSeq = 0;
|
||||
let browserFreezeBlocker = null;
|
||||
const browserProcessHistory = [];
|
||||
const browserFreezeSignalHistory = [];
|
||||
const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] };
|
||||
|
||||
try {
|
||||
@@ -117,17 +120,26 @@ try {
|
||||
await writeHeartbeat({ status: "running" });
|
||||
while (!stopping) {
|
||||
await drainOneCommand();
|
||||
if (browserFreezeBlocker) break;
|
||||
await samplePage("interval");
|
||||
if (browserFreezeBlocker) break;
|
||||
if (maxSamples > 0 && sampleSeq >= maxSamples) {
|
||||
await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples }));
|
||||
break;
|
||||
}
|
||||
await sleep(sampleIntervalMs);
|
||||
}
|
||||
terminalStatus = "completed";
|
||||
await writeHeartbeat({ status: "completed" });
|
||||
await writeManifest({ status: "completed", completedAt: new Date().toISOString() });
|
||||
process.exitCode = 0;
|
||||
if (browserFreezeBlocker) {
|
||||
terminalStatus = "failed";
|
||||
await writeHeartbeat({ status: "failed", blocker: browserFreezeBlocker });
|
||||
await writeManifest({ status: "failed", completedAt: new Date().toISOString(), blocker: browserFreezeBlocker });
|
||||
process.exitCode = browserFreezePolicy.kill.exitCode;
|
||||
} else {
|
||||
terminalStatus = "completed";
|
||||
await writeHeartbeat({ status: "completed" });
|
||||
await writeManifest({ status: "completed", completedAt: new Date().toISOString() });
|
||||
process.exitCode = 0;
|
||||
}
|
||||
} catch (error) {
|
||||
terminalStatus = "failed";
|
||||
await appendJsonl(files.errors, eventRecord("runner-error", { error: errorSummary(error) })).catch(() => {});
|
||||
@@ -196,6 +208,7 @@ async function writeManifest(extra = {}) {
|
||||
pageProvenance: compactPageProvenance(currentPageProvenance),
|
||||
sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerRefreshIntervalMs, observerInitiatedDefault: false, responseBodyReadDefault: false },
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
projectManagement,
|
||||
jsonlRotation,
|
||||
commandDirs: dirs,
|
||||
@@ -247,7 +260,7 @@ function startBrowserProcessMonitor() {
|
||||
let stopped = false;
|
||||
let running = false;
|
||||
const tick = async (reason) => {
|
||||
if (stopped || running) return;
|
||||
if (stopped || running || stopping || browserFreezeBlocker) return;
|
||||
running = true;
|
||||
try {
|
||||
await collectBrowserProcessSample(reason || "interval");
|
||||
@@ -281,7 +294,7 @@ async function collectBrowserProcessSample(reason) {
|
||||
if (observerPage && !observerPage.isClosed()) {
|
||||
pages.push(await collectBrowserPageRuntimeMetrics(observerPage, { pageRole: "observer", targetPageId: observerPageId, pageEpoch: observerPageEpoch }).catch((error) => browserPageRuntimeMetricError("observer", observerPageId, observerPageEpoch, error)));
|
||||
}
|
||||
await appendJsonl(files.browserProcess, eventRecord("browser-process-sample", {
|
||||
const sample = eventRecord("browser-process-sample", {
|
||||
seq: browserProcessMonitorSeq,
|
||||
reason,
|
||||
monitorIntervalMs: Math.max(250, Math.floor(Number(alertThresholds.browserProcessSampleIntervalMs) || 1000)),
|
||||
@@ -290,7 +303,260 @@ async function collectBrowserProcessSample(reason) {
|
||||
growth,
|
||||
pages,
|
||||
valuesRedacted: true,
|
||||
}));
|
||||
});
|
||||
await appendJsonl(files.browserProcess, sample);
|
||||
await enforceBrowserFreezePolicy(sample);
|
||||
}
|
||||
|
||||
async function enforceBrowserFreezePolicy(sample) {
|
||||
if (browserFreezePolicy.enabled !== true || browserFreezeBlocker) return;
|
||||
const processSummary = sample && typeof sample.process === "object" ? sample.process : {};
|
||||
const growth = sample && typeof sample.growth === "object" ? sample.growth : {};
|
||||
const totalRssMb = Number(processSummary.totalRssMb);
|
||||
const processRssMb = Number(processSummary.maxProcessRssMb);
|
||||
const totalGrowthMb = Number(growth.totalRssGrowthMb);
|
||||
const processGrowthMb = Number(growth.maxProcessRssGrowthMb);
|
||||
if (Number.isFinite(totalRssMb) && totalRssMb >= browserFreezePolicy.memory.totalRssBlockerMb) {
|
||||
await triggerBrowserFreezeBlocker({
|
||||
kind: "memory-total-rss",
|
||||
rootCause: "frontend_browser_process_memory_pressure",
|
||||
observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true },
|
||||
threshold: { totalRssBlockerMb: browserFreezePolicy.memory.totalRssBlockerMb, valuesRedacted: true },
|
||||
sample: browserProcessSampleRef(sample),
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (Number.isFinite(processRssMb) && processRssMb >= browserFreezePolicy.memory.processRssBlockerMb) {
|
||||
await triggerBrowserFreezeBlocker({
|
||||
kind: "memory-process-rss",
|
||||
rootCause: "frontend_browser_process_memory_pressure",
|
||||
observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true },
|
||||
threshold: { processRssBlockerMb: browserFreezePolicy.memory.processRssBlockerMb, valuesRedacted: true },
|
||||
sample: browserProcessSampleRef(sample),
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (
|
||||
(Number.isFinite(totalGrowthMb) && totalGrowthMb >= browserFreezePolicy.memory.growthBlockerMb)
|
||||
|| (Number.isFinite(processGrowthMb) && processGrowthMb >= browserFreezePolicy.memory.growthBlockerMb)
|
||||
) {
|
||||
await triggerBrowserFreezeBlocker({
|
||||
kind: "memory-rss-growth",
|
||||
rootCause: "frontend_browser_process_memory_leak_or_unbounded_render_growth",
|
||||
observed: { totalRssMb, processRssMb, totalGrowthMb, processGrowthMb, valuesRedacted: true },
|
||||
threshold: { growthBlockerMb: browserFreezePolicy.memory.growthBlockerMb, windowMs: browserFreezePolicy.blockerWindowMs, valuesRedacted: true },
|
||||
sample: browserProcessSampleRef(sample),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
for (const pageMetric of Array.isArray(sample.pages) ? sample.pages : []) {
|
||||
const responsiveness = pageMetric?.responsiveness && typeof pageMetric.responsiveness === "object" ? pageMetric.responsiveness : {};
|
||||
const responsivenessLatencyMs = Number(responsiveness.latencyMs);
|
||||
if (responsiveness.timeout === true || (Number.isFinite(responsivenessLatencyMs) && responsivenessLatencyMs >= browserFreezePolicy.responsiveness.latencyBlockerMs)) {
|
||||
const signal = recordBrowserFreezeSignal("playwright-responsiveness", sample, pageMetric, {
|
||||
rootCause: "frontend_browser_page_unresponsive_to_playwright",
|
||||
observed: {
|
||||
responsivenessLatencyMs: Number.isFinite(responsivenessLatencyMs) ? responsivenessLatencyMs : null,
|
||||
responsivenessTimeout: responsiveness.timeout === true,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
threshold: {
|
||||
latencyBlockerMs: browserFreezePolicy.responsiveness.latencyBlockerMs,
|
||||
eventBlockerCount: browserFreezePolicy.responsiveness.eventBlockerCount,
|
||||
windowMs: browserFreezePolicy.blockerWindowMs,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
});
|
||||
if (signal.burst.length >= browserFreezePolicy.responsiveness.eventBlockerCount) {
|
||||
await triggerBrowserFreezeBlocker(signal);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const cdp = pageMetric?.cdp && typeof pageMetric.cdp === "object" ? pageMetric.cdp : {};
|
||||
const calls = Array.isArray(cdp.calls) ? cdp.calls : [];
|
||||
const metricTimeoutCalls = calls.filter((call) => call?.timeout === true && call?.method !== "Runtime.evaluate");
|
||||
const sessionTimeoutCount = calls.length === 0 ? Number(cdp.timeoutCount || 0) : 0;
|
||||
const metricTimeoutCount = metricTimeoutCalls.length + (Number.isFinite(sessionTimeoutCount) ? sessionTimeoutCount : 0);
|
||||
if (metricTimeoutCount > 0) {
|
||||
const signal = recordBrowserFreezeSignal("cdp-metrics-timeout", sample, pageMetric, {
|
||||
rootCause: "frontend_browser_cdp_metrics_unresponsive",
|
||||
observed: {
|
||||
cdpMetricsTimeoutCount: metricTimeoutCount,
|
||||
methods: metricTimeoutCalls.map((call) => call.method || "unknown").slice(0, 8),
|
||||
valuesRedacted: true,
|
||||
},
|
||||
threshold: {
|
||||
metricsTimeoutBlockerCount: browserFreezePolicy.cdp.metricsTimeoutBlockerCount,
|
||||
windowMs: browserFreezePolicy.blockerWindowMs,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
});
|
||||
if (signal.burst.length >= browserFreezePolicy.cdp.metricsTimeoutBlockerCount) {
|
||||
await triggerBrowserFreezeBlocker(signal);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function recordBrowserFreezeSignal(kind, sample, pageMetric, detail) {
|
||||
const tsMs = Date.parse(String(sample?.ts || ""));
|
||||
const signal = {
|
||||
kind,
|
||||
ts: sample?.ts ?? new Date().toISOString(),
|
||||
tsMs: Number.isFinite(tsMs) ? tsMs : Date.now(),
|
||||
pageRole: pageMetric?.pageRole ?? null,
|
||||
pageId: pageMetric?.pageId ?? null,
|
||||
pageEpoch: pageMetric?.pageEpoch ?? null,
|
||||
sample: browserProcessSampleRef(sample),
|
||||
page: browserPageMetricRef(pageMetric),
|
||||
...detail,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
browserFreezeSignalHistory.push(signal);
|
||||
const windowMs = browserFreezePolicy.blockerWindowMs;
|
||||
const cutoff = signal.tsMs - windowMs;
|
||||
while (browserFreezeSignalHistory.length > 0 && Number(browserFreezeSignalHistory[0].tsMs || 0) < cutoff) browserFreezeSignalHistory.shift();
|
||||
return {
|
||||
kind,
|
||||
rootCause: detail.rootCause,
|
||||
observed: detail.observed,
|
||||
threshold: detail.threshold,
|
||||
sample: browserProcessSampleRef(sample),
|
||||
page: browserPageMetricRef(pageMetric),
|
||||
burst: browserFreezeSignalHistory.filter((item) => item.kind === kind && item.tsMs >= cutoff).slice(-20),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function triggerBrowserFreezeBlocker(trigger) {
|
||||
if (browserFreezeBlocker) return;
|
||||
stopping = true;
|
||||
terminalStatus = "failed";
|
||||
const base = {
|
||||
id: "browser-freeze-policy-blocker",
|
||||
severity: "red",
|
||||
blocking: true,
|
||||
kind: trigger.kind,
|
||||
summary: "web-probe runner matched YAML browserFreezePolicy and stopped the browser; this run must stay red instead of refreshing or falling back",
|
||||
rootCause: trigger.rootCause,
|
||||
rootCauseStatus: "confirmed-from-runner-browser-freeze-policy",
|
||||
rootCauseConfidence: "high",
|
||||
fallbackAllowed: false,
|
||||
observerRefreshAllowed: false,
|
||||
policySource: "config/hwlab-node-lanes.yaml#webProbe.browserFreezePolicy via UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON",
|
||||
observed: trigger.observed || null,
|
||||
threshold: trigger.threshold || null,
|
||||
sample: trigger.sample || null,
|
||||
page: trigger.page || null,
|
||||
burst: Array.isArray(trigger.burst) ? trigger.burst.slice(0, 20) : [],
|
||||
valuesRedacted: true,
|
||||
};
|
||||
browserFreezeBlocker = { ...base, browserKill: { ok: false, pending: true, valuesRedacted: true }, valuesRedacted: true };
|
||||
const browserKill = browserFreezePolicy.kill.enabled === true
|
||||
? await terminateBrowserForFreezeBlocker(base).catch((error) => ({ ok: false, error: errorSummary(error), valuesRedacted: true }))
|
||||
: { ok: true, skipped: true, reason: "kill-disabled-by-yaml-policy", valuesRedacted: true };
|
||||
browserFreezeBlocker = { ...base, browserKill, valuesRedacted: true };
|
||||
await appendJsonl(files.browserProcess, eventRecord("browser-freeze-blocker", browserFreezeBlocker)).catch(() => {});
|
||||
await appendJsonl(files.errors, eventRecord("browser-freeze-blocker", {
|
||||
...browserFreezeBlocker,
|
||||
error: {
|
||||
name: "BrowserFreezeBlocker",
|
||||
message: "YAML browserFreezePolicy matched: " + String(trigger.kind || "unknown"),
|
||||
details: browserFreezeBlocker,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
})).catch(() => {});
|
||||
await writeHeartbeat({ status: "failed", blocker: browserFreezeBlocker }).catch(() => {});
|
||||
await writeManifest({ status: "failed", blocker: browserFreezeBlocker }).catch(() => {});
|
||||
}
|
||||
|
||||
async function terminateBrowserForFreezeBlocker(blocker) {
|
||||
const childProcess = browser && typeof browser.process === "function" ? browser.process() : null;
|
||||
const pid = Number(childProcess?.pid);
|
||||
if (!Number.isFinite(pid) || pid <= 0) {
|
||||
return { ok: false, reason: "browser-process-unavailable", blockerKind: blocker.kind, valuesRedacted: true };
|
||||
}
|
||||
const result = {
|
||||
ok: false,
|
||||
pid: Math.floor(pid),
|
||||
gracefulSignal: browserFreezePolicy.kill.gracefulSignal,
|
||||
forceSignal: browserFreezePolicy.kill.forceSignal,
|
||||
graceMs: browserFreezePolicy.kill.graceMs,
|
||||
pollIntervalMs: browserFreezePolicy.kill.pollIntervalMs,
|
||||
gracefulSent: false,
|
||||
forceSent: false,
|
||||
exitedAfterGrace: false,
|
||||
exitedAfterForce: false,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
try {
|
||||
childProcess.kill(browserFreezePolicy.kill.gracefulSignal);
|
||||
result.gracefulSent = true;
|
||||
} catch (error) {
|
||||
result.gracefulError = errorSummary(error);
|
||||
}
|
||||
result.exitedAfterGrace = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
|
||||
if (!result.exitedAfterGrace) {
|
||||
try {
|
||||
childProcess.kill(browserFreezePolicy.kill.forceSignal);
|
||||
result.forceSent = true;
|
||||
} catch (error) {
|
||||
result.forceError = errorSummary(error);
|
||||
}
|
||||
result.exitedAfterForce = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
|
||||
}
|
||||
result.ok = result.exitedAfterGrace || result.exitedAfterForce;
|
||||
return result;
|
||||
}
|
||||
|
||||
async function waitForPidExit(pid, timeoutMs, pollIntervalMs) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() <= deadline) {
|
||||
if (!pidAlive(pid)) return true;
|
||||
await sleep(pollIntervalMs);
|
||||
}
|
||||
return !pidAlive(pid);
|
||||
}
|
||||
|
||||
function pidAlive(pid) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function browserProcessSampleRef(sample) {
|
||||
return {
|
||||
ts: sample?.ts ?? null,
|
||||
seq: sample?.seq ?? null,
|
||||
sampleSeq: sample?.sampleSeq ?? null,
|
||||
browserPid: sample?.browserPid ?? null,
|
||||
totalRssMb: sample?.process?.totalRssMb ?? null,
|
||||
maxProcessRssMb: sample?.process?.maxProcessRssMb ?? null,
|
||||
totalRssGrowthMb: sample?.growth?.totalRssGrowthMb ?? null,
|
||||
maxProcessRssGrowthMb: sample?.growth?.maxProcessRssGrowthMb ?? null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function browserPageMetricRef(pageMetric) {
|
||||
const responsiveness = pageMetric?.responsiveness && typeof pageMetric.responsiveness === "object" ? pageMetric.responsiveness : {};
|
||||
const cdp = pageMetric?.cdp && typeof pageMetric.cdp === "object" ? pageMetric.cdp : {};
|
||||
return {
|
||||
pageRole: pageMetric?.pageRole ?? null,
|
||||
pageId: pageMetric?.pageId ?? null,
|
||||
pageEpoch: pageMetric?.pageEpoch ?? null,
|
||||
timeoutMs: pageMetric?.timeoutMs ?? null,
|
||||
responsivenessLatencyMs: responsiveness.latencyMs ?? null,
|
||||
responsivenessTimeout: responsiveness.timeout === true,
|
||||
cdpTimeoutCount: cdp.timeoutCount ?? null,
|
||||
cdpErrorCount: cdp.errorCount ?? null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function browserProcessPid() {
|
||||
@@ -5070,6 +5336,85 @@ function parseAlertThresholds(value) {
|
||||
};
|
||||
}
|
||||
|
||||
function parseBrowserFreezePolicy(value) {
|
||||
if (!value) {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is required; configure config/hwlab-node-lanes.yaml webProbe.browserFreezePolicy for the selected node/lane");
|
||||
}
|
||||
const raw = (() => {
|
||||
try { return JSON.parse(value); } catch (error) { throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON is invalid JSON: " + (error instanceof Error ? error.message : String(error))); }
|
||||
})();
|
||||
const memory = requiredPolicyRecord(raw, "memory", "webProbe.browserFreezePolicy");
|
||||
const responsiveness = requiredPolicyRecord(raw, "responsiveness", "webProbe.browserFreezePolicy");
|
||||
const cdp = requiredPolicyRecord(raw, "cdp", "webProbe.browserFreezePolicy");
|
||||
const kill = requiredPolicyRecord(raw, "kill", "webProbe.browserFreezePolicy");
|
||||
return {
|
||||
enabled: requiredPolicyBoolean(raw, "enabled", "webProbe.browserFreezePolicy"),
|
||||
blockerWindowMs: requiredPolicyPositiveNumber(raw, "blockerWindowMs", "webProbe.browserFreezePolicy"),
|
||||
memory: {
|
||||
totalRssBlockerMb: requiredPolicyPositiveNumber(memory, "totalRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
|
||||
processRssBlockerMb: requiredPolicyPositiveNumber(memory, "processRssBlockerMb", "webProbe.browserFreezePolicy.memory"),
|
||||
growthBlockerMb: requiredPolicyPositiveNumber(memory, "growthBlockerMb", "webProbe.browserFreezePolicy.memory"),
|
||||
},
|
||||
responsiveness: {
|
||||
latencyBlockerMs: requiredPolicyPositiveNumber(responsiveness, "latencyBlockerMs", "webProbe.browserFreezePolicy.responsiveness"),
|
||||
eventBlockerCount: requiredPolicyPositiveNumber(responsiveness, "eventBlockerCount", "webProbe.browserFreezePolicy.responsiveness"),
|
||||
},
|
||||
cdp: {
|
||||
metricsTimeoutBlockerCount: requiredPolicyPositiveNumber(cdp, "metricsTimeoutBlockerCount", "webProbe.browserFreezePolicy.cdp"),
|
||||
},
|
||||
kill: {
|
||||
enabled: requiredPolicyBoolean(kill, "enabled", "webProbe.browserFreezePolicy.kill"),
|
||||
gracefulSignal: requiredPolicySignal(kill, "gracefulSignal", "webProbe.browserFreezePolicy.kill", "SIGTERM"),
|
||||
forceSignal: requiredPolicySignal(kill, "forceSignal", "webProbe.browserFreezePolicy.kill", "SIGKILL"),
|
||||
graceMs: requiredPolicyIntegerInRange(kill, "graceMs", "webProbe.browserFreezePolicy.kill", 1, 120000),
|
||||
pollIntervalMs: requiredPolicyIntegerInRange(kill, "pollIntervalMs", "webProbe.browserFreezePolicy.kill", 1, 10000),
|
||||
exitCode: requiredPolicyIntegerInRange(kill, "exitCode", "webProbe.browserFreezePolicy.kill", 1, 125),
|
||||
},
|
||||
source: "yaml-env",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function requiredPolicyRecord(raw, key, pathLabel) {
|
||||
const value = raw?.[key];
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires object " + pathLabel + "." + key);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredPolicyBoolean(raw, key, pathLabel) {
|
||||
const value = raw?.[key];
|
||||
if (typeof value !== "boolean") {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires boolean " + pathLabel + "." + key);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredPolicyPositiveNumber(raw, key, pathLabel) {
|
||||
const value = Number(raw?.[key]);
|
||||
if (!Number.isFinite(value) || value <= 0) {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires positive number " + pathLabel + "." + key);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredPolicyIntegerInRange(raw, key, pathLabel, min, max) {
|
||||
const value = Number(raw?.[key]);
|
||||
if (!Number.isInteger(value) || value < min || value > max) {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires integer " + pathLabel + "." + key + " between " + min + " and " + max);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredPolicySignal(raw, key, pathLabel, expected) {
|
||||
const value = String(raw?.[key] || "");
|
||||
if (value !== expected) {
|
||||
throw new Error("UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON requires " + pathLabel + "." + key + "=" + expected);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseProjectManagementConfig(value) {
|
||||
if (!value || value === "null") {
|
||||
return {
|
||||
|
||||
@@ -14,7 +14,7 @@ import { runCommand, type CommandResult } from "../command";
|
||||
import { startJob } from "../jobs";
|
||||
import { classifySshTcpPoolFailure } from "../ssh";
|
||||
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
||||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeAuthLoginSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
||||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeAuthLoginSpec, type HwlabRuntimeWebProbeBrowserFreezePolicySpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
||||
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
||||
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
||||
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
||||
@@ -1141,6 +1141,14 @@ export function nodeWebProbeAlertThresholds(spec: HwlabRuntimeLaneSpec): HwlabRu
|
||||
return thresholds;
|
||||
}
|
||||
|
||||
export function nodeWebProbeBrowserFreezePolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeBrowserFreezePolicySpec {
|
||||
const policy = spec.webProbe?.browserFreezePolicy;
|
||||
if (policy === undefined) {
|
||||
throw new Error(`${hwlabRuntimeLaneConfigPath()} node=${spec.nodeId} lane=${spec.lane} requires webProbe.browserFreezePolicy for web-probe observe`);
|
||||
}
|
||||
return policy;
|
||||
}
|
||||
|
||||
export function nodeWebProbeProjectManagementConfig(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeProjectManagementSpec | null {
|
||||
return spec.webProbe?.projectManagement ?? null;
|
||||
}
|
||||
@@ -1311,6 +1319,7 @@ export function runNodeWebProbeObserveStart(
|
||||
const runnerB64Body = runnerB64.match(/.{1,76}/gu)?.join("\n") ?? runnerB64;
|
||||
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
||||
const alertThresholds = nodeWebProbeAlertThresholds(spec);
|
||||
const browserFreezePolicy = nodeWebProbeBrowserFreezePolicy(spec);
|
||||
const projectManagement = nodeWebProbeProjectManagementConfig(spec);
|
||||
const authLogin = nodeWebProbeAuthLoginConfig(spec);
|
||||
const runnerEnvAssignments = [
|
||||
@@ -1332,6 +1341,7 @@ export function runNodeWebProbeObserveStart(
|
||||
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
|
||||
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
||||
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
|
||||
`UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=${shellQuote(JSON.stringify(browserFreezePolicy))}`,
|
||||
`UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`,
|
||||
...(authLogin === null
|
||||
? []
|
||||
@@ -1407,6 +1417,7 @@ export function runNodeWebProbeObserveStart(
|
||||
url: options.url,
|
||||
network: webProbeProxy.summary,
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
projectManagement,
|
||||
targetPath: options.targetPath,
|
||||
id: observerId,
|
||||
@@ -1739,6 +1750,7 @@ function observeRecord(value: unknown): Record<string, unknown> {
|
||||
export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
|
||||
const analyzerB64 = Buffer.from(nodeWebObserveAnalyzerSource(), "utf8").toString("base64");
|
||||
const alertThresholds = nodeWebProbeAlertThresholds(spec);
|
||||
const browserFreezePolicy = nodeWebProbeBrowserFreezePolicy(spec);
|
||||
const projectManagement = nodeWebProbeProjectManagementConfig(spec);
|
||||
const script = [
|
||||
"set -eu",
|
||||
@@ -1758,8 +1770,9 @@ export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOption
|
||||
`UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=${shellQuote(options.analyzeArchivePrefix ?? "")}`,
|
||||
`UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES=${shellQuote(options.analyzeTailSamples === null ? "" : String(options.analyzeTailSamples))}`,
|
||||
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
|
||||
`UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=${shellQuote(JSON.stringify(browserFreezePolicy))}`,
|
||||
`UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`,
|
||||
"UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=\"$UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX\" UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES=\"$UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES\" UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=\"$UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON\" UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=\"$UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON\" node \"$analyzer\" >\"$analysis_stdout\" 2>\"$analysis_stderr\"",
|
||||
"UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=\"$UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX\" UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES=\"$UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES\" UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=\"$UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON\" UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON=\"$UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON\" UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=\"$UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON\" node \"$analyzer\" >\"$analysis_stdout\" 2>\"$analysis_stderr\"",
|
||||
"analyzer_exit=$?",
|
||||
"set -e",
|
||||
"report_json=\"$state_dir/analysis/report.json\"",
|
||||
@@ -2169,6 +2182,7 @@ export function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOption
|
||||
analysis: failureAnalysis,
|
||||
failure: analysisFailure,
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
wrapper: buildWebObserveWrapperForObserveOptions("analyze", options, spec.workspace),
|
||||
result: analysis === null ? compactCommandResultWithStdoutTail(result) : compactCommandResult(result),
|
||||
valuesRedacted: true,
|
||||
|
||||
Reference in New Issue
Block a user