fix: 为 WebProbe 增加物理内存门禁
This commit is contained in:
Codex
@@ -550,6 +550,24 @@ templates:
|
||||
webProbeWorkbench:
|
||||
browserProxyMode: auto
|
||||
defaultOrigin: internal
|
||||
resourcePolicy:
|
||||
memoryStartGuard:
|
||||
metric: MemAvailable
|
||||
source: /proc/meminfo
|
||||
swapExcluded: true
|
||||
thresholdBytes: 4294967296
|
||||
manualComparator: less-than-or-equal
|
||||
sentinelComparator: less-than
|
||||
snapshotTimeoutSeconds: 30
|
||||
observerLifecycle:
|
||||
maxRunSeconds: 3600
|
||||
maxSamples: 720
|
||||
maxScreenshots: 48
|
||||
maxPerformanceEntries: 2000
|
||||
maxBrowserProcessHistoryEntries: 360
|
||||
maxBrowserFreezeSignalHistoryEntries: 120
|
||||
maxResponseBodyBytes: 524288
|
||||
maxWebPerformanceBodyBytes: 65536
|
||||
origins:
|
||||
internal:
|
||||
mode: k8s-service-cluster-ip
|
||||
|
||||
@@ -123,6 +123,28 @@ gc:
|
||||
full: false
|
||||
remote:
|
||||
targets:
|
||||
NC01:
|
||||
hostDockerGc:
|
||||
dockerJsonLogs:
|
||||
enabled: false
|
||||
reason: NC01 主机 Docker 承载主服务运行面，通用远端 GC 不得截断活跃服务日志。
|
||||
buildCachePrune:
|
||||
enabled: false
|
||||
reason: NC01 不是通用构建主机，通用远端 GC 不得清理其活跃 Docker 状态。
|
||||
memoryPressure:
|
||||
processPatterns:
|
||||
- k3s-server
|
||||
- containerd
|
||||
- chrome
|
||||
- chromium
|
||||
- observer-runner
|
||||
- playwright
|
||||
observeStateRoots:
|
||||
- /root/hwlab-v03/.state/web-observe
|
||||
- /root/hwlab/.state/web-observe
|
||||
observeRunDepth: 6
|
||||
observeScanLimit: 2000
|
||||
staleRunMaxAgeHours: 1
|
||||
JD01:
|
||||
hostDockerGc:
|
||||
dockerJsonLogs:
|
||||
|
||||
@@ -569,24 +569,50 @@ def collect_process_pressure(patterns):
|
||||
}
|
||||
|
||||
def collect_memory_snapshot():
|
||||
result = command(["free", "-b"], 5)
|
||||
if result["exitCode"] != 0:
|
||||
return {"ok": False, "error": "free-failed", "command": bounded(result)}
|
||||
memory = {}
|
||||
for line in result["stdout"].splitlines():
|
||||
parts = line.split()
|
||||
if parts and parts[0].rstrip(":") == "Mem" and len(parts) >= 7:
|
||||
memory = {
|
||||
"totalBytes": safe_int(parts[1]),
|
||||
"usedBytes": safe_int(parts[2]),
|
||||
"freeBytes": safe_int(parts[3]),
|
||||
"availableBytes": safe_int(parts[6]),
|
||||
"totalHuman": fmt_bytes(parts[1]),
|
||||
"usedHuman": fmt_bytes(parts[2]),
|
||||
"availableHuman": fmt_bytes(parts[6]),
|
||||
}
|
||||
break
|
||||
return {"ok": bool(memory), "memory": memory, "command": bounded(result)}
|
||||
source = "/proc/meminfo"
|
||||
fields = {}
|
||||
try:
|
||||
with open(source, "r", encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
if ":" not in line:
|
||||
continue
|
||||
key, raw = line.split(":", 1)
|
||||
parts = raw.strip().split()
|
||||
if not parts:
|
||||
continue
|
||||
fields[key] = safe_int(parts[0]) * 1024
|
||||
except OSError as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "proc-meminfo-read-failed",
|
||||
"metric": "MemAvailable",
|
||||
"source": source,
|
||||
"swapExcluded": True,
|
||||
"errorType": type(exc).__name__,
|
||||
}
|
||||
available_bytes = fields.get("MemAvailable")
|
||||
if available_bytes is None:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "memavailable-missing",
|
||||
"metric": "MemAvailable",
|
||||
"source": source,
|
||||
"swapExcluded": True,
|
||||
}
|
||||
total_bytes = fields.get("MemTotal")
|
||||
memory = {
|
||||
"totalBytes": total_bytes,
|
||||
"availableBytes": available_bytes,
|
||||
"totalHuman": fmt_bytes(total_bytes) if total_bytes is not None else None,
|
||||
"availableHuman": fmt_bytes(available_bytes),
|
||||
}
|
||||
return {
|
||||
"ok": True,
|
||||
"metric": "MemAvailable",
|
||||
"source": source,
|
||||
"swapExcluded": True,
|
||||
"memory": memory,
|
||||
}
|
||||
|
||||
def observe_run_record(path, stale_hours):
|
||||
stat = os.stat(path)
|
||||
@@ -1795,6 +1821,23 @@ def remote_policy_install_payload(observed_at):
|
||||
|
||||
def main():
|
||||
observed_at = now_iso()
|
||||
if ACTION == "memory":
|
||||
memory = collect_memory_snapshot()
|
||||
emit_json({
|
||||
"ok": memory.get("ok") is True,
|
||||
"action": "gc remote memory",
|
||||
"providerId": PROVIDER_ID,
|
||||
"dryRun": True,
|
||||
"mutation": False,
|
||||
"observedAt": observed_at,
|
||||
"metric": memory.get("metric"),
|
||||
"source": memory.get("source"),
|
||||
"swapExcluded": memory.get("swapExcluded") is True,
|
||||
"memory": memory.get("memory"),
|
||||
"error": memory.get("error"),
|
||||
"valuesRedacted": True,
|
||||
}, persist_large=False)
|
||||
return 0
|
||||
preflight = cluster_preflight()
|
||||
if ACTION == "policy-plan":
|
||||
emit_json(remote_policy_plan_payload(observed_at), persist_large=False)
|
||||
|
||||
@@ -5,7 +5,7 @@ import { type UniDeskConfig, rootPath } from "./config";
|
||||
import { remoteGcDegradedFailure } from "./gc-remote-degraded";
|
||||
import { runSshCommandCapture } from "./ssh";
|
||||
|
||||
type RemoteGcAction = "plan" | "snapshot" | "trend" | "run" | "status" | "policy-plan" | "policy-install" | "policy-status";
|
||||
type RemoteGcAction = "plan" | "memory" | "snapshot" | "trend" | "run" | "status" | "policy-plan" | "policy-install" | "policy-status";
|
||||
|
||||
interface RemoteGcOptions {
|
||||
confirm: boolean;
|
||||
@@ -89,7 +89,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
|
||||
return {
|
||||
ok: false,
|
||||
error: "gc-remote-provider-required",
|
||||
usage: "bun scripts/cli.ts gc remote <providerId> plan|snapshot|trend|run|status|policy [--confirm]",
|
||||
usage: "bun scripts/cli.ts gc remote <providerId> memory|plan|snapshot|trend|run|status|policy [--confirm]",
|
||||
};
|
||||
}
|
||||
const subaction = action ?? "plan";
|
||||
@@ -124,6 +124,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
|
||||
};
|
||||
}
|
||||
const options = parseRemoteGcOptions(args);
|
||||
if (subaction === "memory") return await runRemoteGc(config, providerId, "memory", options);
|
||||
if (subaction === "plan" || subaction === "dry-run") return await runRemoteGc(config, providerId, "plan", options);
|
||||
if (subaction === "snapshot" || subaction === "growth") return await runRemoteGc(config, providerId, "snapshot", options);
|
||||
if (subaction === "trend") return await runRemoteGc(config, providerId, "trend", options);
|
||||
@@ -146,7 +147,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
|
||||
ok: false,
|
||||
error: "unsupported-gc-remote-action",
|
||||
action: subaction,
|
||||
supportedActions: ["plan", "snapshot", "trend", "run", "status", "policy"],
|
||||
supportedActions: ["memory", "plan", "snapshot", "trend", "run", "status", "policy"],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -148,6 +148,7 @@ export interface HwlabRuntimeWebProbeSpec {
|
||||
readonly browserProxyMode?: "auto" | "direct";
|
||||
readonly playwrightBrowsersPath?: string;
|
||||
readonly defaultOrigin?: HwlabRuntimeWebProbeOriginName;
|
||||
readonly resourcePolicy?: HwlabRuntimeWebProbeResourcePolicySpec;
|
||||
readonly origins?: HwlabRuntimeWebProbeOriginsSpec;
|
||||
readonly authLogin?: HwlabRuntimeWebProbeAuthLoginSpec;
|
||||
readonly alertThresholds?: HwlabRuntimeWebProbeAlertThresholdsSpec;
|
||||
@@ -158,6 +159,32 @@ export interface HwlabRuntimeWebProbeSpec {
|
||||
readonly workbenchTraceReadability?: HwlabRuntimeWebProbeWorkbenchTraceReadabilitySpec;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeMemoryStartGuardSpec {
|
||||
readonly metric: "MemAvailable";
|
||||
readonly source: "/proc/meminfo";
|
||||
readonly swapExcluded: true;
|
||||
readonly thresholdBytes: number;
|
||||
readonly manualComparator: "less-than-or-equal";
|
||||
readonly sentinelComparator: "less-than";
|
||||
readonly snapshotTimeoutSeconds: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeObserverLifecycleSpec {
|
||||
readonly maxRunSeconds: number;
|
||||
readonly maxSamples: number;
|
||||
readonly maxScreenshots: number;
|
||||
readonly maxPerformanceEntries: number;
|
||||
readonly maxBrowserProcessHistoryEntries: number;
|
||||
readonly maxBrowserFreezeSignalHistoryEntries: number;
|
||||
readonly maxResponseBodyBytes: number;
|
||||
readonly maxWebPerformanceBodyBytes: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeResourcePolicySpec {
|
||||
readonly memoryStartGuard: HwlabRuntimeWebProbeMemoryStartGuardSpec;
|
||||
readonly observerLifecycle: HwlabRuntimeWebProbeObserverLifecycleSpec;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeWorkbenchKafkaDebugReplaySpec {
|
||||
readonly enabled: boolean;
|
||||
readonly topic: string;
|
||||
@@ -754,6 +781,11 @@ function booleanField(obj: Record<string, unknown>, key: string, path: string):
|
||||
return value;
|
||||
}
|
||||
|
||||
function literalTrueField(obj: Record<string, unknown>, key: string, path: string): true {
|
||||
if (obj[key] !== true) throw new Error(`${path}.${key} must be true`);
|
||||
return true;
|
||||
}
|
||||
|
||||
function nonNegativeIntegerField(obj: Record<string, unknown>, key: string, path: string): number {
|
||||
const value = obj[key];
|
||||
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) throw new Error(`${path}.${key} must be a non-negative integer`);
|
||||
@@ -1410,6 +1442,7 @@ function webProbeConfig(value: unknown, path: string): HwlabRuntimeWebProbeSpec
|
||||
...(browserProxyMode === undefined ? {} : { browserProxyMode }),
|
||||
...(playwrightBrowsersPath === undefined ? {} : { playwrightBrowsersPath }),
|
||||
...(defaultOrigin === undefined ? {} : { defaultOrigin }),
|
||||
...(raw.resourcePolicy === undefined ? {} : { resourcePolicy: webProbeResourcePolicyConfig(raw.resourcePolicy, `${path}.resourcePolicy`) }),
|
||||
...(origins === undefined ? {} : { origins }),
|
||||
...(raw.authLogin === undefined ? {} : { authLogin: webProbeAuthLoginConfig(raw.authLogin, `${path}.authLogin`) }),
|
||||
...(raw.alertThresholds === undefined ? {} : { alertThresholds: webProbeAlertThresholdsConfig(raw.alertThresholds, `${path}.alertThresholds`) }),
|
||||
@@ -1425,6 +1458,33 @@ function webProbeConfig(value: unknown, path: string): HwlabRuntimeWebProbeSpec
|
||||
};
|
||||
}
|
||||
|
||||
function webProbeResourcePolicyConfig(value: unknown, path: string): HwlabRuntimeWebProbeResourcePolicySpec {
|
||||
const raw = asRecord(value, path);
|
||||
const memory = asRecord(raw.memoryStartGuard, `${path}.memoryStartGuard`);
|
||||
const lifecycle = asRecord(raw.observerLifecycle, `${path}.observerLifecycle`);
|
||||
return {
|
||||
memoryStartGuard: {
|
||||
metric: enumStringField(memory, "metric", `${path}.memoryStartGuard`, ["MemAvailable"] as const),
|
||||
source: enumStringField(memory, "source", `${path}.memoryStartGuard`, ["/proc/meminfo"] as const),
|
||||
swapExcluded: literalTrueField(memory, "swapExcluded", `${path}.memoryStartGuard`),
|
||||
thresholdBytes: boundedIntegerField(memory, "thresholdBytes", `${path}.memoryStartGuard`, 1, Number.MAX_SAFE_INTEGER),
|
||||
manualComparator: enumStringField(memory, "manualComparator", `${path}.memoryStartGuard`, ["less-than-or-equal"] as const),
|
||||
sentinelComparator: enumStringField(memory, "sentinelComparator", `${path}.memoryStartGuard`, ["less-than"] as const),
|
||||
snapshotTimeoutSeconds: boundedIntegerField(memory, "snapshotTimeoutSeconds", `${path}.memoryStartGuard`, 5, 120),
|
||||
},
|
||||
observerLifecycle: {
|
||||
maxRunSeconds: boundedIntegerField(lifecycle, "maxRunSeconds", `${path}.observerLifecycle`, 1, 86_400),
|
||||
maxSamples: boundedIntegerField(lifecycle, "maxSamples", `${path}.observerLifecycle`, 1, 10_000_000),
|
||||
maxScreenshots: boundedIntegerField(lifecycle, "maxScreenshots", `${path}.observerLifecycle`, 1, 10_000),
|
||||
maxPerformanceEntries: boundedIntegerField(lifecycle, "maxPerformanceEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxBrowserProcessHistoryEntries: boundedIntegerField(lifecycle, "maxBrowserProcessHistoryEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxBrowserFreezeSignalHistoryEntries: boundedIntegerField(lifecycle, "maxBrowserFreezeSignalHistoryEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxResponseBodyBytes: boundedIntegerField(lifecycle, "maxResponseBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024),
|
||||
maxWebPerformanceBodyBytes: boundedIntegerField(lifecycle, "maxWebPerformanceBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function webProbeWorkbenchKafkaDebugReplayConfig(value: unknown, path: string): HwlabRuntimeWebProbeWorkbenchKafkaDebugReplaySpec {
|
||||
const raw = asRecord(value, path);
|
||||
const topic = stringField(raw, "topic", path);
|
||||
|
||||
@@ -16,7 +16,7 @@ async function installPagePerformanceProbe(targetPage, pageRole = "control", tar
|
||||
}
|
||||
const buffer = [];
|
||||
const supportedTypes = [];
|
||||
const maxBufferSize = 2000;
|
||||
const maxBufferSize = Math.max(1, Number(input.maxBufferSize || 1));
|
||||
const push = (event) => {
|
||||
buffer.push({
|
||||
probeVersion: 1,
|
||||
@@ -126,9 +126,9 @@ async function installPagePerformanceProbe(targetPage, pageRole = "control", tar
|
||||
};
|
||||
return { ok: true, alreadyInstalled: false, supportedTypes, valuesRedacted: true };
|
||||
};
|
||||
await targetPage.addInitScript(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs });
|
||||
await targetPage.addInitScript(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs, maxBufferSize: maxPerformanceEntries });
|
||||
if (!targetPage.isClosed()) {
|
||||
await withHardTimeout(targetPage.evaluate(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs }), 3000, "installPagePerformanceProbe evaluate exceeded 3000ms");
|
||||
await withHardTimeout(targetPage.evaluate(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs, maxBufferSize: maxPerformanceEntries }), 3000, "installPagePerformanceProbe evaluate exceeded 3000ms");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -68,6 +68,15 @@ async function writeManifest(extra = {}) {
|
||||
observerInitiatedDefault: false,
|
||||
responseBodyReadDefault: false,
|
||||
},
|
||||
resourceLimits: {
|
||||
maxScreenshots,
|
||||
maxPerformanceEntries,
|
||||
maxBrowserProcessHistoryEntries,
|
||||
maxBrowserFreezeSignalHistoryEntries,
|
||||
responseBodyMaxBytes,
|
||||
webPerformanceRequestBodyMaxBytes,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
projectManagement,
|
||||
@@ -100,6 +109,16 @@ async function writeHeartbeat(extra = {}) {
|
||||
maxRunSeconds: maxRunMs > 0 ? Math.ceil(maxRunMs / 1000) : 0,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
resourceLimits: {
|
||||
screenshotCount,
|
||||
maxScreenshots,
|
||||
maxPerformanceEntries,
|
||||
maxBrowserProcessHistoryEntries,
|
||||
maxBrowserFreezeSignalHistoryEntries,
|
||||
responseBodyMaxBytes,
|
||||
webPerformanceRequestBodyMaxBytes,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
lastObserverRefreshAt: Number.isFinite(lastObserverRefreshAtMs) ? new Date(lastObserverRefreshAtMs).toISOString() : null,
|
||||
pageProvenance: compactPageProvenance(currentPageProvenance),
|
||||
sampleSeq,
|
||||
@@ -335,6 +354,7 @@ function recordBrowserFreezeSignal(kind, sample, pageMetric, detail) {
|
||||
const windowMs = browserFreezePolicy.blockerWindowMs;
|
||||
const cutoff = signal.tsMs - windowMs;
|
||||
while (browserFreezeSignalHistory.length > 0 && Number(browserFreezeSignalHistory[0].tsMs || 0) < cutoff) browserFreezeSignalHistory.shift();
|
||||
while (browserFreezeSignalHistory.length > maxBrowserFreezeSignalHistoryEntries) browserFreezeSignalHistory.shift();
|
||||
return {
|
||||
kind,
|
||||
rootCause: detail.rootCause,
|
||||
@@ -639,6 +659,7 @@ function updateBrowserProcessHistory(tsMs, processSummary) {
|
||||
browserProcessHistory.push({ tsMs, totalRssBytes, maxProcessRssBytes });
|
||||
const retentionMs = Math.max(windowMs * 3, 180000);
|
||||
while (browserProcessHistory.length > 0 && tsMs - browserProcessHistory[0].tsMs > retentionMs) browserProcessHistory.shift();
|
||||
while (browserProcessHistory.length > maxBrowserProcessHistoryEntries) browserProcessHistory.shift();
|
||||
const windowStartMs = tsMs - windowMs;
|
||||
const candidates = browserProcessHistory.filter((item) => item.tsMs >= windowStartMs && item.tsMs <= tsMs);
|
||||
const baseline = candidates[0] || browserProcessHistory[0] || null;
|
||||
|
||||
@@ -797,6 +797,23 @@ function digestSessionRail(value) {
|
||||
|
||||
async function captureScreenshot(reason, imageType = "png") {
|
||||
if (!page || page.isClosed()) throw new Error("page is not available for screenshot");
|
||||
if (screenshotCount >= maxScreenshots) {
|
||||
lastScreenshotAtMs = Date.now();
|
||||
return {
|
||||
seq: null,
|
||||
sampleSeq,
|
||||
ts: new Date().toISOString(),
|
||||
kind: "screenshot-skipped",
|
||||
reason,
|
||||
skipped: true,
|
||||
skipReason: "screenshot-cap-reached",
|
||||
screenshotCount,
|
||||
maxScreenshots,
|
||||
pageId,
|
||||
currentUrl: currentPageUrl(),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
if (screenshotCaptureState && screenshotCaptureState.settled !== true) {
|
||||
const ageMs = Date.now() - Number(screenshotCaptureState.startedAtMs || Date.now());
|
||||
const error = new Error("screenshot capture already in progress");
|
||||
@@ -814,6 +831,7 @@ async function captureScreenshot(reason, imageType = "png") {
|
||||
lastScreenshotAtMs = Date.now();
|
||||
throw error;
|
||||
}
|
||||
screenshotCount += 1;
|
||||
artifactSeq += 1;
|
||||
const safeReason = safeId(String(reason || "manual")).slice(0, 40) || "manual";
|
||||
const type = imageType === "jpeg" || imageType === "jpg" ? "jpeg" : "png";
|
||||
@@ -888,7 +906,7 @@ async function summarizeWorkbenchResponseBody(response, request) {
|
||||
const contentType = String(headers["content-type"] || headers["Content-Type"] || "");
|
||||
if (!/json/iu.test(contentType)) return { bodyRead: false, bodyReadSkipped: "non-json", valuesRedacted: true };
|
||||
const contentLength = Number(headers["content-length"] || headers["Content-Length"]);
|
||||
const maxBytes = 512 * 1024;
|
||||
const maxBytes = responseBodyMaxBytes;
|
||||
if (Number.isFinite(contentLength) && contentLength > maxBytes) return { bodyRead: false, bodyReadSkipped: "content-length-too-large", bodyByteCount: contentLength, valuesRedacted: true };
|
||||
const text = await response.text();
|
||||
const byteCount = Buffer.byteLength(text);
|
||||
|
||||
@@ -31,9 +31,14 @@ const targetPath = process.env.UNIDESK_WEB_OBSERVE_TARGET_PATH || "/workbench";
|
||||
const sampleIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS, 5000);
|
||||
const screenshotIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS, 300000);
|
||||
const screenshotCaptureTimeoutMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_CAPTURE_TIMEOUT_MS, 15000, 1000, 120000);
|
||||
const maxSamples = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 0);
|
||||
const maxSamples = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 720, 1, 10000000);
|
||||
const observerRefreshIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS, 180000);
|
||||
const maxRunMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RUN_MS, 0);
|
||||
const maxRunMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RUN_MS, 3600000, 1000, 86400000);
|
||||
const maxScreenshots = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS, 48, 1, 10000);
|
||||
const maxPerformanceEntries = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES, 2000, 1, 100000);
|
||||
const maxBrowserProcessHistoryEntries = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES, 360, 1, 100000);
|
||||
const maxBrowserFreezeSignalHistoryEntries = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES, 120, 1, 100000);
|
||||
const responseBodyMaxBytes = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES, 524288, 1024, 16777216);
|
||||
const viewport = parseViewport(process.env.UNIDESK_WEB_OBSERVE_VIEWPORT || "1440x900");
|
||||
const browserProxyMode = parseBrowserProxyMode(process.env.UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE || "auto");
|
||||
const origin = {
|
||||
@@ -92,6 +97,7 @@ let lastObserverRefreshAtMs = Date.now();
|
||||
let sampleSeq = 0;
|
||||
let commandSeq = 0;
|
||||
let artifactSeq = 0;
|
||||
let screenshotCount = 0;
|
||||
let activeCommandId = null;
|
||||
let activeCommandType = null;
|
||||
let stopping = false;
|
||||
@@ -107,11 +113,22 @@ let heartbeatPulseTimer = null;
|
||||
let browserProcessMonitorStop = null;
|
||||
let browserProcessMonitorSeq = 0;
|
||||
let browserFreezeBlocker = null;
|
||||
let shutdownSignal = null;
|
||||
let browserResourcesClosePromise = null;
|
||||
const browserProcessHistory = [];
|
||||
const browserPageRuntimeBaselines = new Map();
|
||||
const browserFreezeSignalHistory = [];
|
||||
const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] };
|
||||
|
||||
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
||||
process.once(signal, () => {
|
||||
shutdownSignal = signal;
|
||||
stopping = true;
|
||||
terminalStatus = "stopping";
|
||||
void closeBrowserResources("signal:" + signal);
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
if (!password) throw new Error("missing HWLAB_WEB_PASS");
|
||||
await prepareDirs();
|
||||
@@ -152,18 +169,18 @@ try {
|
||||
await writeManifest({ status: "running", auth: publicAuth(auth) });
|
||||
await writeHeartbeat({ status: "running" });
|
||||
while (!stopping) {
|
||||
await drainOneCommand();
|
||||
if (browserFreezeBlocker) break;
|
||||
await samplePage("interval");
|
||||
if (browserFreezeBlocker) break;
|
||||
if (maxSamples > 0 && sampleSeq >= maxSamples) {
|
||||
await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples }));
|
||||
break;
|
||||
}
|
||||
if (maxRunMs > 0 && Date.now() - startedAtMs >= maxRunMs) {
|
||||
await appendJsonl(files.control, controlRecord({ id: "max-run-ms", type: "stop", source: "sampler" }, "completed", { reason: "max-run-ms", maxRunMs, elapsedMs: Date.now() - startedAtMs }));
|
||||
break;
|
||||
}
|
||||
if (maxSamples > 0 && sampleSeq >= maxSamples) {
|
||||
await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples }));
|
||||
break;
|
||||
}
|
||||
await drainOneCommand();
|
||||
if (browserFreezeBlocker) break;
|
||||
await samplePage("interval");
|
||||
if (browserFreezeBlocker) break;
|
||||
await sleep(sampleIntervalMs);
|
||||
}
|
||||
if (browserFreezeBlocker) {
|
||||
@@ -186,7 +203,28 @@ try {
|
||||
} finally {
|
||||
if (browserProcessMonitorStop) browserProcessMonitorStop();
|
||||
if (heartbeatPulseTimer) clearInterval(heartbeatPulseTimer);
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
await closeBrowserResources(shutdownSignal === null ? "finally" : "signal:" + shutdownSignal);
|
||||
}
|
||||
|
||||
async function closeBrowserResources(reason) {
|
||||
if (browserResourcesClosePromise) return browserResourcesClosePromise;
|
||||
browserResourcesClosePromise = (async () => {
|
||||
stopping = true;
|
||||
const closingObserverPage = observerPage;
|
||||
const closingPage = page;
|
||||
const closingContext = context;
|
||||
const closingBrowser = browser;
|
||||
observerPage = null;
|
||||
page = null;
|
||||
context = null;
|
||||
browser = null;
|
||||
if (closingObserverPage && !closingObserverPage.isClosed()) await closingObserverPage.close().catch(() => {});
|
||||
if (closingPage && !closingPage.isClosed()) await closingPage.close().catch(() => {});
|
||||
if (closingContext) await closingContext.close().catch(() => {});
|
||||
if (closingBrowser) await closingBrowser.close().catch(() => {});
|
||||
return { ok: true, reason, valuesRedacted: true };
|
||||
})();
|
||||
return browserResourcesClosePromise;
|
||||
}
|
||||
|
||||
${nodeWebObserveRunnerRuntimeSource()}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import type { CommandResult } from "./command";
|
||||
import { rootPath } from "./config";
|
||||
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
|
||||
import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard, webProbeMemoryStartGuardConfigRef } from "./hwlab-node-web-probe-memory-guard";
|
||||
|
||||
const spec = hwlabRuntimeLaneSpecForNode("v03", "NC01");
|
||||
const thresholdBytes = 4 * 1024 * 1024 * 1024;
|
||||
|
||||
function memoryPayload(availableBytes: number, extra: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return {
|
||||
ok: true,
|
||||
action: "gc remote memory",
|
||||
providerId: "NC01",
|
||||
metric: "MemAvailable",
|
||||
source: "/proc/meminfo",
|
||||
swapExcluded: true,
|
||||
memory: { availableBytes, availableHuman: "fixture", ...extra },
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
test("WebProbe memory threshold is read from the single owning YAML policy", () => {
|
||||
const policy = spec.webProbe?.resourcePolicy?.memoryStartGuard;
|
||||
assert.equal(policy?.thresholdBytes, thresholdBytes);
|
||||
assert.equal(policy?.metric, "MemAvailable");
|
||||
assert.equal(policy?.source, "/proc/meminfo");
|
||||
assert.equal(policy?.swapExcluded, true);
|
||||
assert.equal(policy?.manualComparator, "less-than-or-equal");
|
||||
assert.equal(policy?.sentinelComparator, "less-than");
|
||||
assert.match(webProbeMemoryStartGuardConfigRef, /resourcePolicy\.memoryStartGuard$/u);
|
||||
});
|
||||
|
||||
test("manual start is blocked at exactly 4 GiB and requires controlled GC then MemAvailable recheck", () => {
|
||||
const guard = evaluateWebProbeMemoryGuard(spec, "manual-start", memoryPayload(thresholdBytes));
|
||||
const memory = guard.memory as Record<string, unknown>;
|
||||
const gcNext = guard.gcNext as Record<string, unknown>;
|
||||
|
||||
assert.equal(guard.status, "blocked");
|
||||
assert.equal(guard.startBlocked, true);
|
||||
assert.equal(guard.decision, "gc-required");
|
||||
assert.equal(memory.metric, "MemAvailable");
|
||||
assert.equal(memory.source, "/proc/meminfo");
|
||||
assert.equal(memory.swapExcluded, true);
|
||||
assert.equal(memory.comparator, "less-than-or-equal");
|
||||
assert.equal(memory.thresholdMatched, true);
|
||||
assert.equal(gcNext.requiredBeforeStart, true);
|
||||
assert.match(String(gcNext.planCommand), /gc remote NC01 plan$/u);
|
||||
assert.match(String(gcNext.runCommand), /gc remote NC01 run --confirm$/u);
|
||||
assert.match(String(gcNext.recheckCommand), /gc remote NC01 memory$/u);
|
||||
assert.equal(gcNext.recheckMetric, "MemAvailable");
|
||||
});
|
||||
|
||||
test("sentinel allows exactly 4 GiB but skips one byte below without considering swap", () => {
|
||||
const exact = evaluateWebProbeMemoryGuard(spec, "sentinel-cadence", memoryPayload(thresholdBytes, { swapFreeBytes: 12 * 1024 ** 3 }));
|
||||
const below = evaluateWebProbeMemoryGuard(spec, "sentinel-cadence", memoryPayload(thresholdBytes - 1, { swapFreeBytes: 12 * 1024 ** 3 }));
|
||||
const belowMemory = below.memory as Record<string, unknown>;
|
||||
|
||||
assert.equal(exact.status, "allowed");
|
||||
assert.equal(exact.startBlocked, false);
|
||||
assert.equal((exact.memory as Record<string, unknown>).comparator, "less-than");
|
||||
assert.equal(below.status, "skipped");
|
||||
assert.equal(below.startBlocked, true);
|
||||
assert.equal(below.decision, "skip-observer");
|
||||
assert.equal(belowMemory.availableBytes, thresholdBytes - 1);
|
||||
assert.equal(belowMemory.swapExcluded, true);
|
||||
});
|
||||
|
||||
test("memory guard rejects a snapshot that is not the /proc/meminfo MemAvailable contract", () => {
|
||||
const guard = evaluateWebProbeMemoryGuard(spec, "manual-start", {
|
||||
...memoryPayload(thresholdBytes + 1),
|
||||
source: "cgroup-v2-memory.current",
|
||||
});
|
||||
assert.equal(guard.status, "blocked");
|
||||
assert.equal(guard.reason, "memory-snapshot-contract-invalid");
|
||||
assert.equal(guard.startBlocked, true);
|
||||
});
|
||||
|
||||
test("memory guard executes only the controlled read-only remote memory entry", () => {
|
||||
let observedCommand: string[] = [];
|
||||
let observedTimeoutMs = 0;
|
||||
const execute = (command: string[], cwd: string, options: { timeoutMs?: number }): CommandResult => {
|
||||
observedCommand = command;
|
||||
observedTimeoutMs = options.timeoutMs ?? 0;
|
||||
return {
|
||||
command,
|
||||
cwd,
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ ok: true, command: "gc remote NC01 memory", data: memoryPayload(thresholdBytes + 1) }),
|
||||
stderr: "",
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
};
|
||||
};
|
||||
const guard = runWebProbeMemoryGuard(spec, "manual-start", execute);
|
||||
|
||||
assert.deepEqual(observedCommand, ["bun", "scripts/cli.ts", "gc", "remote", "NC01", "memory"]);
|
||||
assert.equal(observedTimeoutMs, 30_000);
|
||||
assert.equal(guard.status, "allowed");
|
||||
});
|
||||
|
||||
test("remote memory collector reads MemAvailable directly and excludes swap", () => {
|
||||
const source = readFileSync(rootPath("scripts/src/gc-remote-runner.py"), "utf8");
|
||||
const start = source.indexOf("def collect_memory_snapshot():");
|
||||
const end = source.indexOf("\ndef observe_run_record", start);
|
||||
const collector = source.slice(start, end);
|
||||
|
||||
assert.match(collector, /source = "\/proc\/meminfo"/u);
|
||||
assert.match(collector, /fields\.get\("MemAvailable"\)/u);
|
||||
assert.match(collector, /"swapExcluded": True/u);
|
||||
assert.doesNotMatch(collector, /SwapFree|SwapTotal|\["free", "-b"\]/u);
|
||||
});
|
||||
|
||||
test("manual and sentinel gates return before any observer launch path", () => {
|
||||
const manualSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe-actions.ts"), "utf8");
|
||||
const manualStart = manualSource.indexOf("export function runNodeWebProbeObserveStart");
|
||||
const manualGuard = manualSource.indexOf("runWebProbeMemoryGuard(spec, options.memoryGuardMode)", manualStart);
|
||||
const manualReturn = manualSource.indexOf("startBlocked: true", manualGuard);
|
||||
const manualJob = manualSource.indexOf("const jobId =", manualStart);
|
||||
const manualLaunch = manualSource.indexOf("setsid env", manualStart);
|
||||
assert.ok(manualStart >= 0 && manualGuard > manualStart && manualReturn > manualGuard);
|
||||
assert.ok(manualReturn < manualJob && manualJob < manualLaunch);
|
||||
|
||||
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5-observe.ts"), "utf8");
|
||||
const quickVerify = sentinelSource.indexOf("export function runSentinelQuickVerify");
|
||||
const sentinelGuard = sentinelSource.indexOf("runWebProbeMemoryGuard(state.spec, \"sentinel-cadence\")", quickVerify);
|
||||
const sentinelSkip = sentinelSource.indexOf('status: "skipped-wait-next-round"', sentinelGuard);
|
||||
const providerPreflight = sentinelSource.indexOf("runSentinelExactProviderProfilePreflight", quickVerify);
|
||||
const observeStart = sentinelSource.indexOf('"web-probe", "observe", "start"', quickVerify);
|
||||
assert.ok(quickVerify >= 0 && sentinelGuard > quickVerify && sentinelSkip > sentinelGuard);
|
||||
assert.ok(sentinelSkip < providerPreflight && providerPreflight < observeStart);
|
||||
assert.match(sentinelSource.slice(observeStart, observeStart + 800), /--sentinel-cadence/u);
|
||||
|
||||
const serviceSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-service.ts"), "utf8");
|
||||
const plannedStart = serviceSource.indexOf('"bun", "scripts/cli.ts", "web-probe", "observe", "start"');
|
||||
assert.match(serviceSource.slice(plannedStart, plannedStart + 800), /--sentinel-cadence/u);
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
// SPEC: PJ2026-01060508 Web 哨兵。WebProbe 启动前的 YAML-first 物理内存资格判定。
|
||||
import type { CommandResult } from "./command";
|
||||
import { runCommand } from "./command";
|
||||
import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery";
|
||||
import { repoRoot } from "./config";
|
||||
import type { HwlabRuntimeLaneSpec, HwlabRuntimeWebProbeMemoryStartGuardSpec } from "./hwlab-node-lanes";
|
||||
|
||||
export const webProbeMemoryStartGuardConfigRef = "config/hwlab-node-lanes.yaml#templates.hwlabV03.webProbeWorkbench.resourcePolicy.memoryStartGuard";
|
||||
|
||||
export type WebProbeMemoryGuardMode = "manual-start" | "sentinel-cadence";
|
||||
export type WebProbeMemoryGuardStatus = "allowed" | "blocked" | "skipped";
|
||||
|
||||
type MemoryGuardExecutor = (
|
||||
command: string[],
|
||||
cwd: string,
|
||||
options: { timeoutMs?: number },
|
||||
) => CommandResult;
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function memoryPolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeMemoryStartGuardSpec | null {
|
||||
return spec.webProbe?.resourcePolicy?.memoryStartGuard ?? null;
|
||||
}
|
||||
|
||||
function humanBytes(value: number | null): string | null {
|
||||
if (value === null || !Number.isFinite(value)) return null;
|
||||
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
||||
let current = value;
|
||||
let index = 0;
|
||||
while (Math.abs(current) >= 1024 && index < units.length - 1) {
|
||||
current /= 1024;
|
||||
index += 1;
|
||||
}
|
||||
return `${current.toFixed(index === 0 ? 0 : 2)} ${units[index]}`;
|
||||
}
|
||||
|
||||
function controlledGcNext(node: string): Record<string, unknown> {
|
||||
return {
|
||||
automaticGc: false,
|
||||
requiredOrder: ["plan", "run", "recheck"],
|
||||
planCommand: `bun scripts/cli.ts gc remote ${node} plan`,
|
||||
runCommand: `bun scripts/cli.ts gc remote ${node} run --confirm`,
|
||||
recheckCommand: `bun scripts/cli.ts gc remote ${node} memory`,
|
||||
recheckMetric: "MemAvailable",
|
||||
recheckSource: "/proc/meminfo",
|
||||
swapExcluded: true,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function unavailableGuard(spec: HwlabRuntimeLaneSpec, mode: WebProbeMemoryGuardMode, reason: string): Record<string, unknown> {
|
||||
const policy = memoryPolicy(spec);
|
||||
const comparator = policy === null
|
||||
? null
|
||||
: mode === "manual-start"
|
||||
? policy.manualComparator
|
||||
: policy.sentinelComparator;
|
||||
return {
|
||||
ok: false,
|
||||
status: "blocked" satisfies WebProbeMemoryGuardStatus,
|
||||
eligible: false,
|
||||
startBlocked: true,
|
||||
decision: "do-not-start-observer",
|
||||
reason,
|
||||
mode,
|
||||
policyConfigRef: webProbeMemoryStartGuardConfigRef,
|
||||
memory: {
|
||||
metric: policy?.metric ?? "MemAvailable",
|
||||
source: policy?.source ?? "/proc/meminfo",
|
||||
swapExcluded: policy?.swapExcluded ?? true,
|
||||
availableBytes: null,
|
||||
availableHuman: null,
|
||||
thresholdBytes: policy?.thresholdBytes ?? null,
|
||||
thresholdHuman: humanBytes(policy?.thresholdBytes ?? null),
|
||||
comparator,
|
||||
thresholdMatched: null,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
gcNext: controlledGcNext(spec.nodeId),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
export function evaluateWebProbeMemoryGuard(
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
mode: WebProbeMemoryGuardMode,
|
||||
snapshotPayload: unknown,
|
||||
): Record<string, unknown> {
|
||||
const policy = memoryPolicy(spec);
|
||||
if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing");
|
||||
const payload = record(snapshotPayload);
|
||||
const memory = record(payload.memory);
|
||||
const availableBytes = typeof memory.availableBytes === "number" && Number.isSafeInteger(memory.availableBytes) && memory.availableBytes >= 0
|
||||
? memory.availableBytes
|
||||
: null;
|
||||
const metadataMatches = payload.ok === true
|
||||
&& payload.action === "gc remote memory"
|
||||
&& payload.metric === policy.metric
|
||||
&& payload.source === policy.source
|
||||
&& payload.swapExcluded === true;
|
||||
if (!metadataMatches || availableBytes === null) {
|
||||
return unavailableGuard(spec, mode, metadataMatches ? "memavailable-unavailable" : "memory-snapshot-contract-invalid");
|
||||
}
|
||||
const comparator = mode === "manual-start" ? policy.manualComparator : policy.sentinelComparator;
|
||||
const thresholdMatched = mode === "manual-start"
|
||||
? availableBytes <= policy.thresholdBytes
|
||||
: availableBytes < policy.thresholdBytes;
|
||||
const status: WebProbeMemoryGuardStatus = thresholdMatched
|
||||
? mode === "manual-start" ? "blocked" : "skipped"
|
||||
: "allowed";
|
||||
return {
|
||||
ok: status === "allowed",
|
||||
status,
|
||||
eligible: status === "allowed",
|
||||
startBlocked: status !== "allowed",
|
||||
decision: status === "allowed" ? "start-observer" : mode === "manual-start" ? "gc-required" : "skip-observer",
|
||||
reason: status === "allowed" ? null : "memavailable-threshold-matched",
|
||||
mode,
|
||||
policyConfigRef: webProbeMemoryStartGuardConfigRef,
|
||||
memory: {
|
||||
metric: policy.metric,
|
||||
source: policy.source,
|
||||
swapExcluded: policy.swapExcluded,
|
||||
availableBytes,
|
||||
availableHuman: humanBytes(availableBytes),
|
||||
thresholdBytes: policy.thresholdBytes,
|
||||
thresholdHuman: humanBytes(policy.thresholdBytes),
|
||||
comparator,
|
||||
thresholdMatched,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
gcNext: status === "blocked" && mode === "manual-start"
|
||||
? { ...controlledGcNext(spec.nodeId), requiredBeforeStart: true }
|
||||
: { ...controlledGcNext(spec.nodeId), requiredBeforeStart: false },
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
export function runWebProbeMemoryGuard(
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
mode: WebProbeMemoryGuardMode,
|
||||
execute: MemoryGuardExecutor = runCommand,
|
||||
): Record<string, unknown> {
|
||||
const policy = memoryPolicy(spec);
|
||||
if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing");
|
||||
const command = ["bun", "scripts/cli.ts", "gc", "remote", spec.nodeId, "memory"];
|
||||
const result = execute(command, repoRoot, { timeoutMs: policy.snapshotTimeoutSeconds * 1000 });
|
||||
const resolution = resolveCliChildJsonCommandResult({
|
||||
result,
|
||||
requestedStdoutType: `gc remote ${spec.nodeId} memory contract`,
|
||||
acceptParsed: (value) => {
|
||||
const payload = value.action === "gc remote memory" ? value : record(value.data);
|
||||
return payload.action === "gc remote memory" && payload.providerId === spec.nodeId;
|
||||
},
|
||||
});
|
||||
const resolvedPayload = resolution.parsed?.action === "gc remote memory"
|
||||
? resolution.parsed
|
||||
: record(resolution.parsed?.data);
|
||||
const evaluated = evaluateWebProbeMemoryGuard(spec, mode, resolvedPayload);
|
||||
return {
|
||||
...evaluated,
|
||||
snapshot: {
|
||||
action: "gc remote memory",
|
||||
command: command.join(" "),
|
||||
exitCode: result.exitCode,
|
||||
timedOut: result.timedOut,
|
||||
parsed: resolution.parsed !== null,
|
||||
diagnostics: resolution.diagnostics,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -302,5 +302,13 @@ test("web observe runner records finite lifecycle policy and closes the browser"
|
||||
assert.match(source, /sampling:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u);
|
||||
assert.match(source, /observerLifecycle:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u);
|
||||
assert.match(source, /if \(maxRunMs > 0 && Date\.now\(\) - startedAtMs >= maxRunMs\)/u);
|
||||
assert.match(source, /finally \{[\s\S]*?if \(browser\) await browser\.close\(\)\.catch/u);
|
||||
assert.match(source, /const maxScreenshots = boundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS/u);
|
||||
assert.match(source, /const maxPerformanceEntries = boundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES/u);
|
||||
assert.match(source, /while \(browserProcessHistory\.length > maxBrowserProcessHistoryEntries\)/u);
|
||||
assert.match(source, /while \(browserFreezeSignalHistory\.length > maxBrowserFreezeSignalHistoryEntries\)/u);
|
||||
assert.match(source, /if \(screenshotCount >= maxScreenshots\)/u);
|
||||
assert.match(source, /const maxBytes = responseBodyMaxBytes/u);
|
||||
assert.match(source, /process\.once\(signal,[\s\S]*?closeBrowserResources\("signal:" \+ signal\)/u);
|
||||
assert.match(source, /finally \{[\s\S]*?await closeBrowserResources/u);
|
||||
assert.match(source, /closingObserverPage\.close\(\)[\s\S]*?closingPage\.close\(\)[\s\S]*?closingContext\.close\(\)[\s\S]*?closingBrowser\.close\(\)/u);
|
||||
});
|
||||
|
||||
@@ -7,6 +7,7 @@ import type { CommandResult } from "./command";
|
||||
import { runCommand } from "./command";
|
||||
import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery";
|
||||
import { repoRoot, rootPath } from "./config";
|
||||
import { runWebProbeMemoryGuard } from "./hwlab-node-web-probe-memory-guard";
|
||||
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
|
||||
import type { ChildCliResult, CompactCommandResult, SentinelCicdState } from "./hwlab-node-web-sentinel-cicd";
|
||||
import {
|
||||
@@ -125,6 +126,40 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
|
||||
const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds");
|
||||
const scenario = findScenario(state, scenarioId);
|
||||
if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true };
|
||||
const memoryGuard = runWebProbeMemoryGuard(state.spec, "sentinel-cadence");
|
||||
if (memoryGuard.status === "skipped") {
|
||||
return {
|
||||
ok: true,
|
||||
status: "skipped-wait-next-round",
|
||||
decision: "skip-observer",
|
||||
reason: "memavailable-below-threshold",
|
||||
scenarioId,
|
||||
observerCreated: false,
|
||||
runCreated: false,
|
||||
mutation: false,
|
||||
memoryGuard,
|
||||
nextCadence: {
|
||||
action: "wait-for-next-scheduled-cadence",
|
||||
cadence: stringAtNullable(scenario, "cadence"),
|
||||
automaticRetry: true,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
if (memoryGuard.status !== "allowed") {
|
||||
return {
|
||||
ok: false,
|
||||
status: "blocked",
|
||||
reason: "memavailable-preflight-unavailable",
|
||||
scenarioId,
|
||||
observerCreated: false,
|
||||
runCreated: false,
|
||||
mutation: false,
|
||||
memoryGuard,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
const providerProfilePreflight = runSentinelExactProviderProfilePreflight(state, timeoutSeconds, scenario);
|
||||
if (providerProfilePreflight.ok !== true) {
|
||||
return {
|
||||
@@ -214,12 +249,36 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
|
||||
"--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")),
|
||||
"--max-run-seconds", String(hardBudgetSeconds),
|
||||
"--command-timeout-seconds", "55",
|
||||
"--sentinel-cadence",
|
||||
];
|
||||
const viewport = stringAtNullable(scenario, "viewport");
|
||||
if (viewport !== null) startArgs.push("--viewport", viewport);
|
||||
printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) });
|
||||
const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env);
|
||||
steps.push({ phase: "observe-start", ok: started.ok, result: started.result });
|
||||
const startedEnvelope = record(started.parsed);
|
||||
const startedPayload = Object.keys(record(startedEnvelope.data)).length > 0 ? record(startedEnvelope.data) : startedEnvelope;
|
||||
if (startedPayload.status === "skipped-wait-next-round") {
|
||||
return {
|
||||
ok: true,
|
||||
status: "skipped-wait-next-round",
|
||||
decision: "skip-observer",
|
||||
reason: "memavailable-below-threshold-after-cadence-preflight",
|
||||
scenarioId,
|
||||
observerCreated: false,
|
||||
runCreated: false,
|
||||
mutation: false,
|
||||
memoryGuard: startedPayload.memoryGuard ?? null,
|
||||
steps,
|
||||
nextCadence: {
|
||||
action: "wait-for-next-scheduled-cadence",
|
||||
cadence: stringAtNullable(scenario, "cadence"),
|
||||
automaticRetry: true,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? ""));
|
||||
printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() });
|
||||
if (!started.ok || observerId === null) {
|
||||
|
||||
@@ -562,6 +562,7 @@ function buildObserveCommandPlan(config: WebProbeSentinelServiceConfig, scenario
|
||||
"--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")),
|
||||
"--max-run-seconds", String(numberAt(scenario, "maxRunSeconds")),
|
||||
"--command-timeout-seconds", "55",
|
||||
"--sentinel-cadence",
|
||||
];
|
||||
const viewport = stringOrNull(scenario.viewport);
|
||||
if (viewport !== null) startArgv.push("--viewport", viewport);
|
||||
|
||||
@@ -188,6 +188,7 @@ export interface NodeWebProbeObserveOptions extends WebProbeOriginSelection {
|
||||
targetPath: string;
|
||||
viewport: string;
|
||||
browserProxyMode: WebProbeBrowserProxyMode;
|
||||
memoryGuardMode: "manual-start" | "sentinel-cadence";
|
||||
sampleIntervalMs: number;
|
||||
screenshotIntervalMs: number;
|
||||
observerRefreshIntervalMs: number;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { resolveCliChildJsonCommandResult, type CliChildJsonResolution } from "../cli-child-json-recovery";
|
||||
import type { HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
import { runWebProbeMemoryGuard } from "../hwlab-node-web-probe-memory-guard";
|
||||
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
||||
import { withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
||||
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
||||
@@ -461,6 +462,27 @@ export function runNodeWebProbeObserveStart(
|
||||
material: BootstrapAdminPasswordMaterial,
|
||||
credential: Record<string, unknown>,
|
||||
): Record<string, unknown> | RenderedCliResult {
|
||||
const memoryGuard = runWebProbeMemoryGuard(spec, options.memoryGuardMode);
|
||||
if (memoryGuard.status !== "allowed") {
|
||||
const cadenceSkip = options.memoryGuardMode === "sentinel-cadence" && memoryGuard.status === "skipped";
|
||||
return {
|
||||
ok: cadenceSkip,
|
||||
status: cadenceSkip ? "skipped-wait-next-round" : "blocked",
|
||||
failureKind: "web-probe-memory-start-guard",
|
||||
command: `web-probe observe start --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
observerCreated: false,
|
||||
startBlocked: true,
|
||||
decision: cadenceSkip ? "skip-observer" : "gc-required",
|
||||
mutation: false,
|
||||
memoryGuard,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
const observerLifecycle = spec.webProbe?.resourcePolicy?.observerLifecycle;
|
||||
if (observerLifecycle === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML");
|
||||
const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
||||
const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
|
||||
const day = timestamp.slice(0, 8);
|
||||
@@ -494,6 +516,12 @@ export function runNodeWebProbeObserveStart(
|
||||
`UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS=${shellQuote(String(options.observerRefreshIntervalMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_RUN_MS=${shellQuote(String(options.maxRunSeconds > 0 ? options.maxRunSeconds * 1000 : 0))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS=${shellQuote(String(observerLifecycle.maxScreenshots))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES=${shellQuote(String(observerLifecycle.maxPerformanceEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserProcessHistoryEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserFreezeSignalHistoryEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxResponseBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxWebPerformanceBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
|
||||
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
||||
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
|
||||
@@ -592,6 +620,7 @@ export function runNodeWebProbeObserveStart(
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
projectManagement,
|
||||
memoryGuard,
|
||||
targetPath: options.targetPath,
|
||||
id: observerId,
|
||||
degradedReason,
|
||||
|
||||
@@ -183,8 +183,42 @@ test("observe start inherits finite observer lifecycle limits from the owning YA
|
||||
|
||||
assert.equal(options.maxRunSeconds, 3600);
|
||||
assert.equal(options.maxSamples, 720);
|
||||
assert.equal(options.memoryGuardMode, "manual-start");
|
||||
assert.ok(options.maxRunSeconds > 0);
|
||||
assert.ok(options.maxSamples > 0);
|
||||
assert.deepEqual(spec.webProbe?.resourcePolicy?.observerLifecycle, {
|
||||
maxRunSeconds: 3600,
|
||||
maxSamples: 720,
|
||||
maxScreenshots: 48,
|
||||
maxPerformanceEntries: 2000,
|
||||
maxBrowserProcessHistoryEntries: 360,
|
||||
maxBrowserFreezeSignalHistoryEntries: 120,
|
||||
maxResponseBodyBytes: 524288,
|
||||
maxWebPerformanceBodyBytes: 65536,
|
||||
});
|
||||
});
|
||||
|
||||
test("sentinel observe start preserves the cadence comparator through the child CLI", () => {
|
||||
const options = parseNodeWebProbeObserveOptions(
|
||||
["start", "--sentinel-cadence"],
|
||||
"NC01",
|
||||
"v03",
|
||||
spec,
|
||||
null,
|
||||
null,
|
||||
);
|
||||
assert.equal(options.memoryGuardMode, "sentinel-cadence");
|
||||
});
|
||||
|
||||
test("observe start cannot disable or exceed the YAML lifecycle caps", () => {
|
||||
assert.throws(
|
||||
() => parseNodeWebProbeObserveOptions(["start", "--max-run-seconds", "0"], "NC01", "v03", spec, null, null),
|
||||
/must be a positive integer/u,
|
||||
);
|
||||
assert.throws(
|
||||
() => parseNodeWebProbeObserveOptions(["start", "--max-samples", "721"], "NC01", "v03", spec, null, null),
|
||||
/exceeds the owning YAML observer lifecycle cap 720/u,
|
||||
);
|
||||
});
|
||||
|
||||
test("observe start refuses URL-based internal/public selection ambiguity", () => {
|
||||
|
||||
@@ -483,7 +483,7 @@ export function parseNodeWebProbeObserveOptions(
|
||||
"--workspace-root",
|
||||
"--workspace-root-ref",
|
||||
"--root",
|
||||
]), new Set(["--force", "--full", "--raw", "--text-stdin", "--require-composer-ready", "--wait-project-management-ready", "--blocking", "--non-blocking", "--dry-run", "--confirm"]));
|
||||
]), new Set(["--force", "--full", "--raw", "--text-stdin", "--require-composer-ready", "--wait-project-management-ready", "--blocking", "--non-blocking", "--dry-run", "--confirm", "--sentinel-cadence"]));
|
||||
const commandTypeRaw = optionValue(args, "--type") ?? null;
|
||||
const commandType = commandTypeRaw === null ? null : parseNodeWebProbeObserveCommandType(commandTypeRaw);
|
||||
const stateDir = optionValue(args, "--state-dir") ?? indexed?.stateDir ?? null;
|
||||
@@ -528,6 +528,9 @@ export function parseNodeWebProbeObserveOptions(
|
||||
if (observeActionRaw !== "start" && (requestedOrigin !== undefined || customUrl !== undefined || browserProxyModeRaw !== undefined)) {
|
||||
throw new Error("web-probe observe --origin/--url/--browser-proxy-mode is only valid for start; later actions inherit the observer origin");
|
||||
}
|
||||
if (observeActionRaw !== "start" && args.includes("--sentinel-cadence")) {
|
||||
throw new Error("web-probe observe --sentinel-cadence is only valid for start");
|
||||
}
|
||||
const requestedBrowserProxyMode = browserProxyModeRaw === undefined ? undefined : parseWebProbeBrowserProxyMode(browserProxyModeRaw);
|
||||
const selectedOrigin = observeActionRaw === "start"
|
||||
? resolveNodeWebProbeCliOrigin(spec, requestedOrigin, customUrl, requestedBrowserProxyMode)
|
||||
@@ -645,6 +648,11 @@ export function parseNodeWebProbeObserveOptions(
|
||||
throw new Error("owning YAML 未启用 validateWorkbenchTraceReadability");
|
||||
}
|
||||
}
|
||||
const resourcePolicy = spec.webProbe?.resourcePolicy;
|
||||
if (resourcePolicy === undefined) {
|
||||
throw new Error(`web-probe resource policy is missing from ${hwlabRuntimeLaneConfigPath} for ${node}/${lane}`);
|
||||
}
|
||||
const observerLifecycle = resourcePolicy.observerLifecycle;
|
||||
return {
|
||||
action: "observe",
|
||||
observeAction: observeActionRaw,
|
||||
@@ -659,11 +667,12 @@ export function parseNodeWebProbeObserveOptions(
|
||||
viewport: optionValue(args, "--viewport") ?? "1440x900",
|
||||
browserProxyMode: selectedOrigin.browserProxyMode,
|
||||
browserProxyModeSource: selectedOrigin.browserProxyModeSource,
|
||||
memoryGuardMode: args.includes("--sentinel-cadence") ? "sentinel-cadence" : "manual-start",
|
||||
sampleIntervalMs: positiveIntegerOption(args, "--sample-interval-ms", 5000, 600000),
|
||||
screenshotIntervalMs: positiveIntegerOption(args, "--screenshot-interval-ms", 300000, 86_400_000),
|
||||
observerRefreshIntervalMs: positiveIntegerOption(args, "--observer-refresh-interval-ms", 180000, 86_400_000),
|
||||
maxSamples: positiveIntegerOption(args, "--max-samples", 0, 10_000_000),
|
||||
maxRunSeconds: positiveIntegerOption(args, "--max-run-seconds", 0, 86_400),
|
||||
maxSamples: observerLifecycleOption(args, "--max-samples", observerLifecycle.maxSamples),
|
||||
maxRunSeconds: observerLifecycleOption(args, "--max-run-seconds", observerLifecycle.maxRunSeconds),
|
||||
commandTimeoutSeconds: positiveIntegerOption(args, "--command-timeout-seconds", 55, 3600),
|
||||
waitMs: positiveIntegerOption(args, "--wait-ms", 0, 600000),
|
||||
tailLines: positiveIntegerOption(args, "--tail-lines", 5, 200),
|
||||
@@ -728,6 +737,17 @@ export function parseNodeWebProbeObserveOptions(
|
||||
};
|
||||
}
|
||||
|
||||
function observerLifecycleOption(args: string[], name: string, yamlMaximum: number): number {
|
||||
const raw = optionValue(args, name);
|
||||
if (raw === undefined) return yamlMaximum;
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value < 1) throw new Error(`${name} must be a positive integer`);
|
||||
if (value > yamlMaximum) {
|
||||
throw new Error(`${name}=${value} exceeds the owning YAML observer lifecycle cap ${yamlMaximum}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function parseNodeWebProbeObserveCommandType(value: string): NodeWebProbeObserveCommandType {
|
||||
if (
|
||||
value === "login"
|
||||
|
||||
Reference in New Issue
Block a user