From 0c22a05cee83182744f9e7a0b5da028b72059cf0 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 07:10:26 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=B8=BA=20WebProbe=20=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E7=89=A9=E7=90=86=E5=86=85=E5=AD=98=E9=97=A8=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/hwlab-node-lanes.yaml | 18 ++ config/unidesk-cli.yaml | 22 +++ scripts/src/gc-remote-runner.py | 79 ++++++-- scripts/src/gc-remote.ts | 7 +- scripts/src/hwlab-node-lanes.ts | 60 ++++++ ...e-web-observe-runner-performance-source.ts | 6 +- ...-node-web-observe-runner-runtime-source.ts | 21 +++ ...node-web-observe-runner-sampling-source.ts | 20 +- .../hwlab-node-web-observe-runner-source.ts | 60 ++++-- .../hwlab-node-web-probe-memory-guard.test.ts | 140 ++++++++++++++ .../src/hwlab-node-web-probe-memory-guard.ts | 174 ++++++++++++++++++ ...hwlab-node-web-sentinel-p5-observe.test.ts | 10 +- .../src/hwlab-node-web-sentinel-p5-observe.ts | 59 ++++++ .../src/hwlab-node-web-sentinel-service.ts | 1 + scripts/src/hwlab-node/entry.ts | 1 + .../hwlab-node/web-probe-observe-actions.ts | 29 +++ .../web-probe-observe-options.test.ts | 34 ++++ scripts/src/hwlab-node/web-probe-observe.ts | 26 ++- 18 files changed, 727 insertions(+), 40 deletions(-) create mode 100644 scripts/src/hwlab-node-web-probe-memory-guard.test.ts create mode 100644 scripts/src/hwlab-node-web-probe-memory-guard.ts diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index aa831aad..9a6b641f 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -550,6 +550,24 @@ templates: webProbeWorkbench: browserProxyMode: auto defaultOrigin: internal + resourcePolicy: + memoryStartGuard: + metric: MemAvailable + source: /proc/meminfo + swapExcluded: true + thresholdBytes: 4294967296 + manualComparator: less-than-or-equal + sentinelComparator: less-than + snapshotTimeoutSeconds: 30 + observerLifecycle: + maxRunSeconds: 3600 + maxSamples: 720 + maxScreenshots: 48 + maxPerformanceEntries: 2000 + maxBrowserProcessHistoryEntries: 360 + maxBrowserFreezeSignalHistoryEntries: 120 + maxResponseBodyBytes: 524288 + maxWebPerformanceBodyBytes: 65536 origins: internal: mode: k8s-service-cluster-ip diff --git a/config/unidesk-cli.yaml b/config/unidesk-cli.yaml index 936e3217..ff7d85ce 100644 --- a/config/unidesk-cli.yaml +++ b/config/unidesk-cli.yaml @@ -123,6 +123,28 @@ gc: full: false remote: targets: + NC01: + hostDockerGc: + dockerJsonLogs: + enabled: false + reason: NC01 主机 Docker 承载主服务运行面,通用远端 GC 不得截断活跃服务日志。 + buildCachePrune: + enabled: false + reason: NC01 不是通用构建主机,通用远端 GC 不得清理其活跃 Docker 状态。 + memoryPressure: + processPatterns: + - k3s-server + - containerd + - chrome + - chromium + - observer-runner + - playwright + observeStateRoots: + - /root/hwlab-v03/.state/web-observe + - /root/hwlab/.state/web-observe + observeRunDepth: 6 + observeScanLimit: 2000 + staleRunMaxAgeHours: 1 JD01: hostDockerGc: dockerJsonLogs: diff --git a/scripts/src/gc-remote-runner.py b/scripts/src/gc-remote-runner.py index 511d29e0..685a0da8 100644 --- a/scripts/src/gc-remote-runner.py +++ b/scripts/src/gc-remote-runner.py @@ -569,24 +569,50 @@ def collect_process_pressure(patterns): } def collect_memory_snapshot(): - result = command(["free", "-b"], 5) - if result["exitCode"] != 0: - return {"ok": False, "error": "free-failed", "command": bounded(result)} - memory = {} - for line in result["stdout"].splitlines(): - parts = line.split() - if parts and parts[0].rstrip(":") == "Mem" and len(parts) >= 7: - memory = { - "totalBytes": safe_int(parts[1]), - "usedBytes": safe_int(parts[2]), - "freeBytes": safe_int(parts[3]), - "availableBytes": safe_int(parts[6]), - "totalHuman": fmt_bytes(parts[1]), - "usedHuman": fmt_bytes(parts[2]), - "availableHuman": fmt_bytes(parts[6]), - } - break - return {"ok": bool(memory), "memory": memory, "command": bounded(result)} + source = "/proc/meminfo" + fields = {} + try: + with open(source, "r", encoding="utf-8") as handle: + for line in handle: + if ":" not in line: + continue + key, raw = line.split(":", 1) + parts = raw.strip().split() + if not parts: + continue + fields[key] = safe_int(parts[0]) * 1024 + except OSError as exc: + return { + "ok": False, + "error": "proc-meminfo-read-failed", + "metric": "MemAvailable", + "source": source, + "swapExcluded": True, + "errorType": type(exc).__name__, + } + available_bytes = fields.get("MemAvailable") + if available_bytes is None: + return { + "ok": False, + "error": "memavailable-missing", + "metric": "MemAvailable", + "source": source, + "swapExcluded": True, + } + total_bytes = fields.get("MemTotal") + memory = { + "totalBytes": total_bytes, + "availableBytes": available_bytes, + "totalHuman": fmt_bytes(total_bytes) if total_bytes is not None else None, + "availableHuman": fmt_bytes(available_bytes), + } + return { + "ok": True, + "metric": "MemAvailable", + "source": source, + "swapExcluded": True, + "memory": memory, + } def observe_run_record(path, stale_hours): stat = os.stat(path) @@ -1795,6 +1821,23 @@ def remote_policy_install_payload(observed_at): def main(): observed_at = now_iso() + if ACTION == "memory": + memory = collect_memory_snapshot() + emit_json({ + "ok": memory.get("ok") is True, + "action": "gc remote memory", + "providerId": PROVIDER_ID, + "dryRun": True, + "mutation": False, + "observedAt": observed_at, + "metric": memory.get("metric"), + "source": memory.get("source"), + "swapExcluded": memory.get("swapExcluded") is True, + "memory": memory.get("memory"), + "error": memory.get("error"), + "valuesRedacted": True, + }, persist_large=False) + return 0 preflight = cluster_preflight() if ACTION == "policy-plan": emit_json(remote_policy_plan_payload(observed_at), persist_large=False) diff --git a/scripts/src/gc-remote.ts b/scripts/src/gc-remote.ts index c9ec2a6a..cb35601b 100644 --- a/scripts/src/gc-remote.ts +++ b/scripts/src/gc-remote.ts @@ -5,7 +5,7 @@ import { type UniDeskConfig, rootPath } from "./config"; import { remoteGcDegradedFailure } from "./gc-remote-degraded"; import { runSshCommandCapture } from "./ssh"; -type RemoteGcAction = "plan" | "snapshot" | "trend" | "run" | "status" | "policy-plan" | "policy-install" | "policy-status"; +type RemoteGcAction = "plan" | "memory" | "snapshot" | "trend" | "run" | "status" | "policy-plan" | "policy-install" | "policy-status"; interface RemoteGcOptions { confirm: boolean; @@ -89,7 +89,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri return { ok: false, error: "gc-remote-provider-required", - usage: "bun scripts/cli.ts gc remote plan|snapshot|trend|run|status|policy [--confirm]", + usage: "bun scripts/cli.ts gc remote memory|plan|snapshot|trend|run|status|policy [--confirm]", }; } const subaction = action ?? "plan"; @@ -124,6 +124,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri }; } const options = parseRemoteGcOptions(args); + if (subaction === "memory") return await runRemoteGc(config, providerId, "memory", options); if (subaction === "plan" || subaction === "dry-run") return await runRemoteGc(config, providerId, "plan", options); if (subaction === "snapshot" || subaction === "growth") return await runRemoteGc(config, providerId, "snapshot", options); if (subaction === "trend") return await runRemoteGc(config, providerId, "trend", options); @@ -146,7 +147,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri ok: false, error: "unsupported-gc-remote-action", action: subaction, - supportedActions: ["plan", "snapshot", "trend", "run", "status", "policy"], + supportedActions: ["memory", "plan", "snapshot", "trend", "run", "status", "policy"], }; } diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 484fa843..77600997 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -148,6 +148,7 @@ export interface HwlabRuntimeWebProbeSpec { readonly browserProxyMode?: "auto" | "direct"; readonly playwrightBrowsersPath?: string; readonly defaultOrigin?: HwlabRuntimeWebProbeOriginName; + readonly resourcePolicy?: HwlabRuntimeWebProbeResourcePolicySpec; readonly origins?: HwlabRuntimeWebProbeOriginsSpec; readonly authLogin?: HwlabRuntimeWebProbeAuthLoginSpec; readonly alertThresholds?: HwlabRuntimeWebProbeAlertThresholdsSpec; @@ -158,6 +159,32 @@ export interface HwlabRuntimeWebProbeSpec { readonly workbenchTraceReadability?: HwlabRuntimeWebProbeWorkbenchTraceReadabilitySpec; } +export interface HwlabRuntimeWebProbeMemoryStartGuardSpec { + readonly metric: "MemAvailable"; + readonly source: "/proc/meminfo"; + readonly swapExcluded: true; + readonly thresholdBytes: number; + readonly manualComparator: "less-than-or-equal"; + readonly sentinelComparator: "less-than"; + readonly snapshotTimeoutSeconds: number; +} + +export interface HwlabRuntimeWebProbeObserverLifecycleSpec { + readonly maxRunSeconds: number; + readonly maxSamples: number; + readonly maxScreenshots: number; + readonly maxPerformanceEntries: number; + readonly maxBrowserProcessHistoryEntries: number; + readonly maxBrowserFreezeSignalHistoryEntries: number; + readonly maxResponseBodyBytes: number; + readonly maxWebPerformanceBodyBytes: number; +} + +export interface HwlabRuntimeWebProbeResourcePolicySpec { + readonly memoryStartGuard: HwlabRuntimeWebProbeMemoryStartGuardSpec; + readonly observerLifecycle: HwlabRuntimeWebProbeObserverLifecycleSpec; +} + export interface HwlabRuntimeWebProbeWorkbenchKafkaDebugReplaySpec { readonly enabled: boolean; readonly topic: string; @@ -754,6 +781,11 @@ function booleanField(obj: Record, key: string, path: string): return value; } +function literalTrueField(obj: Record, key: string, path: string): true { + if (obj[key] !== true) throw new Error(`${path}.${key} must be true`); + return true; +} + function nonNegativeIntegerField(obj: Record, key: string, path: string): number { const value = obj[key]; if (typeof value !== "number" || !Number.isInteger(value) || value < 0) throw new Error(`${path}.${key} must be a non-negative integer`); @@ -1410,6 +1442,7 @@ function webProbeConfig(value: unknown, path: string): HwlabRuntimeWebProbeSpec ...(browserProxyMode === undefined ? {} : { browserProxyMode }), ...(playwrightBrowsersPath === undefined ? {} : { playwrightBrowsersPath }), ...(defaultOrigin === undefined ? {} : { defaultOrigin }), + ...(raw.resourcePolicy === undefined ? {} : { resourcePolicy: webProbeResourcePolicyConfig(raw.resourcePolicy, `${path}.resourcePolicy`) }), ...(origins === undefined ? {} : { origins }), ...(raw.authLogin === undefined ? {} : { authLogin: webProbeAuthLoginConfig(raw.authLogin, `${path}.authLogin`) }), ...(raw.alertThresholds === undefined ? {} : { alertThresholds: webProbeAlertThresholdsConfig(raw.alertThresholds, `${path}.alertThresholds`) }), @@ -1425,6 +1458,33 @@ function webProbeConfig(value: unknown, path: string): HwlabRuntimeWebProbeSpec }; } +function webProbeResourcePolicyConfig(value: unknown, path: string): HwlabRuntimeWebProbeResourcePolicySpec { + const raw = asRecord(value, path); + const memory = asRecord(raw.memoryStartGuard, `${path}.memoryStartGuard`); + const lifecycle = asRecord(raw.observerLifecycle, `${path}.observerLifecycle`); + return { + memoryStartGuard: { + metric: enumStringField(memory, "metric", `${path}.memoryStartGuard`, ["MemAvailable"] as const), + source: enumStringField(memory, "source", `${path}.memoryStartGuard`, ["/proc/meminfo"] as const), + swapExcluded: literalTrueField(memory, "swapExcluded", `${path}.memoryStartGuard`), + thresholdBytes: boundedIntegerField(memory, "thresholdBytes", `${path}.memoryStartGuard`, 1, Number.MAX_SAFE_INTEGER), + manualComparator: enumStringField(memory, "manualComparator", `${path}.memoryStartGuard`, ["less-than-or-equal"] as const), + sentinelComparator: enumStringField(memory, "sentinelComparator", `${path}.memoryStartGuard`, ["less-than"] as const), + snapshotTimeoutSeconds: boundedIntegerField(memory, "snapshotTimeoutSeconds", `${path}.memoryStartGuard`, 5, 120), + }, + observerLifecycle: { + maxRunSeconds: boundedIntegerField(lifecycle, "maxRunSeconds", `${path}.observerLifecycle`, 1, 86_400), + maxSamples: boundedIntegerField(lifecycle, "maxSamples", `${path}.observerLifecycle`, 1, 10_000_000), + maxScreenshots: boundedIntegerField(lifecycle, "maxScreenshots", `${path}.observerLifecycle`, 1, 10_000), + maxPerformanceEntries: boundedIntegerField(lifecycle, "maxPerformanceEntries", `${path}.observerLifecycle`, 1, 100_000), + maxBrowserProcessHistoryEntries: boundedIntegerField(lifecycle, "maxBrowserProcessHistoryEntries", `${path}.observerLifecycle`, 1, 100_000), + maxBrowserFreezeSignalHistoryEntries: boundedIntegerField(lifecycle, "maxBrowserFreezeSignalHistoryEntries", `${path}.observerLifecycle`, 1, 100_000), + maxResponseBodyBytes: boundedIntegerField(lifecycle, "maxResponseBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024), + maxWebPerformanceBodyBytes: boundedIntegerField(lifecycle, "maxWebPerformanceBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024), + }, + }; +} + function webProbeWorkbenchKafkaDebugReplayConfig(value: unknown, path: string): HwlabRuntimeWebProbeWorkbenchKafkaDebugReplaySpec { const raw = asRecord(value, path); const topic = stringField(raw, "topic", path); diff --git a/scripts/src/hwlab-node-web-observe-runner-performance-source.ts b/scripts/src/hwlab-node-web-observe-runner-performance-source.ts index be0cf785..3da42d54 100644 --- a/scripts/src/hwlab-node-web-observe-runner-performance-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-performance-source.ts @@ -16,7 +16,7 @@ async function installPagePerformanceProbe(targetPage, pageRole = "control", tar } const buffer = []; const supportedTypes = []; - const maxBufferSize = 2000; + const maxBufferSize = Math.max(1, Number(input.maxBufferSize || 1)); const push = (event) => { buffer.push({ probeVersion: 1, @@ -126,9 +126,9 @@ async function installPagePerformanceProbe(targetPage, pageRole = "control", tar }; return { ok: true, alreadyInstalled: false, supportedTypes, valuesRedacted: true }; }; - await targetPage.addInitScript(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs }); + await targetPage.addInitScript(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs, maxBufferSize: maxPerformanceEntries }); if (!targetPage.isClosed()) { - await withHardTimeout(targetPage.evaluate(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs }), 3000, "installPagePerformanceProbe evaluate exceeded 3000ms"); + await withHardTimeout(targetPage.evaluate(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs, maxBufferSize: maxPerformanceEntries }), 3000, "installPagePerformanceProbe evaluate exceeded 3000ms"); } } diff --git a/scripts/src/hwlab-node-web-observe-runner-runtime-source.ts b/scripts/src/hwlab-node-web-observe-runner-runtime-source.ts index c066f3f9..12c18da8 100644 --- a/scripts/src/hwlab-node-web-observe-runner-runtime-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-runtime-source.ts @@ -68,6 +68,15 @@ async function writeManifest(extra = {}) { observerInitiatedDefault: false, responseBodyReadDefault: false, }, + resourceLimits: { + maxScreenshots, + maxPerformanceEntries, + maxBrowserProcessHistoryEntries, + maxBrowserFreezeSignalHistoryEntries, + responseBodyMaxBytes, + webPerformanceRequestBodyMaxBytes, + valuesRedacted: true, + }, alertThresholds, browserFreezePolicy, projectManagement, @@ -100,6 +109,16 @@ async function writeHeartbeat(extra = {}) { maxRunSeconds: maxRunMs > 0 ? Math.ceil(maxRunMs / 1000) : 0, valuesRedacted: true, }, + resourceLimits: { + screenshotCount, + maxScreenshots, + maxPerformanceEntries, + maxBrowserProcessHistoryEntries, + maxBrowserFreezeSignalHistoryEntries, + responseBodyMaxBytes, + webPerformanceRequestBodyMaxBytes, + valuesRedacted: true, + }, lastObserverRefreshAt: Number.isFinite(lastObserverRefreshAtMs) ? new Date(lastObserverRefreshAtMs).toISOString() : null, pageProvenance: compactPageProvenance(currentPageProvenance), sampleSeq, @@ -335,6 +354,7 @@ function recordBrowserFreezeSignal(kind, sample, pageMetric, detail) { const windowMs = browserFreezePolicy.blockerWindowMs; const cutoff = signal.tsMs - windowMs; while (browserFreezeSignalHistory.length > 0 && Number(browserFreezeSignalHistory[0].tsMs || 0) < cutoff) browserFreezeSignalHistory.shift(); + while (browserFreezeSignalHistory.length > maxBrowserFreezeSignalHistoryEntries) browserFreezeSignalHistory.shift(); return { kind, rootCause: detail.rootCause, @@ -639,6 +659,7 @@ function updateBrowserProcessHistory(tsMs, processSummary) { browserProcessHistory.push({ tsMs, totalRssBytes, maxProcessRssBytes }); const retentionMs = Math.max(windowMs * 3, 180000); while (browserProcessHistory.length > 0 && tsMs - browserProcessHistory[0].tsMs > retentionMs) browserProcessHistory.shift(); + while (browserProcessHistory.length > maxBrowserProcessHistoryEntries) browserProcessHistory.shift(); const windowStartMs = tsMs - windowMs; const candidates = browserProcessHistory.filter((item) => item.tsMs >= windowStartMs && item.tsMs <= tsMs); const baseline = candidates[0] || browserProcessHistory[0] || null; diff --git a/scripts/src/hwlab-node-web-observe-runner-sampling-source.ts b/scripts/src/hwlab-node-web-observe-runner-sampling-source.ts index c0b6809d..6c01a99d 100644 --- a/scripts/src/hwlab-node-web-observe-runner-sampling-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-sampling-source.ts @@ -797,6 +797,23 @@ function digestSessionRail(value) { async function captureScreenshot(reason, imageType = "png") { if (!page || page.isClosed()) throw new Error("page is not available for screenshot"); + if (screenshotCount >= maxScreenshots) { + lastScreenshotAtMs = Date.now(); + return { + seq: null, + sampleSeq, + ts: new Date().toISOString(), + kind: "screenshot-skipped", + reason, + skipped: true, + skipReason: "screenshot-cap-reached", + screenshotCount, + maxScreenshots, + pageId, + currentUrl: currentPageUrl(), + valuesRedacted: true, + }; + } if (screenshotCaptureState && screenshotCaptureState.settled !== true) { const ageMs = Date.now() - Number(screenshotCaptureState.startedAtMs || Date.now()); const error = new Error("screenshot capture already in progress"); @@ -814,6 +831,7 @@ async function captureScreenshot(reason, imageType = "png") { lastScreenshotAtMs = Date.now(); throw error; } + screenshotCount += 1; artifactSeq += 1; const safeReason = safeId(String(reason || "manual")).slice(0, 40) || "manual"; const type = imageType === "jpeg" || imageType === "jpg" ? "jpeg" : "png"; @@ -888,7 +906,7 @@ async function summarizeWorkbenchResponseBody(response, request) { const contentType = String(headers["content-type"] || headers["Content-Type"] || ""); if (!/json/iu.test(contentType)) return { bodyRead: false, bodyReadSkipped: "non-json", valuesRedacted: true }; const contentLength = Number(headers["content-length"] || headers["Content-Length"]); - const maxBytes = 512 * 1024; + const maxBytes = responseBodyMaxBytes; if (Number.isFinite(contentLength) && contentLength > maxBytes) return { bodyRead: false, bodyReadSkipped: "content-length-too-large", bodyByteCount: contentLength, valuesRedacted: true }; const text = await response.text(); const byteCount = Buffer.byteLength(text); diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 96467a06..6924940f 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -31,9 +31,14 @@ const targetPath = process.env.UNIDESK_WEB_OBSERVE_TARGET_PATH || "/workbench"; const sampleIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS, 5000); const screenshotIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS, 300000); const screenshotCaptureTimeoutMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_CAPTURE_TIMEOUT_MS, 15000, 1000, 120000); -const maxSamples = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 0); +const maxSamples = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 720, 1, 10000000); const observerRefreshIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS, 180000); -const maxRunMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RUN_MS, 0); +const maxRunMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RUN_MS, 3600000, 1000, 86400000); +const maxScreenshots = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS, 48, 1, 10000); +const maxPerformanceEntries = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES, 2000, 1, 100000); +const maxBrowserProcessHistoryEntries = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES, 360, 1, 100000); +const maxBrowserFreezeSignalHistoryEntries = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES, 120, 1, 100000); +const responseBodyMaxBytes = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES, 524288, 1024, 16777216); const viewport = parseViewport(process.env.UNIDESK_WEB_OBSERVE_VIEWPORT || "1440x900"); const browserProxyMode = parseBrowserProxyMode(process.env.UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE || "auto"); const origin = { @@ -92,6 +97,7 @@ let lastObserverRefreshAtMs = Date.now(); let sampleSeq = 0; let commandSeq = 0; let artifactSeq = 0; +let screenshotCount = 0; let activeCommandId = null; let activeCommandType = null; let stopping = false; @@ -107,11 +113,22 @@ let heartbeatPulseTimer = null; let browserProcessMonitorStop = null; let browserProcessMonitorSeq = 0; let browserFreezeBlocker = null; +let shutdownSignal = null; +let browserResourcesClosePromise = null; const browserProcessHistory = []; const browserPageRuntimeBaselines = new Map(); const browserFreezeSignalHistory = []; const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] }; +for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) { + process.once(signal, () => { + shutdownSignal = signal; + stopping = true; + terminalStatus = "stopping"; + void closeBrowserResources("signal:" + signal); + }); +} + try { if (!password) throw new Error("missing HWLAB_WEB_PASS"); await prepareDirs(); @@ -152,18 +169,18 @@ try { await writeManifest({ status: "running", auth: publicAuth(auth) }); await writeHeartbeat({ status: "running" }); while (!stopping) { - await drainOneCommand(); - if (browserFreezeBlocker) break; - await samplePage("interval"); - if (browserFreezeBlocker) break; - if (maxSamples > 0 && sampleSeq >= maxSamples) { - await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples })); - break; - } if (maxRunMs > 0 && Date.now() - startedAtMs >= maxRunMs) { await appendJsonl(files.control, controlRecord({ id: "max-run-ms", type: "stop", source: "sampler" }, "completed", { reason: "max-run-ms", maxRunMs, elapsedMs: Date.now() - startedAtMs })); break; } + if (maxSamples > 0 && sampleSeq >= maxSamples) { + await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples })); + break; + } + await drainOneCommand(); + if (browserFreezeBlocker) break; + await samplePage("interval"); + if (browserFreezeBlocker) break; await sleep(sampleIntervalMs); } if (browserFreezeBlocker) { @@ -186,7 +203,28 @@ try { } finally { if (browserProcessMonitorStop) browserProcessMonitorStop(); if (heartbeatPulseTimer) clearInterval(heartbeatPulseTimer); - if (browser) await browser.close().catch(() => {}); + await closeBrowserResources(shutdownSignal === null ? "finally" : "signal:" + shutdownSignal); +} + +async function closeBrowserResources(reason) { + if (browserResourcesClosePromise) return browserResourcesClosePromise; + browserResourcesClosePromise = (async () => { + stopping = true; + const closingObserverPage = observerPage; + const closingPage = page; + const closingContext = context; + const closingBrowser = browser; + observerPage = null; + page = null; + context = null; + browser = null; + if (closingObserverPage && !closingObserverPage.isClosed()) await closingObserverPage.close().catch(() => {}); + if (closingPage && !closingPage.isClosed()) await closingPage.close().catch(() => {}); + if (closingContext) await closingContext.close().catch(() => {}); + if (closingBrowser) await closingBrowser.close().catch(() => {}); + return { ok: true, reason, valuesRedacted: true }; + })(); + return browserResourcesClosePromise; } ${nodeWebObserveRunnerRuntimeSource()} diff --git a/scripts/src/hwlab-node-web-probe-memory-guard.test.ts b/scripts/src/hwlab-node-web-probe-memory-guard.test.ts new file mode 100644 index 00000000..0265e83d --- /dev/null +++ b/scripts/src/hwlab-node-web-probe-memory-guard.test.ts @@ -0,0 +1,140 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "bun:test"; + +import type { CommandResult } from "./command"; +import { rootPath } from "./config"; +import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; +import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard, webProbeMemoryStartGuardConfigRef } from "./hwlab-node-web-probe-memory-guard"; + +const spec = hwlabRuntimeLaneSpecForNode("v03", "NC01"); +const thresholdBytes = 4 * 1024 * 1024 * 1024; + +function memoryPayload(availableBytes: number, extra: Record = {}): Record { + return { + ok: true, + action: "gc remote memory", + providerId: "NC01", + metric: "MemAvailable", + source: "/proc/meminfo", + swapExcluded: true, + memory: { availableBytes, availableHuman: "fixture", ...extra }, + valuesRedacted: true, + }; +} + +test("WebProbe memory threshold is read from the single owning YAML policy", () => { + const policy = spec.webProbe?.resourcePolicy?.memoryStartGuard; + assert.equal(policy?.thresholdBytes, thresholdBytes); + assert.equal(policy?.metric, "MemAvailable"); + assert.equal(policy?.source, "/proc/meminfo"); + assert.equal(policy?.swapExcluded, true); + assert.equal(policy?.manualComparator, "less-than-or-equal"); + assert.equal(policy?.sentinelComparator, "less-than"); + assert.match(webProbeMemoryStartGuardConfigRef, /resourcePolicy\.memoryStartGuard$/u); +}); + +test("manual start is blocked at exactly 4 GiB and requires controlled GC then MemAvailable recheck", () => { + const guard = evaluateWebProbeMemoryGuard(spec, "manual-start", memoryPayload(thresholdBytes)); + const memory = guard.memory as Record; + const gcNext = guard.gcNext as Record; + + assert.equal(guard.status, "blocked"); + assert.equal(guard.startBlocked, true); + assert.equal(guard.decision, "gc-required"); + assert.equal(memory.metric, "MemAvailable"); + assert.equal(memory.source, "/proc/meminfo"); + assert.equal(memory.swapExcluded, true); + assert.equal(memory.comparator, "less-than-or-equal"); + assert.equal(memory.thresholdMatched, true); + assert.equal(gcNext.requiredBeforeStart, true); + assert.match(String(gcNext.planCommand), /gc remote NC01 plan$/u); + assert.match(String(gcNext.runCommand), /gc remote NC01 run --confirm$/u); + assert.match(String(gcNext.recheckCommand), /gc remote NC01 memory$/u); + assert.equal(gcNext.recheckMetric, "MemAvailable"); +}); + +test("sentinel allows exactly 4 GiB but skips one byte below without considering swap", () => { + const exact = evaluateWebProbeMemoryGuard(spec, "sentinel-cadence", memoryPayload(thresholdBytes, { swapFreeBytes: 12 * 1024 ** 3 })); + const below = evaluateWebProbeMemoryGuard(spec, "sentinel-cadence", memoryPayload(thresholdBytes - 1, { swapFreeBytes: 12 * 1024 ** 3 })); + const belowMemory = below.memory as Record; + + assert.equal(exact.status, "allowed"); + assert.equal(exact.startBlocked, false); + assert.equal((exact.memory as Record).comparator, "less-than"); + assert.equal(below.status, "skipped"); + assert.equal(below.startBlocked, true); + assert.equal(below.decision, "skip-observer"); + assert.equal(belowMemory.availableBytes, thresholdBytes - 1); + assert.equal(belowMemory.swapExcluded, true); +}); + +test("memory guard rejects a snapshot that is not the /proc/meminfo MemAvailable contract", () => { + const guard = evaluateWebProbeMemoryGuard(spec, "manual-start", { + ...memoryPayload(thresholdBytes + 1), + source: "cgroup-v2-memory.current", + }); + assert.equal(guard.status, "blocked"); + assert.equal(guard.reason, "memory-snapshot-contract-invalid"); + assert.equal(guard.startBlocked, true); +}); + +test("memory guard executes only the controlled read-only remote memory entry", () => { + let observedCommand: string[] = []; + let observedTimeoutMs = 0; + const execute = (command: string[], cwd: string, options: { timeoutMs?: number }): CommandResult => { + observedCommand = command; + observedTimeoutMs = options.timeoutMs ?? 0; + return { + command, + cwd, + exitCode: 0, + stdout: JSON.stringify({ ok: true, command: "gc remote NC01 memory", data: memoryPayload(thresholdBytes + 1) }), + stderr: "", + signal: null, + timedOut: false, + }; + }; + const guard = runWebProbeMemoryGuard(spec, "manual-start", execute); + + assert.deepEqual(observedCommand, ["bun", "scripts/cli.ts", "gc", "remote", "NC01", "memory"]); + assert.equal(observedTimeoutMs, 30_000); + assert.equal(guard.status, "allowed"); +}); + +test("remote memory collector reads MemAvailable directly and excludes swap", () => { + const source = readFileSync(rootPath("scripts/src/gc-remote-runner.py"), "utf8"); + const start = source.indexOf("def collect_memory_snapshot():"); + const end = source.indexOf("\ndef observe_run_record", start); + const collector = source.slice(start, end); + + assert.match(collector, /source = "\/proc\/meminfo"/u); + assert.match(collector, /fields\.get\("MemAvailable"\)/u); + assert.match(collector, /"swapExcluded": True/u); + assert.doesNotMatch(collector, /SwapFree|SwapTotal|\["free", "-b"\]/u); +}); + +test("manual and sentinel gates return before any observer launch path", () => { + const manualSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe-actions.ts"), "utf8"); + const manualStart = manualSource.indexOf("export function runNodeWebProbeObserveStart"); + const manualGuard = manualSource.indexOf("runWebProbeMemoryGuard(spec, options.memoryGuardMode)", manualStart); + const manualReturn = manualSource.indexOf("startBlocked: true", manualGuard); + const manualJob = manualSource.indexOf("const jobId =", manualStart); + const manualLaunch = manualSource.indexOf("setsid env", manualStart); + assert.ok(manualStart >= 0 && manualGuard > manualStart && manualReturn > manualGuard); + assert.ok(manualReturn < manualJob && manualJob < manualLaunch); + + const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5-observe.ts"), "utf8"); + const quickVerify = sentinelSource.indexOf("export function runSentinelQuickVerify"); + const sentinelGuard = sentinelSource.indexOf("runWebProbeMemoryGuard(state.spec, \"sentinel-cadence\")", quickVerify); + const sentinelSkip = sentinelSource.indexOf('status: "skipped-wait-next-round"', sentinelGuard); + const providerPreflight = sentinelSource.indexOf("runSentinelExactProviderProfilePreflight", quickVerify); + const observeStart = sentinelSource.indexOf('"web-probe", "observe", "start"', quickVerify); + assert.ok(quickVerify >= 0 && sentinelGuard > quickVerify && sentinelSkip > sentinelGuard); + assert.ok(sentinelSkip < providerPreflight && providerPreflight < observeStart); + assert.match(sentinelSource.slice(observeStart, observeStart + 800), /--sentinel-cadence/u); + + const serviceSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-service.ts"), "utf8"); + const plannedStart = serviceSource.indexOf('"bun", "scripts/cli.ts", "web-probe", "observe", "start"'); + assert.match(serviceSource.slice(plannedStart, plannedStart + 800), /--sentinel-cadence/u); +}); diff --git a/scripts/src/hwlab-node-web-probe-memory-guard.ts b/scripts/src/hwlab-node-web-probe-memory-guard.ts new file mode 100644 index 00000000..458cb5ef --- /dev/null +++ b/scripts/src/hwlab-node-web-probe-memory-guard.ts @@ -0,0 +1,174 @@ +// SPEC: PJ2026-01060508 Web 哨兵。WebProbe 启动前的 YAML-first 物理内存资格判定。 +import type { CommandResult } from "./command"; +import { runCommand } from "./command"; +import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery"; +import { repoRoot } from "./config"; +import type { HwlabRuntimeLaneSpec, HwlabRuntimeWebProbeMemoryStartGuardSpec } from "./hwlab-node-lanes"; + +export const webProbeMemoryStartGuardConfigRef = "config/hwlab-node-lanes.yaml#templates.hwlabV03.webProbeWorkbench.resourcePolicy.memoryStartGuard"; + +export type WebProbeMemoryGuardMode = "manual-start" | "sentinel-cadence"; +export type WebProbeMemoryGuardStatus = "allowed" | "blocked" | "skipped"; + +type MemoryGuardExecutor = ( + command: string[], + cwd: string, + options: { timeoutMs?: number }, +) => CommandResult; + +function record(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +function memoryPolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeMemoryStartGuardSpec | null { + return spec.webProbe?.resourcePolicy?.memoryStartGuard ?? null; +} + +function humanBytes(value: number | null): string | null { + if (value === null || !Number.isFinite(value)) return null; + const units = ["B", "KiB", "MiB", "GiB", "TiB"]; + let current = value; + let index = 0; + while (Math.abs(current) >= 1024 && index < units.length - 1) { + current /= 1024; + index += 1; + } + return `${current.toFixed(index === 0 ? 0 : 2)} ${units[index]}`; +} + +function controlledGcNext(node: string): Record { + return { + automaticGc: false, + requiredOrder: ["plan", "run", "recheck"], + planCommand: `bun scripts/cli.ts gc remote ${node} plan`, + runCommand: `bun scripts/cli.ts gc remote ${node} run --confirm`, + recheckCommand: `bun scripts/cli.ts gc remote ${node} memory`, + recheckMetric: "MemAvailable", + recheckSource: "/proc/meminfo", + swapExcluded: true, + valuesRedacted: true, + }; +} + +function unavailableGuard(spec: HwlabRuntimeLaneSpec, mode: WebProbeMemoryGuardMode, reason: string): Record { + const policy = memoryPolicy(spec); + const comparator = policy === null + ? null + : mode === "manual-start" + ? policy.manualComparator + : policy.sentinelComparator; + return { + ok: false, + status: "blocked" satisfies WebProbeMemoryGuardStatus, + eligible: false, + startBlocked: true, + decision: "do-not-start-observer", + reason, + mode, + policyConfigRef: webProbeMemoryStartGuardConfigRef, + memory: { + metric: policy?.metric ?? "MemAvailable", + source: policy?.source ?? "/proc/meminfo", + swapExcluded: policy?.swapExcluded ?? true, + availableBytes: null, + availableHuman: null, + thresholdBytes: policy?.thresholdBytes ?? null, + thresholdHuman: humanBytes(policy?.thresholdBytes ?? null), + comparator, + thresholdMatched: null, + valuesRedacted: true, + }, + gcNext: controlledGcNext(spec.nodeId), + valuesRedacted: true, + }; +} + +export function evaluateWebProbeMemoryGuard( + spec: HwlabRuntimeLaneSpec, + mode: WebProbeMemoryGuardMode, + snapshotPayload: unknown, +): Record { + const policy = memoryPolicy(spec); + if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing"); + const payload = record(snapshotPayload); + const memory = record(payload.memory); + const availableBytes = typeof memory.availableBytes === "number" && Number.isSafeInteger(memory.availableBytes) && memory.availableBytes >= 0 + ? memory.availableBytes + : null; + const metadataMatches = payload.ok === true + && payload.action === "gc remote memory" + && payload.metric === policy.metric + && payload.source === policy.source + && payload.swapExcluded === true; + if (!metadataMatches || availableBytes === null) { + return unavailableGuard(spec, mode, metadataMatches ? "memavailable-unavailable" : "memory-snapshot-contract-invalid"); + } + const comparator = mode === "manual-start" ? policy.manualComparator : policy.sentinelComparator; + const thresholdMatched = mode === "manual-start" + ? availableBytes <= policy.thresholdBytes + : availableBytes < policy.thresholdBytes; + const status: WebProbeMemoryGuardStatus = thresholdMatched + ? mode === "manual-start" ? "blocked" : "skipped" + : "allowed"; + return { + ok: status === "allowed", + status, + eligible: status === "allowed", + startBlocked: status !== "allowed", + decision: status === "allowed" ? "start-observer" : mode === "manual-start" ? "gc-required" : "skip-observer", + reason: status === "allowed" ? null : "memavailable-threshold-matched", + mode, + policyConfigRef: webProbeMemoryStartGuardConfigRef, + memory: { + metric: policy.metric, + source: policy.source, + swapExcluded: policy.swapExcluded, + availableBytes, + availableHuman: humanBytes(availableBytes), + thresholdBytes: policy.thresholdBytes, + thresholdHuman: humanBytes(policy.thresholdBytes), + comparator, + thresholdMatched, + valuesRedacted: true, + }, + gcNext: status === "blocked" && mode === "manual-start" + ? { ...controlledGcNext(spec.nodeId), requiredBeforeStart: true } + : { ...controlledGcNext(spec.nodeId), requiredBeforeStart: false }, + valuesRedacted: true, + }; +} + +export function runWebProbeMemoryGuard( + spec: HwlabRuntimeLaneSpec, + mode: WebProbeMemoryGuardMode, + execute: MemoryGuardExecutor = runCommand, +): Record { + const policy = memoryPolicy(spec); + if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing"); + const command = ["bun", "scripts/cli.ts", "gc", "remote", spec.nodeId, "memory"]; + const result = execute(command, repoRoot, { timeoutMs: policy.snapshotTimeoutSeconds * 1000 }); + const resolution = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType: `gc remote ${spec.nodeId} memory contract`, + acceptParsed: (value) => { + const payload = value.action === "gc remote memory" ? value : record(value.data); + return payload.action === "gc remote memory" && payload.providerId === spec.nodeId; + }, + }); + const resolvedPayload = resolution.parsed?.action === "gc remote memory" + ? resolution.parsed + : record(resolution.parsed?.data); + const evaluated = evaluateWebProbeMemoryGuard(spec, mode, resolvedPayload); + return { + ...evaluated, + snapshot: { + action: "gc remote memory", + command: command.join(" "), + exitCode: result.exitCode, + timedOut: result.timedOut, + parsed: resolution.parsed !== null, + diagnostics: resolution.diagnostics, + valuesRedacted: true, + }, + }; +} diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts index dd574a2f..65077da2 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.test.ts @@ -302,5 +302,13 @@ test("web observe runner records finite lifecycle policy and closes the browser" assert.match(source, /sampling:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u); assert.match(source, /observerLifecycle:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u); assert.match(source, /if \(maxRunMs > 0 && Date\.now\(\) - startedAtMs >= maxRunMs\)/u); - assert.match(source, /finally \{[\s\S]*?if \(browser\) await browser\.close\(\)\.catch/u); + assert.match(source, /const maxScreenshots = boundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS/u); + assert.match(source, /const maxPerformanceEntries = boundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES/u); + assert.match(source, /while \(browserProcessHistory\.length > maxBrowserProcessHistoryEntries\)/u); + assert.match(source, /while \(browserFreezeSignalHistory\.length > maxBrowserFreezeSignalHistoryEntries\)/u); + assert.match(source, /if \(screenshotCount >= maxScreenshots\)/u); + assert.match(source, /const maxBytes = responseBodyMaxBytes/u); + assert.match(source, /process\.once\(signal,[\s\S]*?closeBrowserResources\("signal:" \+ signal\)/u); + assert.match(source, /finally \{[\s\S]*?await closeBrowserResources/u); + assert.match(source, /closingObserverPage\.close\(\)[\s\S]*?closingPage\.close\(\)[\s\S]*?closingContext\.close\(\)[\s\S]*?closingBrowser\.close\(\)/u); }); diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts index a434966f..8fa99313 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts @@ -7,6 +7,7 @@ import type { CommandResult } from "./command"; import { runCommand } from "./command"; import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery"; import { repoRoot, rootPath } from "./config"; +import { runWebProbeMemoryGuard } from "./hwlab-node-web-probe-memory-guard"; import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref"; import type { ChildCliResult, CompactCommandResult, SentinelCicdState } from "./hwlab-node-web-sentinel-cicd"; import { @@ -125,6 +126,40 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds"); const scenario = findScenario(state, scenarioId); if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true }; + const memoryGuard = runWebProbeMemoryGuard(state.spec, "sentinel-cadence"); + if (memoryGuard.status === "skipped") { + return { + ok: true, + status: "skipped-wait-next-round", + decision: "skip-observer", + reason: "memavailable-below-threshold", + scenarioId, + observerCreated: false, + runCreated: false, + mutation: false, + memoryGuard, + nextCadence: { + action: "wait-for-next-scheduled-cadence", + cadence: stringAtNullable(scenario, "cadence"), + automaticRetry: true, + valuesRedacted: true, + }, + valuesRedacted: true, + }; + } + if (memoryGuard.status !== "allowed") { + return { + ok: false, + status: "blocked", + reason: "memavailable-preflight-unavailable", + scenarioId, + observerCreated: false, + runCreated: false, + mutation: false, + memoryGuard, + valuesRedacted: true, + }; + } const providerProfilePreflight = runSentinelExactProviderProfilePreflight(state, timeoutSeconds, scenario); if (providerProfilePreflight.ok !== true) { return { @@ -214,12 +249,36 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, "--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")), "--max-run-seconds", String(hardBudgetSeconds), "--command-timeout-seconds", "55", + "--sentinel-cadence", ]; const viewport = stringAtNullable(scenario, "viewport"); if (viewport !== null) startArgs.push("--viewport", viewport); printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) }); const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env); steps.push({ phase: "observe-start", ok: started.ok, result: started.result }); + const startedEnvelope = record(started.parsed); + const startedPayload = Object.keys(record(startedEnvelope.data)).length > 0 ? record(startedEnvelope.data) : startedEnvelope; + if (startedPayload.status === "skipped-wait-next-round") { + return { + ok: true, + status: "skipped-wait-next-round", + decision: "skip-observer", + reason: "memavailable-below-threshold-after-cadence-preflight", + scenarioId, + observerCreated: false, + runCreated: false, + mutation: false, + memoryGuard: startedPayload.memoryGuard ?? null, + steps, + nextCadence: { + action: "wait-for-next-scheduled-cadence", + cadence: stringAtNullable(scenario, "cadence"), + automaticRetry: true, + valuesRedacted: true, + }, + valuesRedacted: true, + }; + } const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? "")); printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() }); if (!started.ok || observerId === null) { diff --git a/scripts/src/hwlab-node-web-sentinel-service.ts b/scripts/src/hwlab-node-web-sentinel-service.ts index 0cc58e6c..31b690b9 100644 --- a/scripts/src/hwlab-node-web-sentinel-service.ts +++ b/scripts/src/hwlab-node-web-sentinel-service.ts @@ -562,6 +562,7 @@ function buildObserveCommandPlan(config: WebProbeSentinelServiceConfig, scenario "--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")), "--max-run-seconds", String(numberAt(scenario, "maxRunSeconds")), "--command-timeout-seconds", "55", + "--sentinel-cadence", ]; const viewport = stringOrNull(scenario.viewport); if (viewport !== null) startArgv.push("--viewport", viewport); diff --git a/scripts/src/hwlab-node/entry.ts b/scripts/src/hwlab-node/entry.ts index be413a64..749918b6 100644 --- a/scripts/src/hwlab-node/entry.ts +++ b/scripts/src/hwlab-node/entry.ts @@ -188,6 +188,7 @@ export interface NodeWebProbeObserveOptions extends WebProbeOriginSelection { targetPath: string; viewport: string; browserProxyMode: WebProbeBrowserProxyMode; + memoryGuardMode: "manual-start" | "sentinel-cadence"; sampleIntervalMs: number; screenshotIntervalMs: number; observerRefreshIntervalMs: number; diff --git a/scripts/src/hwlab-node/web-probe-observe-actions.ts b/scripts/src/hwlab-node/web-probe-observe-actions.ts index d4a81b6e..3a27a43c 100644 --- a/scripts/src/hwlab-node/web-probe-observe-actions.ts +++ b/scripts/src/hwlab-node/web-probe-observe-actions.ts @@ -3,6 +3,7 @@ import { randomBytes } from "node:crypto"; import { resolveCliChildJsonCommandResult, type CliChildJsonResolution } from "../cli-child-json-recovery"; import type { HwlabRuntimeLaneSpec } from "../hwlab-node-lanes"; +import { runWebProbeMemoryGuard } from "../hwlab-node-web-probe-memory-guard"; import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; import { withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render"; import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper"; @@ -461,6 +462,27 @@ export function runNodeWebProbeObserveStart( material: BootstrapAdminPasswordMaterial, credential: Record, ): Record | RenderedCliResult { + const memoryGuard = runWebProbeMemoryGuard(spec, options.memoryGuardMode); + if (memoryGuard.status !== "allowed") { + const cadenceSkip = options.memoryGuardMode === "sentinel-cadence" && memoryGuard.status === "skipped"; + return { + ok: cadenceSkip, + status: cadenceSkip ? "skipped-wait-next-round" : "blocked", + failureKind: "web-probe-memory-start-guard", + command: `web-probe observe start --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + observerCreated: false, + startBlocked: true, + decision: cadenceSkip ? "skip-observer" : "gc-required", + mutation: false, + memoryGuard, + valuesRedacted: true, + }; + } + const observerLifecycle = spec.webProbe?.resourcePolicy?.observerLifecycle; + if (observerLifecycle === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML"); const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`; const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z"); const day = timestamp.slice(0, 8); @@ -494,6 +516,12 @@ export function runNodeWebProbeObserveStart( `UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS=${shellQuote(String(options.observerRefreshIntervalMs))}`, `UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`, `UNIDESK_WEB_OBSERVE_MAX_RUN_MS=${shellQuote(String(options.maxRunSeconds > 0 ? options.maxRunSeconds * 1000 : 0))}`, + `UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS=${shellQuote(String(observerLifecycle.maxScreenshots))}`, + `UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES=${shellQuote(String(observerLifecycle.maxPerformanceEntries))}`, + `UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserProcessHistoryEntries))}`, + `UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserFreezeSignalHistoryEntries))}`, + `UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxResponseBodyBytes))}`, + `UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxWebPerformanceBodyBytes))}`, `UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`, `UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`, `UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`, @@ -592,6 +620,7 @@ export function runNodeWebProbeObserveStart( alertThresholds, browserFreezePolicy, projectManagement, + memoryGuard, targetPath: options.targetPath, id: observerId, degradedReason, diff --git a/scripts/src/hwlab-node/web-probe-observe-options.test.ts b/scripts/src/hwlab-node/web-probe-observe-options.test.ts index f5f2bbea..3f090402 100644 --- a/scripts/src/hwlab-node/web-probe-observe-options.test.ts +++ b/scripts/src/hwlab-node/web-probe-observe-options.test.ts @@ -183,8 +183,42 @@ test("observe start inherits finite observer lifecycle limits from the owning YA assert.equal(options.maxRunSeconds, 3600); assert.equal(options.maxSamples, 720); + assert.equal(options.memoryGuardMode, "manual-start"); assert.ok(options.maxRunSeconds > 0); assert.ok(options.maxSamples > 0); + assert.deepEqual(spec.webProbe?.resourcePolicy?.observerLifecycle, { + maxRunSeconds: 3600, + maxSamples: 720, + maxScreenshots: 48, + maxPerformanceEntries: 2000, + maxBrowserProcessHistoryEntries: 360, + maxBrowserFreezeSignalHistoryEntries: 120, + maxResponseBodyBytes: 524288, + maxWebPerformanceBodyBytes: 65536, + }); +}); + +test("sentinel observe start preserves the cadence comparator through the child CLI", () => { + const options = parseNodeWebProbeObserveOptions( + ["start", "--sentinel-cadence"], + "NC01", + "v03", + spec, + null, + null, + ); + assert.equal(options.memoryGuardMode, "sentinel-cadence"); +}); + +test("observe start cannot disable or exceed the YAML lifecycle caps", () => { + assert.throws( + () => parseNodeWebProbeObserveOptions(["start", "--max-run-seconds", "0"], "NC01", "v03", spec, null, null), + /must be a positive integer/u, + ); + assert.throws( + () => parseNodeWebProbeObserveOptions(["start", "--max-samples", "721"], "NC01", "v03", spec, null, null), + /exceeds the owning YAML observer lifecycle cap 720/u, + ); }); test("observe start refuses URL-based internal/public selection ambiguity", () => { diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index ab554273..e6e16ba6 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -483,7 +483,7 @@ export function parseNodeWebProbeObserveOptions( "--workspace-root", "--workspace-root-ref", "--root", - ]), new Set(["--force", "--full", "--raw", "--text-stdin", "--require-composer-ready", "--wait-project-management-ready", "--blocking", "--non-blocking", "--dry-run", "--confirm"])); + ]), new Set(["--force", "--full", "--raw", "--text-stdin", "--require-composer-ready", "--wait-project-management-ready", "--blocking", "--non-blocking", "--dry-run", "--confirm", "--sentinel-cadence"])); const commandTypeRaw = optionValue(args, "--type") ?? null; const commandType = commandTypeRaw === null ? null : parseNodeWebProbeObserveCommandType(commandTypeRaw); const stateDir = optionValue(args, "--state-dir") ?? indexed?.stateDir ?? null; @@ -528,6 +528,9 @@ export function parseNodeWebProbeObserveOptions( if (observeActionRaw !== "start" && (requestedOrigin !== undefined || customUrl !== undefined || browserProxyModeRaw !== undefined)) { throw new Error("web-probe observe --origin/--url/--browser-proxy-mode is only valid for start; later actions inherit the observer origin"); } + if (observeActionRaw !== "start" && args.includes("--sentinel-cadence")) { + throw new Error("web-probe observe --sentinel-cadence is only valid for start"); + } const requestedBrowserProxyMode = browserProxyModeRaw === undefined ? undefined : parseWebProbeBrowserProxyMode(browserProxyModeRaw); const selectedOrigin = observeActionRaw === "start" ? resolveNodeWebProbeCliOrigin(spec, requestedOrigin, customUrl, requestedBrowserProxyMode) @@ -645,6 +648,11 @@ export function parseNodeWebProbeObserveOptions( throw new Error("owning YAML 未启用 validateWorkbenchTraceReadability"); } } + const resourcePolicy = spec.webProbe?.resourcePolicy; + if (resourcePolicy === undefined) { + throw new Error(`web-probe resource policy is missing from ${hwlabRuntimeLaneConfigPath} for ${node}/${lane}`); + } + const observerLifecycle = resourcePolicy.observerLifecycle; return { action: "observe", observeAction: observeActionRaw, @@ -659,11 +667,12 @@ export function parseNodeWebProbeObserveOptions( viewport: optionValue(args, "--viewport") ?? "1440x900", browserProxyMode: selectedOrigin.browserProxyMode, browserProxyModeSource: selectedOrigin.browserProxyModeSource, + memoryGuardMode: args.includes("--sentinel-cadence") ? "sentinel-cadence" : "manual-start", sampleIntervalMs: positiveIntegerOption(args, "--sample-interval-ms", 5000, 600000), screenshotIntervalMs: positiveIntegerOption(args, "--screenshot-interval-ms", 300000, 86_400_000), observerRefreshIntervalMs: positiveIntegerOption(args, "--observer-refresh-interval-ms", 180000, 86_400_000), - maxSamples: positiveIntegerOption(args, "--max-samples", 0, 10_000_000), - maxRunSeconds: positiveIntegerOption(args, "--max-run-seconds", 0, 86_400), + maxSamples: observerLifecycleOption(args, "--max-samples", observerLifecycle.maxSamples), + maxRunSeconds: observerLifecycleOption(args, "--max-run-seconds", observerLifecycle.maxRunSeconds), commandTimeoutSeconds: positiveIntegerOption(args, "--command-timeout-seconds", 55, 3600), waitMs: positiveIntegerOption(args, "--wait-ms", 0, 600000), tailLines: positiveIntegerOption(args, "--tail-lines", 5, 200), @@ -728,6 +737,17 @@ export function parseNodeWebProbeObserveOptions( }; } +function observerLifecycleOption(args: string[], name: string, yamlMaximum: number): number { + const raw = optionValue(args, name); + if (raw === undefined) return yamlMaximum; + const value = Number(raw); + if (!Number.isInteger(value) || value < 1) throw new Error(`${name} must be a positive integer`); + if (value > yamlMaximum) { + throw new Error(`${name}=${value} exceeds the owning YAML observer lifecycle cap ${yamlMaximum}`); + } + return value; +} + export function parseNodeWebProbeObserveCommandType(value: string): NodeWebProbeObserveCommandType { if ( value === "login"