// SPEC: PJ2026-01060508 Web 哨兵。WebProbe 启动前的 YAML-first 物理内存资格判定。 import type { CommandResult } from "./command"; import { runCommand } from "./command"; import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery"; import { repoRoot } from "./config"; import type { HwlabRuntimeLaneSpec, HwlabRuntimeWebProbeMemoryStartGuardSpec } from "./hwlab-node-lanes"; export const webProbeMemoryStartGuardConfigRef = "config/hwlab-node-lanes.yaml#templates.hwlabV03.webProbeWorkbench.resourcePolicy.memoryStartGuard"; export type WebProbeMemoryGuardMode = "manual-start" | "sentinel-cadence"; export type WebProbeMemoryGuardStatus = "allowed" | "blocked" | "skipped"; type MemoryGuardExecutor = ( command: string[], cwd: string, options: { timeoutMs?: number }, ) => CommandResult; function record(value: unknown): Record { return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; } function memoryPolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeMemoryStartGuardSpec | null { return spec.webProbe?.resourcePolicy?.memoryStartGuard ?? null; } function shellQuote(value: string): string { return `'${value.replace(/'/g, `'"'"'`)}'`; } export function webProbeBrowserEvidenceEnvAssignments(spec: HwlabRuntimeLaneSpec): string[] { const policy = spec.webProbe?.resourcePolicy?.browserEvidence; if (policy === undefined) throw new Error("web-probe browser evidence policy is missing from the owning YAML"); return [ `UNIDESK_WEB_PROBE_SCREENSHOT_MODE=${shellQuote(policy.screenshotMode)}`, `UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES=${shellQuote(String(policy.maxNetworkRequestEntries))}`, `UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_BYTES=${shellQuote(String(policy.maxNetworkRequestBytes))}`, `UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_ENTRIES=${shellQuote(String(policy.maxNetworkFailureEntries))}`, `UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES=${shellQuote(String(policy.maxNetworkFailureBytes))}`, ]; } export function webProbeScreenshotFullPage(spec: HwlabRuntimeLaneSpec, explicitFullPage: boolean): boolean { const policy = spec.webProbe?.resourcePolicy?.browserEvidence; if (policy === undefined) throw new Error("web-probe browser evidence policy is missing from the owning YAML"); return explicitFullPage || policy.screenshotMode !== "viewport"; } export function webProbeGuardedLaunchShell( spec: HwlabRuntimeLaneSpec, mode: WebProbeMemoryGuardMode, command: string, readiness: "child-process" | "structured-output" = "child-process", ): string { const policy = memoryPolicy(spec); if (policy === null) throw new Error("web-probe memory start policy is missing from the owning YAML"); const closeStepTimeoutMs = spec.webProbe?.resourcePolicy?.observerLifecycle.closeStepTimeoutMs; if (closeStepTimeoutMs === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML"); const cleanupPollMilliseconds = policy.launchReadyPollMilliseconds; const cleanupPollSeconds = Math.max(0.05, cleanupPollMilliseconds / 1000); const cleanupPollAttempts = Math.max(1, Math.ceil(closeStepTimeoutMs / cleanupPollMilliseconds)); const comparator = mode === "manual-start" ? "-le" : "-lt"; const common = [ `exec 9>${shellQuote(policy.launchLockPath)}`, `flock -w ${shellQuote(String(policy.launchLockTimeoutSeconds))} 9`, "mem_available_kib=$(awk '$1 == \"MemAvailable:\" { print $2; exit }' /proc/meminfo)", "case \"$mem_available_kib\" in ''|*[!0-9]*) printf '%s\\n' 'web-probe final memory guard could not read MemAvailable' >&2; exit 42;; esac", "mem_available_bytes=$((mem_available_kib * 1024))", `if [ "$mem_available_bytes" ${comparator} ${shellQuote(String(policy.thresholdBytes))} ]; then printf '%s\\n' 'web-probe final memory guard blocked browser launch' >&2; exit 42; fi`, ]; if (readiness === "structured-output") return [ ...common, "launch_output=$(mktemp)", "launch_error=$(mktemp)", "trap 'rm -f \"$launch_output\" \"$launch_error\"' EXIT", `if ! (exec 9>&-; ${command}) >"$launch_output" 2>"$launch_error"; then cat "$launch_output"; cat "$launch_error" >&2; exit 43; fi`, "cat \"$launch_output\"", "cat \"$launch_error\" >&2", `if ! node - "$launch_output" ${shellQuote(String(policy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(policy.launchReadyPollMilliseconds))} <<'UNIDESK_WEB_PROBE_STRUCTURED_READINESS'`, "const fs = require('fs');", "const text = fs.readFileSync(process.argv[2], 'utf8').trim();", "const timeoutMs = Number(process.argv[3]);", "const pollMs = Number(process.argv[4]);", "let value = null; try { value = JSON.parse(text); } catch {}", "const data = value && typeof value === 'object' && value.data && typeof value.data === 'object' ? value.data : value;", "if (!data || typeof data !== 'object' || !data.jobId) process.exit(43);", "const rootPid = [data.workerPid, data.jobPid, data.pid].find((item) => Number.isInteger(item) && item > 0);", "if (!rootPid) process.exit(43);", "const reportedBrowserPid = Number.isInteger(data.browserPid) && data.browserPid > 0 ? data.browserPid : null;", "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", "const browserDescendant = () => {", " const rows = [];", " for (const name of fs.readdirSync('/proc')) {", " if (!/^[0-9]+$/.test(name)) continue;", " try {", " const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');", " const close = stat.lastIndexOf(')');", " const fields = stat.slice(close + 2).split(' ');", " const ppid = Number(fields[1]);", " const executable = fs.readlinkSync('/proc/' + name + '/exe');", " rows.push({ pid: Number(name), ppid, executable });", " } catch {}", " }", " const descendants = new Set([rootPid]);", " let changed = true;", " while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }", " return rows.some((row) => descendants.has(row.pid) && (reportedBrowserPid === null || row.pid === reportedBrowserPid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable));", "};", "(async () => {", " const deadline = Date.now() + timeoutMs;", " while (Date.now() <= deadline) { if (browserDescendant()) process.exit(0); await sleep(pollMs); }", " process.exit(43);", "})().catch(() => process.exit(43));", "UNIDESK_WEB_PROBE_STRUCTURED_READINESS", "then flock -u 9; exit 43; fi", "flock -u 9", ].join("\n"); return [ ...common, `setsid sh -c ${shellQuote(`exec 9>&-\n${command}`)} &`, "browser_job_pid=$!", "cleanup_browser_group() {", " kill -TERM -- \"-$browser_job_pid\" 2>/dev/null || true", " cleanup_attempt=0", ` while kill -0 -- "-$browser_job_pid" 2>/dev/null && [ "$cleanup_attempt" -lt ${cleanupPollAttempts} ]; do cleanup_attempt=$((cleanup_attempt + 1)); sleep ${cleanupPollSeconds}; done`, " if kill -0 -- \"-$browser_job_pid\" 2>/dev/null; then kill -KILL -- \"-$browser_job_pid\" 2>/dev/null || true; fi", " wait \"$browser_job_pid\" 2>/dev/null || true", "}", "trap 'browser_exit=$?; trap - EXIT TERM INT HUP; cleanup_browser_group; exit \"$browser_exit\"' EXIT", "trap 'exit 143' TERM", "trap 'exit 130' INT", "trap 'exit 129' HUP", "browser_ready=false", `browser_ready_deadline=$(( $(date +%s) + ${policy.launchReadyTimeoutSeconds} ))`, "while kill -0 \"$browser_job_pid\" 2>/dev/null; do", " if node - \"$browser_job_pid\" <<'UNIDESK_WEB_PROBE_CHILD_READINESS'", "const fs = require('fs');", "const rootPid = Number(process.argv[2]);", "const rows = [];", "for (const name of fs.readdirSync('/proc')) {", " if (!/^[0-9]+$/.test(name)) continue;", " try {", " const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');", " const close = stat.lastIndexOf(')');", " const fields = stat.slice(close + 2).split(' ');", " rows.push({ pid: Number(name), ppid: Number(fields[1]), executable: fs.readlinkSync('/proc/' + name + '/exe') });", " } catch {}", "}", "const descendants = new Set([rootPid]);", "let changed = true;", "while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }", "process.exit(rows.some((row) => descendants.has(row.pid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable)) ? 0 : 1);", "UNIDESK_WEB_PROBE_CHILD_READINESS", " then browser_ready=true; break; fi", " if [ \"$(date +%s)\" -ge \"$browser_ready_deadline\" ]; then break; fi", ` sleep ${Math.max(0.05, policy.launchReadyPollMilliseconds / 1000)}`, "done", "flock -u 9", "if [ \"$browser_ready\" != true ]; then", " printf '%s\\n' 'web-probe browser launch readiness was not confirmed' >&2", " exit 43", "fi", "set +e", "wait \"$browser_job_pid\"", "browser_exit=$?", "set -e", "exit \"$browser_exit\"", ].join("\n"); } function humanBytes(value: number | null): string | null { if (value === null || !Number.isFinite(value)) return null; const units = ["B", "KiB", "MiB", "GiB", "TiB"]; let current = value; let index = 0; while (Math.abs(current) >= 1024 && index < units.length - 1) { current /= 1024; index += 1; } return `${current.toFixed(index === 0 ? 0 : 2)} ${units[index]}`; } function controlledGcNext(node: string): Record { return { automaticGc: false, requiredOrder: ["plan", "run", "status-until-terminal", "memory-recheck"], scope: "memory-pressure-only", planCommand: `bun scripts/cli.ts gc remote ${node} plan --memory-pressure-only`, runCommand: `bun scripts/cli.ts gc remote ${node} run --confirm --memory-pressure-only`, jobIdSource: "gc remote run result.data.jobId", statusCommandTemplate: `bun scripts/cli.ts gc remote ${node} status --job-id `, terminalStatuses: ["succeeded", "failed", "completed", "blocked"], statusRequiredWhenJobIdReturned: true, recheckCommand: `bun scripts/cli.ts gc remote ${node} memory`, recheckMetric: "MemAvailable", recheckSource: "/proc/meminfo", swapExcluded: true, startRemainsBlockedUntil: ["gc-job-terminal", "MemAvailable-threshold-cleared"], valuesRedacted: true, }; } function unavailableGuard(spec: HwlabRuntimeLaneSpec, mode: WebProbeMemoryGuardMode, reason: string): Record { const policy = memoryPolicy(spec); const comparator = policy === null ? null : mode === "manual-start" ? policy.manualComparator : policy.sentinelComparator; return { ok: false, status: "blocked" satisfies WebProbeMemoryGuardStatus, eligible: false, startBlocked: true, decision: "do-not-start-observer", reason, mode, policyConfigRef: webProbeMemoryStartGuardConfigRef, memory: { metric: policy?.metric ?? "MemAvailable", source: policy?.source ?? "/proc/meminfo", swapExcluded: policy?.swapExcluded ?? true, availableBytes: null, availableHuman: null, thresholdBytes: policy?.thresholdBytes ?? null, thresholdHuman: humanBytes(policy?.thresholdBytes ?? null), comparator, thresholdMatched: null, valuesRedacted: true, }, gcNext: controlledGcNext(spec.nodeId), valuesRedacted: true, }; } export function evaluateWebProbeMemoryGuard( spec: HwlabRuntimeLaneSpec, mode: WebProbeMemoryGuardMode, snapshotPayload: unknown, ): Record { const policy = memoryPolicy(spec); if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing"); const payload = record(snapshotPayload); const memory = record(payload.memory); const availableBytes = typeof memory.availableBytes === "number" && Number.isSafeInteger(memory.availableBytes) && memory.availableBytes >= 0 ? memory.availableBytes : null; const metadataMatches = payload.ok === true && payload.action === "gc remote memory" && payload.metric === policy.metric && payload.source === policy.source && payload.swapExcluded === true; if (!metadataMatches || availableBytes === null) { return unavailableGuard(spec, mode, metadataMatches ? "memavailable-unavailable" : "memory-snapshot-contract-invalid"); } const comparator = mode === "manual-start" ? policy.manualComparator : policy.sentinelComparator; const thresholdMatched = mode === "manual-start" ? availableBytes <= policy.thresholdBytes : availableBytes < policy.thresholdBytes; const status: WebProbeMemoryGuardStatus = thresholdMatched ? mode === "manual-start" ? "blocked" : "skipped" : "allowed"; return { ok: status === "allowed", status, eligible: status === "allowed", startBlocked: status !== "allowed", decision: status === "allowed" ? "start-observer" : mode === "manual-start" ? "gc-required" : "skip-observer", reason: status === "allowed" ? null : "memavailable-threshold-matched", mode, policyConfigRef: webProbeMemoryStartGuardConfigRef, memory: { metric: policy.metric, source: policy.source, swapExcluded: policy.swapExcluded, availableBytes, availableHuman: humanBytes(availableBytes), thresholdBytes: policy.thresholdBytes, thresholdHuman: humanBytes(policy.thresholdBytes), comparator, thresholdMatched, valuesRedacted: true, }, gcNext: status === "blocked" && mode === "manual-start" ? { ...controlledGcNext(spec.nodeId), requiredBeforeStart: true } : { ...controlledGcNext(spec.nodeId), requiredBeforeStart: false }, valuesRedacted: true, }; } export function runWebProbeMemoryGuard( spec: HwlabRuntimeLaneSpec, mode: WebProbeMemoryGuardMode, execute: MemoryGuardExecutor = runCommand, ): Record { const policy = memoryPolicy(spec); if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing"); const command = ["bun", "scripts/cli.ts", "gc", "remote", spec.nodeId, "memory"]; const result = execute(command, repoRoot, { timeoutMs: policy.snapshotTimeoutSeconds * 1000 }); const resolution = resolveCliChildJsonCommandResult({ result, requestedStdoutType: `gc remote ${spec.nodeId} memory contract`, acceptParsed: (value) => { const payload = value.action === "gc remote memory" ? value : record(value.data); return payload.action === "gc remote memory" && payload.providerId === spec.nodeId; }, }); const resolvedPayload = resolution.parsed?.action === "gc remote memory" ? resolution.parsed : record(resolution.parsed?.data); const evaluated = evaluateWebProbeMemoryGuard(spec, mode, resolvedPayload); return { ...evaluated, snapshot: { action: "gc remote memory", command: command.join(" "), exitCode: result.exitCode, timedOut: result.timedOut, parsed: resolution.parsed !== null, diagnostics: resolution.diagnostics, valuesRedacted: true, }, }; }