Codex
325 lines
15 KiB
TypeScript
325 lines
15 KiB
TypeScript
// SPEC: PJ2026-01060508 Web 哨兵。WebProbe 启动前的 YAML-first 物理内存资格判定。
|
|
import type { CommandResult } from "./command";
|
|
import { runCommand } from "./command";
|
|
import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery";
|
|
import { repoRoot } from "./config";
|
|
import type { HwlabRuntimeLaneSpec, HwlabRuntimeWebProbeMemoryStartGuardSpec } from "./hwlab-node-lanes";
|
|
|
|
export const webProbeMemoryStartGuardConfigRef = "config/hwlab-node-lanes.yaml#templates.hwlabV03.webProbeWorkbench.resourcePolicy.memoryStartGuard";
|
|
|
|
export type WebProbeMemoryGuardMode = "manual-start" | "sentinel-cadence";
|
|
export type WebProbeMemoryGuardStatus = "allowed" | "blocked" | "skipped";
|
|
|
|
type MemoryGuardExecutor = (
|
|
command: string[],
|
|
cwd: string,
|
|
options: { timeoutMs?: number },
|
|
) => CommandResult;
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function memoryPolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeMemoryStartGuardSpec | null {
|
|
return spec.webProbe?.resourcePolicy?.memoryStartGuard ?? null;
|
|
}
|
|
|
|
function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/g, `'"'"'`)}'`;
|
|
}
|
|
|
|
export function webProbeBrowserEvidenceEnvAssignments(spec: HwlabRuntimeLaneSpec): string[] {
|
|
const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
|
|
if (policy === undefined) throw new Error("web-probe browser evidence policy is missing from the owning YAML");
|
|
return [
|
|
`UNIDESK_WEB_PROBE_SCREENSHOT_MODE=${shellQuote(policy.screenshotMode)}`,
|
|
`UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES=${shellQuote(String(policy.maxNetworkRequestEntries))}`,
|
|
`UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_BYTES=${shellQuote(String(policy.maxNetworkRequestBytes))}`,
|
|
`UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_ENTRIES=${shellQuote(String(policy.maxNetworkFailureEntries))}`,
|
|
`UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES=${shellQuote(String(policy.maxNetworkFailureBytes))}`,
|
|
];
|
|
}
|
|
|
|
export function webProbeScreenshotFullPage(spec: HwlabRuntimeLaneSpec, explicitFullPage: boolean): boolean {
|
|
const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
|
|
if (policy === undefined) throw new Error("web-probe browser evidence policy is missing from the owning YAML");
|
|
return explicitFullPage || policy.screenshotMode !== "viewport";
|
|
}
|
|
|
|
export function webProbeGuardedLaunchShell(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
mode: WebProbeMemoryGuardMode,
|
|
command: string,
|
|
readiness: "child-process" | "structured-output" = "child-process",
|
|
): string {
|
|
const policy = memoryPolicy(spec);
|
|
if (policy === null) throw new Error("web-probe memory start policy is missing from the owning YAML");
|
|
const closeStepTimeoutMs = spec.webProbe?.resourcePolicy?.observerLifecycle.closeStepTimeoutMs;
|
|
if (closeStepTimeoutMs === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML");
|
|
const cleanupPollMilliseconds = policy.launchReadyPollMilliseconds;
|
|
const cleanupPollSeconds = Math.max(0.05, cleanupPollMilliseconds / 1000);
|
|
const cleanupPollAttempts = Math.max(1, Math.ceil(closeStepTimeoutMs / cleanupPollMilliseconds));
|
|
const comparator = mode === "manual-start" ? "-le" : "-lt";
|
|
const common = [
|
|
`exec 9>${shellQuote(policy.launchLockPath)}`,
|
|
`flock -w ${shellQuote(String(policy.launchLockTimeoutSeconds))} 9`,
|
|
"mem_available_kib=$(awk '$1 == \"MemAvailable:\" { print $2; exit }' /proc/meminfo)",
|
|
"case \"$mem_available_kib\" in ''|*[!0-9]*) printf '%s\\n' 'web-probe final memory guard could not read MemAvailable' >&2; exit 42;; esac",
|
|
"mem_available_bytes=$((mem_available_kib * 1024))",
|
|
`if [ "$mem_available_bytes" ${comparator} ${shellQuote(String(policy.thresholdBytes))} ]; then printf '%s\\n' 'web-probe final memory guard blocked browser launch' >&2; exit 42; fi`,
|
|
];
|
|
if (readiness === "structured-output") return [
|
|
...common,
|
|
"launch_output=$(mktemp)",
|
|
"launch_error=$(mktemp)",
|
|
"trap 'rm -f \"$launch_output\" \"$launch_error\"' EXIT",
|
|
`if ! (exec 9>&-; ${command}) >"$launch_output" 2>"$launch_error"; then cat "$launch_output"; cat "$launch_error" >&2; exit 43; fi`,
|
|
"cat \"$launch_output\"",
|
|
"cat \"$launch_error\" >&2",
|
|
`if ! node - "$launch_output" ${shellQuote(String(policy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(policy.launchReadyPollMilliseconds))} <<'UNIDESK_WEB_PROBE_STRUCTURED_READINESS'`,
|
|
"const fs = require('fs');",
|
|
"const text = fs.readFileSync(process.argv[2], 'utf8').trim();",
|
|
"const timeoutMs = Number(process.argv[3]);",
|
|
"const pollMs = Number(process.argv[4]);",
|
|
"let value = null; try { value = JSON.parse(text); } catch {}",
|
|
"const data = value && typeof value === 'object' && value.data && typeof value.data === 'object' ? value.data : value;",
|
|
"if (!data || typeof data !== 'object' || !data.jobId) process.exit(43);",
|
|
"const rootPid = [data.workerPid, data.jobPid, data.pid].find((item) => Number.isInteger(item) && item > 0);",
|
|
"if (!rootPid) process.exit(43);",
|
|
"const reportedBrowserPid = Number.isInteger(data.browserPid) && data.browserPid > 0 ? data.browserPid : null;",
|
|
"const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
|
|
"const browserDescendant = () => {",
|
|
" const rows = [];",
|
|
" for (const name of fs.readdirSync('/proc')) {",
|
|
" if (!/^[0-9]+$/.test(name)) continue;",
|
|
" try {",
|
|
" const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');",
|
|
" const close = stat.lastIndexOf(')');",
|
|
" const fields = stat.slice(close + 2).split(' ');",
|
|
" const ppid = Number(fields[1]);",
|
|
" const executable = fs.readlinkSync('/proc/' + name + '/exe');",
|
|
" rows.push({ pid: Number(name), ppid, executable });",
|
|
" } catch {}",
|
|
" }",
|
|
" const descendants = new Set([rootPid]);",
|
|
" let changed = true;",
|
|
" while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }",
|
|
" return rows.some((row) => descendants.has(row.pid) && (reportedBrowserPid === null || row.pid === reportedBrowserPid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable));",
|
|
"};",
|
|
"(async () => {",
|
|
" const deadline = Date.now() + timeoutMs;",
|
|
" while (Date.now() <= deadline) { if (browserDescendant()) process.exit(0); await sleep(pollMs); }",
|
|
" process.exit(43);",
|
|
"})().catch(() => process.exit(43));",
|
|
"UNIDESK_WEB_PROBE_STRUCTURED_READINESS",
|
|
"then flock -u 9; exit 43; fi",
|
|
"flock -u 9",
|
|
].join("\n");
|
|
return [
|
|
...common,
|
|
`setsid sh -c ${shellQuote(`exec 9>&-\n${command}`)} &`,
|
|
"browser_job_pid=$!",
|
|
"cleanup_browser_group() {",
|
|
" kill -TERM -- \"-$browser_job_pid\" 2>/dev/null || true",
|
|
" cleanup_attempt=0",
|
|
` while kill -0 -- "-$browser_job_pid" 2>/dev/null && [ "$cleanup_attempt" -lt ${cleanupPollAttempts} ]; do cleanup_attempt=$((cleanup_attempt + 1)); sleep ${cleanupPollSeconds}; done`,
|
|
" if kill -0 -- \"-$browser_job_pid\" 2>/dev/null; then kill -KILL -- \"-$browser_job_pid\" 2>/dev/null || true; fi",
|
|
" wait \"$browser_job_pid\" 2>/dev/null || true",
|
|
"}",
|
|
"trap 'browser_exit=$?; trap - EXIT TERM INT HUP; cleanup_browser_group; exit \"$browser_exit\"' EXIT",
|
|
"trap 'exit 143' TERM",
|
|
"trap 'exit 130' INT",
|
|
"trap 'exit 129' HUP",
|
|
"browser_ready=false",
|
|
`browser_ready_deadline=$(( $(date +%s) + ${policy.launchReadyTimeoutSeconds} ))`,
|
|
"while kill -0 \"$browser_job_pid\" 2>/dev/null; do",
|
|
" if node - \"$browser_job_pid\" <<'UNIDESK_WEB_PROBE_CHILD_READINESS'",
|
|
"const fs = require('fs');",
|
|
"const rootPid = Number(process.argv[2]);",
|
|
"const rows = [];",
|
|
"for (const name of fs.readdirSync('/proc')) {",
|
|
" if (!/^[0-9]+$/.test(name)) continue;",
|
|
" try {",
|
|
" const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');",
|
|
" const close = stat.lastIndexOf(')');",
|
|
" const fields = stat.slice(close + 2).split(' ');",
|
|
" rows.push({ pid: Number(name), ppid: Number(fields[1]), executable: fs.readlinkSync('/proc/' + name + '/exe') });",
|
|
" } catch {}",
|
|
"}",
|
|
"const descendants = new Set([rootPid]);",
|
|
"let changed = true;",
|
|
"while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }",
|
|
"process.exit(rows.some((row) => descendants.has(row.pid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable)) ? 0 : 1);",
|
|
"UNIDESK_WEB_PROBE_CHILD_READINESS",
|
|
" then browser_ready=true; break; fi",
|
|
" if [ \"$(date +%s)\" -ge \"$browser_ready_deadline\" ]; then break; fi",
|
|
` sleep ${Math.max(0.05, policy.launchReadyPollMilliseconds / 1000)}`,
|
|
"done",
|
|
"flock -u 9",
|
|
"if [ \"$browser_ready\" != true ]; then",
|
|
" printf '%s\\n' 'web-probe browser launch readiness was not confirmed' >&2",
|
|
" exit 43",
|
|
"fi",
|
|
"set +e",
|
|
"wait \"$browser_job_pid\"",
|
|
"browser_exit=$?",
|
|
"set -e",
|
|
"exit \"$browser_exit\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function humanBytes(value: number | null): string | null {
|
|
if (value === null || !Number.isFinite(value)) return null;
|
|
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
let current = value;
|
|
let index = 0;
|
|
while (Math.abs(current) >= 1024 && index < units.length - 1) {
|
|
current /= 1024;
|
|
index += 1;
|
|
}
|
|
return `${current.toFixed(index === 0 ? 0 : 2)} ${units[index]}`;
|
|
}
|
|
|
|
function controlledGcNext(node: string): Record<string, unknown> {
|
|
return {
|
|
automaticGc: false,
|
|
requiredOrder: ["plan", "run", "status-until-terminal", "memory-recheck"],
|
|
scope: "memory-pressure-only",
|
|
planCommand: `bun scripts/cli.ts gc remote ${node} plan --memory-pressure-only`,
|
|
runCommand: `bun scripts/cli.ts gc remote ${node} run --confirm --memory-pressure-only`,
|
|
jobIdSource: "gc remote run result.data.jobId",
|
|
statusCommandTemplate: `bun scripts/cli.ts gc remote ${node} status --job-id <job-id>`,
|
|
terminalStatuses: ["succeeded", "failed", "completed", "blocked"],
|
|
statusRequiredWhenJobIdReturned: true,
|
|
recheckCommand: `bun scripts/cli.ts gc remote ${node} memory`,
|
|
recheckMetric: "MemAvailable",
|
|
recheckSource: "/proc/meminfo",
|
|
swapExcluded: true,
|
|
startRemainsBlockedUntil: ["gc-job-terminal", "MemAvailable-threshold-cleared"],
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function unavailableGuard(spec: HwlabRuntimeLaneSpec, mode: WebProbeMemoryGuardMode, reason: string): Record<string, unknown> {
|
|
const policy = memoryPolicy(spec);
|
|
const comparator = policy === null
|
|
? null
|
|
: mode === "manual-start"
|
|
? policy.manualComparator
|
|
: policy.sentinelComparator;
|
|
return {
|
|
ok: false,
|
|
status: "blocked" satisfies WebProbeMemoryGuardStatus,
|
|
eligible: false,
|
|
startBlocked: true,
|
|
decision: "do-not-start-observer",
|
|
reason,
|
|
mode,
|
|
policyConfigRef: webProbeMemoryStartGuardConfigRef,
|
|
memory: {
|
|
metric: policy?.metric ?? "MemAvailable",
|
|
source: policy?.source ?? "/proc/meminfo",
|
|
swapExcluded: policy?.swapExcluded ?? true,
|
|
availableBytes: null,
|
|
availableHuman: null,
|
|
thresholdBytes: policy?.thresholdBytes ?? null,
|
|
thresholdHuman: humanBytes(policy?.thresholdBytes ?? null),
|
|
comparator,
|
|
thresholdMatched: null,
|
|
valuesRedacted: true,
|
|
},
|
|
gcNext: controlledGcNext(spec.nodeId),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
export function evaluateWebProbeMemoryGuard(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
mode: WebProbeMemoryGuardMode,
|
|
snapshotPayload: unknown,
|
|
): Record<string, unknown> {
|
|
const policy = memoryPolicy(spec);
|
|
if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing");
|
|
const payload = record(snapshotPayload);
|
|
const memory = record(payload.memory);
|
|
const availableBytes = typeof memory.availableBytes === "number" && Number.isSafeInteger(memory.availableBytes) && memory.availableBytes >= 0
|
|
? memory.availableBytes
|
|
: null;
|
|
const metadataMatches = payload.ok === true
|
|
&& payload.action === "gc remote memory"
|
|
&& payload.metric === policy.metric
|
|
&& payload.source === policy.source
|
|
&& payload.swapExcluded === true;
|
|
if (!metadataMatches || availableBytes === null) {
|
|
return unavailableGuard(spec, mode, metadataMatches ? "memavailable-unavailable" : "memory-snapshot-contract-invalid");
|
|
}
|
|
const comparator = mode === "manual-start" ? policy.manualComparator : policy.sentinelComparator;
|
|
const thresholdMatched = mode === "manual-start"
|
|
? availableBytes <= policy.thresholdBytes
|
|
: availableBytes < policy.thresholdBytes;
|
|
const status: WebProbeMemoryGuardStatus = thresholdMatched
|
|
? mode === "manual-start" ? "blocked" : "skipped"
|
|
: "allowed";
|
|
return {
|
|
ok: status === "allowed",
|
|
status,
|
|
eligible: status === "allowed",
|
|
startBlocked: status !== "allowed",
|
|
decision: status === "allowed" ? "start-observer" : mode === "manual-start" ? "gc-required" : "skip-observer",
|
|
reason: status === "allowed" ? null : "memavailable-threshold-matched",
|
|
mode,
|
|
policyConfigRef: webProbeMemoryStartGuardConfigRef,
|
|
memory: {
|
|
metric: policy.metric,
|
|
source: policy.source,
|
|
swapExcluded: policy.swapExcluded,
|
|
availableBytes,
|
|
availableHuman: humanBytes(availableBytes),
|
|
thresholdBytes: policy.thresholdBytes,
|
|
thresholdHuman: humanBytes(policy.thresholdBytes),
|
|
comparator,
|
|
thresholdMatched,
|
|
valuesRedacted: true,
|
|
},
|
|
gcNext: status === "blocked" && mode === "manual-start"
|
|
? { ...controlledGcNext(spec.nodeId), requiredBeforeStart: true }
|
|
: { ...controlledGcNext(spec.nodeId), requiredBeforeStart: false },
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
export function runWebProbeMemoryGuard(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
mode: WebProbeMemoryGuardMode,
|
|
execute: MemoryGuardExecutor = runCommand,
|
|
): Record<string, unknown> {
|
|
const policy = memoryPolicy(spec);
|
|
if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing");
|
|
const command = ["bun", "scripts/cli.ts", "gc", "remote", spec.nodeId, "memory"];
|
|
const result = execute(command, repoRoot, { timeoutMs: policy.snapshotTimeoutSeconds * 1000 });
|
|
const resolution = resolveCliChildJsonCommandResult({
|
|
result,
|
|
requestedStdoutType: `gc remote ${spec.nodeId} memory contract`,
|
|
acceptParsed: (value) => {
|
|
const payload = value.action === "gc remote memory" ? value : record(value.data);
|
|
return payload.action === "gc remote memory" && payload.providerId === spec.nodeId;
|
|
},
|
|
});
|
|
const resolvedPayload = resolution.parsed?.action === "gc remote memory"
|
|
? resolution.parsed
|
|
: record(resolution.parsed?.data);
|
|
const evaluated = evaluateWebProbeMemoryGuard(spec, mode, resolvedPayload);
|
|
return {
|
|
...evaluated,
|
|
snapshot: {
|
|
action: "gc remote memory",
|
|
command: command.join(" "),
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
parsed: resolution.parsed !== null,
|
|
diagnostics: resolution.diagnostics,
|
|
valuesRedacted: true,
|
|
},
|
|
};
|
|
}
|