Codex
262 lines
16 KiB
TypeScript
262 lines
16 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { readFileSync } from "node:fs";
|
|
import { test } from "bun:test";
|
|
|
|
import type { CommandResult } from "./command";
|
|
import { rootPath } from "./config";
|
|
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
|
|
import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeMemoryStartGuardConfigRef, webProbeScreenshotFullPage } from "./hwlab-node-web-probe-memory-guard";
|
|
import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source";
|
|
|
|
const spec = hwlabRuntimeLaneSpecForNode("v03", "NC01");
|
|
const thresholdBytes = 4 * 1024 * 1024 * 1024;
|
|
|
|
function memoryPayload(availableBytes: number, extra: Record<string, unknown> = {}): Record<string, unknown> {
|
|
return {
|
|
ok: true,
|
|
action: "gc remote memory",
|
|
providerId: "NC01",
|
|
metric: "MemAvailable",
|
|
source: "/proc/meminfo",
|
|
swapExcluded: true,
|
|
memory: { availableBytes, availableHuman: "fixture", ...extra },
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function exportedFunctionSource(source: string, name: string): string {
|
|
const start = source.indexOf(`export function ${name}(`);
|
|
assert.ok(start >= 0, `missing exported function ${name}`);
|
|
const nextExport = source.indexOf("\nexport function ", start + 1);
|
|
const nextFunction = source.indexOf("\nfunction ", start + 1);
|
|
const next = [nextExport, nextFunction].filter((value) => value >= 0).sort((left, right) => left - right)[0] ?? -1;
|
|
return source.slice(start, next < 0 ? source.length : next);
|
|
}
|
|
|
|
test("WebProbe memory threshold is read from the single owning YAML policy", () => {
|
|
const policy = spec.webProbe?.resourcePolicy?.memoryStartGuard;
|
|
assert.equal(policy?.thresholdBytes, thresholdBytes);
|
|
assert.equal(policy?.metric, "MemAvailable");
|
|
assert.equal(policy?.source, "/proc/meminfo");
|
|
assert.equal(policy?.swapExcluded, true);
|
|
assert.equal(policy?.manualComparator, "less-than-or-equal");
|
|
assert.equal(policy?.sentinelComparator, "less-than");
|
|
assert.match(webProbeMemoryStartGuardConfigRef, /resourcePolicy\.memoryStartGuard$/u);
|
|
});
|
|
|
|
test("browser evidence limits and viewport screenshots are read from owning YAML", () => {
|
|
const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
|
|
assert.deepEqual(policy, {
|
|
screenshotMode: "viewport",
|
|
maxNetworkRequestEntries: 200,
|
|
maxNetworkRequestBytes: 262144,
|
|
maxNetworkFailureEntries: 100,
|
|
maxNetworkFailureBytes: 131072,
|
|
});
|
|
const assignments = webProbeBrowserEvidenceEnvAssignments(spec).join(" ");
|
|
assert.match(assignments, /UNIDESK_WEB_PROBE_SCREENSHOT_MODE='viewport'/u);
|
|
assert.match(assignments, /UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES='200'/u);
|
|
assert.match(assignments, /UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES='131072'/u);
|
|
});
|
|
|
|
test("manual start is blocked at exactly 4 GiB and requires controlled GC then MemAvailable recheck", () => {
|
|
const guard = evaluateWebProbeMemoryGuard(spec, "manual-start", memoryPayload(thresholdBytes));
|
|
const memory = guard.memory as Record<string, unknown>;
|
|
const gcNext = guard.gcNext as Record<string, unknown>;
|
|
|
|
assert.equal(guard.status, "blocked");
|
|
assert.equal(guard.startBlocked, true);
|
|
assert.equal(guard.decision, "gc-required");
|
|
assert.equal(memory.metric, "MemAvailable");
|
|
assert.equal(memory.source, "/proc/meminfo");
|
|
assert.equal(memory.swapExcluded, true);
|
|
assert.equal(memory.comparator, "less-than-or-equal");
|
|
assert.equal(memory.thresholdMatched, true);
|
|
assert.equal(gcNext.requiredBeforeStart, true);
|
|
assert.match(String(gcNext.planCommand), /gc remote NC01 plan --memory-pressure-only$/u);
|
|
assert.match(String(gcNext.runCommand), /gc remote NC01 run --confirm --memory-pressure-only$/u);
|
|
assert.match(String(gcNext.statusCommandTemplate), /gc remote NC01 status --job-id <job-id>$/u);
|
|
assert.match(String(gcNext.recheckCommand), /gc remote NC01 memory$/u);
|
|
assert.equal(gcNext.recheckMetric, "MemAvailable");
|
|
assert.deepEqual(gcNext.requiredOrder, ["plan", "run", "status-until-terminal", "memory-recheck"]);
|
|
assert.deepEqual(gcNext.startRemainsBlockedUntil, ["gc-job-terminal", "MemAvailable-threshold-cleared"]);
|
|
});
|
|
|
|
test("sentinel allows exactly 4 GiB but skips one byte below without considering swap", () => {
|
|
const exact = evaluateWebProbeMemoryGuard(spec, "sentinel-cadence", memoryPayload(thresholdBytes, { swapFreeBytes: 12 * 1024 ** 3 }));
|
|
const below = evaluateWebProbeMemoryGuard(spec, "sentinel-cadence", memoryPayload(thresholdBytes - 1, { swapFreeBytes: 12 * 1024 ** 3 }));
|
|
const belowMemory = below.memory as Record<string, unknown>;
|
|
|
|
assert.equal(exact.status, "allowed");
|
|
assert.equal(exact.startBlocked, false);
|
|
assert.equal((exact.memory as Record<string, unknown>).comparator, "less-than");
|
|
assert.equal(below.status, "skipped");
|
|
assert.equal(below.startBlocked, true);
|
|
assert.equal(below.decision, "skip-observer");
|
|
assert.equal(belowMemory.availableBytes, thresholdBytes - 1);
|
|
assert.equal(belowMemory.swapExcluded, true);
|
|
});
|
|
|
|
test("memory guard rejects a snapshot that is not the /proc/meminfo MemAvailable contract", () => {
|
|
const guard = evaluateWebProbeMemoryGuard(spec, "manual-start", {
|
|
...memoryPayload(thresholdBytes + 1),
|
|
source: "cgroup-v2-memory.current",
|
|
});
|
|
assert.equal(guard.status, "blocked");
|
|
assert.equal(guard.reason, "memory-snapshot-contract-invalid");
|
|
assert.equal(guard.startBlocked, true);
|
|
});
|
|
|
|
test("memory guard executes only the controlled read-only remote memory entry", () => {
|
|
let observedCommand: string[] = [];
|
|
let observedTimeoutMs = 0;
|
|
const execute = (command: string[], cwd: string, options: { timeoutMs?: number }): CommandResult => {
|
|
observedCommand = command;
|
|
observedTimeoutMs = options.timeoutMs ?? 0;
|
|
return {
|
|
command,
|
|
cwd,
|
|
exitCode: 0,
|
|
stdout: JSON.stringify({ ok: true, command: "gc remote NC01 memory", data: memoryPayload(thresholdBytes + 1) }),
|
|
stderr: "",
|
|
signal: null,
|
|
timedOut: false,
|
|
};
|
|
};
|
|
const guard = runWebProbeMemoryGuard(spec, "manual-start", execute);
|
|
|
|
assert.deepEqual(observedCommand, ["bun", "scripts/cli.ts", "gc", "remote", "NC01", "memory"]);
|
|
assert.equal(observedTimeoutMs, 30_000);
|
|
assert.equal(guard.status, "allowed");
|
|
});
|
|
|
|
test("remote memory collector reads MemAvailable directly and excludes swap", () => {
|
|
const source = readFileSync(rootPath("scripts/src/gc-remote-runner.py"), "utf8");
|
|
const start = source.indexOf("def collect_memory_snapshot():");
|
|
const end = source.indexOf("\ndef observe_run_record", start);
|
|
const collector = source.slice(start, end);
|
|
|
|
assert.match(collector, /source = "\/proc\/meminfo"/u);
|
|
assert.match(collector, /fields\.get\("MemAvailable"\)/u);
|
|
assert.match(collector, /"swapExcluded": True/u);
|
|
assert.doesNotMatch(collector, /SwapFree|SwapTotal|\["free", "-b"\]/u);
|
|
});
|
|
|
|
test("manual and sentinel gates return before any observer launch path", () => {
|
|
const manualSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe-actions.ts"), "utf8");
|
|
const manualStart = manualSource.indexOf("export function runNodeWebProbeObserveStart");
|
|
const manualGuard = manualSource.indexOf("runWebProbeMemoryGuard(spec, options.memoryGuardMode)", manualStart);
|
|
const manualReturn = manualSource.indexOf("startBlocked: true", manualGuard);
|
|
const manualJob = manualSource.indexOf("const jobId =", manualStart);
|
|
const manualLaunch = manualSource.indexOf("setsid env", manualStart);
|
|
assert.ok(manualStart >= 0 && manualGuard > manualStart && manualReturn > manualGuard);
|
|
assert.ok(manualReturn < manualJob && manualJob < manualLaunch);
|
|
|
|
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5-observe.ts"), "utf8");
|
|
const quickVerify = sentinelSource.indexOf("export function runSentinelQuickVerify");
|
|
const sentinelGuard = sentinelSource.indexOf("runWebProbeMemoryGuard(state.spec, \"sentinel-cadence\")", quickVerify);
|
|
const sentinelSkip = sentinelSource.indexOf("return recordSkippedQuickVerify", sentinelGuard);
|
|
const providerPreflight = sentinelSource.indexOf("runSentinelExactProviderProfilePreflight", quickVerify);
|
|
const observeStart = sentinelSource.indexOf('"web-probe", "observe", "start"', quickVerify);
|
|
assert.ok(quickVerify >= 0 && sentinelGuard > quickVerify && sentinelSkip > sentinelGuard);
|
|
assert.ok(sentinelSkip < providerPreflight && providerPreflight < observeStart);
|
|
assert.match(sentinelSource.slice(observeStart, observeStart + 800), /--sentinel-cadence/u);
|
|
|
|
const serviceSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-service.ts"), "utf8");
|
|
const plannedStart = serviceSource.indexOf('"bun", "scripts/cli.ts", "web-probe", "observe", "start"');
|
|
assert.match(serviceSource.slice(plannedStart, plannedStart + 800), /--sentinel-cadence/u);
|
|
});
|
|
|
|
test("all browser-launch entries use the shared preflight and final launch guard", () => {
|
|
const probeSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
|
|
const scriptSource = readFileSync(rootPath("scripts/src/hwlab-node/web-observe-scripts.ts"), "utf8");
|
|
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5.ts"), "utf8");
|
|
const parserSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe.ts"), "utf8");
|
|
|
|
const run = exportedFunctionSource(probeSource, "runNodeWebProbe");
|
|
const screenshot = exportedFunctionSource(probeSource, "runNodeWebProbeScreenshot");
|
|
const asyncRun = exportedFunctionSource(probeSource, "runNodeWebProbeAsync");
|
|
const script = exportedFunctionSource(scriptSource, "runNodeWebProbeScript");
|
|
const scriptShell = exportedFunctionSource(scriptSource, "nodeWebProbeScriptRemoteShell");
|
|
const dashboard = exportedFunctionSource(sentinelSource, "probeSentinelDashboardBrowser");
|
|
const parser = exportedFunctionSource(parserSource, "parseNodeWebProbeOptions");
|
|
|
|
assert.doesNotMatch(probeSource, /runNodeWebProbeMemoryGuard/u);
|
|
assert.match(run, /if \(options\.action === "script"\) return runNodeWebProbeScript/u);
|
|
assert.match(run, /if \(options\.commandTimeoutSeconds > 55\) return runNodeWebProbeAsync/u);
|
|
assert.match(run, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start"/u);
|
|
assert.match(screenshot, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start", webProbeScreenshotRemoteScript/u);
|
|
assert.match(asyncRun, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start"[\s\S]+"structured-output"\)/u);
|
|
assert.match(script, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+nodeWebProbeScriptRemoteShell/u);
|
|
assert.match(scriptShell, /webProbeGuardedLaunchShell\(spec, "manual-start"/u);
|
|
assert.match(parser, /if \(actionRaw === "opencode-smoke"\)[\s\S]+action: "script"/u);
|
|
assert.match(dashboard, /runWebProbeMemoryGuard\(state\.spec, "sentinel-cadence"\)[\s\S]+webProbeGuardedLaunchShell\(state\.spec, "sentinel-cadence"/u);
|
|
assert.match(dashboard, /UNIDESK_SENTINEL_DASHBOARD_TRIGGER=.*options\.action === "trigger"/u);
|
|
assert.match(dashboard, /UNIDESK_SENTINEL_DASHBOARD_CAPTURE=.*options\.action === "screenshot"/u);
|
|
});
|
|
|
|
test("read-only and control-plane entries stay outside browser launch guard", () => {
|
|
const dispatchSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
|
|
const dispatchStart = dispatchSource.indexOf("export function runNodeWebProbe(");
|
|
const dispatchEnd = dispatchSource.indexOf("export function runNodeWebProbeScreenshot", dispatchStart);
|
|
const dispatch = dispatchSource.slice(dispatchStart, dispatchEnd);
|
|
const readOnlyReturn = dispatch.indexOf('options.action === "observe" && options.observeAction !== "start"');
|
|
const firstGuard = dispatch.indexOf("runWebProbeMemoryGuard");
|
|
assert.ok(readOnlyReturn >= 0 && firstGuard > readOnlyReturn);
|
|
assert.match(dispatch.slice(0, readOnlyReturn), /options\.action === "sentinel"/u);
|
|
assert.match(dispatch.slice(0, readOnlyReturn), /options\.action === "screenshot"/u);
|
|
|
|
const sentinelDispatch = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-cicd.ts"), "utf8");
|
|
const sentinel = exportedFunctionSource(sentinelDispatch, "runWebProbeSentinelCommand");
|
|
assert.doesNotMatch(sentinel, /runWebProbeMemoryGuard|webProbeGuardedLaunchShell/u);
|
|
assert.match(sentinel, /options\.kind === "dashboard"/u);
|
|
});
|
|
|
|
test("generated runner bounds network evidence and defaults screenshots to viewport", () => {
|
|
const source = nodeWebProbeScriptRunnerSource();
|
|
assert.match(source, /maxNetworkRequestEntries/u);
|
|
assert.match(source, /maxNetworkRequestBytes/u);
|
|
assert.match(source, /maxNetworkFailureEntries/u);
|
|
assert.match(source, /maxNetworkFailureBytes/u);
|
|
assert.match(source, /apiRequestEvidenceDropped/u);
|
|
assert.match(source, /apiFailureEvidenceDropped/u);
|
|
assert.doesNotMatch(source, /fullPage: true/u);
|
|
assert.equal((source.match(/fullPage: screenshotMode !== "viewport"/gu) ?? []).length, 2);
|
|
|
|
const parserSource = exportedFunctionSource(readFileSync(rootPath("scripts/src/hwlab-node/web-probe.ts"), "utf8"), "parseNodeWebProbeOptions");
|
|
assert.match(parserSource, /fullPage: args\.includes\("--full-page"\) && !args\.includes\("--no-full-page"\)/u);
|
|
const screenshotSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
|
|
assert.match(screenshotSource, /webProbeScreenshotFullPage\(spec, options\.fullPage\)/u);
|
|
assert.match(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE === "1"/u);
|
|
assert.doesNotMatch(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE !== "0"/u);
|
|
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5.ts"), "utf8");
|
|
assert.match(sentinelSource, /webProbeScreenshotFullPage\(state\.spec, options\.fullPage\)/u);
|
|
assert.match(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE === "1"/u);
|
|
assert.doesNotMatch(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0"/u);
|
|
assert.equal(webProbeScreenshotFullPage(spec, false), false);
|
|
assert.equal(webProbeScreenshotFullPage({ ...spec, webProbe: { ...spec.webProbe!, resourcePolicy: { ...spec.webProbe!.resourcePolicy!, browserEvidence: { ...spec.webProbe!.resourcePolicy!.browserEvidence!, screenshotMode: "full-page" } } } }, false), true);
|
|
assert.equal(webProbeScreenshotFullPage(spec, true), true);
|
|
});
|
|
|
|
test("final launch guard rechecks MemAvailable under the configured lock", () => {
|
|
const sync = webProbeGuardedLaunchShell(spec, "manual-start", "node runner.mjs");
|
|
const async = webProbeGuardedLaunchShell(spec, "manual-start", "node helper.mjs start", "structured-output");
|
|
assert.match(sync, /flock -w '15'/u);
|
|
assert.match(sync, /MemAvailable:/u);
|
|
assert.match(sync, /mem_available_bytes.*-le '4294967296'/u);
|
|
assert.match(sync, /setsid sh -c/u);
|
|
assert.match(sync, /exec 9>&-/u);
|
|
assert.equal(sync.includes("playwright"), false);
|
|
assert.equal(sync.includes("fs.readlinkSync('/proc/' + name + '/exe')"), true);
|
|
assert.equal(sync.includes("trap 'browser_exit=$?; trap - EXIT TERM INT HUP; cleanup_browser_group"), true);
|
|
assert.equal(sync.includes('kill -TERM -- "-$browser_job_pid"'), true);
|
|
assert.equal(sync.includes('kill -0 -- "-$browser_job_pid"'), true);
|
|
assert.equal(sync.includes('kill -KILL -- "-$browser_job_pid"'), true);
|
|
assert.equal(async.includes("browserCreated === true"), false);
|
|
assert.equal(async.includes("data.workerPid, data.jobPid, data.pid"), true);
|
|
assert.equal(async.includes("[data.browserPid, data.workerPid"), false);
|
|
assert.equal(async.includes("reportedBrowserPid === null || row.pid === reportedBrowserPid"), true);
|
|
assert.equal(async.includes("then flock -u 9; exit 43; fi"), true);
|
|
assert.equal(async.includes("fs.readlinkSync('/proc/' + name + '/exe')"), true);
|
|
});
|