fix: 补齐 WebProbe 浏览器启动门禁
This commit is contained in:
AgentRun Codex
@@ -588,6 +588,12 @@ templates:
|
||||
maxProcessReadConcurrency: 32
|
||||
maxRealtimeTotalEntries: 512
|
||||
maxRealtimeTotalBodyBytes: 4194304
|
||||
browserEvidence:
|
||||
screenshotMode: viewport
|
||||
maxNetworkRequestEntries: 200
|
||||
maxNetworkRequestBytes: 262144
|
||||
maxNetworkFailureEntries: 100
|
||||
maxNetworkFailureBytes: 131072
|
||||
origins:
|
||||
internal:
|
||||
mode: k8s-service-cluster-ip
|
||||
|
||||
@@ -203,6 +203,15 @@ export interface HwlabRuntimeWebProbeObserverLifecycleSpec {
|
||||
export interface HwlabRuntimeWebProbeResourcePolicySpec {
|
||||
readonly memoryStartGuard: HwlabRuntimeWebProbeMemoryStartGuardSpec;
|
||||
readonly observerLifecycle: HwlabRuntimeWebProbeObserverLifecycleSpec;
|
||||
readonly browserEvidence: HwlabRuntimeWebProbeBrowserEvidenceSpec;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeBrowserEvidenceSpec {
|
||||
readonly screenshotMode: "viewport";
|
||||
readonly maxNetworkRequestEntries: number;
|
||||
readonly maxNetworkRequestBytes: number;
|
||||
readonly maxNetworkFailureEntries: number;
|
||||
readonly maxNetworkFailureBytes: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeWorkbenchKafkaDebugReplaySpec {
|
||||
@@ -1482,6 +1491,7 @@ function webProbeResourcePolicyConfig(value: unknown, path: string): HwlabRuntim
|
||||
const raw = asRecord(value, path);
|
||||
const memory = asRecord(raw.memoryStartGuard, `${path}.memoryStartGuard`);
|
||||
const lifecycle = asRecord(raw.observerLifecycle, `${path}.observerLifecycle`);
|
||||
const browserEvidence = asRecord(raw.browserEvidence, `${path}.browserEvidence`);
|
||||
return {
|
||||
memoryStartGuard: {
|
||||
metric: enumStringField(memory, "metric", `${path}.memoryStartGuard`, ["MemAvailable"] as const),
|
||||
@@ -1522,6 +1532,13 @@ function webProbeResourcePolicyConfig(value: unknown, path: string): HwlabRuntim
|
||||
maxRealtimeTotalEntries: boundedIntegerField(lifecycle, "maxRealtimeTotalEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxRealtimeTotalBodyBytes: boundedIntegerField(lifecycle, "maxRealtimeTotalBodyBytes", `${path}.observerLifecycle`, 1024, 64 * 1024 * 1024),
|
||||
},
|
||||
browserEvidence: {
|
||||
screenshotMode: enumStringField(browserEvidence, "screenshotMode", `${path}.browserEvidence`, ["viewport"] as const),
|
||||
maxNetworkRequestEntries: boundedIntegerField(browserEvidence, "maxNetworkRequestEntries", `${path}.browserEvidence`, 1, 100_000),
|
||||
maxNetworkRequestBytes: boundedIntegerField(browserEvidence, "maxNetworkRequestBytes", `${path}.browserEvidence`, 1024, 64 * 1024 * 1024),
|
||||
maxNetworkFailureEntries: boundedIntegerField(browserEvidence, "maxNetworkFailureEntries", `${path}.browserEvidence`, 1, 100_000),
|
||||
maxNetworkFailureBytes: boundedIntegerField(browserEvidence, "maxNetworkFailureBytes", `${path}.browserEvidence`, 1024, 64 * 1024 * 1024),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -5,7 +5,8 @@ import { test } from "bun:test";
|
||||
import type { CommandResult } from "./command";
|
||||
import { rootPath } from "./config";
|
||||
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
|
||||
import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard, webProbeMemoryStartGuardConfigRef } from "./hwlab-node-web-probe-memory-guard";
|
||||
import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeMemoryStartGuardConfigRef } from "./hwlab-node-web-probe-memory-guard";
|
||||
import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source";
|
||||
|
||||
const spec = hwlabRuntimeLaneSpecForNode("v03", "NC01");
|
||||
const thresholdBytes = 4 * 1024 * 1024 * 1024;
|
||||
@@ -23,6 +24,15 @@ function memoryPayload(availableBytes: number, extra: Record<string, unknown> =
|
||||
};
|
||||
}
|
||||
|
||||
function exportedFunctionSource(source: string, name: string): string {
|
||||
const start = source.indexOf(`export function ${name}(`);
|
||||
assert.ok(start >= 0, `missing exported function ${name}`);
|
||||
const nextExport = source.indexOf("\nexport function ", start + 1);
|
||||
const nextFunction = source.indexOf("\nfunction ", start + 1);
|
||||
const next = [nextExport, nextFunction].filter((value) => value >= 0).sort((left, right) => left - right)[0] ?? -1;
|
||||
return source.slice(start, next < 0 ? source.length : next);
|
||||
}
|
||||
|
||||
test("WebProbe memory threshold is read from the single owning YAML policy", () => {
|
||||
const policy = spec.webProbe?.resourcePolicy?.memoryStartGuard;
|
||||
assert.equal(policy?.thresholdBytes, thresholdBytes);
|
||||
@@ -34,6 +44,21 @@ test("WebProbe memory threshold is read from the single owning YAML policy", ()
|
||||
assert.match(webProbeMemoryStartGuardConfigRef, /resourcePolicy\.memoryStartGuard$/u);
|
||||
});
|
||||
|
||||
test("browser evidence limits and viewport screenshots are read from owning YAML", () => {
|
||||
const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
|
||||
assert.deepEqual(policy, {
|
||||
screenshotMode: "viewport",
|
||||
maxNetworkRequestEntries: 200,
|
||||
maxNetworkRequestBytes: 262144,
|
||||
maxNetworkFailureEntries: 100,
|
||||
maxNetworkFailureBytes: 131072,
|
||||
});
|
||||
const assignments = webProbeBrowserEvidenceEnvAssignments(spec).join(" ");
|
||||
assert.match(assignments, /UNIDESK_WEB_PROBE_SCREENSHOT_MODE='viewport'/u);
|
||||
assert.match(assignments, /UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES='200'/u);
|
||||
assert.match(assignments, /UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES='131072'/u);
|
||||
});
|
||||
|
||||
test("manual start is blocked at exactly 4 GiB and requires controlled GC then MemAvailable recheck", () => {
|
||||
const guard = evaluateWebProbeMemoryGuard(spec, "manual-start", memoryPayload(thresholdBytes));
|
||||
const memory = guard.memory as Record<string, unknown>;
|
||||
@@ -141,3 +166,83 @@ test("manual and sentinel gates return before any observer launch path", () => {
|
||||
const plannedStart = serviceSource.indexOf('"bun", "scripts/cli.ts", "web-probe", "observe", "start"');
|
||||
assert.match(serviceSource.slice(plannedStart, plannedStart + 800), /--sentinel-cadence/u);
|
||||
});
|
||||
|
||||
test("all browser-launch entries use the shared preflight and final launch guard", () => {
|
||||
const probeSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
|
||||
const scriptSource = readFileSync(rootPath("scripts/src/hwlab-node/web-observe-scripts.ts"), "utf8");
|
||||
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5.ts"), "utf8");
|
||||
const parserSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe.ts"), "utf8");
|
||||
|
||||
const run = exportedFunctionSource(probeSource, "runNodeWebProbe");
|
||||
const screenshot = exportedFunctionSource(probeSource, "runNodeWebProbeScreenshot");
|
||||
const asyncRun = exportedFunctionSource(probeSource, "runNodeWebProbeAsync");
|
||||
const script = exportedFunctionSource(scriptSource, "runNodeWebProbeScript");
|
||||
const scriptShell = exportedFunctionSource(scriptSource, "nodeWebProbeScriptRemoteShell");
|
||||
const dashboard = exportedFunctionSource(sentinelSource, "probeSentinelDashboardBrowser");
|
||||
const parser = exportedFunctionSource(parserSource, "parseNodeWebProbeOptions");
|
||||
|
||||
assert.doesNotMatch(probeSource, /runNodeWebProbeMemoryGuard/u);
|
||||
assert.match(run, /if \(options\.action === "script"\) return runNodeWebProbeScript/u);
|
||||
assert.match(run, /if \(options\.commandTimeoutSeconds > 55\) return runNodeWebProbeAsync/u);
|
||||
assert.match(run, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start"/u);
|
||||
assert.match(screenshot, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start", webProbeScreenshotRemoteScript/u);
|
||||
assert.match(asyncRun, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start"[\s\S]+"structured-output"\)/u);
|
||||
assert.match(script, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+nodeWebProbeScriptRemoteShell/u);
|
||||
assert.match(scriptShell, /webProbeGuardedLaunchShell\(spec, "manual-start"/u);
|
||||
assert.match(parser, /if \(actionRaw === "opencode-smoke"\)[\s\S]+action: "script"/u);
|
||||
assert.match(dashboard, /runWebProbeMemoryGuard\(state\.spec, "sentinel-cadence"\)[\s\S]+webProbeGuardedLaunchShell\(state\.spec, "sentinel-cadence"/u);
|
||||
assert.match(dashboard, /UNIDESK_SENTINEL_DASHBOARD_TRIGGER=.*options\.action === "trigger"/u);
|
||||
assert.match(dashboard, /UNIDESK_SENTINEL_DASHBOARD_CAPTURE=.*options\.action === "screenshot"/u);
|
||||
});
|
||||
|
||||
test("read-only and control-plane entries stay outside browser launch guard", () => {
|
||||
const dispatchSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
|
||||
const dispatchStart = dispatchSource.indexOf("export function runNodeWebProbe(");
|
||||
const dispatchEnd = dispatchSource.indexOf("export function runNodeWebProbeScreenshot", dispatchStart);
|
||||
const dispatch = dispatchSource.slice(dispatchStart, dispatchEnd);
|
||||
const readOnlyReturn = dispatch.indexOf('options.action === "observe" && options.observeAction !== "start"');
|
||||
const firstGuard = dispatch.indexOf("runWebProbeMemoryGuard");
|
||||
assert.ok(readOnlyReturn >= 0 && firstGuard > readOnlyReturn);
|
||||
assert.match(dispatch.slice(0, readOnlyReturn), /options\.action === "sentinel"/u);
|
||||
assert.match(dispatch.slice(0, readOnlyReturn), /options\.action === "screenshot"/u);
|
||||
|
||||
const sentinelDispatch = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-cicd.ts"), "utf8");
|
||||
const sentinel = exportedFunctionSource(sentinelDispatch, "runWebProbeSentinelCommand");
|
||||
assert.doesNotMatch(sentinel, /runWebProbeMemoryGuard|webProbeGuardedLaunchShell/u);
|
||||
assert.match(sentinel, /options\.kind === "dashboard"/u);
|
||||
});
|
||||
|
||||
test("generated runner bounds network evidence and defaults screenshots to viewport", () => {
|
||||
const source = nodeWebProbeScriptRunnerSource();
|
||||
assert.match(source, /maxNetworkRequestEntries/u);
|
||||
assert.match(source, /maxNetworkRequestBytes/u);
|
||||
assert.match(source, /maxNetworkFailureEntries/u);
|
||||
assert.match(source, /maxNetworkFailureBytes/u);
|
||||
assert.match(source, /apiRequestEvidenceDropped/u);
|
||||
assert.match(source, /apiFailureEvidenceDropped/u);
|
||||
assert.doesNotMatch(source, /fullPage: true/u);
|
||||
assert.equal((source.match(/fullPage: screenshotMode !== "viewport"/gu) ?? []).length, 2);
|
||||
|
||||
const parserSource = exportedFunctionSource(readFileSync(rootPath("scripts/src/hwlab-node/web-probe.ts"), "utf8"), "parseNodeWebProbeOptions");
|
||||
assert.match(parserSource, /fullPage: args\.includes\("--full-page"\) && !args\.includes\("--no-full-page"\)/u);
|
||||
const screenshotSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
|
||||
assert.match(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE === "1"/u);
|
||||
assert.doesNotMatch(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE !== "0"/u);
|
||||
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5.ts"), "utf8");
|
||||
assert.match(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE === "1"/u);
|
||||
assert.doesNotMatch(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0"/u);
|
||||
});
|
||||
|
||||
test("final launch guard rechecks MemAvailable under the configured lock", () => {
|
||||
const sync = webProbeGuardedLaunchShell(spec, "manual-start", "node runner.mjs");
|
||||
const async = webProbeGuardedLaunchShell(spec, "manual-start", "node helper.mjs start", "structured-output");
|
||||
assert.match(sync, /flock -w '15'/u);
|
||||
assert.match(sync, /MemAvailable:/u);
|
||||
assert.match(sync, /mem_available_bytes.*-le '4294967296'/u);
|
||||
assert.match(sync, /setsid sh -c/u);
|
||||
assert.match(sync, /kill -TERM -- "-\$browser_job_pid"/u);
|
||||
assert.match(sync, /kill -KILL -- "-\$browser_job_pid"/u);
|
||||
assert.match(async, /browserCreated === true/u);
|
||||
assert.match(async, /data\.workerPid, data\.jobPid, data\.pid/u);
|
||||
assert.match(async, /\/proc\/.*\/cmdline/u);
|
||||
});
|
||||
|
||||
@@ -24,6 +24,127 @@ function memoryPolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeMemorySta
|
||||
return spec.webProbe?.resourcePolicy?.memoryStartGuard ?? null;
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replace(/'/g, `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
export function webProbeBrowserEvidenceEnvAssignments(spec: HwlabRuntimeLaneSpec): string[] {
|
||||
const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
|
||||
if (policy === undefined) throw new Error("web-probe browser evidence policy is missing from the owning YAML");
|
||||
return [
|
||||
`UNIDESK_WEB_PROBE_SCREENSHOT_MODE=${shellQuote(policy.screenshotMode)}`,
|
||||
`UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES=${shellQuote(String(policy.maxNetworkRequestEntries))}`,
|
||||
`UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_BYTES=${shellQuote(String(policy.maxNetworkRequestBytes))}`,
|
||||
`UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_ENTRIES=${shellQuote(String(policy.maxNetworkFailureEntries))}`,
|
||||
`UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES=${shellQuote(String(policy.maxNetworkFailureBytes))}`,
|
||||
];
|
||||
}
|
||||
|
||||
export function webProbeGuardedLaunchShell(
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
mode: WebProbeMemoryGuardMode,
|
||||
command: string,
|
||||
readiness: "child-process" | "structured-output" = "child-process",
|
||||
): string {
|
||||
const policy = memoryPolicy(spec);
|
||||
if (policy === null) throw new Error("web-probe memory start policy is missing from the owning YAML");
|
||||
const comparator = mode === "manual-start" ? "-le" : "-lt";
|
||||
const common = [
|
||||
`exec 9>${shellQuote(policy.launchLockPath)}`,
|
||||
`flock -w ${shellQuote(String(policy.launchLockTimeoutSeconds))} 9`,
|
||||
"mem_available_kib=$(awk '$1 == \"MemAvailable:\" { print $2; exit }' /proc/meminfo)",
|
||||
"case \"$mem_available_kib\" in ''|*[!0-9]*) printf '%s\\n' 'web-probe final memory guard could not read MemAvailable' >&2; exit 42;; esac",
|
||||
"mem_available_bytes=$((mem_available_kib * 1024))",
|
||||
`if [ "$mem_available_bytes" ${comparator} ${shellQuote(String(policy.thresholdBytes))} ]; then printf '%s\\n' 'web-probe final memory guard blocked browser launch' >&2; exit 42; fi`,
|
||||
];
|
||||
if (readiness === "structured-output") return [
|
||||
...common,
|
||||
"launch_output=$(mktemp)",
|
||||
"launch_error=$(mktemp)",
|
||||
"trap 'rm -f \"$launch_output\" \"$launch_error\"' EXIT",
|
||||
`if ! ${command} >"$launch_output" 2>"$launch_error"; then cat "$launch_output"; cat "$launch_error" >&2; exit 43; fi`,
|
||||
"cat \"$launch_output\"",
|
||||
"cat \"$launch_error\" >&2",
|
||||
`node - "$launch_output" ${shellQuote(String(policy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(policy.launchReadyPollMilliseconds))} <<'UNIDESK_WEB_PROBE_STRUCTURED_READINESS'`,
|
||||
"const fs = require('fs');",
|
||||
"const text = fs.readFileSync(process.argv[2], 'utf8').trim();",
|
||||
"const timeoutMs = Number(process.argv[3]);",
|
||||
"const pollMs = Number(process.argv[4]);",
|
||||
"let value = null; try { value = JSON.parse(text); } catch {}",
|
||||
"const data = value && typeof value === 'object' && value.data && typeof value.data === 'object' ? value.data : value;",
|
||||
"if (!data || typeof data !== 'object' || !data.jobId) process.exit(43);",
|
||||
"if (data.browserCreated === true && Number.isInteger(data.browserPid) && data.browserPid > 0) process.exit(0);",
|
||||
"const rootPid = [data.browserPid, data.workerPid, data.jobPid, data.pid].find((item) => Number.isInteger(item) && item > 0);",
|
||||
"if (!rootPid) process.exit(43);",
|
||||
"const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
|
||||
"const browserDescendant = () => {",
|
||||
" const rows = [];",
|
||||
" for (const name of fs.readdirSync('/proc')) {",
|
||||
" if (!/^[0-9]+$/.test(name)) continue;",
|
||||
" try {",
|
||||
" const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');",
|
||||
" const close = stat.lastIndexOf(')');",
|
||||
" const fields = stat.slice(close + 2).split(' ');",
|
||||
" const ppid = Number(fields[1]);",
|
||||
" const command = fs.readFileSync('/proc/' + name + '/cmdline', 'utf8').replace(/\\0/g, ' ');",
|
||||
" rows.push({ pid: Number(name), ppid, command });",
|
||||
" } catch {}",
|
||||
" }",
|
||||
" const descendants = new Set([rootPid]);",
|
||||
" let changed = true;",
|
||||
" while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }",
|
||||
" return rows.some((row) => descendants.has(row.pid) && /(chromium|chrome|playwright)/i.test(row.command));",
|
||||
"};",
|
||||
"(async () => {",
|
||||
" const deadline = Date.now() + timeoutMs;",
|
||||
" while (Date.now() <= deadline) { if (browserDescendant()) process.exit(0); await sleep(pollMs); }",
|
||||
" process.exit(43);",
|
||||
"})().catch(() => process.exit(43));",
|
||||
"UNIDESK_WEB_PROBE_STRUCTURED_READINESS",
|
||||
"flock -u 9",
|
||||
].join("\n");
|
||||
return [
|
||||
...common,
|
||||
`setsid sh -c ${shellQuote(command)} &`,
|
||||
"browser_job_pid=$!",
|
||||
"browser_ready=false",
|
||||
`browser_ready_deadline=$(( $(date +%s) + ${policy.launchReadyTimeoutSeconds} ))`,
|
||||
"while kill -0 \"$browser_job_pid\" 2>/dev/null; do",
|
||||
" if node - \"$browser_job_pid\" <<'UNIDESK_WEB_PROBE_CHILD_READINESS'",
|
||||
"const fs = require('fs');",
|
||||
"const rootPid = Number(process.argv[2]);",
|
||||
"const rows = [];",
|
||||
"for (const name of fs.readdirSync('/proc')) {",
|
||||
" if (!/^[0-9]+$/.test(name)) continue;",
|
||||
" try {",
|
||||
" const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');",
|
||||
" const close = stat.lastIndexOf(')');",
|
||||
" const fields = stat.slice(close + 2).split(' ');",
|
||||
" rows.push({ pid: Number(name), ppid: Number(fields[1]), command: fs.readFileSync('/proc/' + name + '/cmdline', 'utf8').replace(/\\0/g, ' ') });",
|
||||
" } catch {}",
|
||||
"}",
|
||||
"const descendants = new Set([rootPid]);",
|
||||
"let changed = true;",
|
||||
"while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }",
|
||||
"process.exit(rows.some((row) => descendants.has(row.pid) && /(chromium|chrome|playwright)/i.test(row.command)) ? 0 : 1);",
|
||||
"UNIDESK_WEB_PROBE_CHILD_READINESS",
|
||||
" then browser_ready=true; break; fi",
|
||||
" if [ \"$(date +%s)\" -ge \"$browser_ready_deadline\" ]; then break; fi",
|
||||
` sleep ${Math.max(0.05, policy.launchReadyPollMilliseconds / 1000)}`,
|
||||
"done",
|
||||
"flock -u 9",
|
||||
"if [ \"$browser_ready\" != true ]; then",
|
||||
" kill -TERM -- \"-$browser_job_pid\" 2>/dev/null || true",
|
||||
" sleep 1",
|
||||
" if kill -0 \"$browser_job_pid\" 2>/dev/null; then kill -KILL -- \"-$browser_job_pid\" 2>/dev/null || true; fi",
|
||||
" wait \"$browser_job_pid\" 2>/dev/null || true",
|
||||
" printf '%s\\n' 'web-probe browser launch readiness was not confirmed' >&2",
|
||||
" exit 43",
|
||||
"fi",
|
||||
"wait \"$browser_job_pid\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function humanBytes(value: number | null): string | null {
|
||||
if (value === null || !Number.isFinite(value)) return null;
|
||||
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
||||
|
||||
@@ -20,6 +20,11 @@ const viewport = parseViewport(process.env.UNIDESK_WEB_PROBE_VIEWPORT || "1440x9
|
||||
const browserProxyMode = parseBrowserProxyMode(process.env.UNIDESK_WEB_PROBE_BROWSER_PROXY_MODE || "auto");
|
||||
const playwrightProxy = proxyConfigFromEnv(baseUrl);
|
||||
const chromiumLaunchOptions = chromiumLaunchOptionsForProxy(playwrightProxy);
|
||||
const screenshotMode = requiredEnum(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_MODE, ["viewport"]);
|
||||
const maxNetworkRequestEntries = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES, "UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES");
|
||||
const maxNetworkRequestBytes = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_BYTES, "UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_BYTES");
|
||||
const maxNetworkFailureEntries = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_ENTRIES, "UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_ENTRIES");
|
||||
const maxNetworkFailureBytes = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES, "UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES");
|
||||
const artifactRecords = [];
|
||||
const readinessRecords = [];
|
||||
const stepRecords = [];
|
||||
@@ -1314,36 +1319,54 @@ function createNetworkTracker(targetPage, apiPattern) {
|
||||
const started = Date.now();
|
||||
const requests = [];
|
||||
const failures = [];
|
||||
let requestBytes = 0;
|
||||
let requestDropped = 0;
|
||||
let failureBytes = 0;
|
||||
let failureDropped = 0;
|
||||
const matches = createApiMatcher(apiPattern);
|
||||
const appendBounded = (items, item, kind) => {
|
||||
const bytes = Buffer.byteLength(JSON.stringify(item), "utf8");
|
||||
const entryLimit = kind === "request" ? maxNetworkRequestEntries : maxNetworkFailureEntries;
|
||||
const byteLimit = kind === "request" ? maxNetworkRequestBytes : maxNetworkFailureBytes;
|
||||
const usedBytes = kind === "request" ? requestBytes : failureBytes;
|
||||
if (items.length >= entryLimit || usedBytes + bytes > byteLimit) {
|
||||
if (kind === "request") requestDropped += 1;
|
||||
else failureDropped += 1;
|
||||
return;
|
||||
}
|
||||
items.push(item);
|
||||
if (kind === "request") requestBytes += bytes;
|
||||
else failureBytes += bytes;
|
||||
};
|
||||
const onRequest = (request) => {
|
||||
if (!matches(request.url())) return;
|
||||
requests.push({
|
||||
appendBounded(requests, {
|
||||
method: request.method(),
|
||||
path: urlPath(request.url()),
|
||||
atMs: Date.now() - started,
|
||||
});
|
||||
}, "request");
|
||||
};
|
||||
const onResponse = (response) => {
|
||||
if (!matches(response.url())) return;
|
||||
if (response.status() >= 400) {
|
||||
failures.push({
|
||||
appendBounded(failures, {
|
||||
type: "response",
|
||||
status: response.status(),
|
||||
statusText: response.statusText(),
|
||||
path: urlPath(response.url()),
|
||||
atMs: Date.now() - started,
|
||||
});
|
||||
}, "failure");
|
||||
}
|
||||
};
|
||||
const onRequestFailed = (request) => {
|
||||
if (!matches(request.url())) return;
|
||||
failures.push({
|
||||
appendBounded(failures, {
|
||||
type: "requestfailed",
|
||||
method: request.method(),
|
||||
path: urlPath(request.url()),
|
||||
failureText: request.failure()?.errorText ?? null,
|
||||
atMs: Date.now() - started,
|
||||
});
|
||||
}, "failure");
|
||||
};
|
||||
targetPage.on("request", onRequest);
|
||||
targetPage.on("response", onResponse);
|
||||
@@ -1356,6 +1379,10 @@ function createNetworkTracker(targetPage, apiPattern) {
|
||||
apiFailureCount: failures.length,
|
||||
apiRequests: requests.slice(0, 5),
|
||||
apiFailures: failures.slice(0, 5),
|
||||
apiRequestEvidenceBytes: requestBytes,
|
||||
apiRequestEvidenceDropped: requestDropped,
|
||||
apiFailureEvidenceBytes: failureBytes,
|
||||
apiFailureEvidenceDropped: failureDropped,
|
||||
}),
|
||||
dispose: () => {
|
||||
targetPage.off("request", onRequest);
|
||||
@@ -1476,7 +1503,7 @@ async function captureFailureScreenshot(name = "failure.png") {
|
||||
|
||||
async function screenshotPage(targetPage, name = "screenshot.png", options = {}) {
|
||||
const file = artifactPath(name);
|
||||
await targetPage.screenshot({ path: file, fullPage: true, ...options });
|
||||
await targetPage.screenshot({ path: file, fullPage: screenshotMode !== "viewport", ...options });
|
||||
return recordArtifact(file, "screenshot");
|
||||
}
|
||||
|
||||
@@ -1910,6 +1937,17 @@ function positiveInteger(raw, fallback) {
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredPositiveInteger(raw, name) {
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value <= 0) throw new Error("missing or invalid " + name);
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredEnum(raw, allowed) {
|
||||
if (!allowed.includes(raw)) throw new Error("missing or invalid screenshot mode");
|
||||
return raw;
|
||||
}
|
||||
|
||||
function artifactPath(name) {
|
||||
const safe = String(name || "artifact")
|
||||
.replace(/[^A-Za-z0-9._-]/gu, "_")
|
||||
@@ -1920,7 +1958,7 @@ function artifactPath(name) {
|
||||
|
||||
async function screenshot(name = "screenshot.png", options = {}) {
|
||||
const file = artifactPath(name);
|
||||
await page.screenshot({ path: file, fullPage: true, ...options });
|
||||
await page.screenshot({ path: file, fullPage: screenshotMode !== "viewport", ...options });
|
||||
return recordArtifact(file, "screenshot");
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import { startJob } from "./jobs";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact";
|
||||
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
|
||||
import { runWebProbeMemoryGuard, webProbeGuardedLaunchShell } from "./hwlab-node-web-probe-memory-guard";
|
||||
import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd";
|
||||
import {
|
||||
clipTail,
|
||||
@@ -1144,11 +1145,13 @@ export function runSentinelDashboard(state: SentinelCicdState, options: Extract<
|
||||
}
|
||||
|
||||
export function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): Record<string, unknown> {
|
||||
const memoryGuard = runWebProbeMemoryGuard(state.spec, "sentinel-cadence");
|
||||
if (memoryGuard.status !== "allowed") return memoryGuard;
|
||||
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
|
||||
const dashboardUrl = sentinelDashboardUrl(publicBaseUrl, options.runId, options.errorId);
|
||||
const [widthRaw, heightRaw] = options.viewport.split("x");
|
||||
const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : "";
|
||||
const script = [
|
||||
const browserScript = [
|
||||
"set -eu",
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(dashboardUrl)}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`,
|
||||
@@ -1178,6 +1181,7 @@ export function probeSentinelDashboardBrowser(state: SentinelCicdState, options:
|
||||
"WEB_PROBE_SENTINEL_DASHBOARD_JS",
|
||||
"node \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\"",
|
||||
].join("\n");
|
||||
const script = webProbeGuardedLaunchShell(state.spec, "sentinel-cadence", browserScript);
|
||||
const route = `${state.spec.nodeId}:${state.spec.workspace}`;
|
||||
const job = runWebProbeRemoteArtifactJob({
|
||||
route,
|
||||
@@ -1263,7 +1267,7 @@ const triggerManual = process.env.UNIDESK_SENTINEL_DASHBOARD_TRIGGER === "1";
|
||||
const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440);
|
||||
const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900);
|
||||
const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000);
|
||||
const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0";
|
||||
const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE === "1";
|
||||
const executablePath = process.env.UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH || "";
|
||||
const expectedSentinelId = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID || "";
|
||||
const expectedRoutePrefix = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX || "/";
|
||||
|
||||
@@ -25,6 +25,7 @@ import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapp
|
||||
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
||||
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
||||
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
||||
import { runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell } from "../hwlab-node-web-probe-memory-guard";
|
||||
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
||||
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
@@ -537,7 +538,9 @@ export function runNodeWebProbeScript(
|
||||
const commandLabel = options.commandLabel ?? `web-probe script --node ${options.node} --lane ${options.lane}${webProbeSemanticOriginArgs(options)}`;
|
||||
const commandGovernance = webProbeScriptCommandGovernance(options);
|
||||
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
||||
const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.username ?? secretSpec.bootstrapAdminUsername, material.password ?? "", webProbeProxy, spec.webProbe?.playwrightBrowsersPath);
|
||||
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
|
||||
if (memoryGuard.status !== "allowed") return memoryGuard;
|
||||
const script = nodeWebProbeScriptRemoteShell(options, spec, secretSpec, material.username ?? secretSpec.bootstrapAdminUsername, material.password ?? "", webProbeProxy, spec.webProbe?.playwrightBrowsersPath);
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const commandTimedOut = result.timedOut || result.exitCode === 124;
|
||||
const runPaths = webProbeScriptRunPathsFromStderr(result.stderr);
|
||||
@@ -740,7 +743,7 @@ function webProbeSemanticOriginArgs(options: Pick<NodeWebProbeScriptOptions, "or
|
||||
return options.originName === "internal" || options.originName === "public" ? ` --origin ${options.originName}` : "";
|
||||
}
|
||||
|
||||
export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, username: string, password: string, webProbeProxy: NodeWebProbeHostProxyEnv, playwrightBrowsersPath: string | undefined): string {
|
||||
export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, spec: HwlabRuntimeLaneSpec, secretSpec: RuntimeSecretSpec, username: string, password: string, webProbeProxy: NodeWebProbeHostProxyEnv, playwrightBrowsersPath: string | undefined): string {
|
||||
const userScriptB64 = Buffer.from(options.scriptText, "utf8").toString("base64");
|
||||
const runnerB64 = Buffer.from(nodeWebProbeScriptRunnerSource(), "utf8").toString("base64");
|
||||
return [
|
||||
@@ -756,8 +759,9 @@ export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions
|
||||
"runner=\"$run_dir/runner.mjs\"",
|
||||
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$user_script" ${shellQuote(userScriptB64)}`,
|
||||
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`,
|
||||
[
|
||||
webProbeGuardedLaunchShell(spec, "manual-start", [
|
||||
...webProbeProxy.envAssignments,
|
||||
...webProbeBrowserEvidenceEnvAssignments(spec),
|
||||
...(playwrightBrowsersPath === undefined ? [] : [
|
||||
`PLAYWRIGHT_BROWSERS_PATH=${shellQuote(playwrightBrowsersPath)}`,
|
||||
]),
|
||||
@@ -770,7 +774,7 @@ export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions
|
||||
`UNIDESK_WEB_PROBE_VIEWPORT=${shellQuote(options.viewport)}`,
|
||||
`UNIDESK_WEB_PROBE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
||||
"node \"$runner\"",
|
||||
].join(" "),
|
||||
].join(" ")),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapp
|
||||
import { runWebProbeSentinelCommand, type WebProbeSentinelDashboardAction, type WebProbeSentinelOptions, type WebProbeSentinelReportView, type WebProbeSentinelSourceAuthority, type WebProbeSentinelSourceOverrideOptions } from "../hwlab-node-web-sentinel-cicd";
|
||||
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
||||
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
||||
import { runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell } from "../hwlab-node-web-probe-memory-guard";
|
||||
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
||||
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
||||
import { runWebProbeRemoteArtifactJob } from "../web-probe-remote-artifact";
|
||||
@@ -890,11 +891,13 @@ export function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, un
|
||||
if (options.action === "observe") return runNodeWebProbeObserve(options, spec, secretSpec, material, credential);
|
||||
if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential);
|
||||
if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential);
|
||||
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
|
||||
if (memoryGuard.status !== "allowed") return memoryGuard;
|
||||
const probeArgs = nodeWebProbeRunArgs(options, "run");
|
||||
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
||||
const script = [
|
||||
"set -eu",
|
||||
`${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`].join(" ")} ${probeArgs.map(shellQuote).join(" ")}`,
|
||||
webProbeGuardedLaunchShell(spec, "manual-start", `${[...webProbeProxy.envAssignments, ...webProbeBrowserEvidenceEnvAssignments(spec), `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`].join(" ")} ${probeArgs.map(shellQuote).join(" ")}`),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const probe = compactWebProbeResult(parseJsonObject(result.stdout));
|
||||
@@ -932,8 +935,10 @@ export function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, un
|
||||
}
|
||||
|
||||
export function runNodeWebProbeScreenshot(options: NodeWebProbeScreenshotOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
|
||||
if (memoryGuard.status !== "allowed") return memoryGuard;
|
||||
const route = `${options.node}:${spec.workspace}`;
|
||||
const script = webProbeScreenshotRemoteScript(options);
|
||||
const script = webProbeGuardedLaunchShell(spec, "manual-start", webProbeScreenshotRemoteScript(options));
|
||||
const job = runWebProbeRemoteArtifactJob({
|
||||
route,
|
||||
localDir: options.localDir,
|
||||
@@ -1095,7 +1100,7 @@ const width = Number(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_WIDTH || 1440);
|
||||
const height = Number(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_HEIGHT || 900);
|
||||
const timeout = Number(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_TIMEOUT_MS || 30000);
|
||||
const waitUntil = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_WAIT_UNTIL || "networkidle";
|
||||
const fullPage = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE !== "0";
|
||||
const fullPage = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE === "1";
|
||||
const selector = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_SELECTOR || "";
|
||||
const executablePath = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_EXECUTABLE_PATH || "";
|
||||
|
||||
@@ -1234,9 +1239,11 @@ export function runNodeWebProbeAsync(
|
||||
): Record<string, unknown> {
|
||||
const startArgs = nodeWebProbeRunArgs(options, "start");
|
||||
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
||||
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
|
||||
if (memoryGuard.status !== "allowed") return memoryGuard;
|
||||
const startScript = [
|
||||
"set -eu",
|
||||
`${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password ?? "")}`].join(" ")} ${startArgs.map(shellQuote).join(" ")}`,
|
||||
webProbeGuardedLaunchShell(spec, "manual-start", `${[...webProbeProxy.envAssignments, ...webProbeBrowserEvidenceEnvAssignments(spec), `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password ?? "")}`].join(" ")} ${startArgs.map(shellQuote).join(" ")}`, "structured-output"),
|
||||
].join("\n");
|
||||
const startResult = runTransWorkspaceStdinScript(options.node, spec.workspace, startScript, 55);
|
||||
const start = parseJsonObject(startResult.stdout);
|
||||
|
||||
@@ -2130,7 +2130,7 @@ export function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
||||
name: parseWebProbeScreenshotName(optionValue(args, "--name") ?? `web-probe-${node.toLowerCase()}-${lane}.png`),
|
||||
timeoutMs,
|
||||
waitUntil: parseWebProbeScreenshotWaitUntil(optionValue(args, "--wait-until") ?? "networkidle"),
|
||||
fullPage: !args.includes("--no-full-page"),
|
||||
fullPage: args.includes("--full-page") && !args.includes("--no-full-page"),
|
||||
selector: optionValue(args, "--selector") ?? null,
|
||||
keepRemote: args.includes("--keep-remote"),
|
||||
waitTimeoutMs,
|
||||
|
||||
Reference in New Issue
Block a user