fix: 收敛 WebProbe 物理内存与资源上限
This commit is contained in:
Codex
@@ -3,7 +3,7 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { resolveCliChildJsonCommandResult, type CliChildJsonResolution } from "../cli-child-json-recovery";
|
||||
import type { HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
import { runWebProbeMemoryGuard } from "../hwlab-node-web-probe-memory-guard";
|
||||
import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard } from "../hwlab-node-web-probe-memory-guard";
|
||||
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
||||
import { withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
||||
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
||||
@@ -455,6 +455,134 @@ console.log(JSON.stringify({
|
||||
`;
|
||||
}
|
||||
|
||||
export function nodeWebObserveFinalLaunchGuardSource(): string {
|
||||
return String.raw`
|
||||
const fs = require("fs");
|
||||
const thresholdBytes = Number(process.argv[2]);
|
||||
const comparator = process.argv[3];
|
||||
const mode = process.argv[4];
|
||||
const jobId = process.argv[5];
|
||||
const stateDir = process.argv[6];
|
||||
const source = "/proc/meminfo";
|
||||
|
||||
function finish(reason, availableBytes) {
|
||||
const valid = Number.isSafeInteger(availableBytes) && availableBytes >= 0;
|
||||
const thresholdMatched = valid
|
||||
? comparator === "less-than" ? availableBytes < thresholdBytes : availableBytes <= thresholdBytes
|
||||
: null;
|
||||
const blocked = reason !== null || thresholdMatched === true;
|
||||
const cadenceSkip = blocked && mode === "sentinel-cadence";
|
||||
const payload = {
|
||||
ok: !blocked || cadenceSkip,
|
||||
command: "web-probe-observe start",
|
||||
status: blocked ? cadenceSkip ? "skipped-wait-next-round" : "blocked" : "allowed",
|
||||
reason: reason || (thresholdMatched ? "memavailable-threshold-matched" : null),
|
||||
failureKind: blocked ? "web-probe-final-memory-start-guard" : null,
|
||||
jobId,
|
||||
stateDir,
|
||||
observerCreated: false,
|
||||
browserCreated: false,
|
||||
startBlocked: blocked,
|
||||
decision: blocked ? cadenceSkip ? "skip-observer" : "gc-required" : "start-observer",
|
||||
mutation: blocked,
|
||||
stateArtifactCreated: blocked,
|
||||
memoryGuard: {
|
||||
phase: "remote-launch-critical-section",
|
||||
metric: "MemAvailable",
|
||||
source,
|
||||
swapExcluded: true,
|
||||
availableBytes: valid ? availableBytes : null,
|
||||
thresholdBytes,
|
||||
comparator,
|
||||
thresholdMatched,
|
||||
},
|
||||
observedAt: new Date().toISOString(),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
console.log(JSON.stringify(payload));
|
||||
process.exit(blocked ? 42 : 0);
|
||||
}
|
||||
|
||||
if (!Number.isSafeInteger(thresholdBytes) || thresholdBytes <= 0) finish("memory-threshold-invalid", null);
|
||||
if (comparator !== "less-than" && comparator !== "less-than-or-equal") finish("memory-comparator-invalid", null);
|
||||
let text;
|
||||
try { text = fs.readFileSync(source, "utf8"); } catch { finish("proc-meminfo-read-failed", null); }
|
||||
const match = text.match(/^MemAvailable:\s+(\d+)\s+kB$/m);
|
||||
if (!match) finish("memavailable-missing", null);
|
||||
const availableBytes = Number(match[1]) * 1024;
|
||||
finish(null, availableBytes);
|
||||
`;
|
||||
}
|
||||
|
||||
export function nodeWebObserveLaunchReadinessSource(): string {
|
||||
return String.raw`
|
||||
const fs = require("fs");
|
||||
const runnerPid = Number(process.argv[2]);
|
||||
const stateDir = process.argv[3];
|
||||
const expectedJobId = process.argv[4];
|
||||
const timeoutMs = Number(process.argv[5]);
|
||||
const pollMs = Number(process.argv[6]);
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
||||
const readJson = (name) => { try { return JSON.parse(fs.readFileSync(stateDir + "/" + name, "utf8")); } catch { return null; } };
|
||||
|
||||
function procRows() {
|
||||
const rows = new Map();
|
||||
let names = [];
|
||||
try { names = fs.readdirSync("/proc").filter((name) => /^\d+$/.test(name)); } catch { return rows; }
|
||||
for (const name of names) {
|
||||
try {
|
||||
const stat = fs.readFileSync("/proc/" + name + "/stat", "utf8");
|
||||
const close = stat.lastIndexOf(")");
|
||||
const fields = stat.slice(close + 2).trim().split(/\s+/);
|
||||
const command = fs.readFileSync("/proc/" + name + "/cmdline").toString("utf8").replace(/\0/g, " ");
|
||||
rows.set(Number(name), { pid: Number(name), ppid: Number(fields[1]), command });
|
||||
} catch {}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function descendants(rows) {
|
||||
const selected = new Set([runnerPid]);
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const row of rows.values()) if (!selected.has(row.pid) && selected.has(row.ppid)) { selected.add(row.pid); changed = true; }
|
||||
}
|
||||
return [...selected].map((pid) => rows.get(pid)).filter(Boolean);
|
||||
}
|
||||
|
||||
function matchingStateRecord(name, value) {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const status = String(value.status || "");
|
||||
if (!/^(starting|running|ready)$/i.test(status)) return null;
|
||||
if (String(value.jobId || "") !== expectedJobId || Number(value.pid) !== runnerPid) return null;
|
||||
return { name, status, jobId: String(value.jobId), pid: Number(value.pid) };
|
||||
}
|
||||
|
||||
while (Date.now() <= deadline) {
|
||||
const rows = procRows();
|
||||
const runnerAlive = rows.has(runnerPid);
|
||||
const children = runnerAlive ? descendants(rows) : [];
|
||||
const browserChild = children.find((row) => row.pid !== runnerPid && /(?:^|[ /])(chrome|chromium)(?:[ /]|$)/i.test(row.command));
|
||||
const heartbeat = readJson("heartbeat.json");
|
||||
const manifest = readJson("manifest.json");
|
||||
const stateRecord = matchingStateRecord("heartbeat.json", heartbeat) || matchingStateRecord("manifest.json", manifest);
|
||||
if (runnerAlive && browserChild && stateRecord) {
|
||||
console.log(JSON.stringify({ ok: true, expectedJobId, runnerPid, runnerAlive: true, browserPid: browserChild.pid, stateRecord, readiness: "current-state-and-browser-descendant", observedAt: new Date().toISOString(), valuesRedacted: true }));
|
||||
process.exit(0);
|
||||
}
|
||||
if (!runnerAlive) {
|
||||
console.log(JSON.stringify({ ok: false, reason: "runner-exited-before-ready", expectedJobId, runnerPid, runnerAlive: false, stateRecord, observedAt: new Date().toISOString(), valuesRedacted: true }));
|
||||
process.exit(43);
|
||||
}
|
||||
pause(pollMs);
|
||||
}
|
||||
console.log(JSON.stringify({ ok: false, reason: "launch-readiness-timeout", expectedJobId, runnerPid, runnerAlive: fs.existsSync("/proc/" + runnerPid), observedAt: new Date().toISOString(), valuesRedacted: true }));
|
||||
process.exit(44);
|
||||
`;
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveStart(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
@@ -482,7 +610,9 @@ export function runNodeWebProbeObserveStart(
|
||||
};
|
||||
}
|
||||
const observerLifecycle = spec.webProbe?.resourcePolicy?.observerLifecycle;
|
||||
const finalMemoryPolicy = spec.webProbe?.resourcePolicy?.memoryStartGuard;
|
||||
if (observerLifecycle === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML");
|
||||
if (finalMemoryPolicy === undefined) throw new Error("web-probe memory start policy is missing from the owning YAML");
|
||||
const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
||||
const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
|
||||
const day = timestamp.slice(0, 8);
|
||||
@@ -520,8 +650,24 @@ export function runNodeWebProbeObserveStart(
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES=${shellQuote(String(observerLifecycle.maxPerformanceEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserProcessHistoryEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserFreezeSignalHistoryEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_ENTRIES=${shellQuote(String(observerLifecycle.maxRealtimeEventEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES=${shellQuote(String(observerLifecycle.maxRealtimeEventBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_DOM_EVIDENCE_ROWS=${shellQuote(String(observerLifecycle.maxDomEvidenceRows))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PROCESS_SCAN_ENTRIES=${shellQuote(String(observerLifecycle.maxProcessScanEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_CAPTURE_SECONDS=${shellQuote(String(observerLifecycle.maxPerformanceCaptureSeconds))}`,
|
||||
`UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxResponseBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxWebPerformanceBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_ENTRIES=${shellQuote(String(observerLifecycle.maxJsonlQueueEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_BYTES=${shellQuote(String(observerLifecycle.maxJsonlQueueBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_JSONL_LINE_BYTES=${shellQuote(String(observerLifecycle.maxJsonlLineBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_JSONL_FLUSH_TIMEOUT_MS=${shellQuote(String(observerLifecycle.jsonlFlushTimeoutMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_RESPONSE_BODY_IN_FLIGHT=${shellQuote(String(observerLifecycle.maxResponseBodyInFlight))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PASSIVE_LISTENER_TASKS=${shellQuote(String(observerLifecycle.maxPassiveListenerTasks))}`,
|
||||
`UNIDESK_WEB_OBSERVE_PASSIVE_TASK_FLUSH_TIMEOUT_MS=${shellQuote(String(observerLifecycle.passiveTaskFlushTimeoutMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_CLOSE_STEP_TIMEOUT_MS=${shellQuote(String(observerLifecycle.closeStepTimeoutMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PROCESS_READ_CONCURRENCY=${shellQuote(String(observerLifecycle.maxProcessReadConcurrency))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_ENTRIES=${shellQuote(String(observerLifecycle.maxRealtimeTotalEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_BODY_BYTES=${shellQuote(String(observerLifecycle.maxRealtimeTotalBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
|
||||
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
||||
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
|
||||
@@ -539,6 +685,26 @@ export function runNodeWebProbeObserveStart(
|
||||
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS=${shellQuote(String(authLogin.maxDelayMs))}`,
|
||||
]),
|
||||
].join(" ");
|
||||
const finalComparator = options.memoryGuardMode === "sentinel-cadence"
|
||||
? finalMemoryPolicy.sentinelComparator
|
||||
: finalMemoryPolicy.manualComparator;
|
||||
const finalGuardFailure = (reason: string) => JSON.stringify({
|
||||
ok: false,
|
||||
command: "web-probe-observe start",
|
||||
status: "blocked",
|
||||
reason,
|
||||
failureKind: "web-probe-final-memory-start-guard",
|
||||
jobId,
|
||||
stateDir,
|
||||
observerCreated: false,
|
||||
startBlocked: true,
|
||||
mutation: true,
|
||||
stateArtifactCreated: true,
|
||||
browserCreated: false,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
const launchFailureOutputSource = "const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const guard=read('launch-guard.json'); const readiness=read('launch-ready.json'); const pid=Number(process.argv[3]); const bool=(v)=>v==='true'; console.log(JSON.stringify({ok:false,command:'web-probe-observe start',status:'blocked',reason:'launch-readiness-unconfirmed',failureKind:'web-probe-launch-readiness',jobId:process.argv[2],stateDir:dir,observerCreated:false,browserCreated:false,browserCreationAttempted:true,startBlocked:true,mutation:true,stateArtifactCreated:true,memoryGuard:guard&&guard.memoryGuard||null,finalLaunchGuard:guard,launchReadiness:readiness,cleanup:{attempted:true,processGroupId:pid,termSent:bool(process.argv[4]),killSent:bool(process.argv[5]),aliveAfter:bool(process.argv[6]),completed:!bool(process.argv[6]),observedAt:new Date().toISOString(),valuesRedacted:true},valuesRedacted:true},null,2))";
|
||||
const cleanupPollCount = Math.ceil(finalMemoryPolicy.launchReadyTimeoutSeconds * 1000 / finalMemoryPolicy.launchReadyPollMilliseconds);
|
||||
const script = [
|
||||
"set -eu",
|
||||
`state_dir=${shellQuote(stateDir)}`,
|
||||
@@ -552,16 +718,71 @@ export function runNodeWebProbeObserveStart(
|
||||
"node -e \"const fs=require('fs'); fs.writeFileSync(process.argv[1], Buffer.from(fs.readFileSync(process.argv[2], 'utf8').replace(/\\s+/g, ''), 'base64'))\" \"$runner\" \"$runner_b64\"",
|
||||
"rm -f \"$runner_b64\"",
|
||||
"chmod 700 \"$runner\"",
|
||||
`if command -v setsid >/dev/null 2>&1; then setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & else nohup env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & fi`,
|
||||
`launch_lock=${shellQuote(finalMemoryPolicy.launchLockPath)}`,
|
||||
`launch_guard_result="$state_dir/launch-guard.json"`,
|
||||
`launch_ready_result="$state_dir/launch-ready.json"`,
|
||||
`if ! command -v flock >/dev/null 2>&1; then printf '%s\\n' ${shellQuote(finalGuardFailure("host-flock-unavailable"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
|
||||
`if ! command -v setsid >/dev/null 2>&1; then printf '%s\\n' ${shellQuote(finalGuardFailure("host-setsid-unavailable"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
|
||||
`exec 9>"$launch_lock"`,
|
||||
`if ! flock -w ${shellQuote(String(finalMemoryPolicy.launchLockTimeoutSeconds))} 9; then printf '%s\\n' ${shellQuote(finalGuardFailure("launch-lock-timeout"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
|
||||
"trap 'flock -u 9 2>/dev/null || true' EXIT",
|
||||
"set +e",
|
||||
`node - ${shellQuote(String(finalMemoryPolicy.thresholdBytes))} ${shellQuote(finalComparator)} ${shellQuote(options.memoryGuardMode)} ${shellQuote(jobId)} ${shellQuote(stateDir)} >"$launch_guard_result" <<'UNIDESK_WEB_OBSERVE_FINAL_GUARD'`,
|
||||
nodeWebObserveFinalLaunchGuardSource(),
|
||||
"UNIDESK_WEB_OBSERVE_FINAL_GUARD",
|
||||
"launch_guard_rc=$?",
|
||||
"set -e",
|
||||
"if [ \"$launch_guard_rc\" -ne 0 ]; then cat \"$launch_guard_result\"; flock -u 9; exit 0; fi",
|
||||
`setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null 9>&- &`,
|
||||
"pid=$!",
|
||||
"printf '%s\\n' \"$pid\" >\"$state_dir/pid\"",
|
||||
"sleep 1",
|
||||
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
|
||||
"set +e",
|
||||
`node - "$pid" "$state_dir" ${shellQuote(jobId)} ${shellQuote(String(finalMemoryPolicy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds))} >"$launch_ready_result" <<'UNIDESK_WEB_OBSERVE_LAUNCH_READY'`,
|
||||
nodeWebObserveLaunchReadinessSource(),
|
||||
"UNIDESK_WEB_OBSERVE_LAUNCH_READY",
|
||||
"launch_ready_rc=$?",
|
||||
"set -e",
|
||||
`if [ "$launch_ready_rc" -ne 0 ]; then term_sent=false; if kill -TERM -- -"$pid" 2>/dev/null; then term_sent=true; fi; i=0; while [ "$i" -lt ${shellQuote(String(cleanupPollCount))} ] && kill -0 -- -"$pid" 2>/dev/null; do sleep ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds / 1000))}; i=$((i+1)); done; kill_sent=false; if kill -0 -- -"$pid" 2>/dev/null; then if kill -KILL -- -"$pid" 2>/dev/null; then kill_sent=true; fi; fi; i=0; while [ "$i" -lt ${shellQuote(String(cleanupPollCount))} ] && kill -0 -- -"$pid" 2>/dev/null; do sleep ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds / 1000))}; i=$((i+1)); done; alive_after=false; if kill -0 -- -"$pid" 2>/dev/null; then alive_after=true; fi; node -e ${shellQuote(launchFailureOutputSource)} "$state_dir" ${shellQuote(jobId)} "$pid" "$term_sent" "$kill_sent" "$alive_after"; flock -u 9; trap - EXIT; exit 0; fi`,
|
||||
"flock -u 9",
|
||||
"trap - EXIT",
|
||||
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),finalLaunchGuard:read('launch-guard.json'),launchReadiness:read('launch-ready.json'),observerCreated:true,browserCreated:true,observerMutation:true,stateArtifactMutation:true,statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const startResolution = resolveWebObserveActionJson(result, "start");
|
||||
const started = startResolution.parsed;
|
||||
if (started?.observerCreated === false && started.startBlocked === true) {
|
||||
const finalSnapshot = record(started.memoryGuard);
|
||||
const finalMemoryGuard = evaluateWebProbeMemoryGuard(spec, options.memoryGuardMode, {
|
||||
ok: finalSnapshot.metric === "MemAvailable" && finalSnapshot.source === "/proc/meminfo" && finalSnapshot.swapExcluded === true,
|
||||
action: "gc remote memory",
|
||||
metric: finalSnapshot.metric,
|
||||
source: finalSnapshot.source,
|
||||
swapExcluded: finalSnapshot.swapExcluded,
|
||||
memory: { availableBytes: finalSnapshot.availableBytes },
|
||||
});
|
||||
return {
|
||||
...started,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
initialMemoryGuard: memoryGuard,
|
||||
memoryGuard: finalMemoryGuard,
|
||||
finalLaunchGuard: started.finalLaunchGuard ?? started,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
const startOk = result.exitCode === 0 && started?.ok === true;
|
||||
const allowedFinalSnapshot = record(record(started?.finalLaunchGuard).memoryGuard);
|
||||
const effectiveMemoryGuard = allowedFinalSnapshot.metric === "MemAvailable"
|
||||
? evaluateWebProbeMemoryGuard(spec, options.memoryGuardMode, {
|
||||
ok: true,
|
||||
action: "gc remote memory",
|
||||
metric: allowedFinalSnapshot.metric,
|
||||
source: allowedFinalSnapshot.source,
|
||||
swapExcluded: allowedFinalSnapshot.swapExcluded,
|
||||
memory: { availableBytes: allowedFinalSnapshot.availableBytes },
|
||||
})
|
||||
: memoryGuard;
|
||||
const recovery = startOk
|
||||
? null
|
||||
: readNodeWebProbeObserveRemoteStatus({ ...options, id: jobId, stateDir }, spec, 1, Math.min(options.commandTimeoutSeconds, 30));
|
||||
@@ -620,7 +841,10 @@ export function runNodeWebProbeObserveStart(
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
projectManagement,
|
||||
memoryGuard,
|
||||
initialMemoryGuard: memoryGuard,
|
||||
memoryGuard: effectiveMemoryGuard,
|
||||
finalLaunchGuard: started?.finalLaunchGuard ?? null,
|
||||
launchReadiness: started?.launchReadiness ?? null,
|
||||
targetPath: options.targetPath,
|
||||
id: observerId,
|
||||
degradedReason,
|
||||
|
||||
@@ -189,12 +189,28 @@ test("observe start inherits finite observer lifecycle limits from the owning YA
|
||||
assert.deepEqual(spec.webProbe?.resourcePolicy?.observerLifecycle, {
|
||||
maxRunSeconds: 3600,
|
||||
maxSamples: 720,
|
||||
maxScreenshots: 48,
|
||||
maxPerformanceEntries: 2000,
|
||||
maxScreenshots: 24,
|
||||
maxPerformanceEntries: 1000,
|
||||
maxBrowserProcessHistoryEntries: 360,
|
||||
maxBrowserFreezeSignalHistoryEntries: 120,
|
||||
maxRealtimeEventEntries: 256,
|
||||
maxRealtimeEventBodyBytes: 65536,
|
||||
maxDomEvidenceRows: 1000,
|
||||
maxProcessScanEntries: 8192,
|
||||
maxPerformanceCaptureSeconds: 30,
|
||||
maxResponseBodyBytes: 524288,
|
||||
maxWebPerformanceBodyBytes: 65536,
|
||||
maxJsonlQueueEntries: 512,
|
||||
maxJsonlQueueBytes: 4194304,
|
||||
maxJsonlLineBytes: 262144,
|
||||
jsonlFlushTimeoutMs: 5000,
|
||||
maxResponseBodyInFlight: 2,
|
||||
maxPassiveListenerTasks: 64,
|
||||
passiveTaskFlushTimeoutMs: 5000,
|
||||
closeStepTimeoutMs: 5000,
|
||||
maxProcessReadConcurrency: 32,
|
||||
maxRealtimeTotalEntries: 512,
|
||||
maxRealtimeTotalBodyBytes: 4194304,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user