Merge pull request #1912 from pikasTech/fix/web-probe-memory-guard
fix: 补齐 WebProbe 全入口浏览器启动门禁
This commit is contained in:
@@ -0,0 +1,123 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { rootPath } from "./config";
|
||||
|
||||
const webObserveSource = readFileSync(rootPath("scripts/src/gc-remote-web-observe.py"), "utf8");
|
||||
const runnerSource = readFileSync(rootPath("scripts/src/gc-remote-runner.py"), "utf8");
|
||||
|
||||
function runPythonFixture(body: string): Record<string, unknown> {
|
||||
const fixture = [
|
||||
"import json, os, re, signal, time",
|
||||
"PROVIDER_ID = 'NC01'",
|
||||
"def safe_int(value, fallback=0):",
|
||||
" try: return int(value)",
|
||||
" except (TypeError, ValueError): return fallback",
|
||||
"def config_list(config, key, fallback):",
|
||||
" value = config.get(key, fallback)",
|
||||
" return value if isinstance(value, list) else fallback",
|
||||
"def config_int(config, key, fallback, minimum=None, maximum=None):",
|
||||
" value = config.get(key, fallback)",
|
||||
" return value if isinstance(value, int) and not isinstance(value, bool) else fallback",
|
||||
"def config_float(config, key, fallback, minimum=None, maximum=None): return float(config.get(key, fallback))",
|
||||
"def fmt_bytes(value): return str(value)",
|
||||
"def redact_command_preview(value): return str(value)[:200]",
|
||||
"def observe_run_record(path, stale_hours): return {}",
|
||||
webObserveSource,
|
||||
body,
|
||||
].join("\n");
|
||||
const result = spawnSync("python3", ["-"], { input: fixture, encoding: "utf8" });
|
||||
assert.equal(result.status, 0, result.stderr || result.stdout);
|
||||
return JSON.parse(result.stdout.trim()) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
test("memory-pressure candidate fixtures protect active observers and classify only closed stale/orphan trees", () => {
|
||||
const result = runPythonFixture(String.raw`
|
||||
MEMORY_CONFIG = {
|
||||
"observeStateRoots": ["/tmp/observe"], "minimumReadableObserveRoots": 1,
|
||||
"observeRunDepth": 6, "observeScanLimit": 100, "staleRunMaxAgeHours": 1,
|
||||
"processScanLimit": 100, "processStaleMinAgeSeconds": 3600,
|
||||
"maxMemoryReclaimCandidates": 20, "terminationGraceSeconds": 1,
|
||||
"terminationPollMilliseconds": 50, "observerRootPatterns": ["observer-runner.mjs"],
|
||||
"browserRootPatterns": ["observer-runner.mjs", "cliDaemon.mjs"],
|
||||
"browserProcessPatterns": ["chromium", "/usr/bin/chromium"]
|
||||
}
|
||||
def row(pid, ppid, sid, start, age, command, rss=1024):
|
||||
return {"pid": pid, "ppid": ppid, "sid": sid, "startTicks": start, "ageSeconds": age, "rssBytes": rss, "commandLine": command, "commandPreview": command}
|
||||
processes = {
|
||||
100: row(100, 1, 100, 1000, 7200, "node observer-runner.mjs"),
|
||||
101: row(101, 100, 100, 1001, 7200, "/usr/bin/chromium --type=browser"),
|
||||
200: row(200, 1, 200, 2000, 7200, "node observer-runner.mjs"),
|
||||
300: row(300, 1, 300, 3000, 7200, "node cliDaemon.mjs"),
|
||||
301: row(301, 300, 300, 3001, 7200, "/usr/bin/chromium --type=browser"),
|
||||
}
|
||||
read_proc_process_table = lambda: processes
|
||||
observer_run_records = lambda: [
|
||||
{"pid": 100, "pidAlive": True, "staleLiveSignal": False},
|
||||
{"pid": 200, "pidAlive": True, "staleLiveSignal": True},
|
||||
]
|
||||
candidates = collect_memory_reclaim_candidates()
|
||||
print(json.dumps({"kinds": sorted(item["kind"] for item in candidates), "roots": sorted(item["rootPid"] for item in candidates), "diagnostics": memory_reclaim_scan_diagnostics()}))
|
||||
`);
|
||||
assert.deepEqual(result.kinds, ["browser-orphan-process-tree", "stale-web-observer-process-tree"]);
|
||||
assert.deepEqual(result.roots, [200, 300]);
|
||||
assert.equal((result.diagnostics as Record<string, unknown>).blocked, false);
|
||||
});
|
||||
|
||||
test("memory-pressure process scans fail closed on empty or truncated process tables", () => {
|
||||
const result = runPythonFixture(String.raw`
|
||||
MEMORY_CONFIG = {
|
||||
"observeStateRoots": ["/tmp/observe"], "minimumReadableObserveRoots": 1,
|
||||
"observeRunDepth": 6, "observeScanLimit": 100, "staleRunMaxAgeHours": 1,
|
||||
"processScanLimit": 100, "processStaleMinAgeSeconds": 3600,
|
||||
"maxMemoryReclaimCandidates": 20, "terminationGraceSeconds": 1,
|
||||
"terminationPollMilliseconds": 50, "observerRootPatterns": ["observer-runner.mjs"],
|
||||
"browserRootPatterns": ["cliDaemon.mjs"], "browserProcessPatterns": ["chromium"]
|
||||
}
|
||||
observer_run_records = lambda: []
|
||||
read_proc_process_table = lambda: {}
|
||||
empty = collect_memory_reclaim_candidates()
|
||||
empty_diag = memory_reclaim_scan_diagnostics()
|
||||
def truncated_scan():
|
||||
global _PROCESS_SCAN_TRUNCATED
|
||||
_PROCESS_SCAN_TRUNCATED = True
|
||||
return {1: {"pid": 1, "ppid": 0, "sid": 1, "startTicks": 1, "ageSeconds": 7200, "rssBytes": 1, "commandLine": "node cliDaemon.mjs", "commandPreview": "node cliDaemon.mjs"}}
|
||||
read_proc_process_table = truncated_scan
|
||||
truncated = collect_memory_reclaim_candidates()
|
||||
truncated_diag = memory_reclaim_scan_diagnostics()
|
||||
print(json.dumps({"emptyCount": len(empty), "emptyBlocked": empty_diag["blocked"], "emptyError": empty_diag["processScanError"], "truncatedCount": len(truncated), "truncatedBlocked": truncated_diag["blocked"], "truncatedFlag": truncated_diag["processScanTruncated"]}))
|
||||
`);
|
||||
assert.equal(result.emptyCount, 0);
|
||||
assert.equal(result.emptyBlocked, true);
|
||||
assert.equal(result.emptyError, "proc-process-table-empty");
|
||||
assert.equal(result.truncatedCount, 0);
|
||||
assert.equal(result.truncatedBlocked, true);
|
||||
assert.equal(result.truncatedFlag, true);
|
||||
});
|
||||
|
||||
test("post-TERM identity matching tolerates reparenting but never PID startTicks reuse", () => {
|
||||
const result = runPythonFixture(String.raw`
|
||||
identity = {"pid": 41, "ppid": 10, "sid": 10, "startTicks": 1234}
|
||||
reparented = {41: {"pid": 41, "ppid": 1, "sid": 41, "startTicks": 1234}}
|
||||
reused = {41: {"pid": 41, "ppid": 10, "sid": 10, "startTicks": 9999}}
|
||||
print(json.dumps({
|
||||
"preTermReparented": matching_live_identity(identity, reparented) is not None,
|
||||
"postTermReparented": matching_surviving_identity(identity, reparented) is not None,
|
||||
"postTermReused": matching_surviving_identity(identity, reused) is not None,
|
||||
}))
|
||||
`);
|
||||
assert.equal(result.preTermReparented, false);
|
||||
assert.equal(result.postTermReparented, true);
|
||||
assert.equal(result.postTermReused, false);
|
||||
});
|
||||
|
||||
test("memory-pressure async runner uses a parent-child barrier and locked worker state", () => {
|
||||
assert.match(runnerSource, /os\.pipe\(\)/u);
|
||||
assert.match(runnerSource, /os\.read\(read_descriptor/u);
|
||||
assert.match(runnerSource, /workerStartTicks/u);
|
||||
assert.match(runnerSource, /write_memory_job_state\(/u);
|
||||
assert.match(runnerSource, /workerAlive/u);
|
||||
assert.match(runnerSource, /worker-identity-unavailable|worker-exited-without-terminal-state/u);
|
||||
});
|
||||
+418
-28
@@ -1,8 +1,10 @@
|
||||
import base64
|
||||
import calendar
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -207,8 +209,41 @@ def job_paths(job_id):
|
||||
return {
|
||||
"state": os.path.join(REMOTE_GC_JOB_DIR, "%s.json" % job_id),
|
||||
"log": os.path.join(REMOTE_GC_JOB_DIR, "%s.log" % job_id),
|
||||
"lock": os.path.join(REMOTE_GC_JOB_DIR, "%s.lock" % job_id),
|
||||
}
|
||||
|
||||
def process_identity(pid):
|
||||
try:
|
||||
with open("/proc/%s/stat" % int(pid), "r", encoding="utf-8") as handle:
|
||||
text = handle.read().strip()
|
||||
close = text.rfind(")")
|
||||
fields = text[close + 2:].split() if close >= 0 else []
|
||||
return {"state": fields[0], "startTicks": int(fields[19])} if len(fields) >= 20 else None
|
||||
except (OSError, ValueError, IndexError):
|
||||
return None
|
||||
|
||||
def process_start_ticks(pid):
|
||||
identity = process_identity(pid)
|
||||
return identity.get("startTicks") if identity else None
|
||||
|
||||
def acquire_job_lock(paths):
|
||||
descriptor = os.open(paths["lock"], os.O_CREAT | os.O_RDWR, 0o600)
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX)
|
||||
return descriptor
|
||||
|
||||
def release_job_lock(descriptor):
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
def write_memory_job_state(paths, payload):
|
||||
descriptor = acquire_job_lock(paths)
|
||||
try:
|
||||
write_json_atomic(paths["state"], payload)
|
||||
finally:
|
||||
release_job_lock(descriptor)
|
||||
|
||||
def status_command(job_id):
|
||||
return "bun scripts/cli.ts gc remote %s status --job-id %s" % (PROVIDER_ID, job_id)
|
||||
|
||||
@@ -326,9 +361,31 @@ def remote_gc_job_status():
|
||||
"statePath": paths["state"],
|
||||
"logTail": read_file_tail(paths["log"]),
|
||||
}
|
||||
descriptor = None
|
||||
try:
|
||||
descriptor = acquire_job_lock(paths)
|
||||
with open(paths["state"], "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if payload.get("scope") == "memory-pressure-only" and payload.get("status") == "running":
|
||||
worker_pid = safe_int(payload.get("workerPid"))
|
||||
expected_start_ticks = safe_int(payload.get("workerStartTicks"))
|
||||
actual_identity = process_identity(worker_pid) if worker_pid > 0 else None
|
||||
actual_start_ticks = actual_identity.get("startTicks") if actual_identity else None
|
||||
worker_alive = actual_identity is not None and actual_identity.get("state") not in set(["Z", "X"]) \
|
||||
and actual_start_ticks == expected_start_ticks
|
||||
if not worker_alive:
|
||||
payload.update({
|
||||
"ok": False,
|
||||
"status": "failed",
|
||||
"outcome": "memory-reclaim-worker-exited-without-terminal-state",
|
||||
"workerAlive": False,
|
||||
"workerObservedState": actual_identity.get("state") if actual_identity else None,
|
||||
"workerObservedStartTicks": actual_start_ticks,
|
||||
"finishedAt": now_iso(),
|
||||
})
|
||||
write_json_atomic(paths["state"], payload)
|
||||
else:
|
||||
payload["workerAlive"] = True
|
||||
except Exception as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
@@ -338,6 +395,9 @@ def remote_gc_job_status():
|
||||
"statePath": paths["state"],
|
||||
"logTail": read_file_tail(paths["log"]),
|
||||
}
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
release_job_lock(descriptor)
|
||||
payload["logTail"] = read_file_tail(paths["log"])
|
||||
return payload
|
||||
|
||||
@@ -569,24 +629,50 @@ def collect_process_pressure(patterns):
|
||||
}
|
||||
|
||||
def collect_memory_snapshot():
|
||||
result = command(["free", "-b"], 5)
|
||||
if result["exitCode"] != 0:
|
||||
return {"ok": False, "error": "free-failed", "command": bounded(result)}
|
||||
memory = {}
|
||||
for line in result["stdout"].splitlines():
|
||||
parts = line.split()
|
||||
if parts and parts[0].rstrip(":") == "Mem" and len(parts) >= 7:
|
||||
memory = {
|
||||
"totalBytes": safe_int(parts[1]),
|
||||
"usedBytes": safe_int(parts[2]),
|
||||
"freeBytes": safe_int(parts[3]),
|
||||
"availableBytes": safe_int(parts[6]),
|
||||
"totalHuman": fmt_bytes(parts[1]),
|
||||
"usedHuman": fmt_bytes(parts[2]),
|
||||
"availableHuman": fmt_bytes(parts[6]),
|
||||
}
|
||||
break
|
||||
return {"ok": bool(memory), "memory": memory, "command": bounded(result)}
|
||||
source = "/proc/meminfo"
|
||||
fields = {}
|
||||
try:
|
||||
with open(source, "r", encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
if ":" not in line:
|
||||
continue
|
||||
key, raw = line.split(":", 1)
|
||||
parts = raw.strip().split()
|
||||
if not parts:
|
||||
continue
|
||||
fields[key] = safe_int(parts[0]) * 1024
|
||||
except OSError as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "proc-meminfo-read-failed",
|
||||
"metric": "MemAvailable",
|
||||
"source": source,
|
||||
"swapExcluded": True,
|
||||
"errorType": type(exc).__name__,
|
||||
}
|
||||
available_bytes = fields.get("MemAvailable")
|
||||
if available_bytes is None:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "memavailable-missing",
|
||||
"metric": "MemAvailable",
|
||||
"source": source,
|
||||
"swapExcluded": True,
|
||||
}
|
||||
total_bytes = fields.get("MemTotal")
|
||||
memory = {
|
||||
"totalBytes": total_bytes,
|
||||
"availableBytes": available_bytes,
|
||||
"totalHuman": fmt_bytes(total_bytes) if total_bytes is not None else None,
|
||||
"availableHuman": fmt_bytes(available_bytes),
|
||||
}
|
||||
return {
|
||||
"ok": True,
|
||||
"metric": "MemAvailable",
|
||||
"source": source,
|
||||
"swapExcluded": True,
|
||||
"memory": memory,
|
||||
}
|
||||
|
||||
def observe_run_record(path, stale_hours):
|
||||
stat = os.stat(path)
|
||||
@@ -608,6 +694,14 @@ def observe_run_record(path, stale_hours):
|
||||
pass
|
||||
if pid is not None:
|
||||
break
|
||||
live_freshness_timestamp = None
|
||||
for source in [heartbeat, manifest]:
|
||||
for key in ["updatedAt", "completedAt", "finishedAt", "stoppedAt"]:
|
||||
live_freshness_timestamp = iso_or_epoch_to_epoch(source.get(key))
|
||||
if live_freshness_timestamp is not None:
|
||||
break
|
||||
if live_freshness_timestamp is not None:
|
||||
break
|
||||
timestamp = None
|
||||
for source in [heartbeat, manifest]:
|
||||
for key in ["updatedAt", "completedAt", "finishedAt", "stoppedAt", "startedAt", "createdAt"]:
|
||||
@@ -622,7 +716,8 @@ def observe_run_record(path, stale_hours):
|
||||
status = heartbeat.get("status") or manifest.get("status") or manifest.get("state")
|
||||
alive = pid_alive(pid)
|
||||
terminal = str(status or "").lower() in set(["done", "completed", "complete", "failed", "blocked", "timeout", "timed-out", "stopped", "exited"])
|
||||
stale_signal = (not alive) and age_hours >= float(stale_hours) and (terminal or status is None)
|
||||
age_stale = age_hours >= float(stale_hours)
|
||||
stale_signal = (not alive) and age_stale and (terminal or status is None)
|
||||
return {
|
||||
"id": os.path.basename(path),
|
||||
"path": path,
|
||||
@@ -632,6 +727,9 @@ def observe_run_record(path, stale_hours):
|
||||
"ageHours": round(age_hours, 2),
|
||||
"timestampBasis": "manifest-or-heartbeat" if heartbeat or manifest else "directory-mtime-fallback",
|
||||
"staleSignal": stale_signal,
|
||||
"staleLiveSignal": alive and age_stale and live_freshness_timestamp is not None,
|
||||
"activeSignal": alive and not (age_stale and live_freshness_timestamp is not None),
|
||||
"liveStaleEvidence": "heartbeat-or-manifest-update" if live_freshness_timestamp is not None else None,
|
||||
"classification": "review-only",
|
||||
}
|
||||
|
||||
@@ -647,6 +745,7 @@ def collect_web_observe_summary():
|
||||
}
|
||||
root_rows = []
|
||||
stale_rows = []
|
||||
stale_live_rows = []
|
||||
active_rows = []
|
||||
run_count = 0
|
||||
total_bytes = 0
|
||||
@@ -665,22 +764,18 @@ def collect_web_observe_summary():
|
||||
"activeSignalCount": 0,
|
||||
}
|
||||
if exists:
|
||||
try:
|
||||
children = [os.path.join(root, name) for name in os.listdir(root)]
|
||||
except OSError:
|
||||
children = []
|
||||
for child in children:
|
||||
if not os.path.isdir(child):
|
||||
continue
|
||||
for child in configured_observe_runs(root):
|
||||
try:
|
||||
record = observe_run_record(child, stale_hours)
|
||||
except OSError:
|
||||
continue
|
||||
row["runCount"] += 1
|
||||
run_count += 1
|
||||
if record.get("pidAlive"):
|
||||
if record.get("activeSignal"):
|
||||
row["activeSignalCount"] += 1
|
||||
active_rows.append(record)
|
||||
if record.get("staleLiveSignal"):
|
||||
stale_live_rows.append(record)
|
||||
if record.get("staleSignal"):
|
||||
row["staleSignalCount"] += 1
|
||||
stale_rows.append(record)
|
||||
@@ -697,9 +792,11 @@ def collect_web_observe_summary():
|
||||
"runCount": run_count,
|
||||
"activeSignalCount": len(active_rows),
|
||||
"staleSignalCount": len(stale_rows),
|
||||
"staleLiveSignalCount": len(stale_live_rows),
|
||||
"roots": root_rows,
|
||||
"activeSignals": active_rows[:int(OPTIONS.get("limit") or 50)],
|
||||
"staleSignals": stale_rows[:int(OPTIONS.get("limit") or 50)],
|
||||
"staleLiveSignals": stale_live_rows[:int(OPTIONS.get("limit") or 50)],
|
||||
"policy": "analysis-only; active or stale observe runs must be stopped/retained through controlled observer lifecycle commands, not raw process kill or directory deletion",
|
||||
}
|
||||
|
||||
@@ -967,6 +1064,8 @@ def collect_protected():
|
||||
return result
|
||||
|
||||
def collect_candidates(observed_at):
|
||||
if bool(OPTIONS.get("memoryPressureOnly")):
|
||||
return collect_memory_reclaim_candidates()
|
||||
candidates = []
|
||||
if OPTIONS.get("journal", True):
|
||||
usage = command(["journalctl", "--disk-usage"], 5)
|
||||
@@ -1314,6 +1413,29 @@ def summarize(candidates, returned, disk=None):
|
||||
"target": target_assessment(disk, total),
|
||||
}
|
||||
|
||||
def summarize_memory_candidates(candidates, returned):
|
||||
total_memory = sum(safe_int(item.get("expectedMemoryRssBytes")) for item in candidates)
|
||||
returned_memory = sum(safe_int(item.get("expectedMemoryRssBytes")) for item in returned)
|
||||
by_kind = {}
|
||||
for item in candidates:
|
||||
kind = item.get("kind") or "unknown"
|
||||
bucket = by_kind.setdefault(kind, {"count": 0, "expectedMemoryRssBytes": 0})
|
||||
bucket["count"] += 1
|
||||
bucket["expectedMemoryRssBytes"] += safe_int(item.get("expectedMemoryRssBytes"))
|
||||
bucket["expectedMemoryRss"] = fmt_bytes(bucket["expectedMemoryRssBytes"])
|
||||
return {
|
||||
"scope": "memory-pressure-only",
|
||||
"candidateCount": len(candidates),
|
||||
"returnedCandidateCount": len(returned),
|
||||
"expectedMemoryRssBytes": total_memory,
|
||||
"expectedMemoryRss": fmt_bytes(total_memory),
|
||||
"returnedExpectedMemoryRssBytes": returned_memory,
|
||||
"returnedExpectedMemoryRss": fmt_bytes(returned_memory),
|
||||
"estimatedDiskBytes": 0,
|
||||
"outcome": "memory-reclaim-candidates" if candidates else "no-memory-reclaim",
|
||||
"byKind": by_kind,
|
||||
}
|
||||
|
||||
def assert_tmp_candidate(path):
|
||||
resolved = os.path.abspath(path)
|
||||
if not resolved.startswith("/tmp/"):
|
||||
@@ -1400,6 +1522,8 @@ def execute(candidate):
|
||||
before = du_size(path, 15) or path_size(path)
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
return {"reclaimedBytes": before, "stale": record}
|
||||
if kind in set(["browser-orphan-process-tree", "stale-web-observer-process-tree"]):
|
||||
return execute_memory_process_candidate(candidate)
|
||||
if kind == "k3s-cri-image-prune":
|
||||
return execute_k3s_image_cache_prune()
|
||||
if kind == "host-containerd-cache-prune":
|
||||
@@ -1450,6 +1574,13 @@ def returned_results(results):
|
||||
"path",
|
||||
"status",
|
||||
"estimatedReclaimBytes",
|
||||
"estimatedDiskBytes",
|
||||
"expectedMemoryRssBytes",
|
||||
"reclaimedMemoryBytes",
|
||||
"processCount",
|
||||
"termSignalCount",
|
||||
"killSignalCount",
|
||||
"remainingIdentityCount",
|
||||
"reclaimedBytes",
|
||||
"orphanCount",
|
||||
"selectedOrphanCount",
|
||||
@@ -1468,6 +1599,38 @@ def returned_results(results):
|
||||
return [compact_result(item) for item in (failed + started + succeeded)[:int(OPTIONS.get("resultLimit") or 50)]]
|
||||
|
||||
def plan_payload(observed_at, preflight, protected, candidates, visible):
|
||||
if bool(OPTIONS.get("memoryPressureOnly")):
|
||||
scan = memory_reclaim_scan_diagnostics()
|
||||
summary = summarize_memory_candidates(candidates, visible)
|
||||
if scan.get("blocked"):
|
||||
summary["outcome"] = "memory-reclaim-scan-blocked"
|
||||
return {
|
||||
"ok": scan.get("ok") is True,
|
||||
"action": "gc remote plan",
|
||||
"providerId": PROVIDER_ID,
|
||||
"scope": "memory-pressure-only",
|
||||
"dryRun": True,
|
||||
"mutation": False,
|
||||
"observedAt": observed_at,
|
||||
"options": OPTIONS,
|
||||
"memoryBefore": collect_memory_snapshot(),
|
||||
"summary": summary,
|
||||
"candidateScan": scan,
|
||||
"candidates": visible,
|
||||
"protected": {
|
||||
"activeObserverIntersection": "excluded" if scan.get("ok") is True else "unknown-protected-all",
|
||||
"processIdentity": "pre-term:pid-ppid-sid-startTicks;post-term:pid-startTicks",
|
||||
"treeClosure": "unique-root-ppid-sid-closed",
|
||||
},
|
||||
"policy": {
|
||||
"requiresRunConfirm": True,
|
||||
"planCommand": "bun scripts/cli.ts gc remote %s plan --memory-pressure-only" % PROVIDER_ID,
|
||||
"runCommand": "bun scripts/cli.ts gc remote %s run --confirm --memory-pressure-only" % PROVIDER_ID,
|
||||
"candidateKinds": ["browser-orphan-process-tree", "stale-web-observer-process-tree"],
|
||||
"neverTouches": ["disk candidates", "apt cache", "journal", "tmp", "container images", "active observer process trees"],
|
||||
"termination": "SIGTERM, bounded wait, identity reclose, SIGKILL only same-tree residual",
|
||||
},
|
||||
}
|
||||
disk = df_snapshot()
|
||||
ci_storage = ci_storage_snapshot()
|
||||
memory_pressure = collect_memory_pressure()
|
||||
@@ -1793,8 +1956,235 @@ def remote_policy_install_payload(observed_at):
|
||||
},
|
||||
}
|
||||
|
||||
def update_memory_pressure_job_progress(paths, completed_count, candidate_count, worker_pid, worker_start_ticks):
|
||||
descriptor = acquire_job_lock(paths)
|
||||
try:
|
||||
with open(paths["state"], "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if payload.get("status") != "running" or safe_int(payload.get("workerPid")) != worker_pid \
|
||||
or safe_int(payload.get("workerStartTicks")) != worker_start_ticks:
|
||||
raise RuntimeError("memory pressure job state no longer belongs to this worker")
|
||||
payload.update({
|
||||
"workerHeartbeatAt": now_iso(),
|
||||
"progress": {"completedCount": completed_count, "candidateCount": candidate_count},
|
||||
})
|
||||
write_json_atomic(paths["state"], payload)
|
||||
finally:
|
||||
release_job_lock(descriptor)
|
||||
|
||||
def finish_memory_pressure_job(job_id, candidates, observed_at, worker_pid, worker_start_ticks):
|
||||
paths = job_paths(job_id)
|
||||
memory_before = collect_memory_snapshot()
|
||||
results = []
|
||||
update_memory_pressure_job_progress(paths, 0, len(candidates), worker_pid, worker_start_ticks)
|
||||
for index, candidate in enumerate(candidates):
|
||||
try:
|
||||
execution = execute(candidate)
|
||||
item = dict(candidate)
|
||||
item.update({
|
||||
"status": execution.get("status") or "succeeded",
|
||||
"reclaimedBytes": 0,
|
||||
"reclaimedMemoryBytes": execution.get("reclaimedMemoryBytes"),
|
||||
"termSignalCount": execution.get("termSignalCount"),
|
||||
"killSignalCount": execution.get("killSignalCount"),
|
||||
"remainingIdentityCount": execution.get("remainingIdentityCount"),
|
||||
"remainingIdentities": execution.get("remainingIdentities"),
|
||||
"lateAllowlistedIdentityCount": execution.get("lateAllowlistedIdentityCount"),
|
||||
"memoryBefore": execution.get("memoryBefore"),
|
||||
"memoryAfter": execution.get("memoryAfter"),
|
||||
})
|
||||
results.append(item)
|
||||
except Exception as exc:
|
||||
item = dict(candidate)
|
||||
item.update({"status": "failed", "reclaimedBytes": 0, "reclaimedMemoryBytes": None, "error": str(exc)})
|
||||
results.append(item)
|
||||
update_memory_pressure_job_progress(paths, index + 1, len(candidates), worker_pid, worker_start_ticks)
|
||||
memory_after = collect_memory_snapshot()
|
||||
failed = [item for item in results if item.get("status") == "failed"]
|
||||
returned = returned_results(results)
|
||||
summary = summarize_memory_candidates(candidates, returned)
|
||||
summary.update({
|
||||
"attemptedCount": len(results),
|
||||
"succeededCount": len([item for item in results if item.get("status") == "succeeded"]),
|
||||
"failedCount": len(failed),
|
||||
"actualMemoryAvailableDeltaBytes": (
|
||||
safe_int((memory_after.get("memory") or {}).get("availableBytes"))
|
||||
- safe_int((memory_before.get("memory") or {}).get("availableBytes"))
|
||||
) if memory_before.get("ok") and memory_after.get("ok") else None,
|
||||
})
|
||||
payload = {
|
||||
"ok": len(failed) == 0,
|
||||
"action": "gc remote run",
|
||||
"providerId": PROVIDER_ID,
|
||||
"scope": "memory-pressure-only",
|
||||
"status": "succeeded" if len(failed) == 0 else "failed",
|
||||
"jobId": job_id,
|
||||
"statusCommand": status_command(job_id),
|
||||
"statePath": paths["state"],
|
||||
"dryRun": False,
|
||||
"mutation": len(results) > 0,
|
||||
"observedAt": observed_at,
|
||||
"startedAt": observed_at,
|
||||
"finishedAt": now_iso(),
|
||||
"workerPid": worker_pid,
|
||||
"workerStartTicks": worker_start_ticks,
|
||||
"workerAlive": False,
|
||||
"workerCompleted": True,
|
||||
"workerHeartbeatAt": now_iso(),
|
||||
"progress": {"completedCount": len(candidates), "candidateCount": len(candidates)},
|
||||
"memoryBefore": memory_before,
|
||||
"memoryAfter": memory_after,
|
||||
"summary": summary,
|
||||
"results": returned,
|
||||
}
|
||||
write_memory_job_state(paths, payload)
|
||||
|
||||
def start_memory_pressure_job(observed_at, candidates):
|
||||
job_id = "%s-memory-pressure-%s-%s" % (
|
||||
re.sub(r"[^A-Za-z0-9._-]+", "-", PROVIDER_ID.lower()).strip("-") or "provider",
|
||||
int(time.time()),
|
||||
os.getpid(),
|
||||
)
|
||||
paths = job_paths(job_id)
|
||||
summary = summarize_memory_candidates(candidates, candidates)
|
||||
scan = memory_reclaim_scan_diagnostics()
|
||||
base = {
|
||||
"ok": scan.get("ok") is True,
|
||||
"action": "gc remote run",
|
||||
"providerId": PROVIDER_ID,
|
||||
"scope": "memory-pressure-only",
|
||||
"jobId": job_id,
|
||||
"statusCommand": status_command(job_id),
|
||||
"statePath": paths["state"],
|
||||
"dryRun": False,
|
||||
"mutation": len(candidates) > 0,
|
||||
"observedAt": observed_at,
|
||||
"startedAt": observed_at,
|
||||
"summary": summary,
|
||||
"candidateScan": scan,
|
||||
}
|
||||
if scan.get("blocked"):
|
||||
terminal = dict(base)
|
||||
terminal.update({
|
||||
"status": "blocked",
|
||||
"outcome": "memory-reclaim-scan-blocked",
|
||||
"mutation": False,
|
||||
"finishedAt": now_iso(),
|
||||
"results": [],
|
||||
})
|
||||
write_json_atomic(paths["state"], terminal)
|
||||
return terminal
|
||||
if not candidates:
|
||||
terminal = dict(base)
|
||||
terminal.update({
|
||||
"status": "blocked",
|
||||
"outcome": "no-memory-reclaim-candidates",
|
||||
"mutation": False,
|
||||
"finishedAt": now_iso(),
|
||||
"results": [],
|
||||
})
|
||||
write_json_atomic(paths["state"], terminal)
|
||||
return terminal
|
||||
read_descriptor, write_descriptor = os.pipe()
|
||||
try:
|
||||
child_pid = os.fork()
|
||||
except OSError as exc:
|
||||
os.close(read_descriptor)
|
||||
os.close(write_descriptor)
|
||||
failed = dict(base)
|
||||
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-fork-failed", "error": str(exc), "finishedAt": now_iso()})
|
||||
write_memory_job_state(paths, failed)
|
||||
return failed
|
||||
if child_pid == 0:
|
||||
try:
|
||||
os.close(write_descriptor)
|
||||
released = os.read(read_descriptor, 1)
|
||||
os.close(read_descriptor)
|
||||
if released != b"1":
|
||||
os._exit(1)
|
||||
os.setsid()
|
||||
with open("/dev/null", "rb", buffering=0) as stdin_handle, open(paths["log"], "ab", buffering=0) as log_handle:
|
||||
os.dup2(stdin_handle.fileno(), 0)
|
||||
os.dup2(log_handle.fileno(), 1)
|
||||
os.dup2(log_handle.fileno(), 2)
|
||||
finish_memory_pressure_job(job_id, candidates, observed_at, os.getpid(), process_start_ticks(os.getpid()))
|
||||
except Exception as exc:
|
||||
failed = dict(base)
|
||||
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-worker-failed", "workerPid": os.getpid(), "workerStartTicks": process_start_ticks(os.getpid()), "error": str(exc), "finishedAt": now_iso()})
|
||||
try:
|
||||
write_memory_job_state(paths, failed)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
os._exit(0)
|
||||
os.close(read_descriptor)
|
||||
worker_start_ticks = process_start_ticks(child_pid)
|
||||
if worker_start_ticks is None:
|
||||
os.close(write_descriptor)
|
||||
try:
|
||||
os.kill(child_pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
failed = dict(base)
|
||||
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-worker-identity-unavailable", "workerPid": child_pid, "finishedAt": now_iso()})
|
||||
write_memory_job_state(paths, failed)
|
||||
return failed
|
||||
running = dict(base)
|
||||
running.update({
|
||||
"status": "running",
|
||||
"outcome": "memory-reclaim-started",
|
||||
"workerPid": child_pid,
|
||||
"workerStartTicks": worker_start_ticks,
|
||||
"workerAlive": True,
|
||||
"workerHeartbeatAt": now_iso(),
|
||||
"progress": {"completedCount": 0, "candidateCount": len(candidates)},
|
||||
})
|
||||
try:
|
||||
write_memory_job_state(paths, running)
|
||||
os.write(write_descriptor, b"1")
|
||||
except Exception as exc:
|
||||
try:
|
||||
os.kill(child_pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
failed = dict(base)
|
||||
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-worker-release-failed", "workerPid": child_pid, "workerStartTicks": worker_start_ticks, "error": str(exc), "finishedAt": now_iso()})
|
||||
write_memory_job_state(paths, failed)
|
||||
return failed
|
||||
finally:
|
||||
os.close(write_descriptor)
|
||||
return running
|
||||
|
||||
def main():
|
||||
observed_at = now_iso()
|
||||
if ACTION == "memory":
|
||||
memory = collect_memory_snapshot()
|
||||
emit_json({
|
||||
"ok": memory.get("ok") is True,
|
||||
"action": "gc remote memory",
|
||||
"providerId": PROVIDER_ID,
|
||||
"dryRun": True,
|
||||
"mutation": False,
|
||||
"observedAt": observed_at,
|
||||
"metric": memory.get("metric"),
|
||||
"source": memory.get("source"),
|
||||
"swapExcluded": memory.get("swapExcluded") is True,
|
||||
"memory": memory.get("memory"),
|
||||
"error": memory.get("error"),
|
||||
"valuesRedacted": True,
|
||||
}, persist_large=False)
|
||||
return 0
|
||||
if ACTION == "status" and job_id_or_none():
|
||||
emit_json(remote_gc_job_status(), persist_large=False)
|
||||
return 0
|
||||
if bool(OPTIONS.get("memoryPressureOnly")) and ACTION in set(["plan", "run"]):
|
||||
candidates = collect_memory_reclaim_candidates()
|
||||
visible = visible_items(candidates)
|
||||
if ACTION == "plan":
|
||||
emit_json(plan_payload(observed_at, {"skipped": True, "reason": "memory-pressure-only"}, [], candidates, visible), persist_large=True)
|
||||
else:
|
||||
emit_json(start_memory_pressure_job(observed_at, visible), persist_large=True)
|
||||
return 0
|
||||
preflight = cluster_preflight()
|
||||
if ACTION == "policy-plan":
|
||||
emit_json(remote_policy_plan_payload(observed_at), persist_large=False)
|
||||
@@ -1854,7 +2244,7 @@ def main():
|
||||
})
|
||||
emit_json(snapshot, persist_large=True)
|
||||
return 0
|
||||
protected = collect_protected()
|
||||
protected = [] if bool(OPTIONS.get("memoryPressureOnly")) else collect_protected()
|
||||
candidates = collect_candidates(observed_at)
|
||||
visible = visible_items(candidates)
|
||||
if ACTION == "plan":
|
||||
|
||||
@@ -1,14 +1,63 @@
|
||||
_OBSERVE_RUN_CACHE = {}
|
||||
_OBSERVE_SCAN_TRUNCATED = False
|
||||
_PROCESS_SCAN_TRUNCATED = False
|
||||
_OBSERVE_SCAN_ERROR = None
|
||||
_PROCESS_SCAN_ERROR = None
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = None
|
||||
_OBSERVE_ROOT_DIAGNOSTICS = {"configured": [], "readable": [], "missing": []}
|
||||
|
||||
def configured_observe_roots():
|
||||
roots = config_list(MEMORY_CONFIG, "observeStateRoots", config_list(MEMORY_CONFIG, "webObserveRoots", []))
|
||||
return [os.path.abspath(item) for item in roots if isinstance(item, str) and item.startswith("/")]
|
||||
|
||||
def is_direct_observe_run_path(path):
|
||||
def configured_observe_runs(root):
|
||||
global _OBSERVE_SCAN_TRUNCATED, _OBSERVE_SCAN_ERROR
|
||||
resolved_root = os.path.abspath(root)
|
||||
if resolved_root in _OBSERVE_RUN_CACHE:
|
||||
return list(_OBSERVE_RUN_CACHE[resolved_root])
|
||||
max_depth = config_int(MEMORY_CONFIG, "observeRunDepth", 1, minimum=1, maximum=12)
|
||||
scan_limit = config_int(MEMORY_CONFIG, "observeScanLimit", 1000, minimum=1, maximum=100000)
|
||||
markers = set(["manifest.json", "heartbeat.json", "pid", "observer.pid", "runner.pid"])
|
||||
runs = []
|
||||
scanned = 0
|
||||
if not os.path.isdir(resolved_root) or os.path.islink(resolved_root):
|
||||
return runs
|
||||
def record_walk_error(error):
|
||||
global _OBSERVE_SCAN_ERROR
|
||||
_OBSERVE_SCAN_ERROR = "observe-state-walk-failed:%s" % type(error).__name__
|
||||
|
||||
for current, directories, files in os.walk(resolved_root, topdown=True, followlinks=False, onerror=record_walk_error):
|
||||
relative = os.path.relpath(current, resolved_root)
|
||||
depth = 0 if relative == "." else relative.count(os.sep) + 1
|
||||
directories[:] = sorted(name for name in directories if not os.path.islink(os.path.join(current, name)))
|
||||
if depth >= max_depth:
|
||||
directories[:] = []
|
||||
if depth == 0:
|
||||
continue
|
||||
scanned += 1
|
||||
if scanned > scan_limit:
|
||||
_OBSERVE_SCAN_TRUNCATED = True
|
||||
break
|
||||
if markers.intersection(files):
|
||||
runs.append(os.path.abspath(current))
|
||||
directories[:] = []
|
||||
if not _OBSERVE_SCAN_TRUNCATED:
|
||||
_OBSERVE_RUN_CACHE[resolved_root] = list(runs)
|
||||
return runs
|
||||
|
||||
def is_observe_run_path(path):
|
||||
resolved = os.path.abspath(path)
|
||||
for root in configured_observe_roots():
|
||||
if os.path.dirname(resolved) == root and resolved.startswith(root.rstrip("/") + "/"):
|
||||
prefix = root.rstrip("/") + "/"
|
||||
if resolved.startswith(prefix) and resolved in set(configured_observe_runs(root)):
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_direct_observe_run_path(path):
|
||||
# Kept as a compatibility name for generic artifact retention. Validation is
|
||||
# now marker/depth based because the configured roots contain dated trees.
|
||||
return is_observe_run_path(path)
|
||||
|
||||
def path_has_open_fd(path):
|
||||
resolved = os.path.realpath(path)
|
||||
prefix = resolved.rstrip("/") + "/"
|
||||
@@ -42,8 +91,8 @@ def path_has_open_fd(path):
|
||||
|
||||
def assert_web_observe_candidate(path):
|
||||
resolved = os.path.abspath(path)
|
||||
if not is_direct_observe_run_path(resolved):
|
||||
raise RuntimeError("refusing to remove web-observe path outside configured direct run roots: %s" % path)
|
||||
if not is_observe_run_path(resolved):
|
||||
raise RuntimeError("refusing to remove web-observe path outside configured marker/depth run roots: %s" % path)
|
||||
if os.path.islink(resolved) or not os.path.isdir(resolved):
|
||||
raise RuntimeError("refusing to remove non-directory or symlink web-observe path: %s" % path)
|
||||
stale_hours = config_float(MEMORY_CONFIG, "staleRunMaxAgeHours", 6.0, minimum=0.0)
|
||||
@@ -55,3 +104,423 @@ def assert_web_observe_candidate(path):
|
||||
if path_has_open_fd(resolved):
|
||||
raise RuntimeError("refusing to remove web-observe run with open fd/cwd reference: %s" % path)
|
||||
return record
|
||||
|
||||
def read_proc_process_table():
|
||||
global _PROCESS_SCAN_TRUNCATED, _PROCESS_SCAN_ERROR
|
||||
scan_limit = config_int(MEMORY_CONFIG, "processScanLimit", 16384, minimum=1, maximum=1000000)
|
||||
try:
|
||||
clock_ticks = int(os.sysconf("SC_CLK_TCK"))
|
||||
uptime = float(open("/proc/uptime", "r", encoding="utf-8").read().split()[0])
|
||||
page_size = int(os.sysconf("SC_PAGE_SIZE"))
|
||||
pid_names = sorted((name for name in os.listdir("/proc") if name.isdigit()), key=int)
|
||||
if len(pid_names) > scan_limit:
|
||||
_PROCESS_SCAN_TRUNCATED = True
|
||||
return {}
|
||||
except Exception as exc:
|
||||
_PROCESS_SCAN_ERROR = "proc-process-table-read-failed:%s" % type(exc).__name__
|
||||
return {}
|
||||
rows = {}
|
||||
stat_read_failures = 0
|
||||
for pid_name in pid_names:
|
||||
base = os.path.join("/proc", pid_name)
|
||||
try:
|
||||
with open(os.path.join(base, "stat"), "r", encoding="utf-8") as handle:
|
||||
stat_text = handle.read().strip()
|
||||
close = stat_text.rfind(")")
|
||||
if close < 0:
|
||||
continue
|
||||
fields = stat_text[close + 2:].split()
|
||||
if len(fields) < 22:
|
||||
continue
|
||||
pid = int(pid_name)
|
||||
ppid = int(fields[1])
|
||||
sid = int(fields[3])
|
||||
start_ticks = int(fields[19])
|
||||
try:
|
||||
with open(os.path.join(base, "cmdline"), "rb") as handle:
|
||||
command_line = handle.read(65536).replace(b"\0", b" ").decode("utf-8", errors="replace").strip()
|
||||
except OSError:
|
||||
command_line = ""
|
||||
try:
|
||||
with open(os.path.join(base, "statm"), "r", encoding="utf-8") as handle:
|
||||
rss_bytes = safe_int(handle.read().split()[1]) * page_size
|
||||
except Exception:
|
||||
rss_bytes = 0
|
||||
rows[pid] = {
|
||||
"pid": pid,
|
||||
"ppid": ppid,
|
||||
"sid": sid,
|
||||
"startTicks": start_ticks,
|
||||
"ageSeconds": max(0, int(uptime - (float(start_ticks) / float(clock_ticks)))),
|
||||
"rssBytes": rss_bytes,
|
||||
"commandPreview": redact_command_preview(command_line),
|
||||
"commandLine": command_line,
|
||||
}
|
||||
except (OSError, ValueError, IndexError):
|
||||
stat_read_failures += 1
|
||||
continue
|
||||
if not rows:
|
||||
_PROCESS_SCAN_ERROR = "proc-process-table-empty"
|
||||
elif stat_read_failures > max(32, len(pid_names) // 2):
|
||||
_PROCESS_SCAN_ERROR = "proc-process-stat-read-failure-threshold:%s-of-%s" % (stat_read_failures, len(pid_names))
|
||||
return rows
|
||||
|
||||
def validate_observe_run_markers(path):
|
||||
for name in ["heartbeat.json", "manifest.json"]:
|
||||
marker = os.path.join(path, name)
|
||||
if not os.path.lexists(marker):
|
||||
continue
|
||||
if os.path.islink(marker) or not os.path.isfile(marker):
|
||||
raise OSError("observe marker is not a regular file: %s" % marker)
|
||||
with open(marker, "r", encoding="utf-8") as handle:
|
||||
value = json.load(handle)
|
||||
if not isinstance(value, dict):
|
||||
raise OSError("observe JSON marker is not an object: %s" % marker)
|
||||
for name in ["pid", "observer.pid", "browser.pid", "runner.pid"]:
|
||||
marker = os.path.join(path, name)
|
||||
if not os.path.lexists(marker):
|
||||
continue
|
||||
if os.path.islink(marker) or not os.path.isfile(marker):
|
||||
raise OSError("observe pid marker is not a regular file: %s" % marker)
|
||||
with open(marker, "r", encoding="utf-8") as handle:
|
||||
raw = handle.read().strip()
|
||||
if not re.match(r"^[1-9]\d*$", raw):
|
||||
raise OSError("observe pid marker is invalid: %s" % marker)
|
||||
|
||||
def observer_run_records():
|
||||
global _OBSERVE_SCAN_ERROR, _OBSERVE_ROOT_DIAGNOSTICS
|
||||
stale_hours = config_float(MEMORY_CONFIG, "staleRunMaxAgeHours", 6.0, minimum=0.0)
|
||||
records = []
|
||||
roots = configured_observe_roots()
|
||||
accessible_roots = []
|
||||
missing_roots = []
|
||||
for root in roots:
|
||||
try:
|
||||
root_stat = os.stat(root)
|
||||
if os.path.isdir(root) and not os.path.islink(root):
|
||||
accessible_roots.append(root)
|
||||
elif root_stat:
|
||||
_OBSERVE_SCAN_ERROR = "observe-state-root-not-directory"
|
||||
except FileNotFoundError:
|
||||
missing_roots.append(root)
|
||||
continue
|
||||
except OSError as exc:
|
||||
_OBSERVE_SCAN_ERROR = "observe-state-root-read-failed:%s" % type(exc).__name__
|
||||
minimum_readable = config_int(MEMORY_CONFIG, "minimumReadableObserveRoots", 1, minimum=1, maximum=100)
|
||||
_OBSERVE_ROOT_DIAGNOSTICS = {"configured": list(roots), "readable": list(accessible_roots), "missing": missing_roots}
|
||||
if len(accessible_roots) < minimum_readable:
|
||||
_OBSERVE_SCAN_ERROR = _OBSERVE_SCAN_ERROR or "minimum-readable-observe-roots-not-met:%s-of-%s" % (len(accessible_roots), minimum_readable)
|
||||
for root in accessible_roots:
|
||||
for path in configured_observe_runs(root):
|
||||
try:
|
||||
validate_observe_run_markers(path)
|
||||
records.append(observe_run_record(path, stale_hours))
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
_OBSERVE_SCAN_ERROR = "observe-run-marker-read-failed:%s" % type(exc).__name__
|
||||
continue
|
||||
return records
|
||||
|
||||
def process_descendants(processes, root_pid):
|
||||
selected = set([int(root_pid)])
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for pid, row in processes.items():
|
||||
if pid not in selected and safe_int(row.get("ppid")) in selected:
|
||||
selected.add(pid)
|
||||
changed = True
|
||||
return selected
|
||||
|
||||
def session_closed_tree(processes, sid):
|
||||
members = {pid: row for pid, row in processes.items() if safe_int(row.get("sid")) == int(sid)}
|
||||
if not members:
|
||||
return None
|
||||
roots = [pid for pid, row in members.items() if safe_int(row.get("ppid")) not in members]
|
||||
if len(roots) != 1:
|
||||
return None
|
||||
root_pid = roots[0]
|
||||
if process_descendants(members, root_pid) != set(members.keys()):
|
||||
return None
|
||||
return {"rootPid": root_pid, "members": members}
|
||||
|
||||
def command_matches(row, patterns):
|
||||
haystack = str(row.get("commandLine") or "").lower()
|
||||
return any(str(pattern).lower() in haystack for pattern in patterns if str(pattern))
|
||||
|
||||
def memory_process_candidate_id(kind, root):
|
||||
return "%s:%s:%s:%s" % (kind, safe_int(root.get("sid")), safe_int(root.get("pid")), safe_int(root.get("startTicks")))
|
||||
|
||||
def collect_memory_reclaim_candidates():
|
||||
global _OBSERVE_SCAN_TRUNCATED, _PROCESS_SCAN_TRUNCATED, _OBSERVE_SCAN_ERROR, _PROCESS_SCAN_ERROR, _MEMORY_RECLAIM_CONFIG_ERROR, _OBSERVE_ROOT_DIAGNOSTICS
|
||||
_OBSERVE_SCAN_TRUNCATED = False
|
||||
_PROCESS_SCAN_TRUNCATED = False
|
||||
_OBSERVE_SCAN_ERROR = None
|
||||
_PROCESS_SCAN_ERROR = None
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = None
|
||||
_OBSERVE_ROOT_DIAGNOSTICS = {"configured": [], "readable": [], "missing": []}
|
||||
_OBSERVE_RUN_CACHE.clear()
|
||||
required_keys = [
|
||||
"observeStateRoots", "minimumReadableObserveRoots", "observeRunDepth", "observeScanLimit", "staleRunMaxAgeHours",
|
||||
"processScanLimit", "processStaleMinAgeSeconds", "maxMemoryReclaimCandidates",
|
||||
"terminationGraceSeconds", "terminationPollMilliseconds", "observerRootPatterns",
|
||||
"browserRootPatterns", "browserProcessPatterns",
|
||||
]
|
||||
if any(key not in MEMORY_CONFIG for key in required_keys):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-policy-missing"
|
||||
return []
|
||||
list_keys = ["observeStateRoots", "observerRootPatterns", "browserRootPatterns", "browserProcessPatterns"]
|
||||
integer_keys = [
|
||||
"observeRunDepth", "observeScanLimit", "processScanLimit", "processStaleMinAgeSeconds",
|
||||
"maxMemoryReclaimCandidates", "terminationGraceSeconds", "terminationPollMilliseconds",
|
||||
]
|
||||
if any(not isinstance(MEMORY_CONFIG.get(key), list) or not MEMORY_CONFIG.get(key) for key in list_keys):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-list-invalid"
|
||||
return []
|
||||
if any(any(not isinstance(item, str) or not item.strip() or "\0" in item for item in MEMORY_CONFIG.get(key)) for key in list_keys):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-list-entry-invalid"
|
||||
return []
|
||||
if any(not os.path.isabs(item) for item in MEMORY_CONFIG.get("observeStateRoots")):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-observe-root-not-absolute"
|
||||
return []
|
||||
integer_ranges = {
|
||||
"observeRunDepth": (1, 12),
|
||||
"minimumReadableObserveRoots": (1, 100),
|
||||
"observeScanLimit": (1, 100000),
|
||||
"processScanLimit": (1, 1000000),
|
||||
"processStaleMinAgeSeconds": (60, 604800),
|
||||
"maxMemoryReclaimCandidates": (1, 100),
|
||||
"terminationGraceSeconds": (1, 30),
|
||||
"terminationPollMilliseconds": (50, 1000),
|
||||
}
|
||||
if any(isinstance(MEMORY_CONFIG.get(key), bool) or not isinstance(MEMORY_CONFIG.get(key), int)
|
||||
or MEMORY_CONFIG.get(key) < minimum or MEMORY_CONFIG.get(key) > maximum
|
||||
for key, (minimum, maximum) in integer_ranges.items()):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-integer-invalid"
|
||||
return []
|
||||
if MEMORY_CONFIG.get("minimumReadableObserveRoots") > len(MEMORY_CONFIG.get("observeStateRoots")):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "minimum-readable-observe-roots-exceeds-configured-roots"
|
||||
return []
|
||||
stale_hours = MEMORY_CONFIG.get("staleRunMaxAgeHours")
|
||||
if isinstance(stale_hours, bool) or not isinstance(stale_hours, (int, float)) or stale_hours <= 0 or stale_hours > 8760:
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-stale-age-invalid"
|
||||
return []
|
||||
browser_patterns = config_list(MEMORY_CONFIG, "browserProcessPatterns", [])
|
||||
root_patterns = config_list(MEMORY_CONFIG, "browserRootPatterns", [])
|
||||
observer_patterns = config_list(MEMORY_CONFIG, "observerRootPatterns", [])
|
||||
stale_seconds = config_int(MEMORY_CONFIG, "processStaleMinAgeSeconds", 3600, minimum=60, maximum=604800)
|
||||
if not browser_patterns or not root_patterns or not observer_patterns:
|
||||
return []
|
||||
processes = read_proc_process_table()
|
||||
records = observer_run_records()
|
||||
if not processes and _PROCESS_SCAN_ERROR is None:
|
||||
_PROCESS_SCAN_ERROR = "proc-process-table-empty"
|
||||
if _PROCESS_SCAN_TRUNCATED or _OBSERVE_SCAN_TRUNCATED or _PROCESS_SCAN_ERROR or _OBSERVE_SCAN_ERROR:
|
||||
return []
|
||||
fresh_observer_pids = set()
|
||||
stale_observer_pids = set()
|
||||
for record in records:
|
||||
pid = safe_int(record.get("pid"))
|
||||
if pid <= 0 or pid not in processes:
|
||||
continue
|
||||
if record.get("staleLiveSignal"):
|
||||
stale_observer_pids.add(pid)
|
||||
else:
|
||||
fresh_observer_pids.add(pid)
|
||||
protected_pids = set()
|
||||
for pid in fresh_observer_pids:
|
||||
protected_pids.update(process_descendants(processes, pid))
|
||||
candidates = []
|
||||
for sid in sorted(set(safe_int(row.get("sid")) for row in processes.values() if safe_int(row.get("sid")) > 0)):
|
||||
closed = session_closed_tree(processes, sid)
|
||||
if closed is None:
|
||||
continue
|
||||
members = closed["members"]
|
||||
member_pids = set(members.keys())
|
||||
if member_pids.intersection(protected_pids):
|
||||
continue
|
||||
root = members[closed["rootPid"]]
|
||||
if safe_int(root.get("ageSeconds")) < stale_seconds:
|
||||
continue
|
||||
has_browser = any(command_matches(row, browser_patterns) for row in members.values())
|
||||
intersecting_stale = member_pids.intersection(stale_observer_pids)
|
||||
kind = None
|
||||
reason = None
|
||||
if intersecting_stale and command_matches(root, observer_patterns):
|
||||
kind = "stale-web-observer-process-tree"
|
||||
reason = "stale-observer-heartbeat"
|
||||
elif not member_pids.intersection(set(safe_int(item.get("pid")) for item in records if item.get("pidAlive"))):
|
||||
if command_matches(root, root_patterns) and has_browser:
|
||||
kind = "browser-orphan-process-tree"
|
||||
reason = "browser-session-without-observer-record"
|
||||
if kind is None:
|
||||
continue
|
||||
identities = [
|
||||
{
|
||||
"pid": pid,
|
||||
"ppid": safe_int(row.get("ppid")),
|
||||
"sid": safe_int(row.get("sid")),
|
||||
"startTicks": safe_int(row.get("startTicks")),
|
||||
"rssBytes": safe_int(row.get("rssBytes")),
|
||||
"commandPreview": row.get("commandPreview"),
|
||||
}
|
||||
for pid, row in sorted(members.items())
|
||||
]
|
||||
expected_memory = sum(safe_int(item.get("rssBytes")) for item in identities)
|
||||
candidates.append({
|
||||
"id": memory_process_candidate_id(kind, root),
|
||||
"kind": kind,
|
||||
"risk": "medium",
|
||||
"description": "Terminate a YAML-allowlisted stale WebProbe process session after identity and active-observer revalidation",
|
||||
"reason": reason,
|
||||
"rootPid": safe_int(root.get("pid")),
|
||||
"sid": sid,
|
||||
"rootStartTicks": safe_int(root.get("startTicks")),
|
||||
"processCount": len(identities),
|
||||
"processIdentities": identities,
|
||||
"expectedMemoryRssBytes": expected_memory,
|
||||
"expectedMemoryRss": fmt_bytes(expected_memory),
|
||||
"estimatedDiskBytes": 0,
|
||||
"estimatedReclaimBytes": 0,
|
||||
"activeObserverIntersection": False,
|
||||
"treeClosure": "unique-root-ppid-sid-closed",
|
||||
"action": {"op": "term-wait-kill-exact-process-tree", "allowlist": "yaml-web-probe-memory-pressure"},
|
||||
})
|
||||
max_candidates = config_int(MEMORY_CONFIG, "maxMemoryReclaimCandidates", 20, minimum=1, maximum=100)
|
||||
return sorted(candidates, key=lambda item: safe_int(item.get("expectedMemoryRssBytes")), reverse=True)[:max_candidates]
|
||||
|
||||
def memory_reclaim_scan_diagnostics():
|
||||
blocked = bool(_MEMORY_RECLAIM_CONFIG_ERROR or _OBSERVE_SCAN_TRUNCATED or _PROCESS_SCAN_TRUNCATED or _OBSERVE_SCAN_ERROR or _PROCESS_SCAN_ERROR)
|
||||
return {
|
||||
"ok": not blocked,
|
||||
"blocked": blocked,
|
||||
"configError": _MEMORY_RECLAIM_CONFIG_ERROR,
|
||||
"observeScanTruncated": bool(_OBSERVE_SCAN_TRUNCATED),
|
||||
"processScanTruncated": bool(_PROCESS_SCAN_TRUNCATED),
|
||||
"observeScanError": _OBSERVE_SCAN_ERROR,
|
||||
"processScanError": _PROCESS_SCAN_ERROR,
|
||||
"observeRoots": {
|
||||
"configuredCount": len(_OBSERVE_ROOT_DIAGNOSTICS.get("configured") or []),
|
||||
"readableCount": len(_OBSERVE_ROOT_DIAGNOSTICS.get("readable") or []),
|
||||
"missingCount": len(_OBSERVE_ROOT_DIAGNOSTICS.get("missing") or []),
|
||||
"minimumReadable": safe_int(MEMORY_CONFIG.get("minimumReadableObserveRoots")),
|
||||
"configured": _OBSERVE_ROOT_DIAGNOSTICS.get("configured") or [],
|
||||
"readable": _OBSERVE_ROOT_DIAGNOSTICS.get("readable") or [],
|
||||
"missing": _OBSERVE_ROOT_DIAGNOSTICS.get("missing") or [],
|
||||
},
|
||||
"policySource": "config/unidesk-cli.yaml#gc.remote.targets.%s.memoryPressure" % PROVIDER_ID,
|
||||
}
|
||||
|
||||
def matching_live_identity(identity, processes):
|
||||
pid = safe_int(identity.get("pid"))
|
||||
row = processes.get(pid)
|
||||
return row if row \
|
||||
and safe_int(row.get("startTicks")) == safe_int(identity.get("startTicks")) \
|
||||
and safe_int(row.get("ppid")) == safe_int(identity.get("ppid")) \
|
||||
and safe_int(row.get("sid")) == safe_int(identity.get("sid")) else None
|
||||
|
||||
def matching_surviving_identity(identity, processes):
|
||||
pid = safe_int(identity.get("pid"))
|
||||
row = processes.get(pid)
|
||||
return row if row \
|
||||
and safe_int(row.get("startTicks")) == safe_int(identity.get("startTicks")) else None
|
||||
|
||||
def require_complete_process_scan(processes, phase):
|
||||
if not processes or _PROCESS_SCAN_ERROR or _PROCESS_SCAN_TRUNCATED:
|
||||
raise RuntimeError("memory process identity scan blocked during %s: %s" % (
|
||||
phase,
|
||||
_PROCESS_SCAN_ERROR or "process-scan-truncated",
|
||||
))
|
||||
return processes
|
||||
|
||||
def extend_with_allowlisted_session_members(identities, processes, sid, browser_patterns):
|
||||
known = {(safe_int(item.get("pid")), safe_int(item.get("startTicks"))) for item in identities}
|
||||
added = []
|
||||
for pid, row in processes.items():
|
||||
identity_key = (safe_int(pid), safe_int(row.get("startTicks")))
|
||||
if identity_key in known or safe_int(row.get("sid")) != safe_int(sid) or not command_matches(row, browser_patterns):
|
||||
continue
|
||||
item = {
|
||||
"pid": safe_int(pid),
|
||||
"ppid": safe_int(row.get("ppid")),
|
||||
"sid": safe_int(row.get("sid")),
|
||||
"startTicks": safe_int(row.get("startTicks")),
|
||||
"rssBytes": safe_int(row.get("rssBytes")),
|
||||
"commandPreview": row.get("commandPreview"),
|
||||
"discoveredAfterTerm": True,
|
||||
}
|
||||
identities.append(item)
|
||||
added.append(item)
|
||||
known.add(identity_key)
|
||||
return added
|
||||
|
||||
def execute_memory_process_candidate(candidate):
|
||||
current = next((item for item in collect_memory_reclaim_candidates() if item.get("id") == candidate.get("id")), None)
|
||||
if current is None:
|
||||
raise RuntimeError("memory candidate no longer has the same closed identity or is now protected")
|
||||
before = collect_memory_snapshot()
|
||||
identities = [dict(item) for item in (current.get("processIdentities") or [])]
|
||||
candidate_sid = safe_int(current.get("sid"))
|
||||
browser_patterns = config_list(MEMORY_CONFIG, "browserProcessPatterns", [])
|
||||
late_members = []
|
||||
processes = require_complete_process_scan(read_proc_process_table(), "pre-term-revalidation")
|
||||
live = [item for item in identities if matching_live_identity(item, processes)]
|
||||
for item in sorted(live, key=lambda value: safe_int(value.get("pid")), reverse=True):
|
||||
try:
|
||||
os.kill(safe_int(item.get("pid")), signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
grace_seconds = config_int(MEMORY_CONFIG, "terminationGraceSeconds", 5, minimum=1, maximum=30)
|
||||
poll_ms = config_int(MEMORY_CONFIG, "terminationPollMilliseconds", 200, minimum=50, maximum=1000)
|
||||
deadline = time.time() + grace_seconds
|
||||
residual = []
|
||||
while time.time() < deadline:
|
||||
processes = require_complete_process_scan(read_proc_process_table(), "term-grace-poll")
|
||||
late_members.extend(extend_with_allowlisted_session_members(identities, processes, candidate_sid, browser_patterns))
|
||||
residual = [item for item in identities if matching_surviving_identity(item, processes)]
|
||||
if not residual:
|
||||
break
|
||||
time.sleep(float(poll_ms) / 1000.0)
|
||||
processes = require_complete_process_scan(read_proc_process_table(), "pre-kill-revalidation")
|
||||
late_members.extend(extend_with_allowlisted_session_members(identities, processes, candidate_sid, browser_patterns))
|
||||
residual = [item for item in identities if matching_surviving_identity(item, processes)]
|
||||
for item in sorted(residual, key=lambda value: safe_int(value.get("pid")), reverse=True):
|
||||
try:
|
||||
os.kill(safe_int(item.get("pid")), signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
final_deadline = time.time() + min(2.0, float(grace_seconds))
|
||||
remaining = residual
|
||||
while time.time() < final_deadline:
|
||||
processes = require_complete_process_scan(read_proc_process_table(), "kill-grace-poll")
|
||||
late_members.extend(extend_with_allowlisted_session_members(identities, processes, candidate_sid, browser_patterns))
|
||||
remaining = [item for item in identities if matching_surviving_identity(item, processes)]
|
||||
if not remaining:
|
||||
break
|
||||
time.sleep(float(poll_ms) / 1000.0)
|
||||
after = collect_memory_snapshot()
|
||||
before_available = safe_int((before.get("memory") or {}).get("availableBytes"), None)
|
||||
after_available = safe_int((after.get("memory") or {}).get("availableBytes"), None)
|
||||
return {
|
||||
"status": "failed" if remaining else "succeeded",
|
||||
"reclaimedBytes": 0,
|
||||
"reclaimedMemoryBytes": max(0, after_available - before_available) if before_available is not None and after_available is not None else None,
|
||||
"expectedMemoryRssBytes": current.get("expectedMemoryRssBytes"),
|
||||
"termSignalCount": len(live),
|
||||
"killSignalCount": len(residual),
|
||||
"remainingIdentityCount": len(remaining),
|
||||
"lateAllowlistedIdentityCount": len(late_members),
|
||||
"remainingIdentities": [
|
||||
{
|
||||
"pid": safe_int(item.get("pid")),
|
||||
"startTicks": safe_int(item.get("startTicks")),
|
||||
"sid": safe_int(item.get("sid")),
|
||||
"originalPpid": safe_int(item.get("ppid")),
|
||||
"observedPpid": safe_int((matching_surviving_identity(item, processes) or {}).get("ppid")),
|
||||
}
|
||||
for item in remaining
|
||||
],
|
||||
"memoryBefore": before,
|
||||
"memoryAfter": after,
|
||||
"identityRevalidated": True,
|
||||
"activeObserverIntersection": False,
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import { type UniDeskConfig, rootPath } from "./config";
|
||||
import { remoteGcDegradedFailure } from "./gc-remote-degraded";
|
||||
import { runSshCommandCapture } from "./ssh";
|
||||
|
||||
type RemoteGcAction = "plan" | "snapshot" | "trend" | "run" | "status" | "policy-plan" | "policy-install" | "policy-status";
|
||||
type RemoteGcAction = "plan" | "memory" | "snapshot" | "trend" | "run" | "status" | "policy-plan" | "policy-install" | "policy-status";
|
||||
|
||||
interface RemoteGcOptions {
|
||||
confirm: boolean;
|
||||
@@ -36,6 +36,7 @@ interface RemoteGcOptions {
|
||||
full: boolean;
|
||||
historyLimit: number;
|
||||
saveSnapshot: boolean;
|
||||
memoryPressureOnly: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = {
|
||||
@@ -65,6 +66,7 @@ const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = {
|
||||
full: false,
|
||||
historyLimit: 12,
|
||||
saveSnapshot: true,
|
||||
memoryPressureOnly: false,
|
||||
};
|
||||
|
||||
const GC_CONFIG_RELATIVE_PATH = "config/unidesk-cli.yaml";
|
||||
@@ -89,7 +91,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
|
||||
return {
|
||||
ok: false,
|
||||
error: "gc-remote-provider-required",
|
||||
usage: "bun scripts/cli.ts gc remote <providerId> plan|snapshot|trend|run|status|policy [--confirm]",
|
||||
usage: "bun scripts/cli.ts gc remote <providerId> memory|plan|snapshot|trend|run|status|policy [--confirm]",
|
||||
};
|
||||
}
|
||||
const subaction = action ?? "plan";
|
||||
@@ -124,6 +126,17 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
|
||||
};
|
||||
}
|
||||
const options = parseRemoteGcOptions(args);
|
||||
if (options.memoryPressureOnly && subaction !== "plan" && subaction !== "run") {
|
||||
return {
|
||||
ok: false,
|
||||
error: "gc-remote-memory-pressure-only-action-invalid",
|
||||
action: subaction,
|
||||
supportedActions: ["plan", "run"],
|
||||
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan --memory-pressure-only`,
|
||||
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm --memory-pressure-only`,
|
||||
};
|
||||
}
|
||||
if (subaction === "memory") return await runRemoteGc(config, providerId, "memory", options);
|
||||
if (subaction === "plan" || subaction === "dry-run") return await runRemoteGc(config, providerId, "plan", options);
|
||||
if (subaction === "snapshot" || subaction === "growth") return await runRemoteGc(config, providerId, "snapshot", options);
|
||||
if (subaction === "trend") return await runRemoteGc(config, providerId, "trend", options);
|
||||
@@ -136,8 +149,8 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
|
||||
dryRun: true,
|
||||
mutation: false,
|
||||
requiredFlag: "--confirm",
|
||||
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan`,
|
||||
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm`,
|
||||
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan${options.memoryPressureOnly ? " --memory-pressure-only" : ""}`,
|
||||
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm${options.memoryPressureOnly ? " --memory-pressure-only" : ""}`,
|
||||
};
|
||||
}
|
||||
return await runRemoteGc(config, providerId, "run", options);
|
||||
@@ -146,7 +159,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
|
||||
ok: false,
|
||||
error: "unsupported-gc-remote-action",
|
||||
action: subaction,
|
||||
supportedActions: ["plan", "snapshot", "trend", "run", "status", "policy"],
|
||||
supportedActions: ["memory", "plan", "snapshot", "trend", "run", "status", "policy"],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -238,6 +251,8 @@ function parseRemoteGcOptions(args: string[]): RemoteGcOptions {
|
||||
options.full = true;
|
||||
} else if (arg === "--no-save" || arg === "--no-snapshot-save") {
|
||||
options.saveSnapshot = false;
|
||||
} else if (arg === "--memory-pressure-only") {
|
||||
options.memoryPressureOnly = true;
|
||||
} else {
|
||||
throw new Error(`unknown gc remote option: ${arg}`);
|
||||
}
|
||||
|
||||
@@ -517,6 +517,10 @@ function gcHelp(): unknown {
|
||||
"bun scripts/cli.ts gc policy plan",
|
||||
"bun scripts/cli.ts gc policy install",
|
||||
"bun scripts/cli.ts gc remote G14 plan",
|
||||
"bun scripts/cli.ts gc remote NC01 memory",
|
||||
"bun scripts/cli.ts gc remote NC01 plan --memory-pressure-only",
|
||||
"bun scripts/cli.ts gc remote NC01 run --confirm --memory-pressure-only",
|
||||
"bun scripts/cli.ts gc remote NC01 status --job-id <job-id>",
|
||||
"bun scripts/cli.ts gc remote G14 plan --target-use-percent 50 --include-hwlab-registry",
|
||||
"bun scripts/cli.ts gc remote G14 run --confirm",
|
||||
"bun scripts/cli.ts gc remote G14 status --job-id <id>",
|
||||
@@ -563,6 +567,7 @@ function gcHelp(): unknown {
|
||||
"db-trace run --vacuum-full": "rewrite public.oa_events after deletion so df can reclaim disk; requires maintenance window",
|
||||
"policy plan|install": "render or install journald caps and a daily low-risk gc systemd timer from config/unidesk-cli.yaml#gc.policyTimer",
|
||||
"remote <providerId> plan|run": "run bounded GC through UniDesk SSH passthrough on a provider host; G14 protects HWLAB k3s/runtime/PVC/workspace paths, and HWLAB registry retention is explicit opt-in with workload-ref, digest-closure, recent-tag and per-repo tag protection",
|
||||
"remote <providerId> memory": "只读输出目标节点 /proc/meminfo 的 MemAvailable;不合并 swap、cgroup 或 virtual memory,也不执行 GC mutation",
|
||||
"--no-file-logs|--no-docker-logs|--no-journal|--no-build-cache|--no-tmp|--no-db-summary": "disable one collector",
|
||||
},
|
||||
reference: "docs/reference/gc.md",
|
||||
|
||||
@@ -148,6 +148,7 @@ export interface HwlabRuntimeWebProbeSpec {
|
||||
readonly browserProxyMode?: "auto" | "direct";
|
||||
readonly playwrightBrowsersPath?: string;
|
||||
readonly defaultOrigin?: HwlabRuntimeWebProbeOriginName;
|
||||
readonly resourcePolicy?: HwlabRuntimeWebProbeResourcePolicySpec;
|
||||
readonly origins?: HwlabRuntimeWebProbeOriginsSpec;
|
||||
readonly authLogin?: HwlabRuntimeWebProbeAuthLoginSpec;
|
||||
readonly alertThresholds?: HwlabRuntimeWebProbeAlertThresholdsSpec;
|
||||
@@ -158,6 +159,61 @@ export interface HwlabRuntimeWebProbeSpec {
|
||||
readonly workbenchTraceReadability?: HwlabRuntimeWebProbeWorkbenchTraceReadabilitySpec;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeMemoryStartGuardSpec {
|
||||
readonly metric: "MemAvailable";
|
||||
readonly source: "/proc/meminfo";
|
||||
readonly swapExcluded: true;
|
||||
readonly thresholdBytes: number;
|
||||
readonly manualComparator: "less-than-or-equal";
|
||||
readonly sentinelComparator: "less-than";
|
||||
readonly snapshotTimeoutSeconds: number;
|
||||
readonly launchLockPath: string;
|
||||
readonly launchLockTimeoutSeconds: number;
|
||||
readonly launchReadyTimeoutSeconds: number;
|
||||
readonly launchReadyPollMilliseconds: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeObserverLifecycleSpec {
|
||||
readonly maxRunSeconds: number;
|
||||
readonly maxSamples: number;
|
||||
readonly maxScreenshots: number;
|
||||
readonly maxPerformanceEntries: number;
|
||||
readonly maxBrowserProcessHistoryEntries: number;
|
||||
readonly maxBrowserFreezeSignalHistoryEntries: number;
|
||||
readonly maxRealtimeEventEntries: number;
|
||||
readonly maxRealtimeEventBodyBytes: number;
|
||||
readonly maxDomEvidenceRows: number;
|
||||
readonly maxProcessScanEntries: number;
|
||||
readonly maxPerformanceCaptureSeconds: number;
|
||||
readonly maxResponseBodyBytes: number;
|
||||
readonly maxWebPerformanceBodyBytes: number;
|
||||
readonly maxJsonlQueueEntries: number;
|
||||
readonly maxJsonlQueueBytes: number;
|
||||
readonly maxJsonlLineBytes: number;
|
||||
readonly jsonlFlushTimeoutMs: number;
|
||||
readonly maxResponseBodyInFlight: number;
|
||||
readonly maxPassiveListenerTasks: number;
|
||||
readonly passiveTaskFlushTimeoutMs: number;
|
||||
readonly closeStepTimeoutMs: number;
|
||||
readonly maxProcessReadConcurrency: number;
|
||||
readonly maxRealtimeTotalEntries: number;
|
||||
readonly maxRealtimeTotalBodyBytes: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeResourcePolicySpec {
|
||||
readonly memoryStartGuard: HwlabRuntimeWebProbeMemoryStartGuardSpec;
|
||||
readonly observerLifecycle: HwlabRuntimeWebProbeObserverLifecycleSpec;
|
||||
readonly browserEvidence: HwlabRuntimeWebProbeBrowserEvidenceSpec;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeBrowserEvidenceSpec {
|
||||
readonly screenshotMode: "viewport";
|
||||
readonly maxNetworkRequestEntries: number;
|
||||
readonly maxNetworkRequestBytes: number;
|
||||
readonly maxNetworkFailureEntries: number;
|
||||
readonly maxNetworkFailureBytes: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeWorkbenchKafkaDebugReplaySpec {
|
||||
readonly enabled: boolean;
|
||||
readonly topic: string;
|
||||
@@ -754,6 +810,11 @@ function booleanField(obj: Record<string, unknown>, key: string, path: string):
|
||||
return value;
|
||||
}
|
||||
|
||||
function literalTrueField(obj: Record<string, unknown>, key: string, path: string): true {
|
||||
if (obj[key] !== true) throw new Error(`${path}.${key} must be true`);
|
||||
return true;
|
||||
}
|
||||
|
||||
function nonNegativeIntegerField(obj: Record<string, unknown>, key: string, path: string): number {
|
||||
const value = obj[key];
|
||||
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) throw new Error(`${path}.${key} must be a non-negative integer`);
|
||||
@@ -1410,6 +1471,7 @@ function webProbeConfig(value: unknown, path: string): HwlabRuntimeWebProbeSpec
|
||||
...(browserProxyMode === undefined ? {} : { browserProxyMode }),
|
||||
...(playwrightBrowsersPath === undefined ? {} : { playwrightBrowsersPath }),
|
||||
...(defaultOrigin === undefined ? {} : { defaultOrigin }),
|
||||
...(raw.resourcePolicy === undefined ? {} : { resourcePolicy: webProbeResourcePolicyConfig(raw.resourcePolicy, `${path}.resourcePolicy`) }),
|
||||
...(origins === undefined ? {} : { origins }),
|
||||
...(raw.authLogin === undefined ? {} : { authLogin: webProbeAuthLoginConfig(raw.authLogin, `${path}.authLogin`) }),
|
||||
...(raw.alertThresholds === undefined ? {} : { alertThresholds: webProbeAlertThresholdsConfig(raw.alertThresholds, `${path}.alertThresholds`) }),
|
||||
@@ -1425,6 +1487,61 @@ function webProbeConfig(value: unknown, path: string): HwlabRuntimeWebProbeSpec
|
||||
};
|
||||
}
|
||||
|
||||
function webProbeResourcePolicyConfig(value: unknown, path: string): HwlabRuntimeWebProbeResourcePolicySpec {
|
||||
const raw = asRecord(value, path);
|
||||
const memory = asRecord(raw.memoryStartGuard, `${path}.memoryStartGuard`);
|
||||
const lifecycle = asRecord(raw.observerLifecycle, `${path}.observerLifecycle`);
|
||||
const browserEvidence = asRecord(raw.browserEvidence, `${path}.browserEvidence`);
|
||||
return {
|
||||
memoryStartGuard: {
|
||||
metric: enumStringField(memory, "metric", `${path}.memoryStartGuard`, ["MemAvailable"] as const),
|
||||
source: enumStringField(memory, "source", `${path}.memoryStartGuard`, ["/proc/meminfo"] as const),
|
||||
swapExcluded: literalTrueField(memory, "swapExcluded", `${path}.memoryStartGuard`),
|
||||
thresholdBytes: boundedIntegerField(memory, "thresholdBytes", `${path}.memoryStartGuard`, 1, Number.MAX_SAFE_INTEGER),
|
||||
manualComparator: enumStringField(memory, "manualComparator", `${path}.memoryStartGuard`, ["less-than-or-equal"] as const),
|
||||
sentinelComparator: enumStringField(memory, "sentinelComparator", `${path}.memoryStartGuard`, ["less-than"] as const),
|
||||
snapshotTimeoutSeconds: boundedIntegerField(memory, "snapshotTimeoutSeconds", `${path}.memoryStartGuard`, 5, 120),
|
||||
launchLockPath: absoluteHostPathField(stringField(memory, "launchLockPath", `${path}.memoryStartGuard`), `${path}.memoryStartGuard.launchLockPath`),
|
||||
launchLockTimeoutSeconds: boundedIntegerField(memory, "launchLockTimeoutSeconds", `${path}.memoryStartGuard`, 1, 60),
|
||||
launchReadyTimeoutSeconds: boundedIntegerField(memory, "launchReadyTimeoutSeconds", `${path}.memoryStartGuard`, 1, 30),
|
||||
launchReadyPollMilliseconds: boundedIntegerField(memory, "launchReadyPollMilliseconds", `${path}.memoryStartGuard`, 50, 1000),
|
||||
},
|
||||
observerLifecycle: {
|
||||
maxRunSeconds: boundedIntegerField(lifecycle, "maxRunSeconds", `${path}.observerLifecycle`, 1, 86_400),
|
||||
maxSamples: boundedIntegerField(lifecycle, "maxSamples", `${path}.observerLifecycle`, 1, 10_000_000),
|
||||
maxScreenshots: boundedIntegerField(lifecycle, "maxScreenshots", `${path}.observerLifecycle`, 1, 10_000),
|
||||
maxPerformanceEntries: boundedIntegerField(lifecycle, "maxPerformanceEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxBrowserProcessHistoryEntries: boundedIntegerField(lifecycle, "maxBrowserProcessHistoryEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxBrowserFreezeSignalHistoryEntries: boundedIntegerField(lifecycle, "maxBrowserFreezeSignalHistoryEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxRealtimeEventEntries: boundedIntegerField(lifecycle, "maxRealtimeEventEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxRealtimeEventBodyBytes: boundedIntegerField(lifecycle, "maxRealtimeEventBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024),
|
||||
maxDomEvidenceRows: boundedIntegerField(lifecycle, "maxDomEvidenceRows", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxProcessScanEntries: boundedIntegerField(lifecycle, "maxProcessScanEntries", `${path}.observerLifecycle`, 1, 1_000_000),
|
||||
maxPerformanceCaptureSeconds: boundedIntegerField(lifecycle, "maxPerformanceCaptureSeconds", `${path}.observerLifecycle`, 1, 600),
|
||||
maxResponseBodyBytes: boundedIntegerField(lifecycle, "maxResponseBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024),
|
||||
maxWebPerformanceBodyBytes: boundedIntegerField(lifecycle, "maxWebPerformanceBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024),
|
||||
maxJsonlQueueEntries: boundedIntegerField(lifecycle, "maxJsonlQueueEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxJsonlQueueBytes: boundedIntegerField(lifecycle, "maxJsonlQueueBytes", `${path}.observerLifecycle`, 1024, 64 * 1024 * 1024),
|
||||
maxJsonlLineBytes: boundedIntegerField(lifecycle, "maxJsonlLineBytes", `${path}.observerLifecycle`, 1024, 16 * 1024 * 1024),
|
||||
jsonlFlushTimeoutMs: boundedIntegerField(lifecycle, "jsonlFlushTimeoutMs", `${path}.observerLifecycle`, 100, 60_000),
|
||||
maxResponseBodyInFlight: boundedIntegerField(lifecycle, "maxResponseBodyInFlight", `${path}.observerLifecycle`, 1, 64),
|
||||
maxPassiveListenerTasks: boundedIntegerField(lifecycle, "maxPassiveListenerTasks", `${path}.observerLifecycle`, 1, 10_000),
|
||||
passiveTaskFlushTimeoutMs: boundedIntegerField(lifecycle, "passiveTaskFlushTimeoutMs", `${path}.observerLifecycle`, 100, 60_000),
|
||||
closeStepTimeoutMs: boundedIntegerField(lifecycle, "closeStepTimeoutMs", `${path}.observerLifecycle`, 100, 60_000),
|
||||
maxProcessReadConcurrency: boundedIntegerField(lifecycle, "maxProcessReadConcurrency", `${path}.observerLifecycle`, 1, 256),
|
||||
maxRealtimeTotalEntries: boundedIntegerField(lifecycle, "maxRealtimeTotalEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxRealtimeTotalBodyBytes: boundedIntegerField(lifecycle, "maxRealtimeTotalBodyBytes", `${path}.observerLifecycle`, 1024, 64 * 1024 * 1024),
|
||||
},
|
||||
browserEvidence: {
|
||||
screenshotMode: enumStringField(browserEvidence, "screenshotMode", `${path}.browserEvidence`, ["viewport"] as const),
|
||||
maxNetworkRequestEntries: boundedIntegerField(browserEvidence, "maxNetworkRequestEntries", `${path}.browserEvidence`, 1, 100_000),
|
||||
maxNetworkRequestBytes: boundedIntegerField(browserEvidence, "maxNetworkRequestBytes", `${path}.browserEvidence`, 1024, 64 * 1024 * 1024),
|
||||
maxNetworkFailureEntries: boundedIntegerField(browserEvidence, "maxNetworkFailureEntries", `${path}.browserEvidence`, 1, 100_000),
|
||||
maxNetworkFailureBytes: boundedIntegerField(browserEvidence, "maxNetworkFailureBytes", `${path}.browserEvidence`, 1024, 64 * 1024 * 1024),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function webProbeWorkbenchKafkaDebugReplayConfig(value: unknown, path: string): HwlabRuntimeWebProbeWorkbenchKafkaDebugReplaySpec {
|
||||
const raw = asRecord(value, path);
|
||||
const topic = stringField(raw, "topic", path);
|
||||
|
||||
@@ -379,7 +379,7 @@ async function authenticate(browserContext) {
|
||||
await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item?.retryLabel || retryLabel, retryAttempt: attempt, retryMaxAttempts: maxAttempts, retryDelayMs: item?.retryDelayMs ?? 0, lastStatus: item?.status ?? 0, lastStatusText: item?.statusText ?? "request-error", retryable, cookiePresent: false, retryExhausted: false, lastError: item?.error || null, valuesRedacted: true } }).catch(() => {});
|
||||
if (!retryable) break;
|
||||
}
|
||||
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await sleep(retryDelayMs);
|
||||
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await interruptibleRunnerSleep(retryDelayMs);
|
||||
}
|
||||
const cookieState = await readAuthCookieState(browserContext);
|
||||
const last = attempts[attempts.length - 1] || null;
|
||||
@@ -414,12 +414,13 @@ async function pageAuthLogin(browserContext, loginUrl, credential = { username,
|
||||
headers: { accept: "application/json", "content-type": "application/json" },
|
||||
timeout: authLoginRequestTimeoutMs,
|
||||
});
|
||||
await response.text().catch(() => "");
|
||||
return {
|
||||
const result = {
|
||||
ok: response.ok(),
|
||||
status: response.status(),
|
||||
statusText: response.statusText() || "",
|
||||
};
|
||||
if (typeof response.dispose === "function") await response.dispose().catch(() => {});
|
||||
return result;
|
||||
}
|
||||
|
||||
async function loginAccount(command) {
|
||||
@@ -479,7 +480,7 @@ async function loginAccount(command) {
|
||||
cookieState = await readAuthCookieState(context).catch(() => ({ cookiePresent: false, cookieNames: [] }));
|
||||
if (!retryable) break;
|
||||
}
|
||||
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await sleep(retryDelayMs);
|
||||
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await interruptibleRunnerSleep(retryDelayMs);
|
||||
}
|
||||
response = response ?? { ok: false, status: 0, statusText: "api-login-failed" };
|
||||
cookieState = cookieState ?? await readAuthCookieState(context);
|
||||
@@ -514,7 +515,6 @@ async function logoutAccount(command = {}) {
|
||||
const logoutUrl = new URL("/logout", baseUrl).toString();
|
||||
const response = await page.evaluate(async (input) => {
|
||||
const res = await fetch(input.logoutUrl, { method: "POST", headers: { accept: "application/json" }, credentials: "include" });
|
||||
await res.text().catch(() => "");
|
||||
return { ok: res.ok, status: res.status, statusText: res.statusText || "" };
|
||||
}, { logoutUrl });
|
||||
await context.clearCookies().catch(() => {});
|
||||
@@ -1295,11 +1295,9 @@ async function createSessionFromUi() {
|
||||
const createStatus = createResponse.status();
|
||||
let createPayload = null;
|
||||
let createPayloadError = null;
|
||||
try {
|
||||
createPayload = await createResponse.json();
|
||||
} catch (error) {
|
||||
createPayloadError = errorSummary(error);
|
||||
}
|
||||
const createBody = await boundedResponseJson(createResponse);
|
||||
createPayload = createBody.value;
|
||||
createPayloadError = createBody.error;
|
||||
const createdSessionId = sessionIdFromAgentSessionPayload(createPayload);
|
||||
if (createStatus < 200 || createStatus >= 300 || !createdSessionId) {
|
||||
const error = new Error("newSession did not receive an authoritative session id from POST /v1/agent/sessions");
|
||||
@@ -1753,11 +1751,9 @@ async function sendPrompt(text, options = {}) {
|
||||
}
|
||||
let chatPayload = null;
|
||||
let chatPayloadError = null;
|
||||
try {
|
||||
chatPayload = await chatResponse.json();
|
||||
} catch (error) {
|
||||
chatPayloadError = errorSummary(error);
|
||||
}
|
||||
const chatBody = await boundedResponseJson(chatResponse);
|
||||
chatPayload = chatBody.value;
|
||||
chatPayloadError = chatBody.error;
|
||||
const payloadText = chatPayload ? JSON.stringify(chatPayload) : "";
|
||||
const traceId = payloadText.match(/\btrc_[A-Za-z0-9_-]+\b/u)?.[0] || null;
|
||||
const otelTraceId = typeof chatPayload?.otelTrace?.traceId === "string" && /^[0-9a-f]{32}$/u.test(chatPayload.otelTrace.traceId)
|
||||
|
||||
@@ -148,6 +148,7 @@ async function validateWorkbenchKafkaDebugReplay(command) {
|
||||
try { url = new URL(request.url()); } catch { return; }
|
||||
if (url.pathname === config.productEventsPath) {
|
||||
productEventSourceRequests.push({ method: request.method(), path: url.pathname, phase: requestPhase, valuesRedacted: true });
|
||||
if (productEventSourceRequests.length > maxRealtimeEventEntries) productEventSourceRequests.splice(0, productEventSourceRequests.length - maxRealtimeEventEntries);
|
||||
return;
|
||||
}
|
||||
if (url.pathname !== config.debugEventsPath) return;
|
||||
@@ -160,6 +161,7 @@ async function validateWorkbenchKafkaDebugReplay(command) {
|
||||
phase: requestPhase,
|
||||
valuesRedacted: true
|
||||
});
|
||||
if (requests.length > maxRealtimeEventEntries) requests.splice(0, requests.length - maxRealtimeEventEntries);
|
||||
};
|
||||
try {
|
||||
const controlRecovery = await ensureControlPageResponsiveForCommand("validateWorkbenchKafkaDebugReplay");
|
||||
@@ -386,7 +388,7 @@ async function workbenchKafkaDebugReplayWaitForCleared(panel, countsNode, timeou
|
||||
if (status === "idle" && counts?.appliedCount === 0 && counts?.receivedCount === 0) {
|
||||
return { status, ...counts, valuesRedacted: true };
|
||||
}
|
||||
await sleep(100);
|
||||
await interruptibleRunnerSleep(100);
|
||||
}
|
||||
throw new Error("Workbench Kafka debug panel did not clear to idle 0/0");
|
||||
}
|
||||
@@ -397,7 +399,7 @@ async function workbenchKafkaDebugReplayWaitForTerminal(panel, timeoutMs) {
|
||||
const status = await panel.getAttribute("data-replay-status");
|
||||
if (status === "completed") return status;
|
||||
if (status === "incomplete" || status === "error" || status === "closed") return status;
|
||||
await sleep(200);
|
||||
await interruptibleRunnerSleep(200);
|
||||
}
|
||||
throw new Error("Workbench Kafka debug replay did not reach a terminal panel status within " + timeoutMs + "ms");
|
||||
}
|
||||
@@ -406,7 +408,7 @@ async function workbenchKafkaDebugReplayWaitForTabSelected(tab, timeoutMs) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() <= deadline) {
|
||||
if (await tab.getAttribute("aria-selected") === "true") return true;
|
||||
await sleep(50);
|
||||
await interruptibleRunnerSleep(50);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -514,7 +516,7 @@ async function workbenchKafkaDebugReplayWaitForStablePanel(panel, countsNode, ti
|
||||
stableSinceMs = null;
|
||||
}
|
||||
previousSignature = signature;
|
||||
await sleep(100);
|
||||
await interruptibleRunnerSleep(100);
|
||||
}
|
||||
return {
|
||||
...latest,
|
||||
@@ -586,14 +588,14 @@ async function workbenchKafkaDebugReplayInspectRawWindow(input) {
|
||||
await rawCountsNode.waitFor({ state: "visible", timeout: 5000 });
|
||||
await rawTextNode.waitFor({ state: "visible", timeout: 5000 });
|
||||
await rawContractNode.waitFor({ state: "visible", timeout: 5000 });
|
||||
await sleep(input.config.requestObservationQuietMs);
|
||||
await interruptibleRunnerSleep(input.config.requestObservationQuietMs);
|
||||
const counts = workbenchKafkaDebugReplayRawCounts(await rawCountsNode.textContent());
|
||||
const rawText = await rawTextNode.inputValue();
|
||||
const contractText = String(await rawContractNode.textContent() || "").replace(/\s+/gu, " ").trim();
|
||||
input.setRequestPhase("trace-tab-restore");
|
||||
await traceTab.click({ timeout: 10000 });
|
||||
if (!await workbenchKafkaDebugReplayWaitForTabSelected(traceTab, 5000)) throw new Error("Workbench Trace debug tab did not restore after raw inspection");
|
||||
await sleep(input.config.requestObservationQuietMs);
|
||||
await interruptibleRunnerSleep(input.config.requestObservationQuietMs);
|
||||
input.setRequestPhase("complete");
|
||||
const pageUrlAfter = currentPageUrl();
|
||||
const productRequests = input.productEventSourceRequests.slice(requestStart);
|
||||
|
||||
@@ -16,7 +16,7 @@ async function installPagePerformanceProbe(targetPage, pageRole = "control", tar
|
||||
}
|
||||
const buffer = [];
|
||||
const supportedTypes = [];
|
||||
const maxBufferSize = 2000;
|
||||
const maxBufferSize = Math.max(1, Number(input.maxBufferSize || 1));
|
||||
const push = (event) => {
|
||||
buffer.push({
|
||||
probeVersion: 1,
|
||||
@@ -126,9 +126,9 @@ async function installPagePerformanceProbe(targetPage, pageRole = "control", tar
|
||||
};
|
||||
return { ok: true, alreadyInstalled: false, supportedTypes, valuesRedacted: true };
|
||||
};
|
||||
await targetPage.addInitScript(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs });
|
||||
await targetPage.addInitScript(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs, maxBufferSize: maxPerformanceEntries });
|
||||
if (!targetPage.isClosed()) {
|
||||
await withHardTimeout(targetPage.evaluate(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs }), 3000, "installPagePerformanceProbe evaluate exceeded 3000ms");
|
||||
await withHardTimeout(targetPage.evaluate(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs, maxBufferSize: maxPerformanceEntries }), 3000, "installPagePerformanceProbe evaluate exceeded 3000ms");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,7 +227,7 @@ function compactPerformanceAttribution(item) {
|
||||
}
|
||||
|
||||
async function capturePerformanceProfile(command) {
|
||||
const durationMs = boundedInteger(command.durationMs, 5000, 100, 600000);
|
||||
const durationMs = boundedInteger(command.durationMs, Math.min(5000, maxPerformanceCaptureMs), 100, maxPerformanceCaptureMs);
|
||||
const timeoutMs = Math.max(3000, Math.min(15000, Number(alertThresholds.playwrightResponsivenessRedMs) || 5000));
|
||||
const captureId = safeId(command.label || command.id || ("perf-" + Date.now().toString(36))) || ("perf-" + Date.now().toString(36));
|
||||
const targetPage = page && !page.isClosed() ? page : observerPage && !observerPage.isClosed() ? observerPage : null;
|
||||
@@ -264,7 +264,7 @@ async function capturePerformanceProfile(command) {
|
||||
beforeDrain,
|
||||
valuesRedacted: true,
|
||||
}));
|
||||
await sleep(durationMs);
|
||||
await interruptibleRunnerSleep(durationMs);
|
||||
stopped = await withHardTimeout(session.send("Profiler.stop"), Math.max(timeoutMs, 5000), "Profiler.stop exceeded " + Math.max(timeoutMs, 5000) + "ms");
|
||||
} catch (error) {
|
||||
captureError = error;
|
||||
|
||||
@@ -8,14 +8,26 @@ test("realtime fanout keeps the YAML terminal timeout when durationMs is absent"
|
||||
const build = new Function(
|
||||
"boundedInteger",
|
||||
"sanitize",
|
||||
"maxRealtimeTotalEntries",
|
||||
"maxRealtimeTotalBodyBytes",
|
||||
"maxRealtimeEventEntries",
|
||||
"maxRealtimeEventBodyBytes",
|
||||
`${source}\nreturn realtimeFanoutEffectiveProfile;`,
|
||||
) as (
|
||||
boundedInteger: (value: unknown, fallback: number, min: number, max: number) => number,
|
||||
sanitize: <T>(value: T) => T,
|
||||
maxRealtimeTotalEntries: number,
|
||||
maxRealtimeTotalBodyBytes: number,
|
||||
maxRealtimeEventEntries: number,
|
||||
maxRealtimeEventBodyBytes: number,
|
||||
) => (profile: Record<string, unknown>, durationMs: unknown) => Record<string, unknown>;
|
||||
const effectiveProfile = build(
|
||||
(value, fallback, min, max) => Math.min(max, Math.max(min, Number.isFinite(Number(value)) ? Number(value) : fallback)),
|
||||
(value) => value,
|
||||
512,
|
||||
4 * 1024 * 1024,
|
||||
256,
|
||||
64 * 1024,
|
||||
);
|
||||
const profile = {
|
||||
subscriberCount: 2,
|
||||
@@ -48,6 +60,9 @@ test("realtime fanout keeps the YAML terminal timeout when durationMs is absent"
|
||||
assert.equal(effectiveProfile(profile, undefined).terminalTimeoutMs, 480_000);
|
||||
assert.equal(effectiveProfile(profile, 5_000).terminalTimeoutMs, 5_000);
|
||||
assert.deepEqual(effectiveProfile(profile, null).conditionalEventTypePairs, profile.conditionalEventTypePairs);
|
||||
const budget = effectiveProfile(profile, null).aggregateBudget as Record<string, number>;
|
||||
assert.ok(budget.perArrayEntries * budget.entrySlots <= budget.maxTotalEntries);
|
||||
assert.ok(budget.perEventBodyBytes * budget.bodyRetentionSlots <= budget.maxTotalBodyBytes);
|
||||
});
|
||||
|
||||
test("realtime fanout terminal completion does not require absent optional tool families", () => {
|
||||
@@ -137,12 +152,12 @@ test("realtime fanout validates optional tool events only when they occur", () =
|
||||
test("realtime fanout retries a YAML-bounded transient health navigation", async () => {
|
||||
const source = nodeWebObserveRunnerRealtimeSource();
|
||||
const build = new Function(
|
||||
"sleep",
|
||||
"interruptibleRunnerSleep",
|
||||
"isRetryableNavigationError",
|
||||
"baseUrl",
|
||||
`${source}\nreturn realtimeGotoHealth;`,
|
||||
) as (
|
||||
sleep: (milliseconds: number) => Promise<void>,
|
||||
interruptibleRunnerSleep: (milliseconds: number) => Promise<void>,
|
||||
isRetryableNavigationError: (message: string) => boolean,
|
||||
baseUrl: string,
|
||||
) => (page: { goto: () => Promise<{ status: () => number }> }, timeoutMs: number, profile: Record<string, number>, label: string) => Promise<Record<string, unknown>>;
|
||||
@@ -433,11 +448,11 @@ test("realtime replay joint quiet waits for a late formal Kafka envelope before
|
||||
let kafkaRows = [first];
|
||||
const productRows = [first, late];
|
||||
const build = new Function(
|
||||
"sleep",
|
||||
"interruptibleRunnerSleep",
|
||||
"Date",
|
||||
`${source}\nreturn realtimeWaitReplayedTraceJointQuiet;`,
|
||||
) as (
|
||||
sleep: () => Promise<void>,
|
||||
interruptibleRunnerSleep: () => Promise<void>,
|
||||
date: { now: () => number },
|
||||
) => (
|
||||
debug: Record<string, any>,
|
||||
@@ -495,11 +510,11 @@ test("realtime replay joint quiet rejects a permanent 31/32 split with bounded i
|
||||
const first = replayEnvelope("evt_first", 1);
|
||||
const late = replayEnvelope("evt_late", 2, true);
|
||||
const build = new Function(
|
||||
"sleep",
|
||||
"interruptibleRunnerSleep",
|
||||
"Date",
|
||||
`${source}\nreturn realtimeWaitReplayedTraceJointQuiet;`,
|
||||
) as (
|
||||
sleep: () => Promise<void>,
|
||||
interruptibleRunnerSleep: () => Promise<void>,
|
||||
date: { now: () => number },
|
||||
) => (
|
||||
debug: Record<string, any>,
|
||||
|
||||
@@ -23,6 +23,8 @@ async function validateRealtimeFanout(command) {
|
||||
if (!provider) throw new Error("validateRealtimeFanout requires provider");
|
||||
if (!firstPrompt || !secondPrompt) throw new Error("validateRealtimeFanout requires first and second prompt");
|
||||
const effective = realtimeFanoutEffectiveProfile(profile, command.durationMs);
|
||||
const aggregateCounters = { networkDroppedEntries: 0, valuesRedacted: true };
|
||||
realtimeAggregateLast = { commandId: command.id, profile: profileId, budget: effective.aggregateBudget, counters: aggregateCounters, status: "running", valuesRedacted: true };
|
||||
const reportDir = path.join(stateDir, "artifacts", "realtime-fanout", safeId(command.id));
|
||||
await mkdir(reportDir, { recursive: true, mode: 0o700 });
|
||||
const network = [];
|
||||
@@ -40,7 +42,10 @@ async function validateRealtimeFanout(command) {
|
||||
lastEventId: request.headers()["last-event-id"] || null,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
if (network.length > 1000) network.splice(0, network.length - 1000);
|
||||
if (network.length > effective.aggregateBudget.perArrayEntries) {
|
||||
aggregateCounters.networkDroppedEntries += network.length - effective.aggregateBudget.perArrayEntries;
|
||||
network.splice(0, network.length - effective.aggregateBudget.perArrayEntries);
|
||||
}
|
||||
};
|
||||
page.on("request", requestListener);
|
||||
let subscriberA = null;
|
||||
@@ -130,7 +135,7 @@ async function validateRealtimeFanout(command) {
|
||||
subscriberB2 = await realtimeNewSubscriber(storageState, sessionId, "subscriber-b-reconnected", effective);
|
||||
const bReconnected = await realtimeWaitProductConnected(subscriberB2, effective.barrierTimeoutMs);
|
||||
realtimeAssertConnectedContract(bReconnected, sessionId, "subscriber-b-reconnected", effective, { requireRetainedEvents: effective.expectedProductSse.replayPriorTraceOnReconnect });
|
||||
await sleep(effective.reconnectQuietMs);
|
||||
await interruptibleRunnerSleep(effective.reconnectQuietMs);
|
||||
let reconnectBeforeSecond;
|
||||
if (effective.expectedProductSse.replayPriorTraceOnReconnect) {
|
||||
const firstReplayBeforeSecond = await realtimeWaitReplayedTraceJointQuiet(
|
||||
@@ -258,6 +263,13 @@ async function validateRealtimeFanout(command) {
|
||||
realtimeTurnSummary(first, firstDebug, productA, firstTerminal),
|
||||
realtimeTurnSummary(second, secondDebug, productB2, secondTerminalB),
|
||||
];
|
||||
aggregateCounters.productDroppedEntries = Number(productA.droppedEntries || 0) + Number(productB2.droppedEntries || 0);
|
||||
aggregateCounters.debugDroppedEntries = Number(firstDebug.droppedEntries || 0) + Number(secondDebug.droppedEntries || 0);
|
||||
aggregateCounters.productDroppedBodyEntries = Number(productA.droppedBodyEntries || 0) + Number(productB2.droppedBodyEntries || 0);
|
||||
aggregateCounters.productDroppedBodyBytes = Number(productA.droppedBodyBytes || 0) + Number(productB2.droppedBodyBytes || 0);
|
||||
aggregateCounters.debugDroppedBodyEntries = Number(firstDebug.droppedBodyEntries || 0) + Number(secondDebug.droppedBodyEntries || 0);
|
||||
aggregateCounters.debugDroppedBodyBytes = Number(firstDebug.droppedBodyBytes || 0) + Number(secondDebug.droppedBodyBytes || 0);
|
||||
realtimeAggregateLast = { ...realtimeAggregateLast, status: "completed", counters: aggregateCounters, valuesRedacted: true };
|
||||
Object.assign(report, {
|
||||
ok: true,
|
||||
status: "passed",
|
||||
@@ -290,6 +302,7 @@ async function validateRealtimeFanout(command) {
|
||||
valuesRedacted: true,
|
||||
},
|
||||
forbiddenRequests: { count: forbidden.length, rows: forbidden, valuesRedacted: true },
|
||||
aggregateBudget: { ...effective.aggregateBudget, counters: aggregateCounters, valuesRedacted: true },
|
||||
screenshots: [bBeforeShot, summaryScreenshot, subscriberAScreenshot, subscriberB2Screenshot],
|
||||
warmRunnerReused: first.runnerId && second.runnerId ? first.runnerId === second.runnerId : null,
|
||||
valuesRedacted: true,
|
||||
@@ -326,6 +339,7 @@ async function validateRealtimeFanout(command) {
|
||||
valuesRedacted: true,
|
||||
};
|
||||
} catch (error) {
|
||||
realtimeAggregateLast = { ...realtimeAggregateLast, status: "failed", counters: aggregateCounters, failure: errorSummary(error), valuesRedacted: true };
|
||||
Object.assign(report, {
|
||||
ok: false,
|
||||
status: "failed",
|
||||
@@ -367,6 +381,8 @@ async function validateExistingSessionRefresh(command) {
|
||||
const profile = realtimeFanoutProfiles[profileId];
|
||||
if (!profile || typeof profile !== "object") throw new Error("validateExistingSessionRefresh profile is not declared by the owning YAML: " + profileId);
|
||||
const effective = realtimeFanoutEffectiveProfile(profile, command.durationMs);
|
||||
const aggregateCounters = { networkDroppedEntries: 0, valuesRedacted: true };
|
||||
realtimeAggregateLast = { commandId: command.id, profile: profileId, budget: effective.aggregateBudget, counters: aggregateCounters, status: "running", valuesRedacted: true };
|
||||
if (effective.expectedProductSse.refreshHandoff !== true) throw new Error("validateExistingSessionRefresh requires a YAML profile with refresh handoff enabled");
|
||||
const reportDir = path.join(stateDir, "artifacts", "existing-session-refresh", safeId(command.id));
|
||||
await mkdir(reportDir, { recursive: true, mode: 0o700 });
|
||||
@@ -385,7 +401,10 @@ async function validateExistingSessionRefresh(command) {
|
||||
lastEventId: request.headers()["last-event-id"] || null,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
if (network.length > 1000) network.splice(0, network.length - 1000);
|
||||
if (network.length > effective.aggregateBudget.perArrayEntries) {
|
||||
aggregateCounters.networkDroppedEntries += network.length - effective.aggregateBudget.perArrayEntries;
|
||||
network.splice(0, network.length - effective.aggregateBudget.perArrayEntries);
|
||||
}
|
||||
};
|
||||
const report = {
|
||||
ok: false,
|
||||
@@ -447,7 +466,7 @@ async function validateExistingSessionRefresh(command) {
|
||||
}
|
||||
phase = "live-handoff";
|
||||
await realtimeWaitForSameWorkbenchDom(before, effective.barrierTimeoutMs);
|
||||
await sleep(effective.reconnectQuietMs);
|
||||
await interruptibleRunnerSleep(effective.reconnectQuietMs);
|
||||
const after = await realtimeWorkbenchDomIdentity();
|
||||
realtimeAssertSameWorkbenchDom(before, after);
|
||||
const browserEvidence = await page.evaluate(() => JSON.parse(JSON.stringify(window.__unideskExistingSessionRefresh || {})));
|
||||
@@ -463,11 +482,16 @@ async function validateExistingSessionRefresh(command) {
|
||||
if (mainStreams[0].traceId !== null || mainStreams[0].afterSeq !== null || mainStreams[0].lastEventId !== null) throw new Error("main Workbench product SSE used a trace or cursor scope");
|
||||
if (forbidden.length > 0) throw new Error("forbidden recovery or polling requests observed: " + forbidden.map((item) => item.path).join(","));
|
||||
phase = "evidence-write";
|
||||
const screenshotFile = path.join(reportDir, "existing-session-refresh.png");
|
||||
await page.screenshot({ path: screenshotFile, fullPage: true, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
|
||||
const screenshotMeta = await fileMeta(screenshotFile);
|
||||
screenshot = { kind: "existing-session-refresh-screenshot", path: screenshotFile, ...screenshotMeta, commandId: activeCommandId, valuesRedacted: true };
|
||||
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), ...screenshot });
|
||||
const screenshotSkipped = reserveScreenshot("existing-session-refresh");
|
||||
if (screenshotSkipped !== null) {
|
||||
screenshot = screenshotSkipped;
|
||||
} else {
|
||||
const screenshotFile = path.join(reportDir, "existing-session-refresh.png");
|
||||
await page.screenshot({ path: screenshotFile, fullPage: false, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
|
||||
const screenshotMeta = await fileMeta(screenshotFile);
|
||||
screenshot = { kind: "existing-session-refresh-screenshot", path: screenshotFile, ...screenshotMeta, commandId: activeCommandId, valuesRedacted: true };
|
||||
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), ...screenshot });
|
||||
}
|
||||
const productSse = {
|
||||
requestCount: mainStreams.length,
|
||||
browserSourceCount: browserEvidence.sources.length,
|
||||
@@ -488,10 +512,12 @@ async function validateExistingSessionRefresh(command) {
|
||||
connected: connectedSummary,
|
||||
refreshReplay,
|
||||
productSse,
|
||||
aggregateBudget: { ...effective.aggregateBudget, counters: { ...aggregateCounters, browserDroppedEntries: Number(browserEvidence.droppedEntries || 0), browserDroppedBodyEntries: Number(browserEvidence.droppedBodyEntries || 0), browserDroppedBodyBytes: Number(browserEvidence.droppedBodyBytes || 0) }, valuesRedacted: true },
|
||||
forbiddenRequests: { count: 0, rows: [], valuesRedacted: true },
|
||||
screenshot,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
realtimeAggregateLast = { ...realtimeAggregateLast, status: "completed", counters: { ...aggregateCounters, browserDroppedEntries: Number(browserEvidence.droppedEntries || 0), browserDroppedBodyEntries: Number(browserEvidence.droppedBodyEntries || 0), browserDroppedBodyBytes: Number(browserEvidence.droppedBodyBytes || 0) }, valuesRedacted: true };
|
||||
const artifacts = await realtimeWriteEvidenceArtifacts(reportDir, network, [], "existing-session-refresh");
|
||||
report.artifacts = artifacts;
|
||||
const reportArtifact = await realtimeWriteReport(reportDir, report, "existing-session-refresh-report");
|
||||
@@ -527,6 +553,7 @@ async function validateExistingSessionRefresh(command) {
|
||||
valuesRedacted: true,
|
||||
};
|
||||
} catch (error) {
|
||||
realtimeAggregateLast = { ...realtimeAggregateLast, status: "failed", counters: aggregateCounters, failure: errorSummary(error), valuesRedacted: true };
|
||||
const failure = realtimeExistingRefreshFailure(phase, error);
|
||||
if (!screenshot) {
|
||||
screenshot = await captureScreenshot("existing-session-refresh-failed-" + safeId(command.id))
|
||||
@@ -649,10 +676,17 @@ async function realtimeArmExistingSessionRefreshProbe(targetPage, profile) {
|
||||
let config;
|
||||
try { config = JSON.parse(raw); } catch { return; }
|
||||
const NativeEventSource = window.EventSource;
|
||||
const state = { sources: [], connected: [], businessEventCount: 0, failures: [], errors: [] };
|
||||
const state = { sources: [], connected: [], businessEventCount: 0, failures: [], errors: [], droppedEntries: 0, droppedBodyEntries: 0, droppedBodyBytes: 0 };
|
||||
const maxEntries = Math.max(1, Number(config.maxEntries || 1));
|
||||
const maxEventTextChars = Math.max(1, Number(config.maxEventTextChars || 1));
|
||||
const bodyBytes = (value) => new TextEncoder().encode(String(value || "")).byteLength;
|
||||
const pushBounded = (items, value) => {
|
||||
items.push(value);
|
||||
if (items.length > maxEntries) { state.droppedEntries += items.length - maxEntries; items.splice(0, items.length - maxEntries); }
|
||||
};
|
||||
window.__unideskExistingSessionRefresh = state;
|
||||
if (typeof NativeEventSource !== "function") {
|
||||
state.errors.push({ kind: "eventsource-unavailable" });
|
||||
pushBounded(state.errors, { kind: "eventsource-unavailable" });
|
||||
return;
|
||||
}
|
||||
class ObservedEventSource extends NativeEventSource {
|
||||
@@ -661,17 +695,23 @@ async function realtimeArmExistingSessionRefreshProbe(targetPage, profile) {
|
||||
let pathname = "";
|
||||
try { pathname = new URL(String(url), window.location.href).pathname; } catch {}
|
||||
if (pathname !== config.productEventsPath) return;
|
||||
state.sources.push({ url: String(url), opened: 0 });
|
||||
pushBounded(state.sources, { url: String(url), opened: 0 });
|
||||
const source = state.sources[state.sources.length - 1];
|
||||
this.addEventListener("open", () => { source.opened += 1; });
|
||||
this.addEventListener(config.productReadyEvent, (event) => {
|
||||
try { state.connected.push(JSON.parse(event.data)); } catch { state.errors.push({ kind: "connected-parse" }); }
|
||||
const raw = String(event.data || "");
|
||||
const bytes = bodyBytes(raw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors, { kind: "connected-too-large", dataBytes: bytes }); return; }
|
||||
try { pushBounded(state.connected, JSON.parse(raw)); } catch { pushBounded(state.errors, { kind: "connected-parse" }); }
|
||||
});
|
||||
this.addEventListener(config.businessEvent, () => { state.businessEventCount += 1; });
|
||||
this.addEventListener("workbench.error", (event) => {
|
||||
try { state.failures.push(JSON.parse(event.data)); } catch { state.errors.push({ kind: "workbench-error-parse" }); }
|
||||
const raw = String(event.data || "");
|
||||
const bytes = bodyBytes(raw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors, { kind: "workbench-error-too-large", dataBytes: bytes }); return; }
|
||||
try { pushBounded(state.failures, JSON.parse(raw)); } catch { pushBounded(state.errors, { kind: "workbench-error-parse" }); }
|
||||
});
|
||||
this.addEventListener("error", () => { state.errors.push({ kind: "eventsource-error" }); });
|
||||
this.addEventListener("error", () => { pushBounded(state.errors, { kind: "eventsource-error" }); });
|
||||
}
|
||||
}
|
||||
Object.defineProperty(ObservedEventSource, "CONNECTING", { value: NativeEventSource.CONNECTING });
|
||||
@@ -685,12 +725,16 @@ async function realtimeArmExistingSessionRefreshProbe(targetPage, profile) {
|
||||
productEventsPath: profile.productEventsPath,
|
||||
productReadyEvent: profile.productReadyEvent,
|
||||
businessEvent: profile.businessEvent,
|
||||
maxEntries: profile.aggregateBudget.perArrayEntries,
|
||||
maxEventTextChars: profile.aggregateBudget.perEventBodyBytes,
|
||||
});
|
||||
}
|
||||
|
||||
async function realtimeWorkbenchDomIdentity() {
|
||||
return page.evaluate(() => {
|
||||
const cards = Array.from(document.querySelectorAll("#conversation-list > .message-card"));
|
||||
return page.evaluate((maxRows) => {
|
||||
const cardNodes = document.querySelectorAll("#conversation-list > .message-card");
|
||||
if (cardNodes.length > maxRows) return { overflow: true, messageCount: cardNodes.length, maxRows, identities: [], missingIdentityRows: [], duplicateIdentities: [], traceIds: [], rows: [], valuesRedacted: true };
|
||||
const cards = Array.from(cardNodes);
|
||||
const rows = cards.map((element, index) => ({
|
||||
index,
|
||||
role: element.getAttribute("data-role") || null,
|
||||
@@ -709,10 +753,11 @@ async function realtimeWorkbenchDomIdentity() {
|
||||
rows,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
});
|
||||
}, maxDomEvidenceRows);
|
||||
}
|
||||
|
||||
function realtimeAssertSameWorkbenchDom(before, after) {
|
||||
if (before.overflow === true || after.overflow === true) throw new Error("Workbench DOM identity evidence exceeds the YAML row cap");
|
||||
if (before.missingIdentityRows.length > 0 || after.missingIdentityRows.length > 0) throw new Error("Workbench DOM contains rows without a stable message identity");
|
||||
if (before.duplicateIdentities.length > 0 || after.duplicateIdentities.length > 0) throw new Error("Workbench DOM contains duplicate message identities");
|
||||
if (before.messageCount !== after.messageCount) throw new Error("Workbench message count changed across refresh: " + before.messageCount + " -> " + after.messageCount);
|
||||
@@ -771,6 +816,21 @@ function realtimeFanoutEffectiveProfile(profile, durationMs) {
|
||||
const terminalTimeoutMs = hasDurationOverride
|
||||
? Math.min(Number(profile.terminalTimeoutMs), Math.max(1000, Number(durationMs)))
|
||||
: Number(profile.terminalTimeoutMs);
|
||||
const entrySlots = 17;
|
||||
const eventStreamSlots = 8;
|
||||
const perArrayEntries = Math.max(1, Math.min(maxRealtimeEventEntries, Math.floor(maxRealtimeTotalEntries / entrySlots)));
|
||||
const bodyRetentionSlots = eventStreamSlots * (perArrayEntries + 1);
|
||||
const aggregateBudget = {
|
||||
maxTotalEntries: maxRealtimeTotalEntries,
|
||||
maxTotalBodyBytes: maxRealtimeTotalBodyBytes,
|
||||
entrySlots,
|
||||
eventStreamSlots,
|
||||
perArrayEntries,
|
||||
bodyRetentionSlots,
|
||||
perEventBodyBytes: Math.max(1, Math.min(maxRealtimeEventBodyBytes, Math.floor(maxRealtimeTotalBodyBytes / bodyRetentionSlots))),
|
||||
allocation: "partitioned-across-network-product-debug-active-arrays",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return {
|
||||
...profile,
|
||||
debugStreams,
|
||||
@@ -785,6 +845,7 @@ function realtimeFanoutEffectiveProfile(profile, durationMs) {
|
||||
conditionalEventTypePairs: profile.conditionalEventTypePairs && typeof profile.conditionalEventTypePairs === "object" ? sanitize(profile.conditionalEventTypePairs) : {},
|
||||
expectedProductSse: sanitize(expectedProductSse),
|
||||
expectedKafka: sanitize(expectedKafka),
|
||||
aggregateBudget,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -795,7 +856,14 @@ async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
|
||||
const key = label + "-" + sessionId;
|
||||
await subscriberPage.evaluate(({ key: streamKey, sessionId: scopedSessionId, profile: inputProfile }) => {
|
||||
const root = window.__unideskRealtimeFanoutProducts ??= {};
|
||||
const state = { connected: null, connectedEventCount: 0, openCount: 0, records: [], errors: [], source: null };
|
||||
const state = { connected: null, connectedEventCount: 0, openCount: 0, records: [], errors: [], source: null, droppedEntries: 0, droppedBodyEntries: 0, droppedBodyBytes: 0 };
|
||||
const maxEntries = Math.max(1, Number(inputProfile.maxEntries || 1));
|
||||
const maxEventTextChars = Math.max(1, Number(inputProfile.maxEventTextChars || 1));
|
||||
const bodyBytes = (value) => new TextEncoder().encode(String(value || "")).byteLength;
|
||||
const pushBounded = (items, value) => {
|
||||
items.push(value);
|
||||
if (items.length > maxEntries) { state.droppedEntries += items.length - maxEntries; items.splice(0, items.length - maxEntries); }
|
||||
};
|
||||
root[streamKey] = state;
|
||||
const firstText = (...values) => {
|
||||
for (const value of values) if (typeof value === "string" && value.trim()) return value.trim();
|
||||
@@ -811,12 +879,15 @@ async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
|
||||
state.source = source;
|
||||
source.onopen = () => { state.openCount += 1; };
|
||||
const record = (eventName, raw) => {
|
||||
const textRaw = String(raw || "");
|
||||
const bytes = bodyBytes(textRaw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors, { at: Date.now(), event: "event-too-large", dataBytes: bytes }); return; }
|
||||
let value;
|
||||
try { value = JSON.parse(raw); } catch { return; }
|
||||
try { value = JSON.parse(textRaw); } catch { return; }
|
||||
const nested = value?.event && typeof value.event === "object" ? value.event : {};
|
||||
const context = value?.context && typeof value.context === "object" ? value.context : {};
|
||||
const eventType = firstText(nested.eventType, nested.type, value.eventType, value.type);
|
||||
state.records.push({
|
||||
pushBounded(state.records, {
|
||||
eventName,
|
||||
at: Date.now(),
|
||||
schema: firstText(value.schema),
|
||||
@@ -831,19 +902,21 @@ async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
|
||||
userMessageId: firstText(value.userMessageId, nested.userMessageId, nested.messageId),
|
||||
terminal: value.terminal === true || nested.terminal === true || eventType === "terminal" || eventType === "terminal_status",
|
||||
status: firstText(nested.status, value.status),
|
||||
envelopeFingerprint: fingerprint(raw),
|
||||
envelopeFingerprint: fingerprint(textRaw),
|
||||
valuesRedacted: true,
|
||||
});
|
||||
if (state.records.length > 800) state.records.splice(0, state.records.length - 800);
|
||||
};
|
||||
source.addEventListener(inputProfile.productReadyEvent, (event) => {
|
||||
state.connectedEventCount += 1;
|
||||
try { state.connected = JSON.parse(event.data); } catch { state.connected = { parseError: true }; }
|
||||
const raw = String(event.data || "");
|
||||
const bytes = bodyBytes(raw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; state.connected = { tooLarge: true, dataBytes: bytes }; return; }
|
||||
try { state.connected = JSON.parse(raw); } catch { state.connected = { parseError: true }; }
|
||||
});
|
||||
source.addEventListener(inputProfile.businessEvent, (event) => record(inputProfile.businessEvent, event.data));
|
||||
source.addEventListener("workbench.error", (event) => state.errors.push({ at: Date.now(), event: "workbench.error", dataBytes: String(event.data || "").length }));
|
||||
source.onerror = () => state.errors.push({ at: Date.now(), event: "transport.error" });
|
||||
}, { key, sessionId, profile: { productEventsPath: profile.productEventsPath, productReadyEvent: profile.productReadyEvent, businessEvent: profile.businessEvent } });
|
||||
source.addEventListener("workbench.error", (event) => pushBounded(state.errors, { at: Date.now(), event: "workbench.error", dataBytes: String(event.data || "").length }));
|
||||
source.onerror = () => pushBounded(state.errors, { at: Date.now(), event: "transport.error" });
|
||||
}, { key, sessionId, profile: { productEventsPath: profile.productEventsPath, productReadyEvent: profile.productReadyEvent, businessEvent: profile.businessEvent, maxEntries: profile.aggregateBudget.perArrayEntries, maxEventTextChars: profile.aggregateBudget.perEventBodyBytes } });
|
||||
return { label, key, context: subscriberContext, page: subscriberPage };
|
||||
}
|
||||
|
||||
@@ -872,7 +945,7 @@ async function realtimeGotoHealth(targetPage, timeoutMs, profile, label) {
|
||||
wrapped.details = { ...(wrapped.details || {}), label, attempts, valuesRedacted: true };
|
||||
throw wrapped;
|
||||
}
|
||||
await sleep(profile.navigationRetryDelayMs * attempt);
|
||||
await interruptibleRunnerSleep(profile.navigationRetryDelayMs * attempt);
|
||||
}
|
||||
}
|
||||
throw new Error(label + " health navigation retry exhausted");
|
||||
@@ -943,8 +1016,8 @@ async function realtimeRunTurn(input) {
|
||||
if (!admissionOutcome.response) throw new Error("realtime turn admission response failed: " + String(admissionOutcome.error?.message || admissionOutcome.error || "unknown"));
|
||||
const admission = admissionOutcome.response;
|
||||
if (admission.status() < 200 || admission.status() >= 300) throw new Error("realtime turn admission returned HTTP " + admission.status());
|
||||
let admissionBody = {};
|
||||
try { admissionBody = await admission.json(); } catch {}
|
||||
const admissionBodyResult = await boundedResponseJson(admission);
|
||||
const admissionBody = admissionBodyResult.value || {};
|
||||
const canonicalTraceId = realtimeFirstText(admissionBody?.traceId, admissionBody?.result?.traceId, captured.traceId);
|
||||
if (canonicalTraceId !== captured.traceId) throw new Error("canonical trace changed after ready barrier");
|
||||
const nonTerminal = [];
|
||||
@@ -970,7 +1043,14 @@ async function realtimeRunTurn(input) {
|
||||
async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
|
||||
await debug.page.evaluate(({ key: streamKey, traceId: scopedTraceId, profile: inputProfile }) => {
|
||||
const root = window.__unideskRealtimeFanoutDebug ??= {};
|
||||
const state = { connected: {}, connectedEventCount: {}, openCount: {}, records: {}, errors: {}, sources: {} };
|
||||
const state = { connected: {}, connectedEventCount: {}, openCount: {}, records: {}, errors: {}, sources: {}, droppedEntries: 0, droppedBodyEntries: 0, droppedBodyBytes: 0 };
|
||||
const maxEntries = Math.max(1, Number(inputProfile.maxEntries || 1));
|
||||
const maxEventTextChars = Math.max(1, Number(inputProfile.maxEventTextChars || 1));
|
||||
const bodyBytes = (value) => new TextEncoder().encode(String(value || "")).byteLength;
|
||||
const pushBounded = (items, value) => {
|
||||
items.push(value);
|
||||
if (items.length > maxEntries) { state.droppedEntries += items.length - maxEntries; items.splice(0, items.length - maxEntries); }
|
||||
};
|
||||
root[streamKey] = state;
|
||||
const firstText = (...values) => {
|
||||
for (const value of values) if (typeof value === "string" && value.trim()) return value.trim();
|
||||
@@ -1001,11 +1081,17 @@ async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
|
||||
source.onopen = () => { state.openCount[stream] += 1; };
|
||||
source.addEventListener(inputProfile.debugReadyEvent, (event) => {
|
||||
state.connectedEventCount[stream] += 1;
|
||||
try { state.connected[stream] = JSON.parse(event.data); } catch { state.connected[stream] = { parseError: true }; }
|
||||
const raw = String(event.data || "");
|
||||
const bytes = bodyBytes(raw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; state.connected[stream] = { tooLarge: true, dataBytes: bytes }; return; }
|
||||
try { state.connected[stream] = JSON.parse(raw); } catch { state.connected[stream] = { parseError: true }; }
|
||||
});
|
||||
source.addEventListener("hwlab.kafka.event", (event) => {
|
||||
const raw = String(event.data || "");
|
||||
const bytes = bodyBytes(raw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors[stream], { at: Date.now(), kind: "event-too-large", dataBytes: bytes }); return; }
|
||||
let outer;
|
||||
try { outer = JSON.parse(event.data); } catch { return; }
|
||||
try { outer = JSON.parse(raw); } catch { return; }
|
||||
const value = outer?.value && typeof outer.value === "object" ? outer.value : {};
|
||||
const nested = value?.event && typeof value.event === "object" ? value.event : {};
|
||||
const payload = nested?.payload && typeof nested.payload === "object" ? nested.payload : {};
|
||||
@@ -1017,7 +1103,7 @@ async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
|
||||
: stream === "hwlab"
|
||||
? firstText(nested.eventType, nested.type, value.eventType)
|
||||
: firstText(value.eventType, value.stdio?.method);
|
||||
state.records[stream].push({
|
||||
pushBounded(state.records[stream], {
|
||||
stream,
|
||||
at: Date.now(),
|
||||
topic: firstText(outer.topic),
|
||||
@@ -1042,16 +1128,17 @@ async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
|
||||
envelopeFingerprint: stream === "hwlab" ? fingerprint(JSON.stringify(value)) : null,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
if (state.records[stream].length > 1000) state.records[stream].splice(0, state.records[stream].length - 1000);
|
||||
});
|
||||
source.addEventListener("hwlab.kafka.error", (event) => state.errors[stream].push({ at: Date.now(), dataBytes: String(event.data || "").length }));
|
||||
source.onerror = () => state.errors[stream].push({ at: Date.now(), transport: true });
|
||||
source.addEventListener("hwlab.kafka.error", (event) => pushBounded(state.errors[stream], { at: Date.now(), dataBytes: String(event.data || "").length }));
|
||||
source.onerror = () => pushBounded(state.errors[stream], { at: Date.now(), transport: true });
|
||||
}
|
||||
}, { key, traceId, profile: {
|
||||
debugStreams: profile.debugStreams,
|
||||
debugEventsPath: profile.debugEventsPath,
|
||||
debugReadyEvent: profile.debugReadyEvent,
|
||||
fromBeginning: profile.fromBeginning,
|
||||
maxEntries: profile.aggregateBudget.perArrayEntries,
|
||||
maxEventTextChars: profile.aggregateBudget.perEventBodyBytes,
|
||||
} });
|
||||
}
|
||||
|
||||
@@ -1143,7 +1230,7 @@ async function realtimeWaitTurnStable(debug, key, traceId, subscribers, profile)
|
||||
} else if (Date.now() - stableSince >= profile.terminalQuietMs) {
|
||||
return { debug: debugSnapshot, products: productSnapshots };
|
||||
}
|
||||
await sleep(250);
|
||||
await interruptibleRunnerSleep(250);
|
||||
}
|
||||
throw new Error("realtime debug/product streams did not become quiet after terminal");
|
||||
}
|
||||
@@ -1169,7 +1256,7 @@ async function realtimeProductSnapshot(subscriber) {
|
||||
return subscriber.page.evaluate((key) => {
|
||||
const state = window.__unideskRealtimeFanoutProducts?.[key];
|
||||
if (!state) return { connected: null, connectedEventCount: 0, openCount: 0, records: [], errors: [{ missing: true }] };
|
||||
return { connected: state.connected ? JSON.parse(JSON.stringify(state.connected)) : null, connectedEventCount: state.connectedEventCount, openCount: state.openCount, records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)) };
|
||||
return { connected: state.connected ? JSON.parse(JSON.stringify(state.connected)) : null, connectedEventCount: state.connectedEventCount, openCount: state.openCount, records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)), droppedEntries: Number(state.droppedEntries || 0), droppedBodyEntries: Number(state.droppedBodyEntries || 0), droppedBodyBytes: Number(state.droppedBodyBytes || 0) };
|
||||
}, subscriber.key);
|
||||
}
|
||||
|
||||
@@ -1177,7 +1264,7 @@ async function realtimeDebugSnapshot(debug, key) {
|
||||
return debug.page.evaluate((streamKey) => {
|
||||
const state = window.__unideskRealtimeFanoutDebug?.[streamKey];
|
||||
if (!state) return { connected: {}, connectedEventCount: {}, openCount: {}, records: { stdio: [], agentrun: [], hwlab: [] }, errors: { stdio: [], agentrun: [], hwlab: [] } };
|
||||
return { connected: JSON.parse(JSON.stringify(state.connected)), connectedEventCount: JSON.parse(JSON.stringify(state.connectedEventCount)), openCount: JSON.parse(JSON.stringify(state.openCount)), records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)) };
|
||||
return { connected: JSON.parse(JSON.stringify(state.connected)), connectedEventCount: JSON.parse(JSON.stringify(state.connectedEventCount)), openCount: JSON.parse(JSON.stringify(state.openCount)), records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)), droppedEntries: Number(state.droppedEntries || 0), droppedBodyEntries: Number(state.droppedBodyEntries || 0), droppedBodyBytes: Number(state.droppedBodyBytes || 0) };
|
||||
}, key);
|
||||
}
|
||||
|
||||
@@ -1191,6 +1278,9 @@ async function realtimeAllDebugSnapshots(debug) {
|
||||
openCount: JSON.parse(JSON.stringify(state.openCount || {})),
|
||||
records: JSON.parse(JSON.stringify(state.records || {})),
|
||||
errors: JSON.parse(JSON.stringify(state.errors || {})),
|
||||
droppedEntries: Number(state.droppedEntries || 0),
|
||||
droppedBodyEntries: Number(state.droppedBodyEntries || 0),
|
||||
droppedBodyBytes: Number(state.droppedBodyBytes || 0),
|
||||
}));
|
||||
});
|
||||
}
|
||||
@@ -1282,7 +1372,7 @@ async function realtimeWaitReplayedTraceJointQuiet(debug, key, subscriber, trace
|
||||
if (comparison.equal && terminalSeen && sourceTransportStable && quietForMs >= profile.terminalQuietMs) {
|
||||
return { debug: debugSnapshot, product: productSnapshot, comparison, quietForMs, valuesRedacted: true };
|
||||
}
|
||||
await sleep(250);
|
||||
await interruptibleRunnerSleep(250);
|
||||
}
|
||||
const error = new Error(label + " Kafka source and product replay envelope multisets did not converge before the joint quiet timeout");
|
||||
error.details = {
|
||||
@@ -1589,6 +1679,8 @@ function realtimeTurnSummary(turn, debug, product, terminal) {
|
||||
}
|
||||
|
||||
async function realtimeCaptureSubscriberScreenshot(targetPage, reportDir, name, summary) {
|
||||
const screenshotSkipped = reserveScreenshot("realtime-fanout-" + name);
|
||||
if (screenshotSkipped !== null) return screenshotSkipped;
|
||||
await targetPage.evaluate((value) => {
|
||||
document.documentElement.style.background = "#0b1020";
|
||||
document.body.innerHTML = "";
|
||||
@@ -1598,7 +1690,7 @@ async function realtimeCaptureSubscriberScreenshot(targetPage, reportDir, name,
|
||||
document.body.appendChild(pre);
|
||||
}, summary);
|
||||
const file = path.join(reportDir, name);
|
||||
await targetPage.screenshot({ path: file, fullPage: true, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
|
||||
await targetPage.screenshot({ path: file, fullPage: false, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
|
||||
const meta = await fileMeta(file);
|
||||
const artifact = { kind: "realtime-fanout-screenshot", path: file, name, byteCount: meta.byteCount, sha256: meta.sha256, commandId: activeCommandId, valuesRedacted: true };
|
||||
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), ...artifact });
|
||||
@@ -1679,7 +1771,7 @@ async function realtimePoll(read, timeoutMs, label) {
|
||||
while (Date.now() < deadline) {
|
||||
const value = await read();
|
||||
if (value) return value;
|
||||
await sleep(250);
|
||||
await interruptibleRunnerSleep(250);
|
||||
}
|
||||
throw new Error(label + " timed out after " + timeoutMs + "ms");
|
||||
}
|
||||
|
||||
@@ -57,11 +57,50 @@ async function writeManifest(extra = {}) {
|
||||
navigation: { maxAttempts: navigationMaxAttempts, valuesRedacted: true },
|
||||
pageAuthority: { browser: "chromium", context: "shared-auth", pageMode: "dual-control-observer", controlPageId: pageId, observerPageId, continuityBreaksRecorded: true },
|
||||
pageProvenance: compactPageProvenance(currentPageProvenance),
|
||||
sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerRefreshIntervalMs, observerInitiatedDefault: false, responseBodyReadDefault: false },
|
||||
sampling: {
|
||||
mode: "passive",
|
||||
sampleIntervalMs,
|
||||
screenshotIntervalMs,
|
||||
maxSamples,
|
||||
maxRunMs,
|
||||
maxRunSeconds: maxRunMs > 0 ? Math.ceil(maxRunMs / 1000) : 0,
|
||||
observerRefreshIntervalMs,
|
||||
observerInitiatedDefault: false,
|
||||
responseBodyReadDefault: false,
|
||||
},
|
||||
resourceLimits: {
|
||||
maxScreenshots,
|
||||
maxPerformanceEntries,
|
||||
maxBrowserProcessHistoryEntries,
|
||||
maxBrowserFreezeSignalHistoryEntries,
|
||||
maxRealtimeEventEntries,
|
||||
maxRealtimeEventBodyBytes,
|
||||
maxDomEvidenceRows,
|
||||
maxProcessScanEntries,
|
||||
maxPerformanceCaptureMs,
|
||||
responseBodyMaxBytes,
|
||||
webPerformanceRequestBodyMaxBytes,
|
||||
maxJsonlQueueEntries,
|
||||
maxJsonlQueueBytes,
|
||||
maxJsonlLineBytes,
|
||||
jsonlFlushTimeoutMs,
|
||||
maxResponseBodyInFlight,
|
||||
maxPassiveListenerTasks,
|
||||
passiveTaskFlushTimeoutMs,
|
||||
closeStepTimeoutMs,
|
||||
maxProcessReadConcurrency,
|
||||
maxRealtimeTotalEntries,
|
||||
maxRealtimeTotalBodyBytes,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
projectManagement,
|
||||
jsonlRotation,
|
||||
jsonlWriter: jsonlWriterStats(),
|
||||
responseBodies: responseBodyStats(),
|
||||
passiveTasks: passiveTaskStats(),
|
||||
realtimeAggregate: realtimeAggregateLast,
|
||||
commandDirs: dirs,
|
||||
artifacts: files,
|
||||
safety: { pureClient: true, inboundApi: false, database: false, queueConsumer: false, k8sWorkload: false, valuesRedacted: true, secretValuesPrinted: false },
|
||||
@@ -84,11 +123,47 @@ async function writeHeartbeat(extra = {}) {
|
||||
currentUrl: currentPageUrl(),
|
||||
observerUrl: pageUrl(observerPage),
|
||||
observerRefreshIntervalMs,
|
||||
observerLifecycle: {
|
||||
maxSamples,
|
||||
maxRunMs,
|
||||
maxRunSeconds: maxRunMs > 0 ? Math.ceil(maxRunMs / 1000) : 0,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
resourceLimits: {
|
||||
screenshotCount,
|
||||
maxScreenshots,
|
||||
maxPerformanceEntries,
|
||||
maxBrowserProcessHistoryEntries,
|
||||
maxBrowserFreezeSignalHistoryEntries,
|
||||
maxRealtimeEventEntries,
|
||||
maxRealtimeEventBodyBytes,
|
||||
maxDomEvidenceRows,
|
||||
maxProcessScanEntries,
|
||||
maxPerformanceCaptureMs,
|
||||
responseBodyMaxBytes,
|
||||
webPerformanceRequestBodyMaxBytes,
|
||||
maxJsonlQueueEntries,
|
||||
maxJsonlQueueBytes,
|
||||
maxJsonlLineBytes,
|
||||
jsonlFlushTimeoutMs,
|
||||
maxResponseBodyInFlight,
|
||||
maxPassiveListenerTasks,
|
||||
passiveTaskFlushTimeoutMs,
|
||||
closeStepTimeoutMs,
|
||||
maxProcessReadConcurrency,
|
||||
maxRealtimeTotalEntries,
|
||||
maxRealtimeTotalBodyBytes,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
lastObserverRefreshAt: Number.isFinite(lastObserverRefreshAtMs) ? new Date(lastObserverRefreshAtMs).toISOString() : null,
|
||||
pageProvenance: compactPageProvenance(currentPageProvenance),
|
||||
sampleSeq,
|
||||
commandSeq,
|
||||
activeCommandId,
|
||||
jsonlWriter: jsonlWriterStats(),
|
||||
responseBodies: responseBodyStats(),
|
||||
passiveTasks: passiveTaskStats(),
|
||||
realtimeAggregate: realtimeAggregateLast,
|
||||
updatedAt: new Date().toISOString(),
|
||||
uptimeMs: Date.now() - startedAtMs,
|
||||
...extra,
|
||||
@@ -96,6 +171,22 @@ async function writeHeartbeat(extra = {}) {
|
||||
await writeFile(files.heartbeat, JSON.stringify(heartbeat, null, 2) + "\n", { mode: 0o600 });
|
||||
}
|
||||
|
||||
async function writeTerminalResourceStats(extra = {}) {
|
||||
for (const file of [files.manifest, files.heartbeat]) {
|
||||
let existing = {};
|
||||
try { existing = JSON.parse(await readFile(file, "utf8")); } catch {}
|
||||
await writeFile(file, JSON.stringify({
|
||||
...existing,
|
||||
jsonlWriter: jsonlWriterStats(),
|
||||
responseBodies: responseBodyStats(),
|
||||
passiveTasks: passiveTaskStats(),
|
||||
terminalResources: sanitize(extra),
|
||||
updatedAt: new Date().toISOString(),
|
||||
valuesRedacted: true,
|
||||
}, null, 2) + "\n", { mode: 0o600 });
|
||||
}
|
||||
}
|
||||
|
||||
function startHeartbeatPulse() {
|
||||
const timer = setInterval(() => {
|
||||
if (stopping) return;
|
||||
@@ -319,6 +410,7 @@ function recordBrowserFreezeSignal(kind, sample, pageMetric, detail) {
|
||||
const windowMs = browserFreezePolicy.blockerWindowMs;
|
||||
const cutoff = signal.tsMs - windowMs;
|
||||
while (browserFreezeSignalHistory.length > 0 && Number(browserFreezeSignalHistory[0].tsMs || 0) < cutoff) browserFreezeSignalHistory.shift();
|
||||
while (browserFreezeSignalHistory.length > maxBrowserFreezeSignalHistoryEntries) browserFreezeSignalHistory.shift();
|
||||
return {
|
||||
kind,
|
||||
rootCause: detail.rootCause,
|
||||
@@ -374,62 +466,96 @@ async function triggerBrowserFreezeBlocker(trigger) {
|
||||
}
|
||||
|
||||
async function terminateBrowserForFreezeBlocker(blocker) {
|
||||
const childProcess = browser && typeof browser.process === "function" ? browser.process() : null;
|
||||
const pid = Number(childProcess?.pid);
|
||||
if (!Number.isFinite(pid) || pid <= 0) {
|
||||
return { ok: false, reason: "browser-process-unavailable", blockerKind: blocker.kind, valuesRedacted: true };
|
||||
const result = await terminateTrackedBrowserProcesses("freeze-blocker:" + String(blocker.kind || "unknown"));
|
||||
return { ...result, blockerKind: blocker.kind, valuesRedacted: true };
|
||||
}
|
||||
|
||||
function rememberTrackedBrowserProcesses(processes) {
|
||||
const merged = new Map(trackedBrowserProcessIdentities.map((item) => [item.pid + ":" + item.startTicks, item]));
|
||||
for (const item of Array.isArray(processes) ? processes : []) {
|
||||
if (!Number.isInteger(item.pid) || !Number.isInteger(item.startTicks) || !item.role) continue;
|
||||
merged.set(item.pid + ":" + item.startTicks, {
|
||||
pid: item.pid,
|
||||
ppid: item.ppid,
|
||||
pgid: item.pgid,
|
||||
sid: item.sid,
|
||||
startTicks: item.startTicks,
|
||||
role: item.role,
|
||||
commandHash: item.commandHash || null,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
trackedBrowserProcessIdentities = [...merged.values()].slice(-Math.min(maxProcessScanEntries, 1024));
|
||||
}
|
||||
|
||||
async function matchingTrackedBrowserProcesses() {
|
||||
const rows = await Promise.all(trackedBrowserProcessIdentities.map(async (identity) => {
|
||||
const current = await readOneProcProcessStat(identity.pid).catch(() => null);
|
||||
return current && current.startTicks === identity.startTicks ? { ...identity, currentPpid: current.ppid, currentPgid: current.pgid, currentSid: current.sid } : null;
|
||||
}));
|
||||
return rows.filter(Boolean);
|
||||
}
|
||||
|
||||
async function waitForTrackedBrowserExit(timeoutMs, pollIntervalMs) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let remaining = await matchingTrackedBrowserProcesses();
|
||||
while (remaining.length > 0 && Date.now() <= deadline) {
|
||||
await boundedCleanupSleep(pollIntervalMs);
|
||||
remaining = await matchingTrackedBrowserProcesses();
|
||||
}
|
||||
return remaining;
|
||||
}
|
||||
|
||||
async function terminateTrackedBrowserProcesses(reason, requiredDiscovery = null) {
|
||||
let refreshDiscovery;
|
||||
try {
|
||||
const summary = await collectChromiumProcessSummary(null);
|
||||
refreshDiscovery = { ok: true, processScan: summary.processScan, chromiumProcessCount: summary.chromiumProcessCount, valuesRedacted: true };
|
||||
} catch (error) {
|
||||
refreshDiscovery = { ok: false, processScan: lastProcessScan, error: errorSummary(error), valuesRedacted: true };
|
||||
}
|
||||
const discovery = requiredDiscovery || refreshDiscovery;
|
||||
const identityScanUnavailable = browserLaunchAttempted === true
|
||||
&& (discovery?.ok !== true || discovery?.processScan?.complete !== true);
|
||||
const initial = await matchingTrackedBrowserProcesses();
|
||||
const result = {
|
||||
ok: false,
|
||||
pid: Math.floor(pid),
|
||||
ok: initial.length === 0 && !identityScanUnavailable,
|
||||
reason,
|
||||
browserLaunchAttempted,
|
||||
identityScanUnavailable,
|
||||
requiredDiscovery: discovery,
|
||||
refreshDiscovery,
|
||||
trackedIdentityCount: trackedBrowserProcessIdentities.length,
|
||||
initialLiveCount: initial.length,
|
||||
gracefulSignal: browserFreezePolicy.kill.gracefulSignal,
|
||||
forceSignal: browserFreezePolicy.kill.forceSignal,
|
||||
graceMs: browserFreezePolicy.kill.graceMs,
|
||||
pollIntervalMs: browserFreezePolicy.kill.pollIntervalMs,
|
||||
gracefulSent: false,
|
||||
forceSent: false,
|
||||
exitedAfterGrace: false,
|
||||
exitedAfterForce: false,
|
||||
gracefulSentCount: 0,
|
||||
forceSentCount: 0,
|
||||
remainingIdentityCount: initial.length,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
try {
|
||||
childProcess.kill(browserFreezePolicy.kill.gracefulSignal);
|
||||
result.gracefulSent = true;
|
||||
} catch (error) {
|
||||
result.gracefulError = errorSummary(error);
|
||||
}
|
||||
result.exitedAfterGrace = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
|
||||
if (!result.exitedAfterGrace) {
|
||||
for (const item of [...initial].sort((a, b) => b.pid - a.pid)) {
|
||||
try {
|
||||
childProcess.kill(browserFreezePolicy.kill.forceSignal);
|
||||
result.forceSent = true;
|
||||
} catch (error) {
|
||||
result.forceError = errorSummary(error);
|
||||
}
|
||||
result.exitedAfterForce = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
|
||||
process.kill(item.pid, browserFreezePolicy.kill.gracefulSignal);
|
||||
result.gracefulSentCount += 1;
|
||||
} catch {}
|
||||
}
|
||||
result.ok = result.exitedAfterGrace || result.exitedAfterForce;
|
||||
let remaining = await waitForTrackedBrowserExit(browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
|
||||
for (const item of [...remaining].sort((a, b) => b.pid - a.pid)) {
|
||||
try {
|
||||
process.kill(item.pid, browserFreezePolicy.kill.forceSignal);
|
||||
result.forceSentCount += 1;
|
||||
} catch {}
|
||||
}
|
||||
remaining = await waitForTrackedBrowserExit(browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
|
||||
result.remainingIdentityCount = remaining.length;
|
||||
result.remainingIdentities = remaining.slice(0, 50).map((item) => ({ pid: item.pid, startTicks: item.startTicks, role: item.role, originalPpid: item.ppid, currentPpid: item.currentPpid, originalSid: item.sid, currentSid: item.currentSid, valuesRedacted: true }));
|
||||
result.ok = remaining.length === 0 && !identityScanUnavailable;
|
||||
return result;
|
||||
}
|
||||
|
||||
async function waitForPidExit(pid, timeoutMs, pollIntervalMs) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() <= deadline) {
|
||||
if (!pidAlive(pid)) return true;
|
||||
await sleep(pollIntervalMs);
|
||||
}
|
||||
return !pidAlive(pid);
|
||||
}
|
||||
|
||||
function pidAlive(pid) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function browserProcessSampleRef(sample) {
|
||||
return {
|
||||
ts: sample?.ts ?? null,
|
||||
@@ -496,13 +622,16 @@ async function collectChromiumProcessSummary(browserPid) {
|
||||
queue.push(child);
|
||||
}
|
||||
}
|
||||
const processes = all
|
||||
.filter((item) => descendants.has(item.pid))
|
||||
const detailed = await readProcProcessDetails(all.filter((item) => descendants.has(item.pid)));
|
||||
const processes = detailed
|
||||
.map((item) => ({ ...item, role: classifyChromiumProcess(item) }))
|
||||
.filter((item) => item.role)
|
||||
.map((item) => ({
|
||||
pid: item.pid,
|
||||
ppid: item.ppid,
|
||||
pgid: item.pgid,
|
||||
sid: item.sid,
|
||||
startTicks: item.startTicks,
|
||||
name: item.name || null,
|
||||
role: item.role,
|
||||
rssBytes: item.rssBytes,
|
||||
@@ -512,6 +641,7 @@ async function collectChromiumProcessSummary(browserPid) {
|
||||
valuesRedacted: true,
|
||||
}))
|
||||
.sort((a, b) => Number(b.rssBytes || 0) - Number(a.rssBytes || 0));
|
||||
rememberTrackedBrowserProcesses(processes);
|
||||
const roles = {};
|
||||
for (const item of processes) {
|
||||
const role = item.role || "unknown";
|
||||
@@ -525,6 +655,7 @@ async function collectChromiumProcessSummary(browserPid) {
|
||||
const maxProcess = processes[0] || null;
|
||||
return {
|
||||
sampledRootPids: roots,
|
||||
processScan: lastProcessScan,
|
||||
chromiumProcessCount: processes.length,
|
||||
totalRssBytes,
|
||||
totalRssMb: bytesToMb(totalRssBytes),
|
||||
@@ -541,28 +672,68 @@ async function readProcProcessTable() {
|
||||
let entries = [];
|
||||
try {
|
||||
entries = await readdir("/proc");
|
||||
} catch {
|
||||
return [];
|
||||
} catch (error) {
|
||||
lastProcessScan = { ok: false, complete: false, totalPidEntries: 0, scannedPidEntries: 0, failedPidEntries: 0, truncated: false, readConcurrency: maxProcessReadConcurrency, error: errorSummary(error), valuesRedacted: true };
|
||||
throw new Error("proc-process-table-unavailable");
|
||||
}
|
||||
const numeric = entries.map((name) => Number(name)).filter((pid) => Number.isInteger(pid) && pid > 0);
|
||||
const rows = await Promise.all(numeric.map((pid) => readOneProcProcess(pid).catch(() => null)));
|
||||
return rows.filter(Boolean);
|
||||
const allNumeric = entries.map((name) => Number(name)).filter((pid) => Number.isInteger(pid) && pid > 0).sort((a, b) => b - a);
|
||||
const numeric = allNumeric.slice(0, maxProcessScanEntries);
|
||||
const rows = [];
|
||||
for (let offset = 0; offset < numeric.length; offset += maxProcessReadConcurrency) {
|
||||
const batch = await Promise.all(numeric.slice(offset, offset + maxProcessReadConcurrency).map((pid) => readOneProcProcessStat(pid).catch(() => null)));
|
||||
rows.push(...batch.filter(Boolean));
|
||||
}
|
||||
const failedPidEntries = numeric.length - rows.length;
|
||||
const truncated = numeric.length < allNumeric.length;
|
||||
const unusable = numeric.length > 0 && rows.length === 0;
|
||||
lastProcessScan = {
|
||||
ok: !unusable,
|
||||
complete: !unusable && !truncated,
|
||||
totalPidEntries: allNumeric.length,
|
||||
scannedPidEntries: numeric.length,
|
||||
readablePidEntries: rows.length,
|
||||
failedPidEntries,
|
||||
truncated,
|
||||
readConcurrency: maxProcessReadConcurrency,
|
||||
error: unusable ? "all-proc-stat-reads-failed" : null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
if (unusable) throw new Error("proc-process-table-unreadable");
|
||||
return rows;
|
||||
}
|
||||
|
||||
async function readOneProcProcess(pid) {
|
||||
const [statText, statusText, cmdlineText] = await Promise.all([
|
||||
readFile(path.join("/proc", String(pid), "stat"), "utf8").catch(() => ""),
|
||||
async function readOneProcProcessStat(pid) {
|
||||
const statText = await readFile(path.join("/proc", String(pid), "stat"), "utf8");
|
||||
if (!statText) return null;
|
||||
return { pid, ...procIdentityFromStat(statText) };
|
||||
}
|
||||
|
||||
async function readProcProcessDetails(rows) {
|
||||
const detailed = [];
|
||||
for (let offset = 0; offset < rows.length; offset += maxProcessReadConcurrency) {
|
||||
const batch = await Promise.all(rows.slice(offset, offset + maxProcessReadConcurrency).map((row) => readOneProcProcessDetails(row).catch(() => null)));
|
||||
detailed.push(...batch.filter(Boolean));
|
||||
}
|
||||
return detailed;
|
||||
}
|
||||
|
||||
async function readOneProcProcessDetails(row) {
|
||||
const pid = row.pid;
|
||||
const [statusText, cmdlineText] = await Promise.all([
|
||||
readFile(path.join("/proc", String(pid), "status"), "utf8").catch(() => ""),
|
||||
readFile(path.join("/proc", String(pid), "cmdline"), "utf8").catch(() => ""),
|
||||
]);
|
||||
if (!statText && !statusText) return null;
|
||||
const ppid = procPpidFromStat(statText);
|
||||
if (!statusText && !cmdlineText) return null;
|
||||
const ppid = row.ppid;
|
||||
const name = procStatusField(statusText, "Name") || null;
|
||||
const rssKb = procStatusKb(statusText, "VmRSS");
|
||||
const vmSizeKb = procStatusKb(statusText, "VmSize");
|
||||
return {
|
||||
pid,
|
||||
ppid,
|
||||
pgid: row.pgid,
|
||||
sid: row.sid,
|
||||
startTicks: row.startTicks,
|
||||
name,
|
||||
cmdline: String(cmdlineText || "").replace(/\0/gu, " ").replace(/\s+/gu, " ").trim(),
|
||||
rssBytes: Number.isFinite(rssKb) ? rssKb * 1024 : 0,
|
||||
@@ -570,13 +741,13 @@ async function readOneProcProcess(pid) {
|
||||
};
|
||||
}
|
||||
|
||||
function procPpidFromStat(value) {
|
||||
function procIdentityFromStat(value) {
|
||||
const text = String(value || "");
|
||||
const end = text.lastIndexOf(")");
|
||||
if (end < 0) return null;
|
||||
if (end < 0) return { ppid: null, pgid: null, sid: null, startTicks: null };
|
||||
const parts = text.slice(end + 1).trim().split(/\s+/u);
|
||||
const ppid = Number(parts[1]);
|
||||
return Number.isFinite(ppid) ? ppid : null;
|
||||
const integerAt = (index) => Number.isInteger(Number(parts[index])) ? Number(parts[index]) : null;
|
||||
return { ppid: integerAt(1), pgid: integerAt(2), sid: integerAt(3), startTicks: integerAt(19) };
|
||||
}
|
||||
|
||||
function procStatusField(value, key) {
|
||||
@@ -623,6 +794,7 @@ function updateBrowserProcessHistory(tsMs, processSummary) {
|
||||
browserProcessHistory.push({ tsMs, totalRssBytes, maxProcessRssBytes });
|
||||
const retentionMs = Math.max(windowMs * 3, 180000);
|
||||
while (browserProcessHistory.length > 0 && tsMs - browserProcessHistory[0].tsMs > retentionMs) browserProcessHistory.shift();
|
||||
while (browserProcessHistory.length > maxBrowserProcessHistoryEntries) browserProcessHistory.shift();
|
||||
const windowStartMs = tsMs - windowMs;
|
||||
const candidates = browserProcessHistory.filter((item) => item.tsMs >= windowStartMs && item.tsMs <= tsMs);
|
||||
const baseline = candidates[0] || browserProcessHistory[0] || null;
|
||||
@@ -688,7 +860,13 @@ function applyBrowserPageRuntimeBaseline(result) {
|
||||
const key = [result.pageRole || "unknown", result.pageId || "unknown", Number.isFinite(Number(result.pageEpoch)) ? Number(result.pageEpoch) : 0].join(":");
|
||||
const current = browserPageRuntimeMemorySnapshot(result);
|
||||
const existing = browserPageRuntimeBaselines.get(key);
|
||||
if (!existing && current) browserPageRuntimeBaselines.set(key, { ...current, capturedAt: new Date().toISOString(), source: "first-page-runtime-sample", valuesRedacted: true });
|
||||
if (!existing && current) {
|
||||
const prefix = [result.pageRole || "unknown", result.pageId || "unknown"].join(":") + ":";
|
||||
for (const existingKey of browserPageRuntimeBaselines.keys()) {
|
||||
if (existingKey.startsWith(prefix)) browserPageRuntimeBaselines.delete(existingKey);
|
||||
}
|
||||
browserPageRuntimeBaselines.set(key, { ...current, capturedAt: new Date().toISOString(), source: "first-page-runtime-sample", valuesRedacted: true });
|
||||
}
|
||||
const baseline = browserPageRuntimeBaselines.get(key) || null;
|
||||
result.baseline = baseline ? {
|
||||
capturedAt: baseline.capturedAt,
|
||||
@@ -810,8 +988,10 @@ function bytesToMb(value) {
|
||||
}
|
||||
|
||||
function attachPassiveListeners(targetPage, pageRole = "control", targetPageId = pageId) {
|
||||
void installPagePerformanceProbe(targetPage, pageRole, targetPageId)
|
||||
.catch((error) => appendJsonl(files.errors, eventRecord("performance-probe-install-error", { pageRole, pageId: targetPageId, error: errorSummary(error), valuesRedacted: true })));
|
||||
trackPassiveTask(
|
||||
() => installPagePerformanceProbe(targetPage, pageRole, targetPageId),
|
||||
"performance-probe-install:" + pageRole,
|
||||
);
|
||||
targetPage.on("request", (request) => {
|
||||
const webPerformancePayload = summarizeWebPerformanceRequestPayload(request);
|
||||
void appendJsonl(files.network, eventRecord("request", {
|
||||
@@ -841,19 +1021,10 @@ function attachPassiveListeners(targetPage, pageRole = "control", targetPageId =
|
||||
statusText: response.statusText(),
|
||||
fromServiceWorker: response.fromServiceWorker(),
|
||||
};
|
||||
void (async () => {
|
||||
trackPassiveTask(async () => {
|
||||
const bodyFields = await summarizeWorkbenchResponseBody(response, request);
|
||||
await appendJsonl(files.network, eventRecord("response", { ...base, ...bodyFields }));
|
||||
})().catch((error) => appendJsonl(files.errors, eventRecord("response-body-summary-error", {
|
||||
pageRole,
|
||||
pageId: targetPageId,
|
||||
sampleSeq,
|
||||
commandId: activeCommandId,
|
||||
method: request.method(),
|
||||
url: safeUrl(response.url()),
|
||||
error: errorSummary(error),
|
||||
valuesRedacted: true
|
||||
})));
|
||||
}, "response-body-summary:" + pageRole);
|
||||
});
|
||||
targetPage.on("requestfailed", (request) => {
|
||||
void appendJsonl(files.network, eventRecord("requestfailed", {
|
||||
|
||||
@@ -795,8 +795,32 @@ function digestSessionRail(value) {
|
||||
};
|
||||
}
|
||||
|
||||
function reserveScreenshot(reason) {
|
||||
if (screenshotCount >= maxScreenshots) {
|
||||
lastScreenshotAtMs = Date.now();
|
||||
return {
|
||||
seq: null,
|
||||
sampleSeq,
|
||||
ts: new Date().toISOString(),
|
||||
kind: "screenshot-skipped",
|
||||
reason,
|
||||
skipped: true,
|
||||
skipReason: "screenshot-cap-reached",
|
||||
screenshotCount,
|
||||
maxScreenshots,
|
||||
pageId,
|
||||
currentUrl: currentPageUrl(),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
screenshotCount += 1;
|
||||
return null;
|
||||
}
|
||||
|
||||
async function captureScreenshot(reason, imageType = "png") {
|
||||
if (!page || page.isClosed()) throw new Error("page is not available for screenshot");
|
||||
const skipped = reserveScreenshot(reason);
|
||||
if (skipped !== null) return skipped;
|
||||
if (screenshotCaptureState && screenshotCaptureState.settled !== true) {
|
||||
const ageMs = Date.now() - Number(screenshotCaptureState.startedAtMs || Date.now());
|
||||
const error = new Error("screenshot capture already in progress");
|
||||
@@ -887,10 +911,10 @@ async function summarizeWorkbenchResponseBody(response, request) {
|
||||
const headers = response.headers();
|
||||
const contentType = String(headers["content-type"] || headers["Content-Type"] || "");
|
||||
if (!/json/iu.test(contentType)) return { bodyRead: false, bodyReadSkipped: "non-json", valuesRedacted: true };
|
||||
const contentLength = Number(headers["content-length"] || headers["Content-Length"]);
|
||||
const maxBytes = 512 * 1024;
|
||||
if (Number.isFinite(contentLength) && contentLength > maxBytes) return { bodyRead: false, bodyReadSkipped: "content-length-too-large", bodyByteCount: contentLength, valuesRedacted: true };
|
||||
const text = await response.text();
|
||||
const maxBytes = responseBodyMaxBytes;
|
||||
const body = await boundedResponseText(response, maxBytes);
|
||||
if (body.ok !== true || typeof body.value !== "string") return { bodyRead: false, bodyReadSkipped: body.error?.reason || "bounded-response-read-failed", bodyContract: body.bodyContract, valuesRedacted: true };
|
||||
const text = body.value;
|
||||
const byteCount = Buffer.byteLength(text);
|
||||
if (byteCount > maxBytes) return { bodyRead: true, bodyReadSkipped: "body-too-large", bodyByteCount: byteCount, bodyHash: sha256Text(text), valuesRedacted: true };
|
||||
let parsed = null;
|
||||
|
||||
@@ -15,6 +15,7 @@ import { nodeWebObserveRunnerTraceReadabilitySource } from "./hwlab-node-web-obs
|
||||
export function nodeWebObserveRunnerSource(): string {
|
||||
return String.raw`#!/usr/bin/env node
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { appendFile, mkdir, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
@@ -31,9 +32,30 @@ const targetPath = process.env.UNIDESK_WEB_OBSERVE_TARGET_PATH || "/workbench";
|
||||
const sampleIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS, 5000);
|
||||
const screenshotIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS, 300000);
|
||||
const screenshotCaptureTimeoutMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_CAPTURE_TIMEOUT_MS, 15000, 1000, 120000);
|
||||
const maxSamples = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 0);
|
||||
const maxSamples = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, "UNIDESK_WEB_OBSERVE_MAX_SAMPLES", 1, 10000000);
|
||||
const observerRefreshIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS, 180000);
|
||||
const maxRunMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RUN_MS, 0);
|
||||
const maxRunMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RUN_MS, "UNIDESK_WEB_OBSERVE_MAX_RUN_MS", 1000, 86400000);
|
||||
const maxScreenshots = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS, "UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS", 1, 10000);
|
||||
const maxPerformanceEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES", 1, 100000);
|
||||
const maxBrowserProcessHistoryEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES", 1, 100000);
|
||||
const maxBrowserFreezeSignalHistoryEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES", 1, 100000);
|
||||
const maxRealtimeEventEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_ENTRIES", 1, 100000);
|
||||
const maxRealtimeEventBodyBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES", 1, 16777216);
|
||||
const maxDomEvidenceRows = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_DOM_EVIDENCE_ROWS, "UNIDESK_WEB_OBSERVE_MAX_DOM_EVIDENCE_ROWS", 1, 100000);
|
||||
const maxProcessScanEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PROCESS_SCAN_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_PROCESS_SCAN_ENTRIES", 1, 1000000);
|
||||
const maxPerformanceCaptureMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_CAPTURE_SECONDS, "UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_CAPTURE_SECONDS", 1, 600) * 1000;
|
||||
const responseBodyMaxBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES, "UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES", 1024, 16777216);
|
||||
const maxJsonlQueueEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_ENTRIES", 1, 100000);
|
||||
const maxJsonlQueueBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_BYTES, "UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_BYTES", 1024, 67108864);
|
||||
const maxJsonlLineBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_JSONL_LINE_BYTES, "UNIDESK_WEB_OBSERVE_MAX_JSONL_LINE_BYTES", 1024, 16777216);
|
||||
const jsonlFlushTimeoutMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_JSONL_FLUSH_TIMEOUT_MS, "UNIDESK_WEB_OBSERVE_JSONL_FLUSH_TIMEOUT_MS", 100, 60000);
|
||||
const maxResponseBodyInFlight = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RESPONSE_BODY_IN_FLIGHT, "UNIDESK_WEB_OBSERVE_MAX_RESPONSE_BODY_IN_FLIGHT", 1, 64);
|
||||
const maxPassiveListenerTasks = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PASSIVE_LISTENER_TASKS, "UNIDESK_WEB_OBSERVE_MAX_PASSIVE_LISTENER_TASKS", 1, 10000);
|
||||
const passiveTaskFlushTimeoutMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_PASSIVE_TASK_FLUSH_TIMEOUT_MS, "UNIDESK_WEB_OBSERVE_PASSIVE_TASK_FLUSH_TIMEOUT_MS", 100, 60000);
|
||||
const closeStepTimeoutMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_CLOSE_STEP_TIMEOUT_MS, "UNIDESK_WEB_OBSERVE_CLOSE_STEP_TIMEOUT_MS", 100, 60000);
|
||||
const maxProcessReadConcurrency = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PROCESS_READ_CONCURRENCY, "UNIDESK_WEB_OBSERVE_MAX_PROCESS_READ_CONCURRENCY", 1, 256);
|
||||
const maxRealtimeTotalEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_ENTRIES", 1, 100000);
|
||||
const maxRealtimeTotalBodyBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_BODY_BYTES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_BODY_BYTES", 1024, 67108864);
|
||||
const viewport = parseViewport(process.env.UNIDESK_WEB_OBSERVE_VIEWPORT || "1440x900");
|
||||
const browserProxyMode = parseBrowserProxyMode(process.env.UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE || "auto");
|
||||
const origin = {
|
||||
@@ -44,7 +66,7 @@ const origin = {
|
||||
browserProxyModeSource: process.env.UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE_SOURCE || "default",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
const webPerformanceRequestBodyMaxBytes = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES, 65536, 1024, 1048576);
|
||||
const webPerformanceRequestBodyMaxBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES, "UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES", 1024, 1048576);
|
||||
const authLoginMaxAttempts = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_ATTEMPTS, 6, 1, 20);
|
||||
const authLoginRequestTimeoutMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_REQUEST_TIMEOUT_MS, 30000, 1000, 120000);
|
||||
const authLoginInitialDelayMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_INITIAL_DELAY_MS, 500, 0, 60000);
|
||||
@@ -92,6 +114,7 @@ let lastObserverRefreshAtMs = Date.now();
|
||||
let sampleSeq = 0;
|
||||
let commandSeq = 0;
|
||||
let artifactSeq = 0;
|
||||
let screenshotCount = 0;
|
||||
let activeCommandId = null;
|
||||
let activeCommandType = null;
|
||||
let stopping = false;
|
||||
@@ -107,10 +130,30 @@ let heartbeatPulseTimer = null;
|
||||
let browserProcessMonitorStop = null;
|
||||
let browserProcessMonitorSeq = 0;
|
||||
let browserFreezeBlocker = null;
|
||||
let shutdownSignal = null;
|
||||
let browserResourcesClosePromise = null;
|
||||
let browserLaunchAttempted = false;
|
||||
const browserProcessHistory = [];
|
||||
const browserPageRuntimeBaselines = new Map();
|
||||
const browserFreezeSignalHistory = [];
|
||||
let trackedBrowserProcessIdentities = [];
|
||||
const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] };
|
||||
const jsonlWriterState = { queue: [], queuedBytes: 0, inFlight: 0, pump: null, acceptedEntries: 0, acceptedBytes: 0, writtenEntries: 0, writtenBytes: 0, droppedEntries: 0, droppedBytes: 0, failedEntries: 0, dropsByFile: {} };
|
||||
const responseBodyState = { inFlight: 0, accepted: 0, skippedSaturated: 0, completed: 0, failed: 0 };
|
||||
const passiveTaskState = { tasks: new Set(), accepted: 0, dropped: 0, completed: 0, failed: 0, drainTimedOut: false };
|
||||
const runnerSleepWakes = new Set();
|
||||
let lastProcessScan = { ok: false, complete: false, totalPidEntries: 0, scannedPidEntries: 0, failedPidEntries: 0, truncated: false, readConcurrency: maxProcessReadConcurrency, error: "not-scanned", valuesRedacted: true };
|
||||
let realtimeAggregateLast = null;
|
||||
|
||||
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
||||
process.once(signal, () => {
|
||||
shutdownSignal = signal;
|
||||
stopping = true;
|
||||
terminalStatus = "stopping";
|
||||
interruptRunnerSleep();
|
||||
void closeBrowserResources("signal:" + signal);
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
if (!password) throw new Error("missing HWLAB_WEB_PASS");
|
||||
@@ -122,6 +165,7 @@ try {
|
||||
if (jsonlRotation.files.length > 0) await appendJsonl(files.control, eventRecord("jsonl-rotated", { stamp: jsonlRotation.stamp, archiveDir: path.relative(stateDir, dirs.archive), files: jsonlRotation.files, valuesRedacted: true }));
|
||||
const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href);
|
||||
const { chromium } = await launcher.importPlaywright();
|
||||
browserLaunchAttempted = true;
|
||||
browser = await launcher.launchChromium(chromium, chromiumLaunchOptions);
|
||||
context = await browser.newContext({ viewport, ...(playwrightProxy === null ? {} : { proxy: playwrightProxy }) });
|
||||
page = await context.newPage();
|
||||
@@ -152,19 +196,19 @@ try {
|
||||
await writeManifest({ status: "running", auth: publicAuth(auth) });
|
||||
await writeHeartbeat({ status: "running" });
|
||||
while (!stopping) {
|
||||
await drainOneCommand();
|
||||
if (browserFreezeBlocker) break;
|
||||
await samplePage("interval");
|
||||
if (browserFreezeBlocker) break;
|
||||
if (maxSamples > 0 && sampleSeq >= maxSamples) {
|
||||
await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples }));
|
||||
break;
|
||||
}
|
||||
if (maxRunMs > 0 && Date.now() - startedAtMs >= maxRunMs) {
|
||||
await appendJsonl(files.control, controlRecord({ id: "max-run-ms", type: "stop", source: "sampler" }, "completed", { reason: "max-run-ms", maxRunMs, elapsedMs: Date.now() - startedAtMs }));
|
||||
break;
|
||||
}
|
||||
await sleep(sampleIntervalMs);
|
||||
if (maxSamples > 0 && sampleSeq >= maxSamples) {
|
||||
await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples }));
|
||||
break;
|
||||
}
|
||||
await drainOneCommand();
|
||||
if (browserFreezeBlocker) break;
|
||||
await samplePage("interval");
|
||||
if (browserFreezeBlocker) break;
|
||||
await interruptibleRunnerSleep(sampleIntervalMs);
|
||||
}
|
||||
if (browserFreezeBlocker) {
|
||||
terminalStatus = "failed";
|
||||
@@ -186,7 +230,49 @@ try {
|
||||
} finally {
|
||||
if (browserProcessMonitorStop) browserProcessMonitorStop();
|
||||
if (heartbeatPulseTimer) clearInterval(heartbeatPulseTimer);
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
interruptRunnerSleep();
|
||||
const closeResult = await closeBrowserResources(shutdownSignal === null ? "finally" : "signal:" + shutdownSignal);
|
||||
const passiveTaskDrain = await drainPassiveTasks();
|
||||
const terminalFlush = await flushJsonlQueue();
|
||||
await writeTerminalResourceStats({ closeResult, passiveTaskDrain, terminalFlush }).catch(() => {});
|
||||
}
|
||||
|
||||
async function closeBrowserResources(reason) {
|
||||
if (browserResourcesClosePromise) return browserResourcesClosePromise;
|
||||
browserResourcesClosePromise = (async () => {
|
||||
stopping = true;
|
||||
const closingObserverPage = observerPage;
|
||||
const closingPage = page;
|
||||
const closingContext = context;
|
||||
const closingBrowser = browser;
|
||||
let preCloseIdentityDiscovery;
|
||||
try {
|
||||
const summary = await collectChromiumProcessSummary(null);
|
||||
preCloseIdentityDiscovery = { ok: true, processScan: summary.processScan, chromiumProcessCount: summary.chromiumProcessCount, valuesRedacted: true };
|
||||
} catch (error) {
|
||||
preCloseIdentityDiscovery = { ok: false, processScan: lastProcessScan, error: errorSummary(error), valuesRedacted: true };
|
||||
}
|
||||
observerPage = null;
|
||||
page = null;
|
||||
context = null;
|
||||
browser = null;
|
||||
const steps = [];
|
||||
const closeStep = async (label, operation) => {
|
||||
try {
|
||||
await withHardTimeout(Promise.resolve().then(operation), closeStepTimeoutMs, label + " close exceeded " + closeStepTimeoutMs + "ms");
|
||||
steps.push({ label, ok: true, valuesRedacted: true });
|
||||
} catch (error) {
|
||||
steps.push({ label, ok: false, error: errorSummary(error), valuesRedacted: true });
|
||||
}
|
||||
};
|
||||
if (closingObserverPage && !closingObserverPage.isClosed()) await closeStep("observer-page", () => closingObserverPage.close());
|
||||
if (closingPage && !closingPage.isClosed()) await closeStep("control-page", () => closingPage.close());
|
||||
if (closingContext) await closeStep("browser-context", () => closingContext.close());
|
||||
if (closingBrowser) await closeStep("browser", () => closingBrowser.close());
|
||||
const browserCleanup = await terminateTrackedBrowserProcesses("browser-close-residual", preCloseIdentityDiscovery);
|
||||
return { ok: steps.every((step) => step.ok === true) && browserCleanup.ok === true, reason, closeStepTimeoutMs, steps, browserCleanup, valuesRedacted: true };
|
||||
})();
|
||||
return browserResourcesClosePromise;
|
||||
}
|
||||
|
||||
${nodeWebObserveRunnerRuntimeSource()}
|
||||
|
||||
@@ -300,7 +300,7 @@ async function workbenchTraceReadabilityWaitForCurrentProductCard(timeoutMs, pol
|
||||
if (stableCardCount === 1 && timelineCount === 1) return { card, traceId, scope: lastScope };
|
||||
}
|
||||
}
|
||||
await sleep(pollIntervalMs);
|
||||
await interruptibleRunnerSleep(pollIntervalMs);
|
||||
}
|
||||
const error = new Error("在 owning YAML 指定的 " + timeoutMs + "ms 内未找到当前新运行中的产品 Trace 卡片");
|
||||
error.details = { scope: lastScope, valuesRedacted: true };
|
||||
@@ -313,7 +313,7 @@ async function workbenchTraceReadabilityWaitForDisclosureOpen(card, traceId, tim
|
||||
while (Date.now() <= deadline) {
|
||||
latest = await workbenchTraceReadabilitySnapshot(card, traceId);
|
||||
if (latest.disclosure.open === true) return latest;
|
||||
await sleep(pollIntervalMs);
|
||||
await interruptibleRunnerSleep(pollIntervalMs);
|
||||
}
|
||||
return latest;
|
||||
}
|
||||
@@ -422,7 +422,7 @@ async function workbenchTraceReadabilityWaitForReadableRunning(card, traceId, ti
|
||||
if (workbenchTraceReadabilityTerminalStatus(latest.status)) {
|
||||
return { observed: false, reason: "terminal-before-readable-running-sample", elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
|
||||
}
|
||||
await sleep(pollIntervalMs);
|
||||
await interruptibleRunnerSleep(pollIntervalMs);
|
||||
}
|
||||
return { observed: false, reason: "readable-running-timeout", elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
|
||||
}
|
||||
@@ -436,7 +436,7 @@ async function workbenchTraceReadabilityWaitForTerminal(card, traceId, timeoutMs
|
||||
if (workbenchTraceReadabilityTerminalStatus(latest.status)) {
|
||||
return { observed: true, status: latest.status, elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
|
||||
}
|
||||
await sleep(pollIntervalMs);
|
||||
await interruptibleRunnerSleep(pollIntervalMs);
|
||||
}
|
||||
return { observed: false, status: latest.status, elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
|
||||
}
|
||||
@@ -466,7 +466,7 @@ async function workbenchTraceReadabilityWaitForStableTerminal(card, traceId, qui
|
||||
stableSinceMs = null;
|
||||
}
|
||||
previousSignature = signature;
|
||||
await sleep(pollIntervalMs);
|
||||
await interruptibleRunnerSleep(pollIntervalMs);
|
||||
}
|
||||
return { observed: workbenchTraceReadabilityTerminalStatus(latest.status), stable: false, sampleCount, elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
|
||||
}
|
||||
|
||||
@@ -2,13 +2,221 @@
|
||||
// Responsibility: Runner low-level IO, URL, YAML-derived policy parsing, redaction, and timeout utility source fragment.
|
||||
|
||||
export function nodeWebObserveRunnerUtilitySource(): string {
|
||||
return String.raw`async function appendJsonl(file, value) {
|
||||
await appendFile(file, JSON.stringify(sanitize(value)) + "\n", { mode: 0o600 });
|
||||
return String.raw`function appendJsonl(file, value) {
|
||||
let line;
|
||||
try {
|
||||
line = JSON.stringify(sanitize(value)) + "\n";
|
||||
} catch {
|
||||
recordJsonlDrop(file, 0);
|
||||
return Promise.resolve({ ok: false, dropped: true, reason: "json-encode-failed", valuesRedacted: true });
|
||||
}
|
||||
const bytes = Buffer.byteLength(line);
|
||||
if (bytes > maxJsonlLineBytes) {
|
||||
recordJsonlDrop(file, bytes);
|
||||
return Promise.resolve({ ok: false, dropped: true, reason: "entry-too-large", bytes, maxJsonlLineBytes, valuesRedacted: true });
|
||||
}
|
||||
if (jsonlWriterState.queue.length >= maxJsonlQueueEntries || jsonlWriterState.queuedBytes + bytes > maxJsonlQueueBytes) {
|
||||
recordJsonlDrop(file, bytes);
|
||||
return Promise.resolve({ ok: false, dropped: true, reason: "queue-saturated", bytes, valuesRedacted: true });
|
||||
}
|
||||
jsonlWriterState.acceptedEntries += 1;
|
||||
jsonlWriterState.acceptedBytes += bytes;
|
||||
jsonlWriterState.queuedBytes += bytes;
|
||||
const completion = new Promise((resolve) => jsonlWriterState.queue.push({ file, line, bytes, resolve }));
|
||||
startJsonlPump();
|
||||
return completion;
|
||||
}
|
||||
|
||||
function recordJsonlDrop(file, bytes) {
|
||||
jsonlWriterState.droppedEntries += 1;
|
||||
jsonlWriterState.droppedBytes += Math.max(0, Number(bytes) || 0);
|
||||
const key = path.basename(String(file || "unknown")).slice(0, 80);
|
||||
jsonlWriterState.dropsByFile[key] = (jsonlWriterState.dropsByFile[key] || 0) + 1;
|
||||
}
|
||||
|
||||
function startJsonlPump() {
|
||||
if (jsonlWriterState.pump) return jsonlWriterState.pump;
|
||||
jsonlWriterState.pump = (async () => {
|
||||
while (jsonlWriterState.queue.length > 0) {
|
||||
const entry = jsonlWriterState.queue.shift();
|
||||
jsonlWriterState.queuedBytes = Math.max(0, jsonlWriterState.queuedBytes - entry.bytes);
|
||||
jsonlWriterState.inFlight = 1;
|
||||
try {
|
||||
await appendFile(entry.file, entry.line, { mode: 0o600 });
|
||||
jsonlWriterState.writtenEntries += 1;
|
||||
jsonlWriterState.writtenBytes += entry.bytes;
|
||||
entry.resolve({ ok: true, dropped: false, valuesRedacted: true });
|
||||
} catch (error) {
|
||||
jsonlWriterState.failedEntries += 1;
|
||||
entry.resolve({ ok: false, dropped: false, error: errorSummary(error), valuesRedacted: true });
|
||||
} finally {
|
||||
jsonlWriterState.inFlight = 0;
|
||||
}
|
||||
}
|
||||
})().finally(() => {
|
||||
jsonlWriterState.pump = null;
|
||||
if (jsonlWriterState.queue.length > 0) startJsonlPump();
|
||||
});
|
||||
return jsonlWriterState.pump;
|
||||
}
|
||||
|
||||
function jsonlWriterStats() {
|
||||
return {
|
||||
queueEntries: jsonlWriterState.queue.length,
|
||||
queueBytes: jsonlWriterState.queuedBytes,
|
||||
inFlight: jsonlWriterState.inFlight,
|
||||
maxQueueEntries: maxJsonlQueueEntries,
|
||||
maxQueueBytes: maxJsonlQueueBytes,
|
||||
maxLineBytes: maxJsonlLineBytes,
|
||||
acceptedEntries: jsonlWriterState.acceptedEntries,
|
||||
acceptedBytes: jsonlWriterState.acceptedBytes,
|
||||
writtenEntries: jsonlWriterState.writtenEntries,
|
||||
writtenBytes: jsonlWriterState.writtenBytes,
|
||||
droppedEntries: jsonlWriterState.droppedEntries,
|
||||
droppedBytes: jsonlWriterState.droppedBytes,
|
||||
failedEntries: jsonlWriterState.failedEntries,
|
||||
dropsByFile: { ...jsonlWriterState.dropsByFile },
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function flushJsonlQueue() {
|
||||
try {
|
||||
await withHardTimeout((async () => {
|
||||
while (jsonlWriterState.queue.length > 0 || jsonlWriterState.inFlight > 0 || jsonlWriterState.pump) {
|
||||
if (jsonlWriterState.pump) await jsonlWriterState.pump;
|
||||
else if (jsonlWriterState.queue.length > 0) await startJsonlPump();
|
||||
else break;
|
||||
}
|
||||
})(), jsonlFlushTimeoutMs, "jsonl terminal flush exceeded " + jsonlFlushTimeoutMs + "ms");
|
||||
return { ok: true, timedOut: false, stats: jsonlWriterStats(), valuesRedacted: true };
|
||||
} catch (error) {
|
||||
return { ok: false, timedOut: true, error: errorSummary(error), stats: jsonlWriterStats(), valuesRedacted: true };
|
||||
}
|
||||
}
|
||||
|
||||
async function fileMeta(file) {
|
||||
const [buffer, stats] = await Promise.all([readFile(file), stat(file)]);
|
||||
return { byteCount: stats.size, sha256: "sha256:" + createHash("sha256").update(buffer).digest("hex") };
|
||||
const digestPromise = new Promise((resolve, reject) => {
|
||||
const digest = createHash("sha256");
|
||||
const stream = createReadStream(file);
|
||||
stream.on("data", (chunk) => digest.update(chunk));
|
||||
stream.once("error", reject);
|
||||
stream.once("end", () => resolve("sha256:" + digest.digest("hex")));
|
||||
});
|
||||
const [sha256, stats] = await Promise.all([digestPromise, stat(file)]);
|
||||
return { byteCount: stats.size, sha256 };
|
||||
}
|
||||
|
||||
function responseBodyReadContract(response, maxBytes = responseBodyMaxBytes) {
|
||||
let headers = {};
|
||||
try { headers = response?.headers?.() || {}; } catch {}
|
||||
const contentEncoding = String(headers["content-encoding"] || headers["Content-Encoding"] || "").trim().toLowerCase();
|
||||
if (contentEncoding && contentEncoding !== "identity") {
|
||||
return { ok: false, reason: "encoded-body-size-unbounded", contentEncoding: truncate(contentEncoding, 80), contentLength: null, maxBytes, valuesRedacted: true };
|
||||
}
|
||||
const rawContentLength = headers["content-length"] || headers["Content-Length"];
|
||||
const contentLength = Number(rawContentLength);
|
||||
if (!Number.isFinite(contentLength) || contentLength < 0) {
|
||||
return { ok: false, reason: "content-length-unavailable", contentLength: null, maxBytes, valuesRedacted: true };
|
||||
}
|
||||
if (contentLength > maxBytes) {
|
||||
return { ok: false, reason: "content-length-too-large", contentLength, maxBytes, valuesRedacted: true };
|
||||
}
|
||||
return { ok: true, reason: null, contentLength, maxBytes, valuesRedacted: true };
|
||||
}
|
||||
|
||||
async function boundedResponseJson(response, maxBytes = responseBodyMaxBytes) {
|
||||
const bodyContract = responseBodyReadContract(response, maxBytes);
|
||||
if (bodyContract.ok !== true) return { ok: false, value: null, error: bodyContract, bodyContract, valuesRedacted: true };
|
||||
const permit = tryAcquireResponseBodyPermit();
|
||||
if (!permit) return { ok: false, value: null, error: { reason: "response-body-inflight-saturated", valuesRedacted: true }, bodyContract, valuesRedacted: true };
|
||||
try {
|
||||
const value = await response.json();
|
||||
responseBodyState.completed += 1;
|
||||
return { ok: true, value, error: null, bodyContract, valuesRedacted: true };
|
||||
} catch (error) {
|
||||
responseBodyState.failed += 1;
|
||||
return { ok: false, value: null, error: errorSummary(error), bodyContract, valuesRedacted: true };
|
||||
} finally {
|
||||
permit.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function boundedResponseText(response, maxBytes = responseBodyMaxBytes) {
|
||||
const bodyContract = responseBodyReadContract(response, maxBytes);
|
||||
if (bodyContract.ok !== true) return { ok: false, value: null, error: bodyContract, bodyContract, valuesRedacted: true };
|
||||
const permit = tryAcquireResponseBodyPermit();
|
||||
if (!permit) return { ok: false, value: null, error: { reason: "response-body-inflight-saturated", valuesRedacted: true }, bodyContract, valuesRedacted: true };
|
||||
try {
|
||||
const value = await response.text();
|
||||
responseBodyState.completed += 1;
|
||||
return { ok: true, value, error: null, bodyContract, valuesRedacted: true };
|
||||
} catch (error) {
|
||||
responseBodyState.failed += 1;
|
||||
return { ok: false, value: null, error: errorSummary(error), bodyContract, valuesRedacted: true };
|
||||
} finally {
|
||||
permit.release();
|
||||
}
|
||||
}
|
||||
|
||||
function tryAcquireResponseBodyPermit() {
|
||||
if (responseBodyState.inFlight >= maxResponseBodyInFlight) {
|
||||
responseBodyState.skippedSaturated += 1;
|
||||
return null;
|
||||
}
|
||||
responseBodyState.inFlight += 1;
|
||||
responseBodyState.accepted += 1;
|
||||
let released = false;
|
||||
return { release() { if (released) return; released = true; responseBodyState.inFlight = Math.max(0, responseBodyState.inFlight - 1); } };
|
||||
}
|
||||
|
||||
function responseBodyStats() {
|
||||
return { ...responseBodyState, maxInFlight: maxResponseBodyInFlight, valuesRedacted: true };
|
||||
}
|
||||
|
||||
function trackPassiveTask(factory, label) {
|
||||
if (stopping || passiveTaskState.tasks.size >= maxPassiveListenerTasks) {
|
||||
passiveTaskState.dropped += 1;
|
||||
return false;
|
||||
}
|
||||
passiveTaskState.accepted += 1;
|
||||
let task;
|
||||
task = Promise.resolve()
|
||||
.then(factory)
|
||||
.then(() => { passiveTaskState.completed += 1; })
|
||||
.catch(async (error) => {
|
||||
passiveTaskState.failed += 1;
|
||||
await appendJsonl(files.errors, eventRecord("passive-task-error", { label, error: errorSummary(error), valuesRedacted: true })).catch(() => {});
|
||||
})
|
||||
.finally(() => passiveTaskState.tasks.delete(task));
|
||||
passiveTaskState.tasks.add(task);
|
||||
task.catch(() => {});
|
||||
return true;
|
||||
}
|
||||
|
||||
function passiveTaskStats() {
|
||||
return {
|
||||
inFlight: passiveTaskState.tasks.size,
|
||||
maxInFlight: maxPassiveListenerTasks,
|
||||
accepted: passiveTaskState.accepted,
|
||||
dropped: passiveTaskState.dropped,
|
||||
completed: passiveTaskState.completed,
|
||||
failed: passiveTaskState.failed,
|
||||
drainTimedOut: passiveTaskState.drainTimedOut,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function drainPassiveTasks() {
|
||||
try {
|
||||
await withHardTimeout((async () => {
|
||||
while (passiveTaskState.tasks.size > 0) await Promise.allSettled([...passiveTaskState.tasks]);
|
||||
})(), passiveTaskFlushTimeoutMs, "passive task drain exceeded " + passiveTaskFlushTimeoutMs + "ms");
|
||||
return { ok: true, timedOut: false, stats: passiveTaskStats(), valuesRedacted: true };
|
||||
} catch (error) {
|
||||
passiveTaskState.drainTimedOut = true;
|
||||
return { ok: false, timedOut: true, error: errorSummary(error), stats: passiveTaskStats(), valuesRedacted: true };
|
||||
}
|
||||
}
|
||||
|
||||
function currentPageUrl() {
|
||||
@@ -77,6 +285,14 @@ function boundedInteger(value, fallback, min, max) {
|
||||
return integer;
|
||||
}
|
||||
|
||||
function requiredBoundedInteger(value, name, min, max) {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < min || parsed > max) {
|
||||
throw new Error(name + " must come from the owning YAML and be an integer between " + min + " and " + max);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function positiveNumber(value, fallback) {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
@@ -306,9 +522,33 @@ function errorSummary(error) {
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return interruptibleRunnerSleep(ms);
|
||||
}
|
||||
|
||||
function boundedCleanupSleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
|
||||
}
|
||||
|
||||
function interruptibleRunnerSleep(ms) {
|
||||
if (stopping) return Promise.resolve();
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
runnerSleepWakes.delete(wake);
|
||||
resolve();
|
||||
}, Math.max(0, ms));
|
||||
const wake = () => {
|
||||
clearTimeout(timer);
|
||||
runnerSleepWakes.delete(wake);
|
||||
resolve();
|
||||
};
|
||||
runnerSleepWakes.add(wake);
|
||||
});
|
||||
}
|
||||
|
||||
function interruptRunnerSleep() {
|
||||
for (const wake of [...runnerSleepWakes]) wake();
|
||||
}
|
||||
|
||||
function withHardTimeout(promise, timeoutMs, message) {
|
||||
const guarded = Promise.resolve(promise);
|
||||
let timer = null;
|
||||
|
||||
@@ -1285,12 +1285,10 @@ async function launchWorkbenchFromTask(command) {
|
||||
}, { timeout: 30000 }).then(async (response) => {
|
||||
let chatPayload = null;
|
||||
let chatPayloadError = null;
|
||||
try {
|
||||
chatPayload = await response.json();
|
||||
} catch (error) {
|
||||
chatPayloadError = errorSummary(error);
|
||||
}
|
||||
const headers = response.headers();
|
||||
const chatBody = await boundedResponseJson(response);
|
||||
chatPayload = chatBody.value;
|
||||
chatPayloadError = chatBody.error;
|
||||
return {
|
||||
observed: true,
|
||||
status: response.status(),
|
||||
@@ -1316,11 +1314,9 @@ async function launchWorkbenchFromTask(command) {
|
||||
const launchTraceHeader = typeof headers["x-hwlab-otel-trace-id"] === "string" ? headers["x-hwlab-otel-trace-id"] : null;
|
||||
let payload = null;
|
||||
let responseParseError = null;
|
||||
try {
|
||||
payload = await launchResponse.json();
|
||||
} catch (error) {
|
||||
responseParseError = errorSummary(error);
|
||||
}
|
||||
const launchBody = await boundedResponseJson(launchResponse);
|
||||
payload = launchBody.value;
|
||||
responseParseError = launchBody.error;
|
||||
const sessionId = sessionIdFromAgentSessionPayload(payload);
|
||||
const workbenchUrl = safeUrlPath(payload?.workbenchUrl || payload?.url || "");
|
||||
const contractVersion = typeof payload?.contractVersion === "string" ? payload.contractVersion : null;
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import type { CommandResult } from "./command";
|
||||
import { rootPath } from "./config";
|
||||
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
|
||||
import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeMemoryStartGuardConfigRef, webProbeScreenshotFullPage } from "./hwlab-node-web-probe-memory-guard";
|
||||
import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source";
|
||||
|
||||
const spec = hwlabRuntimeLaneSpecForNode("v03", "NC01");
|
||||
const thresholdBytes = 4 * 1024 * 1024 * 1024;
|
||||
|
||||
function memoryPayload(availableBytes: number, extra: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return {
|
||||
ok: true,
|
||||
action: "gc remote memory",
|
||||
providerId: "NC01",
|
||||
metric: "MemAvailable",
|
||||
source: "/proc/meminfo",
|
||||
swapExcluded: true,
|
||||
memory: { availableBytes, availableHuman: "fixture", ...extra },
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function exportedFunctionSource(source: string, name: string): string {
|
||||
const start = source.indexOf(`export function ${name}(`);
|
||||
assert.ok(start >= 0, `missing exported function ${name}`);
|
||||
const nextExport = source.indexOf("\nexport function ", start + 1);
|
||||
const nextFunction = source.indexOf("\nfunction ", start + 1);
|
||||
const next = [nextExport, nextFunction].filter((value) => value >= 0).sort((left, right) => left - right)[0] ?? -1;
|
||||
return source.slice(start, next < 0 ? source.length : next);
|
||||
}
|
||||
|
||||
test("WebProbe memory threshold is read from the single owning YAML policy", () => {
|
||||
const policy = spec.webProbe?.resourcePolicy?.memoryStartGuard;
|
||||
assert.equal(policy?.thresholdBytes, thresholdBytes);
|
||||
assert.equal(policy?.metric, "MemAvailable");
|
||||
assert.equal(policy?.source, "/proc/meminfo");
|
||||
assert.equal(policy?.swapExcluded, true);
|
||||
assert.equal(policy?.manualComparator, "less-than-or-equal");
|
||||
assert.equal(policy?.sentinelComparator, "less-than");
|
||||
assert.match(webProbeMemoryStartGuardConfigRef, /resourcePolicy\.memoryStartGuard$/u);
|
||||
});
|
||||
|
||||
test("browser evidence limits and viewport screenshots are read from owning YAML", () => {
|
||||
const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
|
||||
assert.deepEqual(policy, {
|
||||
screenshotMode: "viewport",
|
||||
maxNetworkRequestEntries: 200,
|
||||
maxNetworkRequestBytes: 262144,
|
||||
maxNetworkFailureEntries: 100,
|
||||
maxNetworkFailureBytes: 131072,
|
||||
});
|
||||
const assignments = webProbeBrowserEvidenceEnvAssignments(spec).join(" ");
|
||||
assert.match(assignments, /UNIDESK_WEB_PROBE_SCREENSHOT_MODE='viewport'/u);
|
||||
assert.match(assignments, /UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES='200'/u);
|
||||
assert.match(assignments, /UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES='131072'/u);
|
||||
});
|
||||
|
||||
test("manual start is blocked at exactly 4 GiB and requires controlled GC then MemAvailable recheck", () => {
|
||||
const guard = evaluateWebProbeMemoryGuard(spec, "manual-start", memoryPayload(thresholdBytes));
|
||||
const memory = guard.memory as Record<string, unknown>;
|
||||
const gcNext = guard.gcNext as Record<string, unknown>;
|
||||
|
||||
assert.equal(guard.status, "blocked");
|
||||
assert.equal(guard.startBlocked, true);
|
||||
assert.equal(guard.decision, "gc-required");
|
||||
assert.equal(memory.metric, "MemAvailable");
|
||||
assert.equal(memory.source, "/proc/meminfo");
|
||||
assert.equal(memory.swapExcluded, true);
|
||||
assert.equal(memory.comparator, "less-than-or-equal");
|
||||
assert.equal(memory.thresholdMatched, true);
|
||||
assert.equal(gcNext.requiredBeforeStart, true);
|
||||
assert.match(String(gcNext.planCommand), /gc remote NC01 plan --memory-pressure-only$/u);
|
||||
assert.match(String(gcNext.runCommand), /gc remote NC01 run --confirm --memory-pressure-only$/u);
|
||||
assert.match(String(gcNext.statusCommandTemplate), /gc remote NC01 status --job-id <job-id>$/u);
|
||||
assert.match(String(gcNext.recheckCommand), /gc remote NC01 memory$/u);
|
||||
assert.equal(gcNext.recheckMetric, "MemAvailable");
|
||||
assert.deepEqual(gcNext.requiredOrder, ["plan", "run", "status-until-terminal", "memory-recheck"]);
|
||||
assert.deepEqual(gcNext.startRemainsBlockedUntil, ["gc-job-terminal", "MemAvailable-threshold-cleared"]);
|
||||
});
|
||||
|
||||
test("sentinel allows exactly 4 GiB but skips one byte below without considering swap", () => {
|
||||
const exact = evaluateWebProbeMemoryGuard(spec, "sentinel-cadence", memoryPayload(thresholdBytes, { swapFreeBytes: 12 * 1024 ** 3 }));
|
||||
const below = evaluateWebProbeMemoryGuard(spec, "sentinel-cadence", memoryPayload(thresholdBytes - 1, { swapFreeBytes: 12 * 1024 ** 3 }));
|
||||
const belowMemory = below.memory as Record<string, unknown>;
|
||||
|
||||
assert.equal(exact.status, "allowed");
|
||||
assert.equal(exact.startBlocked, false);
|
||||
assert.equal((exact.memory as Record<string, unknown>).comparator, "less-than");
|
||||
assert.equal(below.status, "skipped");
|
||||
assert.equal(below.startBlocked, true);
|
||||
assert.equal(below.decision, "skip-observer");
|
||||
assert.equal(belowMemory.availableBytes, thresholdBytes - 1);
|
||||
assert.equal(belowMemory.swapExcluded, true);
|
||||
});
|
||||
|
||||
test("memory guard rejects a snapshot that is not the /proc/meminfo MemAvailable contract", () => {
|
||||
const guard = evaluateWebProbeMemoryGuard(spec, "manual-start", {
|
||||
...memoryPayload(thresholdBytes + 1),
|
||||
source: "cgroup-v2-memory.current",
|
||||
});
|
||||
assert.equal(guard.status, "blocked");
|
||||
assert.equal(guard.reason, "memory-snapshot-contract-invalid");
|
||||
assert.equal(guard.startBlocked, true);
|
||||
});
|
||||
|
||||
test("memory guard executes only the controlled read-only remote memory entry", () => {
|
||||
let observedCommand: string[] = [];
|
||||
let observedTimeoutMs = 0;
|
||||
const execute = (command: string[], cwd: string, options: { timeoutMs?: number }): CommandResult => {
|
||||
observedCommand = command;
|
||||
observedTimeoutMs = options.timeoutMs ?? 0;
|
||||
return {
|
||||
command,
|
||||
cwd,
|
||||
exitCode: 0,
|
||||
stdout: JSON.stringify({ ok: true, command: "gc remote NC01 memory", data: memoryPayload(thresholdBytes + 1) }),
|
||||
stderr: "",
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
};
|
||||
};
|
||||
const guard = runWebProbeMemoryGuard(spec, "manual-start", execute);
|
||||
|
||||
assert.deepEqual(observedCommand, ["bun", "scripts/cli.ts", "gc", "remote", "NC01", "memory"]);
|
||||
assert.equal(observedTimeoutMs, 30_000);
|
||||
assert.equal(guard.status, "allowed");
|
||||
});
|
||||
|
||||
test("remote memory collector reads MemAvailable directly and excludes swap", () => {
|
||||
const source = readFileSync(rootPath("scripts/src/gc-remote-runner.py"), "utf8");
|
||||
const start = source.indexOf("def collect_memory_snapshot():");
|
||||
const end = source.indexOf("\ndef observe_run_record", start);
|
||||
const collector = source.slice(start, end);
|
||||
|
||||
assert.match(collector, /source = "\/proc\/meminfo"/u);
|
||||
assert.match(collector, /fields\.get\("MemAvailable"\)/u);
|
||||
assert.match(collector, /"swapExcluded": True/u);
|
||||
assert.doesNotMatch(collector, /SwapFree|SwapTotal|\["free", "-b"\]/u);
|
||||
});
|
||||
|
||||
test("manual and sentinel gates return before any observer launch path", () => {
|
||||
const manualSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe-actions.ts"), "utf8");
|
||||
const manualStart = manualSource.indexOf("export function runNodeWebProbeObserveStart");
|
||||
const manualGuard = manualSource.indexOf("runWebProbeMemoryGuard(spec, options.memoryGuardMode)", manualStart);
|
||||
const manualReturn = manualSource.indexOf("startBlocked: true", manualGuard);
|
||||
const manualJob = manualSource.indexOf("const jobId =", manualStart);
|
||||
const manualLaunch = manualSource.indexOf("setsid env", manualStart);
|
||||
assert.ok(manualStart >= 0 && manualGuard > manualStart && manualReturn > manualGuard);
|
||||
assert.ok(manualReturn < manualJob && manualJob < manualLaunch);
|
||||
|
||||
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5-observe.ts"), "utf8");
|
||||
const quickVerify = sentinelSource.indexOf("export function runSentinelQuickVerify");
|
||||
const sentinelGuard = sentinelSource.indexOf("runWebProbeMemoryGuard(state.spec, \"sentinel-cadence\")", quickVerify);
|
||||
const sentinelSkip = sentinelSource.indexOf("return recordSkippedQuickVerify", sentinelGuard);
|
||||
const providerPreflight = sentinelSource.indexOf("runSentinelExactProviderProfilePreflight", quickVerify);
|
||||
const observeStart = sentinelSource.indexOf('"web-probe", "observe", "start"', quickVerify);
|
||||
assert.ok(quickVerify >= 0 && sentinelGuard > quickVerify && sentinelSkip > sentinelGuard);
|
||||
assert.ok(sentinelSkip < providerPreflight && providerPreflight < observeStart);
|
||||
assert.match(sentinelSource.slice(observeStart, observeStart + 800), /--sentinel-cadence/u);
|
||||
|
||||
const serviceSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-service.ts"), "utf8");
|
||||
const plannedStart = serviceSource.indexOf('"bun", "scripts/cli.ts", "web-probe", "observe", "start"');
|
||||
assert.match(serviceSource.slice(plannedStart, plannedStart + 800), /--sentinel-cadence/u);
|
||||
});
|
||||
|
||||
test("all browser-launch entries use the shared preflight and final launch guard", () => {
|
||||
const probeSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
|
||||
const scriptSource = readFileSync(rootPath("scripts/src/hwlab-node/web-observe-scripts.ts"), "utf8");
|
||||
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5.ts"), "utf8");
|
||||
const parserSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe.ts"), "utf8");
|
||||
|
||||
const run = exportedFunctionSource(probeSource, "runNodeWebProbe");
|
||||
const screenshot = exportedFunctionSource(probeSource, "runNodeWebProbeScreenshot");
|
||||
const asyncRun = exportedFunctionSource(probeSource, "runNodeWebProbeAsync");
|
||||
const script = exportedFunctionSource(scriptSource, "runNodeWebProbeScript");
|
||||
const scriptShell = exportedFunctionSource(scriptSource, "nodeWebProbeScriptRemoteShell");
|
||||
const dashboard = exportedFunctionSource(sentinelSource, "probeSentinelDashboardBrowser");
|
||||
const parser = exportedFunctionSource(parserSource, "parseNodeWebProbeOptions");
|
||||
|
||||
assert.doesNotMatch(probeSource, /runNodeWebProbeMemoryGuard/u);
|
||||
assert.match(run, /if \(options\.action === "script"\) return runNodeWebProbeScript/u);
|
||||
assert.match(run, /if \(options\.commandTimeoutSeconds > 55\) return runNodeWebProbeAsync/u);
|
||||
assert.match(run, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start"/u);
|
||||
assert.match(screenshot, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start", webProbeScreenshotRemoteScript/u);
|
||||
assert.match(asyncRun, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start"[\s\S]+"structured-output"\)/u);
|
||||
assert.match(script, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+nodeWebProbeScriptRemoteShell/u);
|
||||
assert.match(scriptShell, /webProbeGuardedLaunchShell\(spec, "manual-start"/u);
|
||||
assert.match(parser, /if \(actionRaw === "opencode-smoke"\)[\s\S]+action: "script"/u);
|
||||
assert.match(dashboard, /runWebProbeMemoryGuard\(state\.spec, "sentinel-cadence"\)[\s\S]+webProbeGuardedLaunchShell\(state\.spec, "sentinel-cadence"/u);
|
||||
assert.match(dashboard, /UNIDESK_SENTINEL_DASHBOARD_TRIGGER=.*options\.action === "trigger"/u);
|
||||
assert.match(dashboard, /UNIDESK_SENTINEL_DASHBOARD_CAPTURE=.*options\.action === "screenshot"/u);
|
||||
});
|
||||
|
||||
test("read-only and control-plane entries stay outside browser launch guard", () => {
|
||||
const dispatchSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
|
||||
const dispatchStart = dispatchSource.indexOf("export function runNodeWebProbe(");
|
||||
const dispatchEnd = dispatchSource.indexOf("export function runNodeWebProbeScreenshot", dispatchStart);
|
||||
const dispatch = dispatchSource.slice(dispatchStart, dispatchEnd);
|
||||
const readOnlyReturn = dispatch.indexOf('options.action === "observe" && options.observeAction !== "start"');
|
||||
const firstGuard = dispatch.indexOf("runWebProbeMemoryGuard");
|
||||
assert.ok(readOnlyReturn >= 0 && firstGuard > readOnlyReturn);
|
||||
assert.match(dispatch.slice(0, readOnlyReturn), /options\.action === "sentinel"/u);
|
||||
assert.match(dispatch.slice(0, readOnlyReturn), /options\.action === "screenshot"/u);
|
||||
|
||||
const sentinelDispatch = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-cicd.ts"), "utf8");
|
||||
const sentinel = exportedFunctionSource(sentinelDispatch, "runWebProbeSentinelCommand");
|
||||
assert.doesNotMatch(sentinel, /runWebProbeMemoryGuard|webProbeGuardedLaunchShell/u);
|
||||
assert.match(sentinel, /options\.kind === "dashboard"/u);
|
||||
});
|
||||
|
||||
test("generated runner bounds network evidence and defaults screenshots to viewport", () => {
|
||||
const source = nodeWebProbeScriptRunnerSource();
|
||||
assert.match(source, /maxNetworkRequestEntries/u);
|
||||
assert.match(source, /maxNetworkRequestBytes/u);
|
||||
assert.match(source, /maxNetworkFailureEntries/u);
|
||||
assert.match(source, /maxNetworkFailureBytes/u);
|
||||
assert.match(source, /apiRequestEvidenceDropped/u);
|
||||
assert.match(source, /apiFailureEvidenceDropped/u);
|
||||
assert.doesNotMatch(source, /fullPage: true/u);
|
||||
assert.equal((source.match(/fullPage: screenshotMode !== "viewport"/gu) ?? []).length, 2);
|
||||
|
||||
const parserSource = exportedFunctionSource(readFileSync(rootPath("scripts/src/hwlab-node/web-probe.ts"), "utf8"), "parseNodeWebProbeOptions");
|
||||
assert.match(parserSource, /fullPage: args\.includes\("--full-page"\) && !args\.includes\("--no-full-page"\)/u);
|
||||
const screenshotSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
|
||||
assert.match(screenshotSource, /webProbeScreenshotFullPage\(spec, options\.fullPage\)/u);
|
||||
assert.match(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE === "1"/u);
|
||||
assert.doesNotMatch(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE !== "0"/u);
|
||||
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5.ts"), "utf8");
|
||||
assert.match(sentinelSource, /webProbeScreenshotFullPage\(state\.spec, options\.fullPage\)/u);
|
||||
assert.match(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE === "1"/u);
|
||||
assert.doesNotMatch(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0"/u);
|
||||
assert.equal(webProbeScreenshotFullPage(spec, false), false);
|
||||
assert.equal(webProbeScreenshotFullPage({ ...spec, webProbe: { ...spec.webProbe!, resourcePolicy: { ...spec.webProbe!.resourcePolicy!, browserEvidence: { ...spec.webProbe!.resourcePolicy!.browserEvidence!, screenshotMode: "full-page" } } } }, false), true);
|
||||
assert.equal(webProbeScreenshotFullPage(spec, true), true);
|
||||
});
|
||||
|
||||
test("final launch guard rechecks MemAvailable under the configured lock", () => {
|
||||
const sync = webProbeGuardedLaunchShell(spec, "manual-start", "node runner.mjs");
|
||||
const async = webProbeGuardedLaunchShell(spec, "manual-start", "node helper.mjs start", "structured-output");
|
||||
assert.match(sync, /flock -w '15'/u);
|
||||
assert.match(sync, /MemAvailable:/u);
|
||||
assert.match(sync, /mem_available_bytes.*-le '4294967296'/u);
|
||||
assert.match(sync, /setsid sh -c/u);
|
||||
assert.match(sync, /exec 9>&-/u);
|
||||
assert.equal(sync.includes("playwright"), false);
|
||||
assert.equal(sync.includes("fs.readlinkSync('/proc/' + name + '/exe')"), true);
|
||||
assert.equal(sync.includes("trap 'browser_exit=$?; trap - EXIT TERM INT HUP; cleanup_browser_group"), true);
|
||||
assert.equal(sync.includes('kill -TERM -- "-$browser_job_pid"'), true);
|
||||
assert.equal(sync.includes('kill -0 -- "-$browser_job_pid"'), true);
|
||||
assert.equal(sync.includes('kill -KILL -- "-$browser_job_pid"'), true);
|
||||
assert.equal(async.includes("browserCreated === true"), false);
|
||||
assert.equal(async.includes("data.workerPid, data.jobPid, data.pid"), true);
|
||||
assert.equal(async.includes("[data.browserPid, data.workerPid"), false);
|
||||
assert.equal(async.includes("reportedBrowserPid === null || row.pid === reportedBrowserPid"), true);
|
||||
assert.equal(async.includes("then flock -u 9; exit 43; fi"), true);
|
||||
assert.equal(async.includes("fs.readlinkSync('/proc/' + name + '/exe')"), true);
|
||||
});
|
||||
@@ -0,0 +1,324 @@
|
||||
// SPEC: PJ2026-01060508 Web 哨兵。WebProbe 启动前的 YAML-first 物理内存资格判定。
|
||||
import type { CommandResult } from "./command";
|
||||
import { runCommand } from "./command";
|
||||
import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery";
|
||||
import { repoRoot } from "./config";
|
||||
import type { HwlabRuntimeLaneSpec, HwlabRuntimeWebProbeMemoryStartGuardSpec } from "./hwlab-node-lanes";
|
||||
|
||||
export const webProbeMemoryStartGuardConfigRef = "config/hwlab-node-lanes.yaml#templates.hwlabV03.webProbeWorkbench.resourcePolicy.memoryStartGuard";
|
||||
|
||||
export type WebProbeMemoryGuardMode = "manual-start" | "sentinel-cadence";
|
||||
export type WebProbeMemoryGuardStatus = "allowed" | "blocked" | "skipped";
|
||||
|
||||
type MemoryGuardExecutor = (
|
||||
command: string[],
|
||||
cwd: string,
|
||||
options: { timeoutMs?: number },
|
||||
) => CommandResult;
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function memoryPolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeMemoryStartGuardSpec | null {
|
||||
return spec.webProbe?.resourcePolicy?.memoryStartGuard ?? null;
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replace(/'/g, `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
export function webProbeBrowserEvidenceEnvAssignments(spec: HwlabRuntimeLaneSpec): string[] {
|
||||
const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
|
||||
if (policy === undefined) throw new Error("web-probe browser evidence policy is missing from the owning YAML");
|
||||
return [
|
||||
`UNIDESK_WEB_PROBE_SCREENSHOT_MODE=${shellQuote(policy.screenshotMode)}`,
|
||||
`UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES=${shellQuote(String(policy.maxNetworkRequestEntries))}`,
|
||||
`UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_BYTES=${shellQuote(String(policy.maxNetworkRequestBytes))}`,
|
||||
`UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_ENTRIES=${shellQuote(String(policy.maxNetworkFailureEntries))}`,
|
||||
`UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES=${shellQuote(String(policy.maxNetworkFailureBytes))}`,
|
||||
];
|
||||
}
|
||||
|
||||
export function webProbeScreenshotFullPage(spec: HwlabRuntimeLaneSpec, explicitFullPage: boolean): boolean {
|
||||
const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
|
||||
if (policy === undefined) throw new Error("web-probe browser evidence policy is missing from the owning YAML");
|
||||
return explicitFullPage || policy.screenshotMode !== "viewport";
|
||||
}
|
||||
|
||||
export function webProbeGuardedLaunchShell(
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
mode: WebProbeMemoryGuardMode,
|
||||
command: string,
|
||||
readiness: "child-process" | "structured-output" = "child-process",
|
||||
): string {
|
||||
const policy = memoryPolicy(spec);
|
||||
if (policy === null) throw new Error("web-probe memory start policy is missing from the owning YAML");
|
||||
const closeStepTimeoutMs = spec.webProbe?.resourcePolicy?.observerLifecycle.closeStepTimeoutMs;
|
||||
if (closeStepTimeoutMs === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML");
|
||||
const cleanupPollMilliseconds = policy.launchReadyPollMilliseconds;
|
||||
const cleanupPollSeconds = Math.max(0.05, cleanupPollMilliseconds / 1000);
|
||||
const cleanupPollAttempts = Math.max(1, Math.ceil(closeStepTimeoutMs / cleanupPollMilliseconds));
|
||||
const comparator = mode === "manual-start" ? "-le" : "-lt";
|
||||
const common = [
|
||||
`exec 9>${shellQuote(policy.launchLockPath)}`,
|
||||
`flock -w ${shellQuote(String(policy.launchLockTimeoutSeconds))} 9`,
|
||||
"mem_available_kib=$(awk '$1 == \"MemAvailable:\" { print $2; exit }' /proc/meminfo)",
|
||||
"case \"$mem_available_kib\" in ''|*[!0-9]*) printf '%s\\n' 'web-probe final memory guard could not read MemAvailable' >&2; exit 42;; esac",
|
||||
"mem_available_bytes=$((mem_available_kib * 1024))",
|
||||
`if [ "$mem_available_bytes" ${comparator} ${shellQuote(String(policy.thresholdBytes))} ]; then printf '%s\\n' 'web-probe final memory guard blocked browser launch' >&2; exit 42; fi`,
|
||||
];
|
||||
if (readiness === "structured-output") return [
|
||||
...common,
|
||||
"launch_output=$(mktemp)",
|
||||
"launch_error=$(mktemp)",
|
||||
"trap 'rm -f \"$launch_output\" \"$launch_error\"' EXIT",
|
||||
`if ! (exec 9>&-; ${command}) >"$launch_output" 2>"$launch_error"; then cat "$launch_output"; cat "$launch_error" >&2; exit 43; fi`,
|
||||
"cat \"$launch_output\"",
|
||||
"cat \"$launch_error\" >&2",
|
||||
`if ! node - "$launch_output" ${shellQuote(String(policy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(policy.launchReadyPollMilliseconds))} <<'UNIDESK_WEB_PROBE_STRUCTURED_READINESS'`,
|
||||
"const fs = require('fs');",
|
||||
"const text = fs.readFileSync(process.argv[2], 'utf8').trim();",
|
||||
"const timeoutMs = Number(process.argv[3]);",
|
||||
"const pollMs = Number(process.argv[4]);",
|
||||
"let value = null; try { value = JSON.parse(text); } catch {}",
|
||||
"const data = value && typeof value === 'object' && value.data && typeof value.data === 'object' ? value.data : value;",
|
||||
"if (!data || typeof data !== 'object' || !data.jobId) process.exit(43);",
|
||||
"const rootPid = [data.workerPid, data.jobPid, data.pid].find((item) => Number.isInteger(item) && item > 0);",
|
||||
"if (!rootPid) process.exit(43);",
|
||||
"const reportedBrowserPid = Number.isInteger(data.browserPid) && data.browserPid > 0 ? data.browserPid : null;",
|
||||
"const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
|
||||
"const browserDescendant = () => {",
|
||||
" const rows = [];",
|
||||
" for (const name of fs.readdirSync('/proc')) {",
|
||||
" if (!/^[0-9]+$/.test(name)) continue;",
|
||||
" try {",
|
||||
" const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');",
|
||||
" const close = stat.lastIndexOf(')');",
|
||||
" const fields = stat.slice(close + 2).split(' ');",
|
||||
" const ppid = Number(fields[1]);",
|
||||
" const executable = fs.readlinkSync('/proc/' + name + '/exe');",
|
||||
" rows.push({ pid: Number(name), ppid, executable });",
|
||||
" } catch {}",
|
||||
" }",
|
||||
" const descendants = new Set([rootPid]);",
|
||||
" let changed = true;",
|
||||
" while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }",
|
||||
" return rows.some((row) => descendants.has(row.pid) && (reportedBrowserPid === null || row.pid === reportedBrowserPid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable));",
|
||||
"};",
|
||||
"(async () => {",
|
||||
" const deadline = Date.now() + timeoutMs;",
|
||||
" while (Date.now() <= deadline) { if (browserDescendant()) process.exit(0); await sleep(pollMs); }",
|
||||
" process.exit(43);",
|
||||
"})().catch(() => process.exit(43));",
|
||||
"UNIDESK_WEB_PROBE_STRUCTURED_READINESS",
|
||||
"then flock -u 9; exit 43; fi",
|
||||
"flock -u 9",
|
||||
].join("\n");
|
||||
return [
|
||||
...common,
|
||||
`setsid sh -c ${shellQuote(`exec 9>&-\n${command}`)} &`,
|
||||
"browser_job_pid=$!",
|
||||
"cleanup_browser_group() {",
|
||||
" kill -TERM -- \"-$browser_job_pid\" 2>/dev/null || true",
|
||||
" cleanup_attempt=0",
|
||||
` while kill -0 -- "-$browser_job_pid" 2>/dev/null && [ "$cleanup_attempt" -lt ${cleanupPollAttempts} ]; do cleanup_attempt=$((cleanup_attempt + 1)); sleep ${cleanupPollSeconds}; done`,
|
||||
" if kill -0 -- \"-$browser_job_pid\" 2>/dev/null; then kill -KILL -- \"-$browser_job_pid\" 2>/dev/null || true; fi",
|
||||
" wait \"$browser_job_pid\" 2>/dev/null || true",
|
||||
"}",
|
||||
"trap 'browser_exit=$?; trap - EXIT TERM INT HUP; cleanup_browser_group; exit \"$browser_exit\"' EXIT",
|
||||
"trap 'exit 143' TERM",
|
||||
"trap 'exit 130' INT",
|
||||
"trap 'exit 129' HUP",
|
||||
"browser_ready=false",
|
||||
`browser_ready_deadline=$(( $(date +%s) + ${policy.launchReadyTimeoutSeconds} ))`,
|
||||
"while kill -0 \"$browser_job_pid\" 2>/dev/null; do",
|
||||
" if node - \"$browser_job_pid\" <<'UNIDESK_WEB_PROBE_CHILD_READINESS'",
|
||||
"const fs = require('fs');",
|
||||
"const rootPid = Number(process.argv[2]);",
|
||||
"const rows = [];",
|
||||
"for (const name of fs.readdirSync('/proc')) {",
|
||||
" if (!/^[0-9]+$/.test(name)) continue;",
|
||||
" try {",
|
||||
" const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');",
|
||||
" const close = stat.lastIndexOf(')');",
|
||||
" const fields = stat.slice(close + 2).split(' ');",
|
||||
" rows.push({ pid: Number(name), ppid: Number(fields[1]), executable: fs.readlinkSync('/proc/' + name + '/exe') });",
|
||||
" } catch {}",
|
||||
"}",
|
||||
"const descendants = new Set([rootPid]);",
|
||||
"let changed = true;",
|
||||
"while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }",
|
||||
"process.exit(rows.some((row) => descendants.has(row.pid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable)) ? 0 : 1);",
|
||||
"UNIDESK_WEB_PROBE_CHILD_READINESS",
|
||||
" then browser_ready=true; break; fi",
|
||||
" if [ \"$(date +%s)\" -ge \"$browser_ready_deadline\" ]; then break; fi",
|
||||
` sleep ${Math.max(0.05, policy.launchReadyPollMilliseconds / 1000)}`,
|
||||
"done",
|
||||
"flock -u 9",
|
||||
"if [ \"$browser_ready\" != true ]; then",
|
||||
" printf '%s\\n' 'web-probe browser launch readiness was not confirmed' >&2",
|
||||
" exit 43",
|
||||
"fi",
|
||||
"set +e",
|
||||
"wait \"$browser_job_pid\"",
|
||||
"browser_exit=$?",
|
||||
"set -e",
|
||||
"exit \"$browser_exit\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function humanBytes(value: number | null): string | null {
|
||||
if (value === null || !Number.isFinite(value)) return null;
|
||||
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
||||
let current = value;
|
||||
let index = 0;
|
||||
while (Math.abs(current) >= 1024 && index < units.length - 1) {
|
||||
current /= 1024;
|
||||
index += 1;
|
||||
}
|
||||
return `${current.toFixed(index === 0 ? 0 : 2)} ${units[index]}`;
|
||||
}
|
||||
|
||||
function controlledGcNext(node: string): Record<string, unknown> {
|
||||
return {
|
||||
automaticGc: false,
|
||||
requiredOrder: ["plan", "run", "status-until-terminal", "memory-recheck"],
|
||||
scope: "memory-pressure-only",
|
||||
planCommand: `bun scripts/cli.ts gc remote ${node} plan --memory-pressure-only`,
|
||||
runCommand: `bun scripts/cli.ts gc remote ${node} run --confirm --memory-pressure-only`,
|
||||
jobIdSource: "gc remote run result.data.jobId",
|
||||
statusCommandTemplate: `bun scripts/cli.ts gc remote ${node} status --job-id <job-id>`,
|
||||
terminalStatuses: ["succeeded", "failed", "completed", "blocked"],
|
||||
statusRequiredWhenJobIdReturned: true,
|
||||
recheckCommand: `bun scripts/cli.ts gc remote ${node} memory`,
|
||||
recheckMetric: "MemAvailable",
|
||||
recheckSource: "/proc/meminfo",
|
||||
swapExcluded: true,
|
||||
startRemainsBlockedUntil: ["gc-job-terminal", "MemAvailable-threshold-cleared"],
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function unavailableGuard(spec: HwlabRuntimeLaneSpec, mode: WebProbeMemoryGuardMode, reason: string): Record<string, unknown> {
|
||||
const policy = memoryPolicy(spec);
|
||||
const comparator = policy === null
|
||||
? null
|
||||
: mode === "manual-start"
|
||||
? policy.manualComparator
|
||||
: policy.sentinelComparator;
|
||||
return {
|
||||
ok: false,
|
||||
status: "blocked" satisfies WebProbeMemoryGuardStatus,
|
||||
eligible: false,
|
||||
startBlocked: true,
|
||||
decision: "do-not-start-observer",
|
||||
reason,
|
||||
mode,
|
||||
policyConfigRef: webProbeMemoryStartGuardConfigRef,
|
||||
memory: {
|
||||
metric: policy?.metric ?? "MemAvailable",
|
||||
source: policy?.source ?? "/proc/meminfo",
|
||||
swapExcluded: policy?.swapExcluded ?? true,
|
||||
availableBytes: null,
|
||||
availableHuman: null,
|
||||
thresholdBytes: policy?.thresholdBytes ?? null,
|
||||
thresholdHuman: humanBytes(policy?.thresholdBytes ?? null),
|
||||
comparator,
|
||||
thresholdMatched: null,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
gcNext: controlledGcNext(spec.nodeId),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
export function evaluateWebProbeMemoryGuard(
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
mode: WebProbeMemoryGuardMode,
|
||||
snapshotPayload: unknown,
|
||||
): Record<string, unknown> {
|
||||
const policy = memoryPolicy(spec);
|
||||
if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing");
|
||||
const payload = record(snapshotPayload);
|
||||
const memory = record(payload.memory);
|
||||
const availableBytes = typeof memory.availableBytes === "number" && Number.isSafeInteger(memory.availableBytes) && memory.availableBytes >= 0
|
||||
? memory.availableBytes
|
||||
: null;
|
||||
const metadataMatches = payload.ok === true
|
||||
&& payload.action === "gc remote memory"
|
||||
&& payload.metric === policy.metric
|
||||
&& payload.source === policy.source
|
||||
&& payload.swapExcluded === true;
|
||||
if (!metadataMatches || availableBytes === null) {
|
||||
return unavailableGuard(spec, mode, metadataMatches ? "memavailable-unavailable" : "memory-snapshot-contract-invalid");
|
||||
}
|
||||
const comparator = mode === "manual-start" ? policy.manualComparator : policy.sentinelComparator;
|
||||
const thresholdMatched = mode === "manual-start"
|
||||
? availableBytes <= policy.thresholdBytes
|
||||
: availableBytes < policy.thresholdBytes;
|
||||
const status: WebProbeMemoryGuardStatus = thresholdMatched
|
||||
? mode === "manual-start" ? "blocked" : "skipped"
|
||||
: "allowed";
|
||||
return {
|
||||
ok: status === "allowed",
|
||||
status,
|
||||
eligible: status === "allowed",
|
||||
startBlocked: status !== "allowed",
|
||||
decision: status === "allowed" ? "start-observer" : mode === "manual-start" ? "gc-required" : "skip-observer",
|
||||
reason: status === "allowed" ? null : "memavailable-threshold-matched",
|
||||
mode,
|
||||
policyConfigRef: webProbeMemoryStartGuardConfigRef,
|
||||
memory: {
|
||||
metric: policy.metric,
|
||||
source: policy.source,
|
||||
swapExcluded: policy.swapExcluded,
|
||||
availableBytes,
|
||||
availableHuman: humanBytes(availableBytes),
|
||||
thresholdBytes: policy.thresholdBytes,
|
||||
thresholdHuman: humanBytes(policy.thresholdBytes),
|
||||
comparator,
|
||||
thresholdMatched,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
gcNext: status === "blocked" && mode === "manual-start"
|
||||
? { ...controlledGcNext(spec.nodeId), requiredBeforeStart: true }
|
||||
: { ...controlledGcNext(spec.nodeId), requiredBeforeStart: false },
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
export function runWebProbeMemoryGuard(
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
mode: WebProbeMemoryGuardMode,
|
||||
execute: MemoryGuardExecutor = runCommand,
|
||||
): Record<string, unknown> {
|
||||
const policy = memoryPolicy(spec);
|
||||
if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing");
|
||||
const command = ["bun", "scripts/cli.ts", "gc", "remote", spec.nodeId, "memory"];
|
||||
const result = execute(command, repoRoot, { timeoutMs: policy.snapshotTimeoutSeconds * 1000 });
|
||||
const resolution = resolveCliChildJsonCommandResult({
|
||||
result,
|
||||
requestedStdoutType: `gc remote ${spec.nodeId} memory contract`,
|
||||
acceptParsed: (value) => {
|
||||
const payload = value.action === "gc remote memory" ? value : record(value.data);
|
||||
return payload.action === "gc remote memory" && payload.providerId === spec.nodeId;
|
||||
},
|
||||
});
|
||||
const resolvedPayload = resolution.parsed?.action === "gc remote memory"
|
||||
? resolution.parsed
|
||||
: record(resolution.parsed?.data);
|
||||
const evaluated = evaluateWebProbeMemoryGuard(spec, mode, resolvedPayload);
|
||||
return {
|
||||
...evaluated,
|
||||
snapshot: {
|
||||
action: "gc remote memory",
|
||||
command: command.join(" "),
|
||||
exitCode: result.exitCode,
|
||||
timedOut: result.timedOut,
|
||||
parsed: resolution.parsed !== null,
|
||||
diagnostics: resolution.diagnostics,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -20,6 +20,11 @@ const viewport = parseViewport(process.env.UNIDESK_WEB_PROBE_VIEWPORT || "1440x9
|
||||
const browserProxyMode = parseBrowserProxyMode(process.env.UNIDESK_WEB_PROBE_BROWSER_PROXY_MODE || "auto");
|
||||
const playwrightProxy = proxyConfigFromEnv(baseUrl);
|
||||
const chromiumLaunchOptions = chromiumLaunchOptionsForProxy(playwrightProxy);
|
||||
const screenshotMode = requiredEnum(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_MODE, ["viewport"]);
|
||||
const maxNetworkRequestEntries = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES, "UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES");
|
||||
const maxNetworkRequestBytes = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_BYTES, "UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_BYTES");
|
||||
const maxNetworkFailureEntries = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_ENTRIES, "UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_ENTRIES");
|
||||
const maxNetworkFailureBytes = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES, "UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES");
|
||||
const artifactRecords = [];
|
||||
const readinessRecords = [];
|
||||
const stepRecords = [];
|
||||
@@ -106,6 +111,7 @@ try {
|
||||
});
|
||||
process.exitCode = 2;
|
||||
} finally {
|
||||
if (context) await context.close().catch(() => {});
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
}
|
||||
|
||||
@@ -1314,36 +1320,54 @@ function createNetworkTracker(targetPage, apiPattern) {
|
||||
const started = Date.now();
|
||||
const requests = [];
|
||||
const failures = [];
|
||||
let requestBytes = 0;
|
||||
let requestDropped = 0;
|
||||
let failureBytes = 0;
|
||||
let failureDropped = 0;
|
||||
const matches = createApiMatcher(apiPattern);
|
||||
const appendBounded = (items, item, kind) => {
|
||||
const bytes = Buffer.byteLength(JSON.stringify(item), "utf8");
|
||||
const entryLimit = kind === "request" ? maxNetworkRequestEntries : maxNetworkFailureEntries;
|
||||
const byteLimit = kind === "request" ? maxNetworkRequestBytes : maxNetworkFailureBytes;
|
||||
const usedBytes = kind === "request" ? requestBytes : failureBytes;
|
||||
if (items.length >= entryLimit || usedBytes + bytes > byteLimit) {
|
||||
if (kind === "request") requestDropped += 1;
|
||||
else failureDropped += 1;
|
||||
return;
|
||||
}
|
||||
items.push(item);
|
||||
if (kind === "request") requestBytes += bytes;
|
||||
else failureBytes += bytes;
|
||||
};
|
||||
const onRequest = (request) => {
|
||||
if (!matches(request.url())) return;
|
||||
requests.push({
|
||||
appendBounded(requests, {
|
||||
method: request.method(),
|
||||
path: urlPath(request.url()),
|
||||
atMs: Date.now() - started,
|
||||
});
|
||||
}, "request");
|
||||
};
|
||||
const onResponse = (response) => {
|
||||
if (!matches(response.url())) return;
|
||||
if (response.status() >= 400) {
|
||||
failures.push({
|
||||
appendBounded(failures, {
|
||||
type: "response",
|
||||
status: response.status(),
|
||||
statusText: response.statusText(),
|
||||
path: urlPath(response.url()),
|
||||
atMs: Date.now() - started,
|
||||
});
|
||||
}, "failure");
|
||||
}
|
||||
};
|
||||
const onRequestFailed = (request) => {
|
||||
if (!matches(request.url())) return;
|
||||
failures.push({
|
||||
appendBounded(failures, {
|
||||
type: "requestfailed",
|
||||
method: request.method(),
|
||||
path: urlPath(request.url()),
|
||||
failureText: request.failure()?.errorText ?? null,
|
||||
atMs: Date.now() - started,
|
||||
});
|
||||
}, "failure");
|
||||
};
|
||||
targetPage.on("request", onRequest);
|
||||
targetPage.on("response", onResponse);
|
||||
@@ -1356,6 +1380,10 @@ function createNetworkTracker(targetPage, apiPattern) {
|
||||
apiFailureCount: failures.length,
|
||||
apiRequests: requests.slice(0, 5),
|
||||
apiFailures: failures.slice(0, 5),
|
||||
apiRequestEvidenceBytes: requestBytes,
|
||||
apiRequestEvidenceDropped: requestDropped,
|
||||
apiFailureEvidenceBytes: failureBytes,
|
||||
apiFailureEvidenceDropped: failureDropped,
|
||||
}),
|
||||
dispose: () => {
|
||||
targetPage.off("request", onRequest);
|
||||
@@ -1476,7 +1504,7 @@ async function captureFailureScreenshot(name = "failure.png") {
|
||||
|
||||
async function screenshotPage(targetPage, name = "screenshot.png", options = {}) {
|
||||
const file = artifactPath(name);
|
||||
await targetPage.screenshot({ path: file, fullPage: true, ...options });
|
||||
await targetPage.screenshot({ path: file, fullPage: screenshotMode !== "viewport", ...options });
|
||||
return recordArtifact(file, "screenshot");
|
||||
}
|
||||
|
||||
@@ -1910,6 +1938,17 @@ function positiveInteger(raw, fallback) {
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredPositiveInteger(raw, name) {
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value <= 0) throw new Error("missing or invalid " + name);
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredEnum(raw, allowed) {
|
||||
if (!allowed.includes(raw)) throw new Error("missing or invalid screenshot mode");
|
||||
return raw;
|
||||
}
|
||||
|
||||
function artifactPath(name) {
|
||||
const safe = String(name || "artifact")
|
||||
.replace(/[^A-Za-z0-9._-]/gu, "_")
|
||||
@@ -1920,7 +1959,7 @@ function artifactPath(name) {
|
||||
|
||||
async function screenshot(name = "screenshot.png", options = {}) {
|
||||
const file = artifactPath(name);
|
||||
await page.screenshot({ path: file, fullPage: true, ...options });
|
||||
await page.screenshot({ path: file, fullPage: screenshotMode !== "viewport", ...options });
|
||||
return recordArtifact(file, "screenshot");
|
||||
}
|
||||
|
||||
|
||||
@@ -439,6 +439,7 @@ function knownWebProbeFindingIds(): string[] {
|
||||
"scripts/src/hwlab-node-web-observe-analyzer-window-page-source.ts",
|
||||
"scripts/src/hwlab-node-web-observe-analyzer-workbench-triad-source.ts",
|
||||
"scripts/src/hwlab-node-web-sentinel-p5-observe.ts",
|
||||
"scripts/src/hwlab-node-web-sentinel-p5-observe-analysis.ts",
|
||||
]) {
|
||||
let source = "";
|
||||
try {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,7 +6,17 @@ import { join } from "node:path";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
|
||||
import { classifyQuickVerifyCleanupStep, quickVerifyCleanupFindings, readAnalysisSummaryFromWorkspace, sentinelServiceProxyInvocation } from "./hwlab-node-web-sentinel-p5-observe";
|
||||
import { nodeWebObserveRunnerSource } from "./hwlab-node-web-observe-runner-source";
|
||||
import {
|
||||
classifyQuickVerifyCleanupStep,
|
||||
createQuickVerifyObserverLifecycle,
|
||||
quickVerifyCleanupFindings,
|
||||
quickVerifyUnhandledExceptionSummary,
|
||||
readAnalysisSummaryFromWorkspace,
|
||||
sentinelChildStartMemorySkip,
|
||||
sentinelServiceProxyInvocation,
|
||||
stopQuickVerifyObserverAfterPartialStart,
|
||||
} from "./hwlab-node-web-sentinel-p5-observe";
|
||||
import { createWebProbeSentinelService } from "./hwlab-node-web-sentinel-service";
|
||||
|
||||
test("sentinel service proxy executes its script through trans standard input", () => {
|
||||
@@ -60,6 +70,7 @@ test("recorded sentinel run is available through the latest report index", () =>
|
||||
views: { summary: { renderedText: "indexed summary" } },
|
||||
findingCount: 0,
|
||||
artifactCount: 0,
|
||||
updatedAt: "2026-07-13T00:00:00.000Z",
|
||||
valuesRedacted: true,
|
||||
});
|
||||
const latest = service.report("summary", null);
|
||||
@@ -91,6 +102,7 @@ test("latest report selects the newest stored run without a report SHA", () => {
|
||||
views: { summary: { renderedText: "old indexed summary" } },
|
||||
findingCount: 0,
|
||||
artifactCount: 0,
|
||||
updatedAt: "2026-07-13T00:00:00.000Z",
|
||||
valuesRedacted: true,
|
||||
});
|
||||
service.recordRun({
|
||||
@@ -102,6 +114,7 @@ test("latest report selects the newest stored run without a report SHA", () => {
|
||||
findings: [{ id: "quick-verify-observer-start-failed", severity: "error", summary: "observer start failed" }],
|
||||
findingCount: 1,
|
||||
artifactCount: 0,
|
||||
updatedAt: "2026-07-13T00:00:01.000Z",
|
||||
valuesRedacted: true,
|
||||
});
|
||||
|
||||
@@ -116,7 +129,6 @@ test("latest report selects the newest stored run without a report SHA", () => {
|
||||
rmSync(stateRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function forceStopStep(observer: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
phase: "observe-stop-after-terminal",
|
||||
@@ -224,3 +236,146 @@ test("quick verify cleanup emits evidence-gap code when stopped state cannot be
|
||||
assert.equal(findings[0]?.id, "observer-cleanup-evidence-missing");
|
||||
assert.match(String(findings[0]?.evidenceSummary), /missing=aliveAfter/u);
|
||||
});
|
||||
|
||||
test("quick verify observer lifecycle stops once on success, failure, and exception fallback", () => {
|
||||
const cases = [
|
||||
{ name: "success", initialPhase: "observe-stop-after-terminal", expectedPhase: "observe-stop-after-terminal" },
|
||||
{ name: "failure", initialPhase: "observe-stop-after-failure", expectedPhase: "observe-stop-after-failure" },
|
||||
{ name: "exception-fallback", initialPhase: null, expectedPhase: "observe-stop-after-finally" },
|
||||
] as const;
|
||||
|
||||
for (const scenario of cases) {
|
||||
const calls: { observerId: string; phase: string }[] = [];
|
||||
const lifecycle = createQuickVerifyObserverLifecycle("webobs-fixture", (observerId, phase) => {
|
||||
calls.push({ observerId, phase });
|
||||
return { phase, ok: true, observerId };
|
||||
});
|
||||
if (scenario.initialPhase !== null) lifecycle.stop(scenario.initialPhase);
|
||||
const cleanupStep = lifecycle.stop("observe-stop-after-finally");
|
||||
|
||||
assert.deepEqual(calls, [{ observerId: "webobs-fixture", phase: scenario.expectedPhase }], scenario.name);
|
||||
assert.equal(cleanupStep.phase, scenario.expectedPhase, scenario.name);
|
||||
assert.strictEqual(cleanupStep, lifecycle.cleanupStep, scenario.name);
|
||||
}
|
||||
});
|
||||
|
||||
test("quick verify observer lifecycle retries a failed stop again from the bounded finally fallback", () => {
|
||||
let calls = 0;
|
||||
const lifecycle = createQuickVerifyObserverLifecycle("webobs-stop-throws", () => {
|
||||
calls += 1;
|
||||
throw new Error("sensitive-stop-error");
|
||||
});
|
||||
|
||||
const first = lifecycle.stop("observe-stop-after-failure");
|
||||
const fallback = lifecycle.stop("observe-stop-after-finally");
|
||||
|
||||
assert.equal(calls, 4);
|
||||
assert.notStrictEqual(first, fallback);
|
||||
assert.equal(first.failure, "observer-stop-unhandled-exception");
|
||||
assert.equal(Array.isArray(fallback.stopAttempts), true);
|
||||
assert.equal((fallback.stopAttempts as unknown[]).length, 4);
|
||||
assert.equal(JSON.stringify(first).includes("sensitive-stop-error"), false);
|
||||
});
|
||||
|
||||
test("quick verify observer lifecycle retries a failed force-stop and caches the successful fallback", () => {
|
||||
let calls = 0;
|
||||
const lifecycle = createQuickVerifyObserverLifecycle("webobs-stop-retry", (_observerId, phase) => {
|
||||
calls += 1;
|
||||
return { phase, ok: calls === 2, observerId: "webobs-stop-retry" };
|
||||
});
|
||||
const first = lifecycle.stop("observe-stop-after-failure");
|
||||
const cached = lifecycle.stop("observe-stop-after-finally");
|
||||
assert.equal(calls, 2);
|
||||
assert.equal(first.ok, true);
|
||||
assert.strictEqual(first, cached);
|
||||
assert.equal(Array.isArray(first.stopAttempts), true);
|
||||
});
|
||||
|
||||
test("quick verify cleans a partial start only when start output contains an observer id", () => {
|
||||
const calls: { observerId: string; phase: string }[] = [];
|
||||
const stopObserver = (observerId: string, phase: string) => {
|
||||
calls.push({ observerId, phase });
|
||||
return { phase, ok: true, observerId };
|
||||
};
|
||||
|
||||
assert.equal(stopQuickVerifyObserverAfterPartialStart(null, stopObserver), null);
|
||||
const cleanupStep = stopQuickVerifyObserverAfterPartialStart("webobs-partial", stopObserver);
|
||||
|
||||
assert.deepEqual(calls, [{ observerId: "webobs-partial", phase: "observe-stop-after-start-failure" }]);
|
||||
assert.equal(cleanupStep?.ok, true);
|
||||
});
|
||||
|
||||
test("quick verify unexpected exception evidence is bounded and redacted", () => {
|
||||
const secret = "sensitive-upstream-message";
|
||||
const summary = quickVerifyUnhandledExceptionSummary(new Error(secret));
|
||||
const serialized = JSON.stringify(summary);
|
||||
|
||||
assert.equal(summary.errorName, "Error");
|
||||
assert.match(String(summary.errorMessageSha256), /^[a-f0-9]{64}$/u);
|
||||
assert.equal(summary.errorMessageTruncated, false);
|
||||
assert.equal(serialized.includes(secret), false);
|
||||
});
|
||||
|
||||
test("sentinel preserves a child launch-time memory race as skipped instead of observe-start-failed", () => {
|
||||
const skip = sentinelChildStartMemorySkip({
|
||||
ok: true,
|
||||
command: "web-probe observe start",
|
||||
data: {
|
||||
ok: true,
|
||||
status: "skipped-wait-next-round",
|
||||
observerCreated: false,
|
||||
memoryGuard: {
|
||||
mode: "sentinel-cadence",
|
||||
status: "skipped",
|
||||
memory: {
|
||||
metric: "MemAvailable",
|
||||
source: "/proc/meminfo",
|
||||
swapExcluded: true,
|
||||
availableBytes: 4 * 1024 ** 3 - 1,
|
||||
thresholdBytes: 4 * 1024 ** 3,
|
||||
comparator: "less-than",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(skip?.status, "skipped-wait-next-round");
|
||||
assert.equal(skip?.decision, "skip-observer");
|
||||
assert.equal(skip?.observerCreated, false);
|
||||
assert.equal((skip?.memoryGuard as Record<string, unknown>).mode, "sentinel-cadence");
|
||||
assert.equal(sentinelChildStartMemorySkip({ data: { status: "blocked", observerCreated: false } }), null);
|
||||
});
|
||||
|
||||
test("web observe runner records finite lifecycle policy and closes the browser", () => {
|
||||
const source = nodeWebObserveRunnerSource();
|
||||
|
||||
assert.match(source, /sampling:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u);
|
||||
assert.match(source, /observerLifecycle:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u);
|
||||
assert.match(source, /if \(maxRunMs > 0 && Date\.now\(\) - startedAtMs >= maxRunMs\)/u);
|
||||
assert.match(source, /const maxScreenshots = requiredBoundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS/u);
|
||||
assert.match(source, /const maxPerformanceEntries = requiredBoundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES/u);
|
||||
assert.match(source, /const maxRealtimeEventBodyBytes = requiredBoundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES/u);
|
||||
assert.match(source, /while \(browserProcessHistory\.length > maxBrowserProcessHistoryEntries\)/u);
|
||||
assert.match(source, /while \(browserFreezeSignalHistory\.length > maxBrowserFreezeSignalHistoryEntries\)/u);
|
||||
assert.match(source, /if \(screenshotCount >= maxScreenshots\)/u);
|
||||
assert.match(source, /const maxBytes = responseBodyMaxBytes/u);
|
||||
assert.match(source, /function responseBodyReadContract\(response, maxBytes = responseBodyMaxBytes\)/u);
|
||||
assert.match(source, /contentEncoding[\s\S]*?reason: "encoded-body-size-unbounded"/u);
|
||||
assert.match(source, /!Number\.isFinite\(contentLength\)[\s\S]*?reason: "content-length-unavailable"/u);
|
||||
assert.match(source, /bodyReadSkipped: body\.error\?\.reason \|\| "bounded-response-read-failed"/u);
|
||||
assert.match(source, /\.slice\(0, maxProcessScanEntries\)/u);
|
||||
assert.match(source, /if \(items\.length > maxEntries\)[\s\S]*?items\.splice/u);
|
||||
assert.match(source, /process\.once\(signal,[\s\S]*?closeBrowserResources\("signal:" \+ signal\)/u);
|
||||
assert.match(source, /finally \{[\s\S]*?await closeBrowserResources/u);
|
||||
assert.match(source, /closingObserverPage\.close\(\)[\s\S]*?closingPage\.close\(\)[\s\S]*?closingContext\.close\(\)[\s\S]*?closingBrowser\.close\(\)/u);
|
||||
|
||||
const directory = mkdtempSync(join(tmpdir(), "web-observe-runner-check-"));
|
||||
try {
|
||||
const runner = join(directory, "observer-runner.mjs");
|
||||
writeFileSync(runner, source);
|
||||
const syntax = spawnSync("node", ["--check", runner], { encoding: "utf8" });
|
||||
assert.equal(syntax.status, 0, syntax.stderr);
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,6 +9,7 @@ import { startJob } from "./jobs";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact";
|
||||
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
|
||||
import { runWebProbeMemoryGuard, webProbeGuardedLaunchShell, webProbeScreenshotFullPage } from "./hwlab-node-web-probe-memory-guard";
|
||||
import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd";
|
||||
import {
|
||||
clipTail,
|
||||
@@ -1144,11 +1145,13 @@ export function runSentinelDashboard(state: SentinelCicdState, options: Extract<
|
||||
}
|
||||
|
||||
export function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): Record<string, unknown> {
|
||||
const memoryGuard = runWebProbeMemoryGuard(state.spec, "sentinel-cadence");
|
||||
if (memoryGuard.status !== "allowed") return memoryGuard;
|
||||
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
|
||||
const dashboardUrl = sentinelDashboardUrl(publicBaseUrl, options.runId, options.errorId);
|
||||
const [widthRaw, heightRaw] = options.viewport.split("x");
|
||||
const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : "";
|
||||
const script = [
|
||||
const browserScript = [
|
||||
"set -eu",
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(dashboardUrl)}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`,
|
||||
@@ -1157,7 +1160,7 @@ export function probeSentinelDashboardBrowser(state: SentinelCicdState, options:
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(webProbeScreenshotFullPage(state.spec, options.fullPage) ? "1" : "0")}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID=${shellQuote(state.sentinelId)}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX=${shellQuote(stringAtNullable(state.publicExposure, "routePrefix") ?? "/")}`,
|
||||
`export UNIDESK_SENTINEL_DASHBOARD_RUN_ID=${shellQuote(options.runId ?? "")}`,
|
||||
@@ -1178,6 +1181,7 @@ export function probeSentinelDashboardBrowser(state: SentinelCicdState, options:
|
||||
"WEB_PROBE_SENTINEL_DASHBOARD_JS",
|
||||
"node \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\"",
|
||||
].join("\n");
|
||||
const script = webProbeGuardedLaunchShell(state.spec, "sentinel-cadence", browserScript);
|
||||
const route = `${state.spec.nodeId}:${state.spec.workspace}`;
|
||||
const job = runWebProbeRemoteArtifactJob({
|
||||
route,
|
||||
@@ -1263,7 +1267,7 @@ const triggerManual = process.env.UNIDESK_SENTINEL_DASHBOARD_TRIGGER === "1";
|
||||
const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440);
|
||||
const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900);
|
||||
const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000);
|
||||
const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0";
|
||||
const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE === "1";
|
||||
const executablePath = process.env.UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH || "";
|
||||
const expectedSentinelId = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID || "";
|
||||
const expectedRoutePrefix = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX || "/";
|
||||
@@ -1275,12 +1279,15 @@ if (!url) throw new Error("missing dashboard URL");
|
||||
const consoleMessages = [];
|
||||
const pageErrors = [];
|
||||
const requestFailures = [];
|
||||
const browser = await chromium.launch({
|
||||
let browser = null;
|
||||
let context = null;
|
||||
try {
|
||||
browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: ["--disable-gpu", "--no-sandbox"],
|
||||
...(executablePath ? { executablePath } : {}),
|
||||
});
|
||||
const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
|
||||
context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
|
||||
const page = await context.newPage();
|
||||
page.on("console", (message) => {
|
||||
if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) });
|
||||
@@ -1677,8 +1684,10 @@ console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({
|
||||
valuesRedacted: true,
|
||||
}));
|
||||
|
||||
await context.close().catch(() => {});
|
||||
await browser.close().catch(() => {});
|
||||
} finally {
|
||||
if (context) await context.close().catch(() => {});
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
|
||||
@@ -562,6 +562,7 @@ function buildObserveCommandPlan(config: WebProbeSentinelServiceConfig, scenario
|
||||
"--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")),
|
||||
"--max-run-seconds", String(numberAt(scenario, "maxRunSeconds")),
|
||||
"--command-timeout-seconds", "55",
|
||||
"--sentinel-cadence",
|
||||
];
|
||||
const viewport = stringOrNull(scenario.viewport);
|
||||
if (viewport !== null) startArgv.push("--viewport", viewport);
|
||||
@@ -2727,7 +2728,7 @@ function reportRunView(config: WebProbeSentinelServiceConfig, db: Database, view
|
||||
return { ok: false, error: "unsupported-report-view", view, valuesRedacted: true };
|
||||
}
|
||||
const row = runId === null
|
||||
? db.query("SELECT * FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? ORDER BY updated_at DESC LIMIT 1").get(config.sentinelId, config.node, config.lane) as Record<string, unknown> | null
|
||||
? db.query("SELECT * FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? ORDER BY updated_at DESC, rowid DESC LIMIT 1").get(config.sentinelId, config.node, config.lane) as Record<string, unknown> | null
|
||||
: readRunRow(config, db, runId);
|
||||
if (row === null) return { ok: false, error: "report-run-missing", runId, view, valuesRedacted: true };
|
||||
const selectedRunId = stringOrNull(row.id);
|
||||
|
||||
@@ -188,6 +188,7 @@ export interface NodeWebProbeObserveOptions extends WebProbeOriginSelection {
|
||||
targetPath: string;
|
||||
viewport: string;
|
||||
browserProxyMode: WebProbeBrowserProxyMode;
|
||||
memoryGuardMode: "manual-start" | "sentinel-cadence";
|
||||
sampleIntervalMs: number;
|
||||
screenshotIntervalMs: number;
|
||||
observerRefreshIntervalMs: number;
|
||||
|
||||
@@ -25,6 +25,7 @@ import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapp
|
||||
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
||||
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
||||
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
||||
import { runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell } from "../hwlab-node-web-probe-memory-guard";
|
||||
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
||||
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
@@ -537,7 +538,9 @@ export function runNodeWebProbeScript(
|
||||
const commandLabel = options.commandLabel ?? `web-probe script --node ${options.node} --lane ${options.lane}${webProbeSemanticOriginArgs(options)}`;
|
||||
const commandGovernance = webProbeScriptCommandGovernance(options);
|
||||
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
||||
const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.username ?? secretSpec.bootstrapAdminUsername, material.password ?? "", webProbeProxy, spec.webProbe?.playwrightBrowsersPath);
|
||||
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
|
||||
if (memoryGuard.status !== "allowed") return memoryGuard;
|
||||
const script = nodeWebProbeScriptRemoteShell(options, spec, secretSpec, material.username ?? secretSpec.bootstrapAdminUsername, material.password ?? "", webProbeProxy, spec.webProbe?.playwrightBrowsersPath);
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const commandTimedOut = result.timedOut || result.exitCode === 124;
|
||||
const runPaths = webProbeScriptRunPathsFromStderr(result.stderr);
|
||||
@@ -740,7 +743,7 @@ function webProbeSemanticOriginArgs(options: Pick<NodeWebProbeScriptOptions, "or
|
||||
return options.originName === "internal" || options.originName === "public" ? ` --origin ${options.originName}` : "";
|
||||
}
|
||||
|
||||
export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, username: string, password: string, webProbeProxy: NodeWebProbeHostProxyEnv, playwrightBrowsersPath: string | undefined): string {
|
||||
export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, spec: HwlabRuntimeLaneSpec, secretSpec: RuntimeSecretSpec, username: string, password: string, webProbeProxy: NodeWebProbeHostProxyEnv, playwrightBrowsersPath: string | undefined): string {
|
||||
const userScriptB64 = Buffer.from(options.scriptText, "utf8").toString("base64");
|
||||
const runnerB64 = Buffer.from(nodeWebProbeScriptRunnerSource(), "utf8").toString("base64");
|
||||
return [
|
||||
@@ -756,8 +759,9 @@ export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions
|
||||
"runner=\"$run_dir/runner.mjs\"",
|
||||
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$user_script" ${shellQuote(userScriptB64)}`,
|
||||
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`,
|
||||
[
|
||||
webProbeGuardedLaunchShell(spec, "manual-start", [
|
||||
...webProbeProxy.envAssignments,
|
||||
...webProbeBrowserEvidenceEnvAssignments(spec),
|
||||
...(playwrightBrowsersPath === undefined ? [] : [
|
||||
`PLAYWRIGHT_BROWSERS_PATH=${shellQuote(playwrightBrowsersPath)}`,
|
||||
]),
|
||||
@@ -770,7 +774,7 @@ export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions
|
||||
`UNIDESK_WEB_PROBE_VIEWPORT=${shellQuote(options.viewport)}`,
|
||||
`UNIDESK_WEB_PROBE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
||||
"node \"$runner\"",
|
||||
].join(" "),
|
||||
].join(" ")),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { resolveCliChildJsonCommandResult, type CliChildJsonResolution } from "../cli-child-json-recovery";
|
||||
import type { HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard } from "../hwlab-node-web-probe-memory-guard";
|
||||
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
||||
import { withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
||||
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
||||
@@ -454,6 +455,134 @@ console.log(JSON.stringify({
|
||||
`;
|
||||
}
|
||||
|
||||
export function nodeWebObserveFinalLaunchGuardSource(): string {
|
||||
return String.raw`
|
||||
const fs = require("fs");
|
||||
const thresholdBytes = Number(process.argv[2]);
|
||||
const comparator = process.argv[3];
|
||||
const mode = process.argv[4];
|
||||
const jobId = process.argv[5];
|
||||
const stateDir = process.argv[6];
|
||||
const source = "/proc/meminfo";
|
||||
|
||||
function finish(reason, availableBytes) {
|
||||
const valid = Number.isSafeInteger(availableBytes) && availableBytes >= 0;
|
||||
const thresholdMatched = valid
|
||||
? comparator === "less-than" ? availableBytes < thresholdBytes : availableBytes <= thresholdBytes
|
||||
: null;
|
||||
const blocked = reason !== null || thresholdMatched === true;
|
||||
const cadenceSkip = blocked && mode === "sentinel-cadence";
|
||||
const payload = {
|
||||
ok: !blocked || cadenceSkip,
|
||||
command: "web-probe-observe start",
|
||||
status: blocked ? cadenceSkip ? "skipped-wait-next-round" : "blocked" : "allowed",
|
||||
reason: reason || (thresholdMatched ? "memavailable-threshold-matched" : null),
|
||||
failureKind: blocked ? "web-probe-final-memory-start-guard" : null,
|
||||
jobId,
|
||||
stateDir,
|
||||
observerCreated: false,
|
||||
browserCreated: false,
|
||||
startBlocked: blocked,
|
||||
decision: blocked ? cadenceSkip ? "skip-observer" : "gc-required" : "start-observer",
|
||||
mutation: blocked,
|
||||
stateArtifactCreated: blocked,
|
||||
memoryGuard: {
|
||||
phase: "remote-launch-critical-section",
|
||||
metric: "MemAvailable",
|
||||
source,
|
||||
swapExcluded: true,
|
||||
availableBytes: valid ? availableBytes : null,
|
||||
thresholdBytes,
|
||||
comparator,
|
||||
thresholdMatched,
|
||||
},
|
||||
observedAt: new Date().toISOString(),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
console.log(JSON.stringify(payload));
|
||||
process.exit(blocked ? 42 : 0);
|
||||
}
|
||||
|
||||
if (!Number.isSafeInteger(thresholdBytes) || thresholdBytes <= 0) finish("memory-threshold-invalid", null);
|
||||
if (comparator !== "less-than" && comparator !== "less-than-or-equal") finish("memory-comparator-invalid", null);
|
||||
let text;
|
||||
try { text = fs.readFileSync(source, "utf8"); } catch { finish("proc-meminfo-read-failed", null); }
|
||||
const match = text.match(/^MemAvailable:\s+(\d+)\s+kB$/m);
|
||||
if (!match) finish("memavailable-missing", null);
|
||||
const availableBytes = Number(match[1]) * 1024;
|
||||
finish(null, availableBytes);
|
||||
`;
|
||||
}
|
||||
|
||||
export function nodeWebObserveLaunchReadinessSource(): string {
|
||||
return String.raw`
|
||||
const fs = require("fs");
|
||||
const runnerPid = Number(process.argv[2]);
|
||||
const stateDir = process.argv[3];
|
||||
const expectedJobId = process.argv[4];
|
||||
const timeoutMs = Number(process.argv[5]);
|
||||
const pollMs = Number(process.argv[6]);
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
||||
const readJson = (name) => { try { return JSON.parse(fs.readFileSync(stateDir + "/" + name, "utf8")); } catch { return null; } };
|
||||
|
||||
function procRows() {
|
||||
const rows = new Map();
|
||||
let names = [];
|
||||
try { names = fs.readdirSync("/proc").filter((name) => /^\d+$/.test(name)); } catch { return rows; }
|
||||
for (const name of names) {
|
||||
try {
|
||||
const stat = fs.readFileSync("/proc/" + name + "/stat", "utf8");
|
||||
const close = stat.lastIndexOf(")");
|
||||
const fields = stat.slice(close + 2).trim().split(/\s+/);
|
||||
const command = fs.readFileSync("/proc/" + name + "/cmdline").toString("utf8").replace(/\0/g, " ");
|
||||
rows.set(Number(name), { pid: Number(name), ppid: Number(fields[1]), command });
|
||||
} catch {}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function descendants(rows) {
|
||||
const selected = new Set([runnerPid]);
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const row of rows.values()) if (!selected.has(row.pid) && selected.has(row.ppid)) { selected.add(row.pid); changed = true; }
|
||||
}
|
||||
return [...selected].map((pid) => rows.get(pid)).filter(Boolean);
|
||||
}
|
||||
|
||||
function matchingStateRecord(name, value) {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const status = String(value.status || "");
|
||||
if (!/^(starting|running|ready)$/i.test(status)) return null;
|
||||
if (String(value.jobId || "") !== expectedJobId || Number(value.pid) !== runnerPid) return null;
|
||||
return { name, status, jobId: String(value.jobId), pid: Number(value.pid) };
|
||||
}
|
||||
|
||||
while (Date.now() <= deadline) {
|
||||
const rows = procRows();
|
||||
const runnerAlive = rows.has(runnerPid);
|
||||
const children = runnerAlive ? descendants(rows) : [];
|
||||
const browserChild = children.find((row) => row.pid !== runnerPid && /(?:^|[ /])(chrome|chromium)(?:[ /]|$)/i.test(row.command));
|
||||
const heartbeat = readJson("heartbeat.json");
|
||||
const manifest = readJson("manifest.json");
|
||||
const stateRecord = matchingStateRecord("heartbeat.json", heartbeat) || matchingStateRecord("manifest.json", manifest);
|
||||
if (runnerAlive && browserChild && stateRecord) {
|
||||
console.log(JSON.stringify({ ok: true, expectedJobId, runnerPid, runnerAlive: true, browserPid: browserChild.pid, stateRecord, readiness: "current-state-and-browser-descendant", observedAt: new Date().toISOString(), valuesRedacted: true }));
|
||||
process.exit(0);
|
||||
}
|
||||
if (!runnerAlive) {
|
||||
console.log(JSON.stringify({ ok: false, reason: "runner-exited-before-ready", expectedJobId, runnerPid, runnerAlive: false, stateRecord, observedAt: new Date().toISOString(), valuesRedacted: true }));
|
||||
process.exit(43);
|
||||
}
|
||||
pause(pollMs);
|
||||
}
|
||||
console.log(JSON.stringify({ ok: false, reason: "launch-readiness-timeout", expectedJobId, runnerPid, runnerAlive: fs.existsSync("/proc/" + runnerPid), observedAt: new Date().toISOString(), valuesRedacted: true }));
|
||||
process.exit(44);
|
||||
`;
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveStart(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
@@ -461,6 +590,29 @@ export function runNodeWebProbeObserveStart(
|
||||
material: BootstrapAdminPasswordMaterial,
|
||||
credential: Record<string, unknown>,
|
||||
): Record<string, unknown> | RenderedCliResult {
|
||||
const memoryGuard = runWebProbeMemoryGuard(spec, options.memoryGuardMode);
|
||||
if (memoryGuard.status !== "allowed") {
|
||||
const cadenceSkip = options.memoryGuardMode === "sentinel-cadence" && memoryGuard.status === "skipped";
|
||||
return {
|
||||
ok: cadenceSkip,
|
||||
status: cadenceSkip ? "skipped-wait-next-round" : "blocked",
|
||||
failureKind: "web-probe-memory-start-guard",
|
||||
command: `web-probe observe start --node ${options.node} --lane ${options.lane}`,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
observerCreated: false,
|
||||
startBlocked: true,
|
||||
decision: cadenceSkip ? "skip-observer" : "gc-required",
|
||||
mutation: false,
|
||||
memoryGuard,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
const observerLifecycle = spec.webProbe?.resourcePolicy?.observerLifecycle;
|
||||
const finalMemoryPolicy = spec.webProbe?.resourcePolicy?.memoryStartGuard;
|
||||
if (observerLifecycle === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML");
|
||||
if (finalMemoryPolicy === undefined) throw new Error("web-probe memory start policy is missing from the owning YAML");
|
||||
const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
||||
const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
|
||||
const day = timestamp.slice(0, 8);
|
||||
@@ -494,6 +646,28 @@ export function runNodeWebProbeObserveStart(
|
||||
`UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS=${shellQuote(String(options.observerRefreshIntervalMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_RUN_MS=${shellQuote(String(options.maxRunSeconds > 0 ? options.maxRunSeconds * 1000 : 0))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS=${shellQuote(String(observerLifecycle.maxScreenshots))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES=${shellQuote(String(observerLifecycle.maxPerformanceEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserProcessHistoryEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserFreezeSignalHistoryEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_ENTRIES=${shellQuote(String(observerLifecycle.maxRealtimeEventEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES=${shellQuote(String(observerLifecycle.maxRealtimeEventBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_DOM_EVIDENCE_ROWS=${shellQuote(String(observerLifecycle.maxDomEvidenceRows))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PROCESS_SCAN_ENTRIES=${shellQuote(String(observerLifecycle.maxProcessScanEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_CAPTURE_SECONDS=${shellQuote(String(observerLifecycle.maxPerformanceCaptureSeconds))}`,
|
||||
`UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxResponseBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxWebPerformanceBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_ENTRIES=${shellQuote(String(observerLifecycle.maxJsonlQueueEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_BYTES=${shellQuote(String(observerLifecycle.maxJsonlQueueBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_JSONL_LINE_BYTES=${shellQuote(String(observerLifecycle.maxJsonlLineBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_JSONL_FLUSH_TIMEOUT_MS=${shellQuote(String(observerLifecycle.jsonlFlushTimeoutMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_RESPONSE_BODY_IN_FLIGHT=${shellQuote(String(observerLifecycle.maxResponseBodyInFlight))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PASSIVE_LISTENER_TASKS=${shellQuote(String(observerLifecycle.maxPassiveListenerTasks))}`,
|
||||
`UNIDESK_WEB_OBSERVE_PASSIVE_TASK_FLUSH_TIMEOUT_MS=${shellQuote(String(observerLifecycle.passiveTaskFlushTimeoutMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_CLOSE_STEP_TIMEOUT_MS=${shellQuote(String(observerLifecycle.closeStepTimeoutMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PROCESS_READ_CONCURRENCY=${shellQuote(String(observerLifecycle.maxProcessReadConcurrency))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_ENTRIES=${shellQuote(String(observerLifecycle.maxRealtimeTotalEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_BODY_BYTES=${shellQuote(String(observerLifecycle.maxRealtimeTotalBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
|
||||
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
||||
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
|
||||
@@ -511,6 +685,26 @@ export function runNodeWebProbeObserveStart(
|
||||
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS=${shellQuote(String(authLogin.maxDelayMs))}`,
|
||||
]),
|
||||
].join(" ");
|
||||
const finalComparator = options.memoryGuardMode === "sentinel-cadence"
|
||||
? finalMemoryPolicy.sentinelComparator
|
||||
: finalMemoryPolicy.manualComparator;
|
||||
const finalGuardFailure = (reason: string) => JSON.stringify({
|
||||
ok: false,
|
||||
command: "web-probe-observe start",
|
||||
status: "blocked",
|
||||
reason,
|
||||
failureKind: "web-probe-final-memory-start-guard",
|
||||
jobId,
|
||||
stateDir,
|
||||
observerCreated: false,
|
||||
startBlocked: true,
|
||||
mutation: true,
|
||||
stateArtifactCreated: true,
|
||||
browserCreated: false,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
const launchFailureOutputSource = "const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const guard=read('launch-guard.json'); const readiness=read('launch-ready.json'); const pid=Number(process.argv[3]); const bool=(v)=>v==='true'; console.log(JSON.stringify({ok:false,command:'web-probe-observe start',status:'blocked',reason:'launch-readiness-unconfirmed',failureKind:'web-probe-launch-readiness',jobId:process.argv[2],stateDir:dir,observerCreated:false,browserCreated:false,browserCreationAttempted:true,startBlocked:true,mutation:true,stateArtifactCreated:true,memoryGuard:guard&&guard.memoryGuard||null,finalLaunchGuard:guard,launchReadiness:readiness,cleanup:{attempted:true,processGroupId:pid,termSent:bool(process.argv[4]),killSent:bool(process.argv[5]),aliveAfter:bool(process.argv[6]),completed:!bool(process.argv[6]),observedAt:new Date().toISOString(),valuesRedacted:true},valuesRedacted:true},null,2))";
|
||||
const cleanupPollCount = Math.ceil(finalMemoryPolicy.launchReadyTimeoutSeconds * 1000 / finalMemoryPolicy.launchReadyPollMilliseconds);
|
||||
const script = [
|
||||
"set -eu",
|
||||
`state_dir=${shellQuote(stateDir)}`,
|
||||
@@ -524,16 +718,71 @@ export function runNodeWebProbeObserveStart(
|
||||
"node -e \"const fs=require('fs'); fs.writeFileSync(process.argv[1], Buffer.from(fs.readFileSync(process.argv[2], 'utf8').replace(/\\s+/g, ''), 'base64'))\" \"$runner\" \"$runner_b64\"",
|
||||
"rm -f \"$runner_b64\"",
|
||||
"chmod 700 \"$runner\"",
|
||||
`if command -v setsid >/dev/null 2>&1; then setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & else nohup env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & fi`,
|
||||
`launch_lock=${shellQuote(finalMemoryPolicy.launchLockPath)}`,
|
||||
`launch_guard_result="$state_dir/launch-guard.json"`,
|
||||
`launch_ready_result="$state_dir/launch-ready.json"`,
|
||||
`if ! command -v flock >/dev/null 2>&1; then printf '%s\\n' ${shellQuote(finalGuardFailure("host-flock-unavailable"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
|
||||
`if ! command -v setsid >/dev/null 2>&1; then printf '%s\\n' ${shellQuote(finalGuardFailure("host-setsid-unavailable"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
|
||||
`exec 9>"$launch_lock"`,
|
||||
`if ! flock -w ${shellQuote(String(finalMemoryPolicy.launchLockTimeoutSeconds))} 9; then printf '%s\\n' ${shellQuote(finalGuardFailure("launch-lock-timeout"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
|
||||
"trap 'flock -u 9 2>/dev/null || true' EXIT",
|
||||
"set +e",
|
||||
`node - ${shellQuote(String(finalMemoryPolicy.thresholdBytes))} ${shellQuote(finalComparator)} ${shellQuote(options.memoryGuardMode)} ${shellQuote(jobId)} ${shellQuote(stateDir)} >"$launch_guard_result" <<'UNIDESK_WEB_OBSERVE_FINAL_GUARD'`,
|
||||
nodeWebObserveFinalLaunchGuardSource(),
|
||||
"UNIDESK_WEB_OBSERVE_FINAL_GUARD",
|
||||
"launch_guard_rc=$?",
|
||||
"set -e",
|
||||
"if [ \"$launch_guard_rc\" -ne 0 ]; then cat \"$launch_guard_result\"; flock -u 9; exit 0; fi",
|
||||
`setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null 9>&- &`,
|
||||
"pid=$!",
|
||||
"printf '%s\\n' \"$pid\" >\"$state_dir/pid\"",
|
||||
"sleep 1",
|
||||
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
|
||||
"set +e",
|
||||
`node - "$pid" "$state_dir" ${shellQuote(jobId)} ${shellQuote(String(finalMemoryPolicy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds))} >"$launch_ready_result" <<'UNIDESK_WEB_OBSERVE_LAUNCH_READY'`,
|
||||
nodeWebObserveLaunchReadinessSource(),
|
||||
"UNIDESK_WEB_OBSERVE_LAUNCH_READY",
|
||||
"launch_ready_rc=$?",
|
||||
"set -e",
|
||||
`if [ "$launch_ready_rc" -ne 0 ]; then term_sent=false; if kill -TERM -- -"$pid" 2>/dev/null; then term_sent=true; fi; i=0; while [ "$i" -lt ${shellQuote(String(cleanupPollCount))} ] && kill -0 -- -"$pid" 2>/dev/null; do sleep ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds / 1000))}; i=$((i+1)); done; kill_sent=false; if kill -0 -- -"$pid" 2>/dev/null; then if kill -KILL -- -"$pid" 2>/dev/null; then kill_sent=true; fi; fi; i=0; while [ "$i" -lt ${shellQuote(String(cleanupPollCount))} ] && kill -0 -- -"$pid" 2>/dev/null; do sleep ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds / 1000))}; i=$((i+1)); done; alive_after=false; if kill -0 -- -"$pid" 2>/dev/null; then alive_after=true; fi; node -e ${shellQuote(launchFailureOutputSource)} "$state_dir" ${shellQuote(jobId)} "$pid" "$term_sent" "$kill_sent" "$alive_after"; flock -u 9; trap - EXIT; exit 0; fi`,
|
||||
"flock -u 9",
|
||||
"trap - EXIT",
|
||||
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),finalLaunchGuard:read('launch-guard.json'),launchReadiness:read('launch-ready.json'),observerCreated:true,browserCreated:true,observerMutation:true,stateArtifactMutation:true,statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const startResolution = resolveWebObserveActionJson(result, "start");
|
||||
const started = startResolution.parsed;
|
||||
if (started?.observerCreated === false && started.startBlocked === true) {
|
||||
const finalSnapshot = record(started.memoryGuard);
|
||||
const finalMemoryGuard = evaluateWebProbeMemoryGuard(spec, options.memoryGuardMode, {
|
||||
ok: finalSnapshot.metric === "MemAvailable" && finalSnapshot.source === "/proc/meminfo" && finalSnapshot.swapExcluded === true,
|
||||
action: "gc remote memory",
|
||||
metric: finalSnapshot.metric,
|
||||
source: finalSnapshot.source,
|
||||
swapExcluded: finalSnapshot.swapExcluded,
|
||||
memory: { availableBytes: finalSnapshot.availableBytes },
|
||||
});
|
||||
return {
|
||||
...started,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
initialMemoryGuard: memoryGuard,
|
||||
memoryGuard: finalMemoryGuard,
|
||||
finalLaunchGuard: started.finalLaunchGuard ?? started,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
const startOk = result.exitCode === 0 && started?.ok === true;
|
||||
const allowedFinalSnapshot = record(record(started?.finalLaunchGuard).memoryGuard);
|
||||
const effectiveMemoryGuard = allowedFinalSnapshot.metric === "MemAvailable"
|
||||
? evaluateWebProbeMemoryGuard(spec, options.memoryGuardMode, {
|
||||
ok: true,
|
||||
action: "gc remote memory",
|
||||
metric: allowedFinalSnapshot.metric,
|
||||
source: allowedFinalSnapshot.source,
|
||||
swapExcluded: allowedFinalSnapshot.swapExcluded,
|
||||
memory: { availableBytes: allowedFinalSnapshot.availableBytes },
|
||||
})
|
||||
: memoryGuard;
|
||||
const recovery = startOk
|
||||
? null
|
||||
: readNodeWebProbeObserveRemoteStatus({ ...options, id: jobId, stateDir }, spec, 1, Math.min(options.commandTimeoutSeconds, 30));
|
||||
@@ -592,6 +841,10 @@ export function runNodeWebProbeObserveStart(
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
projectManagement,
|
||||
initialMemoryGuard: memoryGuard,
|
||||
memoryGuard: effectiveMemoryGuard,
|
||||
finalLaunchGuard: started?.finalLaunchGuard ?? null,
|
||||
launchReadiness: started?.launchReadiness ?? null,
|
||||
targetPath: options.targetPath,
|
||||
id: observerId,
|
||||
degradedReason,
|
||||
|
||||
@@ -171,6 +171,72 @@ test("observe start selects the YAML public origin semantically", () => {
|
||||
assert.equal(options.browserProxyModeSource, "yaml-origin");
|
||||
});
|
||||
|
||||
test("observe start inherits finite observer lifecycle limits from the owning YAML", () => {
|
||||
const options = parseNodeWebProbeObserveOptions(
|
||||
["start"],
|
||||
"NC01",
|
||||
"v03",
|
||||
spec,
|
||||
null,
|
||||
null,
|
||||
);
|
||||
|
||||
assert.equal(options.maxRunSeconds, 3600);
|
||||
assert.equal(options.maxSamples, 720);
|
||||
assert.equal(options.memoryGuardMode, "manual-start");
|
||||
assert.ok(options.maxRunSeconds > 0);
|
||||
assert.ok(options.maxSamples > 0);
|
||||
assert.deepEqual(spec.webProbe?.resourcePolicy?.observerLifecycle, {
|
||||
maxRunSeconds: 3600,
|
||||
maxSamples: 720,
|
||||
maxScreenshots: 24,
|
||||
maxPerformanceEntries: 1000,
|
||||
maxBrowserProcessHistoryEntries: 360,
|
||||
maxBrowserFreezeSignalHistoryEntries: 120,
|
||||
maxRealtimeEventEntries: 256,
|
||||
maxRealtimeEventBodyBytes: 65536,
|
||||
maxDomEvidenceRows: 1000,
|
||||
maxProcessScanEntries: 8192,
|
||||
maxPerformanceCaptureSeconds: 30,
|
||||
maxResponseBodyBytes: 524288,
|
||||
maxWebPerformanceBodyBytes: 65536,
|
||||
maxJsonlQueueEntries: 512,
|
||||
maxJsonlQueueBytes: 4194304,
|
||||
maxJsonlLineBytes: 262144,
|
||||
jsonlFlushTimeoutMs: 5000,
|
||||
maxResponseBodyInFlight: 2,
|
||||
maxPassiveListenerTasks: 64,
|
||||
passiveTaskFlushTimeoutMs: 5000,
|
||||
closeStepTimeoutMs: 5000,
|
||||
maxProcessReadConcurrency: 32,
|
||||
maxRealtimeTotalEntries: 512,
|
||||
maxRealtimeTotalBodyBytes: 4194304,
|
||||
});
|
||||
});
|
||||
|
||||
test("sentinel observe start preserves the cadence comparator through the child CLI", () => {
|
||||
const options = parseNodeWebProbeObserveOptions(
|
||||
["start", "--sentinel-cadence"],
|
||||
"NC01",
|
||||
"v03",
|
||||
spec,
|
||||
null,
|
||||
null,
|
||||
);
|
||||
assert.equal(options.memoryGuardMode, "sentinel-cadence");
|
||||
});
|
||||
|
||||
test("observe start cannot disable or exceed the YAML lifecycle caps", () => {
|
||||
assert.throws(
|
||||
() => parseNodeWebProbeObserveOptions(["start", "--max-run-seconds", "0"], "NC01", "v03", spec, null, null),
|
||||
/must be a positive integer/u,
|
||||
);
|
||||
assert.throws(
|
||||
() => parseNodeWebProbeObserveOptions(["start", "--max-samples", "721"], "NC01", "v03", spec, null, null),
|
||||
/exceeds the owning YAML observer lifecycle cap 720/u,
|
||||
);
|
||||
});
|
||||
|
||||
test("observe start refuses URL-based internal/public selection ambiguity", () => {
|
||||
assert.throws(
|
||||
() => parseNodeWebProbeObserveOptions(
|
||||
|
||||
@@ -26,6 +26,7 @@ import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapp
|
||||
import { runWebProbeSentinelCommand, type WebProbeSentinelDashboardAction, type WebProbeSentinelOptions, type WebProbeSentinelReportView, type WebProbeSentinelSourceAuthority, type WebProbeSentinelSourceOverrideOptions } from "../hwlab-node-web-sentinel-cicd";
|
||||
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
||||
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
||||
import { runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeScreenshotFullPage } from "../hwlab-node-web-probe-memory-guard";
|
||||
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
||||
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
||||
import { runWebProbeRemoteArtifactJob } from "../web-probe-remote-artifact";
|
||||
@@ -483,7 +484,7 @@ export function parseNodeWebProbeObserveOptions(
|
||||
"--workspace-root",
|
||||
"--workspace-root-ref",
|
||||
"--root",
|
||||
]), new Set(["--force", "--full", "--raw", "--text-stdin", "--require-composer-ready", "--wait-project-management-ready", "--blocking", "--non-blocking", "--dry-run", "--confirm"]));
|
||||
]), new Set(["--force", "--full", "--raw", "--text-stdin", "--require-composer-ready", "--wait-project-management-ready", "--blocking", "--non-blocking", "--dry-run", "--confirm", "--sentinel-cadence"]));
|
||||
const commandTypeRaw = optionValue(args, "--type") ?? null;
|
||||
const commandType = commandTypeRaw === null ? null : parseNodeWebProbeObserveCommandType(commandTypeRaw);
|
||||
const stateDir = optionValue(args, "--state-dir") ?? indexed?.stateDir ?? null;
|
||||
@@ -528,6 +529,9 @@ export function parseNodeWebProbeObserveOptions(
|
||||
if (observeActionRaw !== "start" && (requestedOrigin !== undefined || customUrl !== undefined || browserProxyModeRaw !== undefined)) {
|
||||
throw new Error("web-probe observe --origin/--url/--browser-proxy-mode is only valid for start; later actions inherit the observer origin");
|
||||
}
|
||||
if (observeActionRaw !== "start" && args.includes("--sentinel-cadence")) {
|
||||
throw new Error("web-probe observe --sentinel-cadence is only valid for start");
|
||||
}
|
||||
const requestedBrowserProxyMode = browserProxyModeRaw === undefined ? undefined : parseWebProbeBrowserProxyMode(browserProxyModeRaw);
|
||||
const selectedOrigin = observeActionRaw === "start"
|
||||
? resolveNodeWebProbeCliOrigin(spec, requestedOrigin, customUrl, requestedBrowserProxyMode)
|
||||
@@ -645,6 +649,11 @@ export function parseNodeWebProbeObserveOptions(
|
||||
throw new Error("owning YAML 未启用 validateWorkbenchTraceReadability");
|
||||
}
|
||||
}
|
||||
const resourcePolicy = spec.webProbe?.resourcePolicy;
|
||||
if (resourcePolicy === undefined) {
|
||||
throw new Error(`web-probe resource policy is missing from ${hwlabRuntimeLaneConfigPath} for ${node}/${lane}`);
|
||||
}
|
||||
const observerLifecycle = resourcePolicy.observerLifecycle;
|
||||
return {
|
||||
action: "observe",
|
||||
observeAction: observeActionRaw,
|
||||
@@ -659,11 +668,12 @@ export function parseNodeWebProbeObserveOptions(
|
||||
viewport: optionValue(args, "--viewport") ?? "1440x900",
|
||||
browserProxyMode: selectedOrigin.browserProxyMode,
|
||||
browserProxyModeSource: selectedOrigin.browserProxyModeSource,
|
||||
memoryGuardMode: args.includes("--sentinel-cadence") ? "sentinel-cadence" : "manual-start",
|
||||
sampleIntervalMs: positiveIntegerOption(args, "--sample-interval-ms", 5000, 600000),
|
||||
screenshotIntervalMs: positiveIntegerOption(args, "--screenshot-interval-ms", 300000, 86_400_000),
|
||||
observerRefreshIntervalMs: positiveIntegerOption(args, "--observer-refresh-interval-ms", 180000, 86_400_000),
|
||||
maxSamples: positiveIntegerOption(args, "--max-samples", 0, 10_000_000),
|
||||
maxRunSeconds: positiveIntegerOption(args, "--max-run-seconds", 0, 86_400),
|
||||
maxSamples: observerLifecycleOption(args, "--max-samples", observerLifecycle.maxSamples),
|
||||
maxRunSeconds: observerLifecycleOption(args, "--max-run-seconds", observerLifecycle.maxRunSeconds),
|
||||
commandTimeoutSeconds: positiveIntegerOption(args, "--command-timeout-seconds", 55, 3600),
|
||||
waitMs: positiveIntegerOption(args, "--wait-ms", 0, 600000),
|
||||
tailLines: positiveIntegerOption(args, "--tail-lines", 5, 200),
|
||||
@@ -728,6 +738,17 @@ export function parseNodeWebProbeObserveOptions(
|
||||
};
|
||||
}
|
||||
|
||||
function observerLifecycleOption(args: string[], name: string, yamlMaximum: number): number {
|
||||
const raw = optionValue(args, name);
|
||||
if (raw === undefined) return yamlMaximum;
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value < 1) throw new Error(`${name} must be a positive integer`);
|
||||
if (value > yamlMaximum) {
|
||||
throw new Error(`${name}=${value} exceeds the owning YAML observer lifecycle cap ${yamlMaximum}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function parseNodeWebProbeObserveCommandType(value: string): NodeWebProbeObserveCommandType {
|
||||
if (
|
||||
value === "login"
|
||||
@@ -870,11 +891,13 @@ export function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, un
|
||||
if (options.action === "observe") return runNodeWebProbeObserve(options, spec, secretSpec, material, credential);
|
||||
if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential);
|
||||
if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential);
|
||||
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
|
||||
if (memoryGuard.status !== "allowed") return memoryGuard;
|
||||
const probeArgs = nodeWebProbeRunArgs(options, "run");
|
||||
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
||||
const script = [
|
||||
"set -eu",
|
||||
`${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`].join(" ")} ${probeArgs.map(shellQuote).join(" ")}`,
|
||||
webProbeGuardedLaunchShell(spec, "manual-start", `${[...webProbeProxy.envAssignments, ...webProbeBrowserEvidenceEnvAssignments(spec), `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`].join(" ")} ${probeArgs.map(shellQuote).join(" ")}`),
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const probe = compactWebProbeResult(parseJsonObject(result.stdout));
|
||||
@@ -912,8 +935,10 @@ export function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, un
|
||||
}
|
||||
|
||||
export function runNodeWebProbeScreenshot(options: NodeWebProbeScreenshotOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
|
||||
if (memoryGuard.status !== "allowed") return memoryGuard;
|
||||
const route = `${options.node}:${spec.workspace}`;
|
||||
const script = webProbeScreenshotRemoteScript(options);
|
||||
const script = webProbeGuardedLaunchShell(spec, "manual-start", webProbeScreenshotRemoteScript(options, spec));
|
||||
const job = runWebProbeRemoteArtifactJob({
|
||||
route,
|
||||
localDir: options.localDir,
|
||||
@@ -1038,7 +1063,7 @@ function compactWebProbeScreenshotCleanup(value: unknown): Record<string, unknow
|
||||
};
|
||||
}
|
||||
|
||||
function webProbeScreenshotRemoteScript(options: NodeWebProbeScreenshotOptions): string {
|
||||
function webProbeScreenshotRemoteScript(options: NodeWebProbeScreenshotOptions, spec: HwlabRuntimeLaneSpec): string {
|
||||
const [widthRaw, heightRaw] = options.viewport.split("x");
|
||||
return [
|
||||
"set -eu",
|
||||
@@ -1048,7 +1073,7 @@ function webProbeScreenshotRemoteScript(options: NodeWebProbeScreenshotOptions):
|
||||
`export UNIDESK_WEB_PROBE_SCREENSHOT_HEIGHT=${shellQuote(heightRaw ?? "900")}`,
|
||||
`export UNIDESK_WEB_PROBE_SCREENSHOT_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
|
||||
`export UNIDESK_WEB_PROBE_SCREENSHOT_WAIT_UNTIL=${shellQuote(options.waitUntil)}`,
|
||||
`export UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`,
|
||||
`export UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE=${shellQuote(webProbeScreenshotFullPage(spec, options.fullPage) ? "1" : "0")}`,
|
||||
`export UNIDESK_WEB_PROBE_SCREENSHOT_SELECTOR=${shellQuote(options.selector ?? "")}`,
|
||||
"if command -v chromium >/dev/null 2>&1; then",
|
||||
" export UNIDESK_WEB_PROBE_SCREENSHOT_EXECUTABLE_PATH=$(command -v chromium)",
|
||||
@@ -1075,7 +1100,7 @@ const width = Number(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_WIDTH || 1440);
|
||||
const height = Number(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_HEIGHT || 900);
|
||||
const timeout = Number(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_TIMEOUT_MS || 30000);
|
||||
const waitUntil = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_WAIT_UNTIL || "networkidle";
|
||||
const fullPage = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE !== "0";
|
||||
const fullPage = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE === "1";
|
||||
const selector = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_SELECTOR || "";
|
||||
const executablePath = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_EXECUTABLE_PATH || "";
|
||||
|
||||
@@ -1088,8 +1113,11 @@ const launchOptions = {
|
||||
args: ["--disable-gpu", "--no-sandbox"],
|
||||
...(executablePath ? { executablePath } : {}),
|
||||
};
|
||||
const browser = await chromium.launch(launchOptions);
|
||||
const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
|
||||
let browser = null;
|
||||
let context = null;
|
||||
try {
|
||||
browser = await chromium.launch(launchOptions);
|
||||
context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
|
||||
const page = await context.newPage();
|
||||
page.on("console", (message) => {
|
||||
if (consoleMessages.length < 20) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 240) });
|
||||
@@ -1099,7 +1127,6 @@ page.on("requestfailed", (request) => {
|
||||
});
|
||||
|
||||
let status = null;
|
||||
try {
|
||||
const response = await page.goto(url, { timeout, waitUntil });
|
||||
status = response?.status() ?? null;
|
||||
await page.waitForTimeout(350);
|
||||
@@ -1168,8 +1195,8 @@ try {
|
||||
valuesRedacted: true,
|
||||
}));
|
||||
} finally {
|
||||
await context.close().catch(() => {});
|
||||
await browser.close().catch(() => {});
|
||||
if (context) await context.close().catch(() => {});
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
}
|
||||
`;
|
||||
}
|
||||
@@ -1214,9 +1241,11 @@ export function runNodeWebProbeAsync(
|
||||
): Record<string, unknown> {
|
||||
const startArgs = nodeWebProbeRunArgs(options, "start");
|
||||
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
|
||||
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
|
||||
if (memoryGuard.status !== "allowed") return memoryGuard;
|
||||
const startScript = [
|
||||
"set -eu",
|
||||
`${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password ?? "")}`].join(" ")} ${startArgs.map(shellQuote).join(" ")}`,
|
||||
webProbeGuardedLaunchShell(spec, "manual-start", `${[...webProbeProxy.envAssignments, ...webProbeBrowserEvidenceEnvAssignments(spec), `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password ?? "")}`].join(" ")} ${startArgs.map(shellQuote).join(" ")}`, "structured-output"),
|
||||
].join("\n");
|
||||
const startResult = runTransWorkspaceStdinScript(options.node, spec.workspace, startScript, 55);
|
||||
const start = parseJsonObject(startResult.stdout);
|
||||
|
||||
@@ -2130,7 +2130,7 @@ export function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
||||
name: parseWebProbeScreenshotName(optionValue(args, "--name") ?? `web-probe-${node.toLowerCase()}-${lane}.png`),
|
||||
timeoutMs,
|
||||
waitUntil: parseWebProbeScreenshotWaitUntil(optionValue(args, "--wait-until") ?? "networkidle"),
|
||||
fullPage: !args.includes("--no-full-page"),
|
||||
fullPage: args.includes("--full-page") && !args.includes("--no-full-page"),
|
||||
selector: optionValue(args, "--selector") ?? null,
|
||||
keepRemote: args.includes("--keep-remote"),
|
||||
waitTimeoutMs,
|
||||
|
||||
Reference in New Issue
Block a user